diff --git a/datafusion/common/src/hash_utils.rs b/datafusion/common/src/hash_utils.rs index 1301d373834bc..616db912efc57 100644 --- a/datafusion/common/src/hash_utils.rs +++ b/datafusion/common/src/hash_utils.rs @@ -98,6 +98,26 @@ pub fn combine_hashes(l: u64, r: u64) -> u64 { /// to this size. const MAX_BUFFER_SIZE: usize = 524_288; +#[cfg(not(feature = "force_hash_collisions"))] +fn dictionary_nulls(array: &dyn AnyDictionaryArray) -> (bool, bool) { + ( + array.keys().null_count() != 0, + array.values().null_count() != 0, + ) +} + +#[cfg(not(feature = "force_hash_collisions"))] +fn dictionary_overwrites_hash_buffer(array: &dyn Array) -> bool { + array + .as_any_dictionary_opt() + .is_some_and(|array| dictionary_nulls(array) == (false, false)) +} + +#[cfg(feature = "force_hash_collisions")] +fn dictionary_overwrites_hash_buffer(_array: &dyn Array) -> bool { + false +} + thread_local! { /// Thread-local buffer for hash computations to avoid repeated allocations. /// The buffer is reused across calls and truncated if it exceeds MAX_BUFFER_SIZE. @@ -160,8 +180,14 @@ where let mut buffer = cell.try_borrow_mut() .map_err(|_| _internal_datafusion_err!("with_hashes cannot be called reentrantly on the same thread"))?; - // Ensure buffer has sufficient length, clearing old values - buffer.clear(); + // A dictionary with non-null keys and values overwrites every output + // slot, so retain the initialized storage instead of zeroing it first. + let reuse_buffer = iter + .peek() + .is_some_and(|array| dictionary_overwrites_hash_buffer(array.as_dyn_array())); + if !reuse_buffer { + buffer.clear(); + } buffer.resize(required_size, 0); // Create hashes in the buffer - this consumes the iterator @@ -544,6 +570,63 @@ fn hash_generic_byte_view_array( } } +/// Scatters non-null dictionary hashes in independent lanes to overlap key-indexed loads. +#[cfg(not(feature = "force_hash_collisions"))] +#[inline(always)] +fn scatter_non_null_dictionary( + array: &DictionaryArray, + dict_hashes: &[u64], + hashes: &mut [u64], +) { + const LANES: usize = 8; + let keys = array.keys().values(); + assert_eq!( + dict_hashes.len(), + array.values().len(), + "dictionary hashes must cover every dictionary value" + ); + assert!( + hashes.len() <= keys.len(), + "dictionary scatter requires at least one key per hash slot" + ); + let keys = &keys[..hashes.len()]; + + let (key_chunks, key_tail) = keys.as_chunks::(); + let (hash_chunks, hash_tail) = hashes.as_chunks_mut::(); + for (keys, hashes) in key_chunks.iter().zip(hash_chunks) { + let gathered: [u64; LANES] = std::array::from_fn(|lane| unsafe { + let key = keys[lane]; + // SAFETY: Dictionary keys are in bounds for values, which has the + // same length as dict_hashes. + *dict_hashes.get_unchecked(key.as_usize()) + }); + if MULTI_COL { + for (hash, dict_hash) in hashes.iter_mut().zip(gathered) { + update_dictionary_hash::(hash, dict_hash); + } + } else { + *hashes = gathered; + } + } + + for (hash, key) in hash_tail.iter_mut().zip(key_tail) { + // SAFETY: Dictionary keys are in bounds for values, which has the + // same length as dict_hashes. + let dict_hash = unsafe { *dict_hashes.get_unchecked(key.as_usize()) }; + update_dictionary_hash::(hash, dict_hash); + } +} + +#[cfg(not(feature = "force_hash_collisions"))] +#[inline(always)] +fn update_dictionary_hash(hash: &mut u64, dict_hash: u64) { + *hash = if MULTI_COL { + combine_hashes(dict_hash, *hash) + } else { + dict_hash + }; +} + /// Scatter precomputed dictionary value hashes to key positions. /// /// Uses const generics to eliminate runtime branching in the hot loop: @@ -562,6 +645,10 @@ fn hash_dictionary_scatter< dict_hashes: &[u64], hashes_buffer: &mut [u64], ) { + if !HAS_NULL_KEYS && !HAS_NULL_VALUES { + scatter_non_null_dictionary::(array, dict_hashes, hashes_buffer); + return; + } let dict_values = array.values(); let scatter_row = |hash: &mut u64, key: &K::Native| { let idx = key.as_usize(); @@ -623,8 +710,7 @@ fn dispatch_dictionary_scatter( hashes_buffer: &mut [u64], multi_col: bool, ) { - let has_null_keys = array.keys().null_count() != 0; - let has_null_values = array.values().null_count() != 0; + let (has_null_keys, has_null_values) = dictionary_nulls(array); match (has_null_keys, has_null_values, multi_col) { (false, false, false) => hash_dictionary_scatter::( @@ -2570,6 +2656,123 @@ mod tests { assert_eq!(result, expected_hashes); } + #[test] + #[cfg(not(feature = "force_hash_collisions"))] + fn test_non_null_dictionary_hash_fast_path() { + check_non_null_dictionary_hash_fast_path::(); + check_non_null_dictionary_hash_fast_path::(); + check_non_null_dictionary_hash_fast_path::(); + + let random_state = RandomState::with_seed(0); + let keys = vec![2, 0, 1, 2, 1, 0, 2, 0, 1, 2, 0]; + let null_values: ArrayRef = Arc::new( + DictionaryArray::::try_new( + Int32Array::from(keys), + Arc::new(StringArray::from(vec![Some("a"), None, Some("c")])), + ) + .unwrap(), + ); + let null_keys: ArrayRef = Arc::new( + DictionaryArray::::try_new( + Int32Array::from(vec![ + Some(2), + None, + Some(1), + Some(2), + None, + Some(0), + Some(2), + None, + Some(1), + Some(2), + Some(0), + ]), + Arc::new(StringArray::from(vec!["a", "b", "c"])), + ) + .unwrap(), + ); + + let assert_matches = |dictionary: &ArrayRef| { + assert!(!dictionary_overwrites_hash_buffer(dictionary.as_ref())); + let strings = arrow::compute::cast(dictionary, &DataType::Utf8).unwrap(); + let mut expected = vec![0; dictionary.len()]; + create_hashes([strings], &random_state, &mut expected).unwrap(); + with_hashes([dictionary], &random_state, |hashes| { + assert_eq!(hashes, expected); + Ok(()) + }) + .unwrap(); + }; + + assert_matches(&null_values); + assert_matches(&null_keys); + } + + #[test] + #[cfg(feature = "force_hash_collisions")] + fn test_force_hash_collisions_clears_dictionary_buffer() { + let dictionary = DictionaryArray::::try_new( + Int32Array::from(vec![0, 1, 0]), + Arc::new(StringArray::from(vec!["alpha", "beta"])), + ) + .unwrap(); + assert!(!dictionary_overwrites_hash_buffer(&dictionary)); + } + + #[cfg(not(feature = "force_hash_collisions"))] + fn check_non_null_dictionary_hash_fast_path() { + const OFFSET: usize = 3; + const LEN: usize = 13; + + let random_state = RandomState::with_seed(0); + let keys = (0..OFFSET + LEN) + .map(|row| K::Native::usize_as((row * 3 + 1) % 4)) + .collect::>(); + let keys = + PrimitiveArray::::new(keys, Some(NullBuffer::new_valid(OFFSET + LEN))); + let dictionary: ArrayRef = Arc::new( + DictionaryArray::::try_new( + keys, + Arc::new(StringArray::from(vec!["alpha", "beta", "gamma", "delta"])), + ) + .unwrap() + .slice(OFFSET, LEN), + ); + assert!(dictionary_overwrites_hash_buffer(dictionary.as_ref())); + let decoded = arrow::compute::cast(&dictionary, &DataType::Utf8).unwrap(); + let prefix: ArrayRef = Arc::new(Int32Array::from_iter_values(0..LEN as i32)); + + let mut expected = vec![0; LEN]; + create_hashes([&decoded], &random_state, &mut expected).unwrap(); + let mut actual = vec![0; LEN]; + create_hashes([&dictionary], &random_state, &mut actual).unwrap(); + assert_eq!(actual, expected); + + let mut short = vec![0; LEN - 1]; + create_hashes([&dictionary], &random_state, &mut short).unwrap(); + assert_eq!(short, expected[..LEN - 1]); + + let dirty: ArrayRef = + Arc::new(Int32Array::from_iter_values(100..100 + LEN as i32)); + with_hashes([&dirty], &random_state, |_| Ok(())).unwrap(); + with_hashes([&dictionary], &random_state, |hashes| { + assert_eq!(hashes, expected); + Ok(()) + }) + .unwrap(); + + let mut expected = vec![0; LEN]; + create_hashes([&prefix, &decoded], &random_state, &mut expected).unwrap(); + let mut actual = vec![0; LEN]; + create_hashes([&prefix, &dictionary], &random_state, &mut actual).unwrap(); + assert_eq!(actual, expected); + with_hashes([&prefix, &dictionary], &random_state, |hashes| { + assert_eq!(hashes, expected); + Ok(()) + }) + .unwrap(); + } + #[test] fn test_with_hashes_multi_column() { let int_array: ArrayRef = Arc::new(Int32Array::from(vec![1, 2, 3]));