Repository navigation
perf: speed up non-null dictionary hash scatter #27025
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
8c0a4ee
cb54f47
c7d9b3a
0a7c4da
b41d408
90f551c
17f967b
1150cfc
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -98,6 +98,26 @@ pub fn combine_hashes(l: u64, r: u64) -> u64 { | |
| /// to this size. | ||
| const MAX_BUFFER_SIZE: usize = 524_288; | ||
|
|
||
| #[cfg(not(feature = "force_hash_collisions"))] | ||
| fn dictionary_nulls(array: &dyn AnyDictionaryArray) -> (bool, bool) { | ||
| ( | ||
| array.keys().null_count() != 0, | ||
| array.values().null_count() != 0, | ||
| ) | ||
| } | ||
|
|
||
| #[cfg(not(feature = "force_hash_collisions"))] | ||
| fn dictionary_overwrites_hash_buffer(array: &dyn Array) -> bool { | ||
|
Rich-T-kid marked this conversation as resolved.
Resolved
|
||
| array | ||
| .as_any_dictionary_opt() | ||
| .is_some_and(|array| dictionary_nulls(array) == (false, false)) | ||
| } | ||
|
|
||
| #[cfg(feature = "force_hash_collisions")] | ||
| fn dictionary_overwrites_hash_buffer(_array: &dyn Array) -> bool { | ||
| false | ||
| } | ||
|
|
||
| thread_local! { | ||
| /// Thread-local buffer for hash computations to avoid repeated allocations. | ||
| /// The buffer is reused across calls and truncated if it exceeds MAX_BUFFER_SIZE. | ||
|
|
@@ -160,8 +180,14 @@ where | |
| let mut buffer = cell.try_borrow_mut() | ||
| .map_err(|_| _internal_datafusion_err!("with_hashes cannot be called reentrantly on the same thread"))?; | ||
|
|
||
| // Ensure buffer has sufficient length, clearing old values | ||
| buffer.clear(); | ||
| // A dictionary with non-null keys and values overwrites every output | ||
| // slot, so retain the initialized storage instead of zeroing it first. | ||
| let reuse_buffer = iter | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. The reuse gate checks nulls on the dictionary before compaction, but the scatter runs on the compacted dictionary. A dictionary whose only null values are never referenced by a key still gets cleared, even though the fast path would overwrite every slot.
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. with_hashes |
||
| .peek() | ||
| .is_some_and(|array| dictionary_overwrites_hash_buffer(array.as_dyn_array())); | ||
| if !reuse_buffer { | ||
| buffer.clear(); | ||
| } | ||
| buffer.resize(required_size, 0); | ||
|
|
||
| // Create hashes in the buffer - this consumes the iterator | ||
|
|
@@ -544,6 +570,63 @@ fn hash_generic_byte_view_array<T: ByteViewType>( | |
| } | ||
| } | ||
|
|
||
| /// Scatters non-null dictionary hashes in independent lanes to overlap key-indexed loads. | ||
| #[cfg(not(feature = "force_hash_collisions"))] | ||
| #[inline(always)] | ||
| fn scatter_non_null_dictionary<K: ArrowDictionaryKeyType, const MULTI_COL: bool>( | ||
|
Rich-T-kid marked this conversation as resolved.
Resolved
|
||
| array: &DictionaryArray<K>, | ||
| dict_hashes: &[u64], | ||
| hashes: &mut [u64], | ||
| ) { | ||
| const LANES: usize = 8; | ||
| let keys = array.keys().values(); | ||
| assert_eq!( | ||
| dict_hashes.len(), | ||
| array.values().len(), | ||
| "dictionary hashes must cover every dictionary value" | ||
| ); | ||
| assert!( | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. The no-null dictionary fast path now uses |
||
| hashes.len() <= keys.len(), | ||
| "dictionary scatter requires at least one key per hash slot" | ||
| ); | ||
| let keys = &keys[..hashes.len()]; | ||
|
|
||
| let (key_chunks, key_tail) = keys.as_chunks::<LANES>(); | ||
|
Rich-T-kid marked this conversation as resolved.
Resolved
|
||
| let (hash_chunks, hash_tail) = hashes.as_chunks_mut::<LANES>(); | ||
| for (keys, hashes) in key_chunks.iter().zip(hash_chunks) { | ||
| let gathered: [u64; LANES] = std::array::from_fn(|lane| unsafe { | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
|
||
| let key = keys[lane]; | ||
| // SAFETY: Dictionary keys are in bounds for values, which has the | ||
| // same length as dict_hashes. | ||
| *dict_hashes.get_unchecked(key.as_usize()) | ||
|
Rich-T-kid marked this conversation as resolved.
Resolved
|
||
| }); | ||
| if MULTI_COL { | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. The MULTI_COL branch of the 8-lane loop calls |
||
| for (hash, dict_hash) in hashes.iter_mut().zip(gathered) { | ||
| update_dictionary_hash::<MULTI_COL>(hash, dict_hash); | ||
| } | ||
| } else { | ||
| *hashes = gathered; | ||
| } | ||
| } | ||
|
|
||
| for (hash, key) in hash_tail.iter_mut().zip(key_tail) { | ||
|
Rich-T-kid marked this conversation as resolved.
Resolved
|
||
| // SAFETY: Dictionary keys are in bounds for values, which has the | ||
| // same length as dict_hashes. | ||
| let dict_hash = unsafe { *dict_hashes.get_unchecked(key.as_usize()) }; | ||
| update_dictionary_hash::<MULTI_COL>(hash, dict_hash); | ||
| } | ||
| } | ||
|
|
||
| #[cfg(not(feature = "force_hash_collisions"))] | ||
| #[inline(always)] | ||
| fn update_dictionary_hash<const MULTI_COL: bool>(hash: &mut u64, dict_hash: u64) { | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
|
||
| *hash = if MULTI_COL { | ||
| combine_hashes(dict_hash, *hash) | ||
| } else { | ||
| dict_hash | ||
| }; | ||
| } | ||
|
|
||
| /// Scatter precomputed dictionary value hashes to key positions. | ||
| /// | ||
| /// Uses const generics to eliminate runtime branching in the hot loop: | ||
|
|
@@ -562,6 +645,10 @@ fn hash_dictionary_scatter< | |
| dict_hashes: &[u64], | ||
| hashes_buffer: &mut [u64], | ||
| ) { | ||
| if !HAS_NULL_KEYS && !HAS_NULL_VALUES { | ||
| scatter_non_null_dictionary::<K, MULTI_COL>(array, dict_hashes, hashes_buffer); | ||
| return; | ||
|
Rich-T-kid marked this conversation as resolved.
Resolved
|
||
| } | ||
| let dict_values = array.values(); | ||
| let scatter_row = |hash: &mut u64, key: &K::Native| { | ||
| let idx = key.as_usize(); | ||
|
|
@@ -623,8 +710,7 @@ fn dispatch_dictionary_scatter<K: ArrowDictionaryKeyType>( | |
| hashes_buffer: &mut [u64], | ||
| multi_col: bool, | ||
| ) { | ||
| let has_null_keys = array.keys().null_count() != 0; | ||
| let has_null_values = array.values().null_count() != 0; | ||
| let (has_null_keys, has_null_values) = dictionary_nulls(array); | ||
|
|
||
| match (has_null_keys, has_null_values, multi_col) { | ||
| (false, false, false) => hash_dictionary_scatter::<K, false, false, false>( | ||
|
|
@@ -2570,6 +2656,123 @@ mod tests { | |
| assert_eq!(result, expected_hashes); | ||
| } | ||
|
|
||
| #[test] | ||
| #[cfg(not(feature = "force_hash_collisions"))] | ||
|
Rich-T-kid marked this conversation as resolved.
Resolved
|
||
| fn test_non_null_dictionary_hash_fast_path() { | ||
|
Rich-T-kid marked this conversation as resolved.
Resolved
|
||
| check_non_null_dictionary_hash_fast_path::<UInt8Type>(); | ||
| check_non_null_dictionary_hash_fast_path::<UInt16Type>(); | ||
| check_non_null_dictionary_hash_fast_path::<Int64Type>(); | ||
|
|
||
| let random_state = RandomState::with_seed(0); | ||
| let keys = vec![2, 0, 1, 2, 1, 0, 2, 0, 1, 2, 0]; | ||
| let null_values: ArrayRef = Arc::new( | ||
| DictionaryArray::<Int32Type>::try_new( | ||
| Int32Array::from(keys), | ||
| Arc::new(StringArray::from(vec![Some("a"), None, Some("c")])), | ||
| ) | ||
| .unwrap(), | ||
| ); | ||
| let null_keys: ArrayRef = Arc::new( | ||
| DictionaryArray::<Int32Type>::try_new( | ||
| Int32Array::from(vec![ | ||
| Some(2), | ||
| None, | ||
| Some(1), | ||
| Some(2), | ||
| None, | ||
| Some(0), | ||
| Some(2), | ||
| None, | ||
| Some(1), | ||
| Some(2), | ||
| Some(0), | ||
| ]), | ||
| Arc::new(StringArray::from(vec!["a", "b", "c"])), | ||
| ) | ||
| .unwrap(), | ||
| ); | ||
|
|
||
| let assert_matches = |dictionary: &ArrayRef| { | ||
| assert!(!dictionary_overwrites_hash_buffer(dictionary.as_ref())); | ||
| let strings = arrow::compute::cast(dictionary, &DataType::Utf8).unwrap(); | ||
| let mut expected = vec![0; dictionary.len()]; | ||
| create_hashes([strings], &random_state, &mut expected).unwrap(); | ||
| with_hashes([dictionary], &random_state, |hashes| { | ||
| assert_eq!(hashes, expected); | ||
| Ok(()) | ||
| }) | ||
| .unwrap(); | ||
| }; | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. The |
||
|
|
||
| assert_matches(&null_values); | ||
| assert_matches(&null_keys); | ||
| } | ||
|
|
||
| #[test] | ||
| #[cfg(feature = "force_hash_collisions")] | ||
| fn test_force_hash_collisions_clears_dictionary_buffer() { | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
|
||
| let dictionary = DictionaryArray::<Int32Type>::try_new( | ||
| Int32Array::from(vec![0, 1, 0]), | ||
| Arc::new(StringArray::from(vec!["alpha", "beta"])), | ||
| ) | ||
| .unwrap(); | ||
| assert!(!dictionary_overwrites_hash_buffer(&dictionary)); | ||
| } | ||
|
|
||
| #[cfg(not(feature = "force_hash_collisions"))] | ||
| fn check_non_null_dictionary_hash_fast_path<K: ArrowDictionaryKeyType>() { | ||
| const OFFSET: usize = 3; | ||
| const LEN: usize = 13; | ||
|
|
||
| let random_state = RandomState::with_seed(0); | ||
| let keys = (0..OFFSET + LEN) | ||
| .map(|row| K::Native::usize_as((row * 3 + 1) % 4)) | ||
| .collect::<ScalarBuffer<_>>(); | ||
| let keys = | ||
| PrimitiveArray::<K>::new(keys, Some(NullBuffer::new_valid(OFFSET + LEN))); | ||
| let dictionary: ArrayRef = Arc::new( | ||
| DictionaryArray::<K>::try_new( | ||
| keys, | ||
| Arc::new(StringArray::from(vec!["alpha", "beta", "gamma", "delta"])), | ||
| ) | ||
| .unwrap() | ||
| .slice(OFFSET, LEN), | ||
| ); | ||
| assert!(dictionary_overwrites_hash_buffer(dictionary.as_ref())); | ||
| let decoded = arrow::compute::cast(&dictionary, &DataType::Utf8).unwrap(); | ||
| let prefix: ArrayRef = Arc::new(Int32Array::from_iter_values(0..LEN as i32)); | ||
|
|
||
| let mut expected = vec![0; LEN]; | ||
| create_hashes([&decoded], &random_state, &mut expected).unwrap(); | ||
| let mut actual = vec![0; LEN]; | ||
| create_hashes([&dictionary], &random_state, &mut actual).unwrap(); | ||
| assert_eq!(actual, expected); | ||
|
|
||
| let mut short = vec![0; LEN - 1]; | ||
| create_hashes([&dictionary], &random_state, &mut short).unwrap(); | ||
| assert_eq!(short, expected[..LEN - 1]); | ||
|
|
||
| let dirty: ArrayRef = | ||
| Arc::new(Int32Array::from_iter_values(100..100 + LEN as i32)); | ||
| with_hashes([&dirty], &random_state, |_| Ok(())).unwrap(); | ||
| with_hashes([&dictionary], &random_state, |hashes| { | ||
| assert_eq!(hashes, expected); | ||
| Ok(()) | ||
| }) | ||
| .unwrap(); | ||
|
|
||
| let mut expected = vec![0; LEN]; | ||
| create_hashes([&prefix, &decoded], &random_state, &mut expected).unwrap(); | ||
| let mut actual = vec![0; LEN]; | ||
| create_hashes([&prefix, &dictionary], &random_state, &mut actual).unwrap(); | ||
| assert_eq!(actual, expected); | ||
| with_hashes([&prefix, &dictionary], &random_state, |hashes| { | ||
| assert_eq!(hashes, expected); | ||
| Ok(()) | ||
| }) | ||
| .unwrap(); | ||
| } | ||
|
|
||
| #[test] | ||
| fn test_with_hashes_multi_column() { | ||
| let int_array: ArrayRef = Arc::new(Int32Array::from(vec![1, 2, 3])); | ||
|
|
||
Uh oh!
There was an error while loading. Please reload this page.