Skip to content
Open
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
211 changes: 207 additions & 4 deletions datafusion/common/src/hash_utils.rs
Comment thread
Rich-T-kid marked this conversation as resolved.
Resolved
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,26 @@ pub fn combine_hashes(l: u64, r: u64) -> u64 {
/// to this size.
const MAX_BUFFER_SIZE: usize = 524_288;

#[cfg(not(feature = "force_hash_collisions"))]
fn dictionary_nulls(array: &dyn AnyDictionaryArray) -> (bool, bool) {
(
array.keys().null_count() != 0,
array.values().null_count() != 0,
)
}

#[cfg(not(feature = "force_hash_collisions"))]
fn dictionary_overwrites_hash_buffer(array: &dyn Array) -> bool {
Comment thread
Rich-T-kid marked this conversation as resolved.
Resolved
array
.as_any_dictionary_opt()
.is_some_and(|array| dictionary_nulls(array) == (false, false))
}

#[cfg(feature = "force_hash_collisions")]
fn dictionary_overwrites_hash_buffer(_array: &dyn Array) -> bool {
false
}

thread_local! {
/// Thread-local buffer for hash computations to avoid repeated allocations.
/// The buffer is reused across calls and truncated if it exceeds MAX_BUFFER_SIZE.
Expand Down Expand Up @@ -160,8 +180,14 @@ where
let mut buffer = cell.try_borrow_mut()
.map_err(|_| _internal_datafusion_err!("with_hashes cannot be called reentrantly on the same thread"))?;

// Ensure buffer has sufficient length, clearing old values
buffer.clear();
// A dictionary with non-null keys and values overwrites every output
// slot, so retain the initialized storage instead of zeroing it first.
let reuse_buffer = iter

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The reuse gate checks nulls on the dictionary before compaction, but the scatter runs on the compacted dictionary. A dictionary whose only null values are never referenced by a key still gets cleared, even though the fast path would overwrite every slot.

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

with_hashespeeks the iterator a second time inside the TLS closure to computereuse_buffer. It could be computed in the same match iter.peek()that already readsrequired_size`.

.peek()
.is_some_and(|array| dictionary_overwrites_hash_buffer(array.as_dyn_array()));
if !reuse_buffer {
buffer.clear();
}
buffer.resize(required_size, 0);

// Create hashes in the buffer - this consumes the iterator
Expand Down Expand Up @@ -544,6 +570,63 @@ fn hash_generic_byte_view_array<T: ByteViewType>(
}
}

/// Scatters non-null dictionary hashes in independent lanes to overlap key-indexed loads.
#[cfg(not(feature = "force_hash_collisions"))]
#[inline(always)]
fn scatter_non_null_dictionary<K: ArrowDictionaryKeyType, const MULTI_COL: bool>(
Comment thread
Rich-T-kid marked this conversation as resolved.
Resolved
array: &DictionaryArray<K>,
dict_hashes: &[u64],
hashes: &mut [u64],
) {
const LANES: usize = 8;
let keys = array.keys().values();
assert_eq!(
dict_hashes.len(),
array.values().len(),
"dictionary hashes must cover every dictionary value"
);
assert!(

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The no-null dictionary fast path now uses assert!(hashes.len() <= keys.len()), which differs both from the dictionary null paths in the same function (min(len) at line 669, zip at line 700) and from every other hasher (assert_eq!).

hashes.len() <= keys.len(),
"dictionary scatter requires at least one key per hash slot"
);
let keys = &keys[..hashes.len()];

let (key_chunks, key_tail) = keys.as_chunks::<LANES>();
Comment thread
Rich-T-kid marked this conversation as resolved.
Resolved
let (hash_chunks, hash_tail) = hashes.as_chunks_mut::<LANES>();
for (keys, hashes) in key_chunks.iter().zip(hash_chunks) {
let gathered: [u64; LANES] = std::array::from_fn(|lane| unsafe {

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

scatter_non_null_dictionary is a safe fn that reads dict_hashes.get_unchecked(key.as_usize()) with no check on the key. Its SAFETY comment relies on dictionary keys being in bounds, which only holds for validated arrays. Also, the unsafe block wraps the whole closure, including the safe keys[lane] index.

let key = keys[lane];
// SAFETY: Dictionary keys are in bounds for values, which has the
// same length as dict_hashes.
*dict_hashes.get_unchecked(key.as_usize())
Comment thread
Rich-T-kid marked this conversation as resolved.
Resolved
});
if MULTI_COL {

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The MULTI_COL branch of the 8-lane loop calls update_dictionary_hash::<MULTI_COL>, which branches on MULTI_COL again. The if MULTI_COL { .. } else { *hashes = gathered } split plus the helper adds layers without changing what the code does.

for (hash, dict_hash) in hashes.iter_mut().zip(gathered) {
update_dictionary_hash::<MULTI_COL>(hash, dict_hash);
}
} else {
*hashes = gathered;
}
}

for (hash, key) in hash_tail.iter_mut().zip(key_tail) {
Comment thread
Rich-T-kid marked this conversation as resolved.
Resolved
// SAFETY: Dictionary keys are in bounds for values, which has the
// same length as dict_hashes.
let dict_hash = unsafe { *dict_hashes.get_unchecked(key.as_usize()) };
update_dictionary_hash::<MULTI_COL>(hash, dict_hash);
}
}

#[cfg(not(feature = "force_hash_collisions"))]
#[inline(always)]
fn update_dictionary_hash<const MULTI_COL: bool>(hash: &mut u64, dict_hash: u64) {

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

update_dictionary_hash repeats the MULTI_COL combine logic that already exists inline in scatter_row in hash_dictionary_scatter (lines 658-662).

*hash = if MULTI_COL {
combine_hashes(dict_hash, *hash)
} else {
dict_hash
};
}

/// Scatter precomputed dictionary value hashes to key positions.
///
/// Uses const generics to eliminate runtime branching in the hot loop:
Expand All @@ -562,6 +645,10 @@ fn hash_dictionary_scatter<
dict_hashes: &[u64],
hashes_buffer: &mut [u64],
) {
if !HAS_NULL_KEYS && !HAS_NULL_VALUES {
scatter_non_null_dictionary::<K, MULTI_COL>(array, dict_hashes, hashes_buffer);
return;
Comment thread
Rich-T-kid marked this conversation as resolved.
Resolved
}
let dict_values = array.values();
let scatter_row = |hash: &mut u64, key: &K::Native| {
let idx = key.as_usize();
Expand Down Expand Up @@ -623,8 +710,7 @@ fn dispatch_dictionary_scatter<K: ArrowDictionaryKeyType>(
hashes_buffer: &mut [u64],
multi_col: bool,
) {
let has_null_keys = array.keys().null_count() != 0;
let has_null_values = array.values().null_count() != 0;
let (has_null_keys, has_null_values) = dictionary_nulls(array);

match (has_null_keys, has_null_values, multi_col) {
(false, false, false) => hash_dictionary_scatter::<K, false, false, false>(
Expand Down Expand Up @@ -2570,6 +2656,123 @@ mod tests {
assert_eq!(result, expected_hashes);
}

#[test]
#[cfg(not(feature = "force_hash_collisions"))]
Comment thread
Rich-T-kid marked this conversation as resolved.
Resolved
fn test_non_null_dictionary_hash_fast_path() {
Comment thread
Rich-T-kid marked this conversation as resolved.
Resolved
check_non_null_dictionary_hash_fast_path::<UInt8Type>();
check_non_null_dictionary_hash_fast_path::<UInt16Type>();
check_non_null_dictionary_hash_fast_path::<Int64Type>();

let random_state = RandomState::with_seed(0);
let keys = vec![2, 0, 1, 2, 1, 0, 2, 0, 1, 2, 0];
let null_values: ArrayRef = Arc::new(
DictionaryArray::<Int32Type>::try_new(
Int32Array::from(keys),
Arc::new(StringArray::from(vec![Some("a"), None, Some("c")])),
)
.unwrap(),
);
let null_keys: ArrayRef = Arc::new(
DictionaryArray::<Int32Type>::try_new(
Int32Array::from(vec![
Some(2),
None,
Some(1),
Some(2),
None,
Some(0),
Some(2),
None,
Some(1),
Some(2),
Some(0),
]),
Arc::new(StringArray::from(vec!["a", "b", "c"])),
)
.unwrap(),
);

let assert_matches = |dictionary: &ArrayRef| {
assert!(!dictionary_overwrites_hash_buffer(dictionary.as_ref()));
let strings = arrow::compute::cast(dictionary, &DataType::Utf8).unwrap();
let mut expected = vec![0; dictionary.len()];
create_hashes([strings], &random_state, &mut expected).unwrap();
with_hashes([dictionary], &random_state, |hashes| {
assert_eq!(hashes, expected);
Ok(())
})
.unwrap();
};

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The assert_matches test of the clearing path for null dictionaries never dirties the thread-local buffer itself. It only catches a missing clear() because earlier check_non_null_dictionary_hash_fast_path calls happened to leave stale data in the buffer.


assert_matches(&null_values);
assert_matches(&null_keys);
}

#[test]
#[cfg(feature = "force_hash_collisions")]
fn test_force_hash_collisions_clears_dictionary_buffer() {

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

test_force_hash_collisions_clears_dictionary_buffer only checks that a cfg stub hard-coded to false returns false. It never calls with_hashes, so it tests nothing about clearing the buffer.

let dictionary = DictionaryArray::<Int32Type>::try_new(
Int32Array::from(vec![0, 1, 0]),
Arc::new(StringArray::from(vec!["alpha", "beta"])),
)
.unwrap();
assert!(!dictionary_overwrites_hash_buffer(&dictionary));
}

#[cfg(not(feature = "force_hash_collisions"))]
fn check_non_null_dictionary_hash_fast_path<K: ArrowDictionaryKeyType>() {
const OFFSET: usize = 3;
const LEN: usize = 13;

let random_state = RandomState::with_seed(0);
let keys = (0..OFFSET + LEN)
.map(|row| K::Native::usize_as((row * 3 + 1) % 4))
.collect::<ScalarBuffer<_>>();
let keys =
PrimitiveArray::<K>::new(keys, Some(NullBuffer::new_valid(OFFSET + LEN)));
let dictionary: ArrayRef = Arc::new(
DictionaryArray::<K>::try_new(
keys,
Arc::new(StringArray::from(vec!["alpha", "beta", "gamma", "delta"])),
)
.unwrap()
.slice(OFFSET, LEN),
);
assert!(dictionary_overwrites_hash_buffer(dictionary.as_ref()));
let decoded = arrow::compute::cast(&dictionary, &DataType::Utf8).unwrap();
let prefix: ArrayRef = Arc::new(Int32Array::from_iter_values(0..LEN as i32));

let mut expected = vec![0; LEN];
create_hashes([&decoded], &random_state, &mut expected).unwrap();
let mut actual = vec![0; LEN];
create_hashes([&dictionary], &random_state, &mut actual).unwrap();
assert_eq!(actual, expected);

let mut short = vec![0; LEN - 1];
create_hashes([&dictionary], &random_state, &mut short).unwrap();
assert_eq!(short, expected[..LEN - 1]);

let dirty: ArrayRef =
Arc::new(Int32Array::from_iter_values(100..100 + LEN as i32));
with_hashes([&dirty], &random_state, |_| Ok(())).unwrap();
with_hashes([&dictionary], &random_state, |hashes| {
assert_eq!(hashes, expected);
Ok(())
})
.unwrap();

let mut expected = vec![0; LEN];
create_hashes([&prefix, &decoded], &random_state, &mut expected).unwrap();
let mut actual = vec![0; LEN];
create_hashes([&prefix, &dictionary], &random_state, &mut actual).unwrap();
assert_eq!(actual, expected);
with_hashes([&prefix, &dictionary], &random_state, |hashes| {
assert_eq!(hashes, expected);
Ok(())
})
.unwrap();
}

#[test]
fn test_with_hashes_multi_column() {
let int_array: ArrayRef = Arc::new(Int32Array::from(vec![1, 2, 3]));
Expand Down
Loading