diff --git a/datafusion/common/src/hash_utils.rs b/datafusion/common/src/hash_utils.rs index 1301d373834bc..8659e1346650f 100644 --- a/datafusion/common/src/hash_utils.rs +++ b/datafusion/common/src/hash_utils.rs @@ -670,6 +670,51 @@ fn dispatch_dictionary_scatter( } } +/// Scatters precomputed dictionary value hashes through `array`'s keys. +/// +/// # Public Only for Internal Use: +/// +/// This is not a public API and is for internal use only; see [API policy] for details. +/// +/// [API policy]: https://datafusion.apache.org/contributor-guide/api-health.html#internal-public-apis +#[doc(hidden)] +#[cfg(not(feature = "force_hash_collisions"))] +pub fn scatter_dictionary_hashes( + array: &DictionaryArray, + dict_hashes: &[u64], + hashes_buffer: &mut [u64], + multi_col: bool, +) -> Result<()> { + if dict_hashes.len() != array.values().len() { + return _internal_err!( + "Expected {} dictionary hashes, got {}", + array.values().len(), + dict_hashes.len() + ); + } + dispatch_dictionary_scatter(array, dict_hashes, hashes_buffer, multi_col); + Ok(()) +} + +/// Hashes one column into `hashes_buffer` exactly as [`create_hashes`] does +/// for that column; `rehash` is `true` for every column after the first. +/// +/// # Public Only for Internal Use: +/// +/// This is not a public API and is for internal use only; see [API policy] for details. +/// +/// [API policy]: https://datafusion.apache.org/contributor-guide/api-health.html#internal-public-apis +#[doc(hidden)] +#[cfg(not(feature = "force_hash_collisions"))] +pub fn hash_column( + array: &dyn Array, + random_state: &impl HashState, + hashes_buffer: &mut [u64], + rehash: bool, +) -> Result<()> { + hash_single_array(array, random_state, hashes_buffer, rehash) +} + /// Hash the values in a dictionary array. #[cfg(not(feature = "force_hash_collisions"))] fn hash_dictionary( diff --git a/datafusion/physical-plan/src/aggregates/group_values/multi_group_by/dictionary.rs b/datafusion/physical-plan/src/aggregates/group_values/multi_group_by/dictionary.rs index 3726c3ea2fb23..5fe02e721c925 100644 --- a/datafusion/physical-plan/src/aggregates/group_values/multi_group_by/dictionary.rs +++ b/datafusion/physical-plan/src/aggregates/group_values/multi_group_by/dictionary.rs @@ -54,6 +54,11 @@ pub struct DictionaryGroupValuesColumn /// were built for. A matching live array reuses both caches without /// retaining an input dictionary after its batch is released. cached_values: Option>, + /// Whether the outer table guarantees each append creates a new group, so + /// inner slots and group indices are identical. + values_are_group_unique: bool, + #[cfg(test)] + hash_builds: usize, _phantom: PhantomData, } @@ -71,6 +76,9 @@ impl DictionaryGroupValuesColumn { val_to_inner: Vec::default(), val_hashes: Vec::default(), cached_values: None, + values_are_group_unique: false, + #[cfg(test)] + hash_builds: 0, _phantom: PhantomData, } } @@ -157,6 +165,10 @@ impl DictionaryGroupValuesColumn { } fn hash_values(&mut self, values: &ArrayRef) { + #[cfg(test)] + { + self.hash_builds += 1; + } self.cached_values = None; self.val_hashes.clear(); self.val_hashes.resize(values.len(), 0); @@ -171,7 +183,8 @@ impl DictionaryGroupValuesColumn { fn is_cached(&self, values: &ArrayRef) -> bool { self.cached_values .as_ref() - .is_some_and(|cached| std::ptr::addr_eq(cached.as_ptr(), Arc::as_ptr(values))) + .and_then(Weak::upgrade) + .is_some_and(|cached| Arc::ptr_eq(&cached, values)) } /// Makes `val_hashes` and `val_to_inner` correspond to `dict_values`, @@ -192,6 +205,17 @@ impl DictionaryGroupValuesColumn { self.cached_values = Some(Arc::downgrade(dict_values)); } + fn insert_value(&mut self, dict_values: &ArrayRef, val_idx: usize) -> Result { + if self.values_are_group_unique { + let slot = self.group_to_inner.len(); + debug_assert_eq!(slot, self.inner.len()); + self.inner.append_val(dict_values, val_idx)?; + Ok(slot) + } else { + self.find_or_insert_value(dict_values, val_idx, self.val_hashes[val_idx]) + } + } + fn find_or_insert_value( &mut self, dict_values: &ArrayRef, @@ -331,13 +355,61 @@ impl GroupColumn Some(val_idx) => { let dict_values = dict.values(); self.sync_value_cache(dict_values); - self.find_or_insert_value(dict_values, val_idx, self.val_hashes[val_idx])? + self.insert_value(dict_values, val_idx)? } }; self.group_to_inner.push(inner_slot); self.check_key_overflow() } + fn dictionary_hashes<'a>( + &'a mut self, + values: &ArrayRef, + group_unique: bool, + ) -> Option<&'a [u64]> { + self.values_are_group_unique = group_unique; + self.sync_value_cache(values); + Some(&self.val_hashes) + } + + fn try_map_dictionary_groups( + &self, + array: &ArrayRef, + groups: &mut Vec, + ) -> bool { + if !self.values_are_group_unique { + return false; + } + + let dictionary = array.as_dictionary::(); + if !self.is_cached(dictionary.values()) { + return false; + } + + let keys = dictionary.keys(); + let raw_keys = keys.values(); + let nulls = keys.nulls(); + groups.clear(); + for row in 0..dictionary.len() { + let group = if nulls.is_some_and(|nulls| nulls.is_null(row)) { + self.null_inner_slot.unwrap_or(usize::MAX) + } else { + self.val_to_inner[raw_keys[row].as_usize()] + }; + if group == usize::MAX { + groups.clear(); + return false; + } + groups.push(group); + } + true + } + + #[cfg(test)] + fn dictionary_hash_builds(&self) -> usize { + self.hash_builds + } + fn vectorized_equal_to( &mut self, lhs_rows: &[usize], @@ -452,11 +524,7 @@ impl GroupColumn self.val_to_inner[val_idx] = if dict_values.is_null(val_idx) { self.find_or_insert_null()? } else { - self.find_or_insert_value( - dict_values, - val_idx, - self.val_hashes[val_idx], - )? + self.insert_value(dict_values, val_idx)? }; } self.group_to_inner.push(self.val_to_inner[val_idx]); @@ -472,11 +540,7 @@ impl GroupColumn self.val_to_inner[val_idx] = if dict_values.is_null(val_idx) { self.find_or_insert_null()? } else { - self.find_or_insert_value( - dict_values, - val_idx, - self.val_hashes[val_idx], - )? + self.insert_value(dict_values, val_idx)? }; } self.val_to_inner[val_idx] @@ -571,34 +635,28 @@ impl GroupColumn let emitted: ArrayRef = Arc::new(DictionaryArray::::new(emitted_keys, compact_emit_values)); - // Null deferred to last so null_inner_slot is always the highest index - // and check_key_overflow can subtract it without a false overflow. + // Preserve group order so inner slots remain equal to group indices + // when this is the only grouping column. let remaining = self.group_to_inner[n..].to_vec(); let mut old_to_new = vec![usize::MAX; old_inner_len]; let mut new_to_old: Vec = Vec::new(); - let mut null_old_slot: Option = None; for &old in &remaining { - if all_inner_values.is_null(old) { - if null_old_slot.is_none() { - null_old_slot = Some(old); - } - continue; - } if old_to_new[old] == usize::MAX { old_to_new[old] = new_to_old.len(); new_to_old.push(old); } } - if let Some(old) = null_old_slot { - old_to_new[old] = new_to_old.len(); - new_to_old.push(old); - } self.value_dedup = HashTable::new(); self.value_dedup_size = 0; self.null_inner_slot = None; - self.hash_values(&all_inner_values); + if self.values_are_group_unique { + self.cached_values = None; + self.val_hashes.clear(); + } else { + self.hash_values(&all_inner_values); + } for (new_slot, &old_slot) in new_to_old.iter().enumerate() { if all_inner_values.is_null(old_slot) { @@ -610,11 +668,13 @@ impl GroupColumn self.inner .append_val(&all_inner_values, old_slot) .expect("append value failed in take_n"); - self.value_dedup.insert_accounted( - (self.val_hashes[old_slot], new_slot), - |&(entry_hash, _)| entry_hash, - &mut self.value_dedup_size, - ); + if !self.values_are_group_unique { + self.value_dedup.insert_accounted( + (self.val_hashes[old_slot], new_slot), + |&(entry_hash, _)| entry_hash, + &mut self.value_dedup_size, + ); + } } } diff --git a/datafusion/physical-plan/src/aggregates/group_values/multi_group_by/mod.rs b/datafusion/physical-plan/src/aggregates/group_values/multi_group_by/mod.rs index 26200b4a03ef1..d0e387ce9309e 100644 --- a/datafusion/physical-plan/src/aggregates/group_values/multi_group_by/mod.rs +++ b/datafusion/physical-plan/src/aggregates/group_values/multi_group_by/mod.rs @@ -25,6 +25,8 @@ use crate::aggregates::group_values::multi_group_by::{ fixed_size_binary::FixedSizeBinaryGroupValueBuilder, primitive::PrimitiveGroupValueBuilder, row_backed::RowsGroupColumn, }; +#[cfg(not(feature = "force_hash_collisions"))] +use arrow::array::AsArray; use arrow::array::{ Array, ArrayRef, BooleanBufferBuilder, downcast_primitive, new_empty_array, }; @@ -32,9 +34,14 @@ use arrow::datatypes::{ BinaryViewType, DataType, Field, Int8Type, Int16Type, Int32Type, Int64Type, Schema, SchemaRef, StringViewType, UInt8Type, UInt16Type, UInt32Type, UInt64Type, }; +#[cfg(not(feature = "force_hash_collisions"))] +use arrow::downcast_dictionary_array; use datafusion_common::float_semantics::{self, has_float_leaf}; -use datafusion_common::hash_utils::RandomState; -use datafusion_common::hash_utils::create_hashes; +use datafusion_common::hash_utils::{RandomState, create_hashes}; +#[cfg(not(feature = "force_hash_collisions"))] +use datafusion_common::hash_utils::{hash_column, scatter_dictionary_hashes}; +#[cfg(not(feature = "force_hash_collisions"))] +use datafusion_common::utils::MAX_DICTIONARY_VALUES_PER_ROW; use datafusion_common::{Result, not_impl_err}; use datafusion_execution::memory_pool::proxy::{HashTableAllocExt, VecAllocExt}; use datafusion_expr::{EmitTo, GroupSelection}; @@ -82,6 +89,31 @@ pub trait GroupColumn: Send + Sync { /// The vectorized version `append_val` fn vectorized_append(&mut self, array: &ArrayRef, rows: &[usize]) -> Result<()>; + /// Returns hashes cached for a dictionary values array. + fn dictionary_hashes<'a>( + &'a mut self, + _values: &ArrayRef, + _group_unique: bool, + ) -> Option<&'a [u64]> { + None + } + + /// Maps dictionary keys directly to group indices when all referenced + /// values are already cached. + fn try_map_dictionary_groups( + &self, + _array: &ArrayRef, + _groups: &mut Vec, + ) -> bool { + false + } + + /// Number of times dictionary value hashes were rebuilt. + #[cfg(test)] + fn dictionary_hash_builds(&self) -> usize { + 0 + } + /// Returns the number of rows stored in this builder fn len(&self) -> usize; @@ -216,8 +248,11 @@ pub struct GroupValuesColumn { /// reused buffer to store hashes hashes_buffer: Vec, - /// Random state for creating hashes + /// Random state for creating hashes. random_state: RandomState, + + /// Whether any group column is dictionary encoded. + has_dictionary: bool, } /// Buffers to store intermediate results in `vectorized_append` @@ -288,6 +323,10 @@ impl GroupValuesColumn { pub fn try_new(schema: SchemaRef) -> Result { let map = HashTable::with_capacity(0); let group_values = Self::build_group_columns(&schema)?; + let has_dictionary = schema + .fields() + .iter() + .any(|field| matches!(field.data_type(), DataType::Dictionary(_, _))); Ok(Self { schema, map, @@ -298,9 +337,51 @@ impl GroupValuesColumn { group_values, hashes_buffer: Default::default(), random_state: crate::aggregates::AGGREGATION_HASH_SEED, + has_dictionary: !STREAMING && has_dictionary, }) } + #[cfg(not(feature = "force_hash_collisions"))] + fn create_dictionary_hashes( + &mut self, + cols: &[ArrayRef], + hashes: &mut [u64], + ) -> Result<()> { + let group_unique = self.group_values.len() == 1; + + for (index, (array, group_value)) in + cols.iter().zip(&mut self.group_values).enumerate() + { + let rehash = index != 0; + let cached = array.as_any_dictionary_opt().and_then(|dictionary| { + let values = dictionary.values(); + (values.len() + <= MAX_DICTIONARY_VALUES_PER_ROW.saturating_mul(array.len()) + && (group_unique || values.len() > array.len())) + .then_some(values) + }); + + if let Some(values) = cached { + let value_hashes = group_value + .dictionary_hashes(values, group_unique) + .expect("dictionary arrays use dictionary group columns"); + let dictionary = array.as_ref(); + downcast_dictionary_array! { + dictionary => scatter_dictionary_hashes( + dictionary, + value_hashes, + hashes, + rehash, + )?, + _ => unreachable!(), + } + } else { + hash_column(array.as_ref(), &self.random_state, hashes, rehash)?; + } + } + Ok(()) + } + /// Build one fresh [`GroupColumn`] per field in the schema. /// /// Used at construction time (`try_new`) and to repopulate the column @@ -474,6 +555,13 @@ impl GroupValuesColumn { let mut batch_hashes = mem::take(&mut self.hashes_buffer); batch_hashes.clear(); batch_hashes.resize(n_rows, 0); + #[cfg(not(feature = "force_hash_collisions"))] + if self.has_dictionary { + self.create_dictionary_hashes(cols, &mut batch_hashes)?; + } else { + create_hashes(cols, &self.random_state, &mut batch_hashes)?; + } + #[cfg(feature = "force_hash_collisions")] create_hashes(cols, &self.random_state, &mut batch_hashes)?; // General steps for one round `vectorized equal_to & append`: @@ -1067,6 +1155,12 @@ impl GroupValues for GroupValuesColumn { // `try_new` and the reset points in `emit` / `clear_shrink` keep // `self.group_values` populated with one builder per schema field, // so no lazy initialization is needed here. + if self.has_dictionary + && self.group_values.len() == 1 + && self.group_values[0].try_map_dictionary_groups(&cols[0], groups) + { + return Ok(()); + } if !STREAMING { self.vectorized_intern(cols, groups) } else { @@ -1251,7 +1345,6 @@ impl GroupValues for GroupValuesColumn { self.map_size = self.map.capacity() * size_of::<(u64, usize)>(); self.hashes_buffer.clear(); self.hashes_buffer.shrink_to(num_rows); - // Such structures are only used in `non-streaming` case if !STREAMING { self.group_index_lists.clear(); @@ -1287,12 +1380,12 @@ mod tests { use std::{collections::HashMap, sync::Arc}; use arrow::array::{ - Array, ArrayRef, DurationMicrosecondArray, FixedSizeBinaryArray, Float16Array, - Int32Array, Int64Array, PrimitiveArray, RecordBatch, StringArray, - StringViewArray, UInt32Array, + Array, ArrayRef, AsArray, DictionaryArray, DurationMicrosecondArray, + FixedSizeBinaryArray, Float16Array, Int8Array, Int32Array, Int64Array, + PrimitiveArray, RecordBatch, StringArray, StringViewArray, UInt32Array, }; use arrow::datatypes::{ - DataType, Decimal32Type, Decimal64Type, Field, Schema, SchemaRef, + DataType, Decimal32Type, Decimal64Type, Field, Int8Type, Schema, SchemaRef, }; use arrow::{ compute::{concat_batches, take}, @@ -2083,9 +2176,6 @@ mod tests { // in the values array even when there are more than 128 groups total. #[test] fn multi_col_groupby_dict_many_groups_two_values() { - use arrow::array::{AsArray, DictionaryArray, Int8Array}; - use arrow::datatypes::Int8Type; - let n_groups = 129_usize; let dict_vocab: ArrayRef = Arc::new(StringArray::from(vec!["cat", "dog"])); @@ -2116,6 +2206,15 @@ mod tests { let mut gv = GroupValuesColumn::::try_new(Arc::clone(&schema)).unwrap(); gv.intern(&[labels, categories], &mut vec![]).unwrap(); + let reordered: ArrayRef = Arc::new(StringArray::from(vec!["dog", "cat"])); + let categories: ArrayRef = Arc::new(DictionaryArray::::new( + Int8Array::from(vec![0, 1]), + reordered, + )); + let labels: ArrayRef = Arc::new(StringArray::from(vec!["g1", "g0"])); + let mut groups = vec![]; + gv.intern(&[labels, categories], &mut groups).unwrap(); + assert_eq!(groups, [1, 0]); let out = gv.emit(EmitTo::All).unwrap(); assert_eq!(out[0].len(), n_groups); @@ -2128,6 +2227,269 @@ mod tests { assert_eq!(out[1].as_dictionary::().values().len(), 2); } + #[test] + #[cfg(not(feature = "force_hash_collisions"))] + fn multi_dictionary_hash_cache_tracks_values() { + let schema = Arc::new(Schema::new(vec![ + Field::new("label", DataType::Int32, false), + Field::new( + "category", + DataType::Dictionary(Box::new(DataType::Int8), Box::new(DataType::Utf8)), + true, + ), + ])); + let dictionary = |keys, values| -> ArrayRef { + Arc::new(DictionaryArray::::new( + Int8Array::from(keys), + values, + )) + }; + let first_values: ArrayRef = Arc::new(StringArray::from(vec![ + Some("cat"), + Some("dog"), + None, + Some("owl"), + ])); + let first = vec![ + Arc::new(Int32Array::from(vec![0, 1])) as ArrayRef, + dictionary(vec![Some(0), Some(1)], Arc::clone(&first_values)), + ]; + + let mut group_values = GroupValuesColumn::::try_new(schema).unwrap(); + let mut groups = vec![]; + group_values.intern(&first, &mut groups).unwrap(); + assert_eq!(groups, [0, 1]); + assert_eq!(group_values.group_values[1].dictionary_hash_builds(), 1); + + let reordered = vec![ + Arc::new(Int32Array::from(vec![0, 1])) as ArrayRef, + dictionary(vec![Some(1), Some(0)], Arc::clone(&first_values)), + ]; + group_values.intern(&reordered, &mut groups).unwrap(); + assert_eq!(groups, [2, 3]); + assert_eq!(group_values.group_values[1].dictionary_hash_builds(), 1); + + let new_values: ArrayRef = Arc::new(StringArray::from(vec![ + Some("dog"), + Some("cat"), + None, + Some("owl"), + ])); + let changed = vec![ + Arc::new(Int32Array::from(vec![0, 1])) as ArrayRef, + dictionary(vec![Some(0), Some(1)], Arc::clone(&new_values)), + ]; + group_values.intern(&changed, &mut groups).unwrap(); + assert_eq!(groups, [2, 3]); + assert_eq!(group_values.group_values[1].dictionary_hash_builds(), 2); + + let nulls = vec![ + Arc::new(Int32Array::from(vec![2, 2])) as ArrayRef, + dictionary(vec![None, Some(2)], new_values), + ]; + group_values.intern(&nulls, &mut groups).unwrap(); + assert_eq!(groups, [4, 4]); + } + + #[test] + #[cfg(not(feature = "force_hash_collisions"))] + fn single_dictionary_group_cache_tracks_values_and_emits() { + let schema = Arc::new(Schema::new(vec![Field::new( + "category", + DataType::Dictionary(Box::new(DataType::Int8), Box::new(DataType::Utf8)), + true, + )])); + let mut group_values = GroupValuesColumn::::try_new(schema).unwrap(); + let first_values: ArrayRef = + Arc::new(StringArray::from(vec![Some("cat"), Some("dog"), None])); + let dictionary = |keys, values| -> ArrayRef { + Arc::new(DictionaryArray::::new( + Int8Array::from(keys), + values, + )) + }; + + let mut groups = vec![]; + let first = dictionary( + vec![Some(0), Some(1), None, Some(2)], + Arc::clone(&first_values), + ); + group_values.intern(&[first], &mut groups).unwrap(); + assert_eq!(groups, vec![0, 1, 2, 2]); + + // Another batch sharing the values array maps keys directly to groups. + let cached_batch = dictionary( + vec![Some(1), Some(0), Some(2), None], + Arc::clone(&first_values), + ); + assert!( + group_values.group_values[0] + .try_map_dictionary_groups(&cached_batch, &mut groups) + ); + assert_eq!(groups, vec![1, 0, 2, 2]); + + let reordered_values: ArrayRef = + Arc::new(StringArray::from(vec![Some("dog"), Some("cat"), None])); + let reordered = dictionary( + vec![Some(0), Some(1), Some(2)], + Arc::clone(&reordered_values), + ); + group_values.intern(&[reordered], &mut groups).unwrap(); + assert_eq!(groups, vec![1, 0, 2]); + + group_values.emit(EmitTo::First(1)).unwrap(); + let after_emit = dictionary(vec![Some(0), Some(2), Some(1)], reordered_values); + group_values.intern(&[after_emit], &mut groups).unwrap(); + assert_eq!(groups, vec![0, 1, 2]); + } + + #[test] + #[cfg(not(feature = "force_hash_collisions"))] + fn single_dictionary_shared_values_new_key_falls_back() { + let schema = Arc::new(Schema::new(vec![Field::new( + "category", + DataType::Dictionary(Box::new(DataType::Int8), Box::new(DataType::Utf8)), + true, + )])); + let mut group_values = GroupValuesColumn::::try_new(schema).unwrap(); + let values: ArrayRef = Arc::new(StringArray::from(vec!["a", "b", "c"])); + let dictionary = |keys: Vec| -> ArrayRef { + Arc::new(DictionaryArray::::new( + Int8Array::from(keys), + Arc::clone(&values), + )) + }; + + let mut groups = vec![]; + group_values + .intern(&[dictionary(vec![0, 1])], &mut groups) + .unwrap(); + assert_eq!(groups, [0, 1]); + + group_values + .intern(&[dictionary(vec![1, 2, 0])], &mut groups) + .unwrap(); + assert_eq!(groups, [1, 2, 0]); + assert_eq!(group_values.len(), 3); + } + + #[test] + fn single_dictionary_direct_mapping_after_emit_first_with_null_group() { + let schema = Arc::new(Schema::new(vec![Field::new( + "d", + DataType::Dictionary(Box::new(DataType::Int8), Box::new(DataType::Utf8)), + true, + )])); + let mut group_values = GroupValuesColumn::::try_new(schema).unwrap(); + let values: ArrayRef = Arc::new(StringArray::from(vec!["a", "b"])); + let dictionary = |keys: Vec>| -> ArrayRef { + Arc::new(DictionaryArray::::new( + Int8Array::from(keys), + Arc::clone(&values), + )) + }; + + let mut groups = vec![]; + group_values + .intern(&[dictionary(vec![Some(0), None, Some(1)])], &mut groups) + .unwrap(); + assert_eq!(groups, [0, 1, 2]); + + group_values.emit(EmitTo::First(1)).unwrap(); + group_values + .intern(&[dictionary(vec![Some(1), None])], &mut groups) + .unwrap(); + assert_eq!(groups, [1, 0]); + group_values + .intern(&[dictionary(vec![Some(1), None])], &mut groups) + .unwrap(); + assert_eq!(groups, [1, 0]); + } + + #[test] + fn single_dictionary_direct_mapping_after_oversized_batches_and_emit_first() { + let schema = Arc::new(Schema::new(vec![Field::new( + "d", + DataType::Dictionary(Box::new(DataType::Int16), Box::new(DataType::Utf8)), + true, + )])); + let mut group_values = GroupValuesColumn::::try_new(schema).unwrap(); + let mut strings: Vec = (0..200).map(|i| format!("v{i}")).collect(); + strings[0] = "a".into(); + strings[1] = "b".into(); + let large_values: ArrayRef = Arc::new(StringArray::from(strings)); + let dictionary = |keys: Vec>, values: &ArrayRef| -> ArrayRef { + Arc::new(DictionaryArray::::new( + arrow::array::Int16Array::from(keys), + Arc::clone(values), + )) + }; + + let mut groups = vec![]; + group_values + .intern( + &[dictionary(vec![Some(0), None, Some(1)], &large_values)], + &mut groups, + ) + .unwrap(); + assert_eq!(groups, [0, 1, 2]); + + group_values.emit(EmitTo::First(1)).unwrap(); + let small_values: ArrayRef = Arc::new(StringArray::from(vec!["b"])); + group_values + .intern( + &[dictionary(vec![Some(0), None], &small_values)], + &mut groups, + ) + .unwrap(); + assert_eq!(groups, [1, 0]); + group_values + .intern( + &[dictionary(vec![Some(0), None], &small_values)], + &mut groups, + ) + .unwrap(); + assert_eq!(groups, [1, 0]); + } + + #[test] + fn dictionary_hash_paths_agree_on_existing_groups() { + let schema = Arc::new(Schema::new(vec![ + Field::new( + "d", + DataType::Dictionary(Box::new(DataType::Int8), Box::new(DataType::Utf8)), + true, + ), + Field::new("i", DataType::Int32, true), + ])); + let mut group_values = GroupValuesColumn::::try_new(schema).unwrap(); + let mut groups = vec![]; + + let small_values: ArrayRef = Arc::new(StringArray::from(vec!["a", "b"])); + let dictionary: ArrayRef = Arc::new(DictionaryArray::::new( + Int8Array::from(vec![0, 1, 0, 1]), + small_values, + )); + let integers: ArrayRef = + Arc::new(Int32Array::from(vec![Some(1), Some(2), None, Some(3)])); + group_values + .intern(&[dictionary, integers], &mut groups) + .unwrap(); + assert_eq!(groups, [0, 1, 2, 3]); + + let large_values: ArrayRef = Arc::new(StringArray::from(vec!["a", "b", "c"])); + let dictionary: ArrayRef = Arc::new(DictionaryArray::::new( + Int8Array::from(vec![0, 0]), + large_values, + )); + let integers: ArrayRef = Arc::new(Int32Array::from(vec![Some(1), None])); + group_values + .intern(&[dictionary, integers], &mut groups) + .unwrap(); + assert_eq!(groups, [0, 2]); + assert_eq!(group_values.len(), 4); + } + #[test] fn test_intern_for_vectorized_group_values() { let data_set = VectorizedTestDataSet::new();