Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
e3f28e6
fix: align Spark collect aggregate element nullability
goutamadwant Aug 28, 2026
ddbadd3
fix: handle encoded values in non-nullable list builder
goutamadwant Aug 30, 2026
4f792b3
fix: normalize Spark collect retractions and retained input rows
goutamadwant Sep 14, 2026
735a4b3
fix: normalize Spark collect encoded values
goutamadwant Sep 19, 2026
d7d26bd
fix: bound collect encoded normalization
goutamadwant Sep 19, 2026
eb0a76d
fix: harden collect encoded normalization
goutamadwant Sep 19, 2026
3cf0870
fix: handle inactive sparse union variants
goutamadwant Sep 19, 2026
cdba3e3
fix: preserve nullable run values during collect normalization
goutamadwant Sep 19, 2026
985de98
fix: preserve nullable dictionary values during collect normalization
goutamadwant Sep 19, 2026
ae8c9c0
fix: compact exact encoded collect values
goutamadwant Sep 19, 2026
bfa35cc
fix: compact nested collect wrappers
goutamadwant Sep 19, 2026
209b774
Merge remote-tracking branch 'origin/main' into fix-spark-collect-nul…
goutamadwant Sep 19, 2026
d65cf5f
fix: preserve fixed-size-list default shape
goutamadwant Sep 19, 2026
f83f0f4
fix: harden Spark collect normalization
goutamadwant Sep 19, 2026
22feb73
test: cover nested dictionary collect SQL paths
goutamadwant Sep 21, 2026
1b489fe
Merge main into fix-spark-collect-nullability
goutamadwant Sep 24, 2026
b259eba
Merge main into fix-spark-collect-nullability
goutamadwant Sep 25, 2026
4351242
Merge main and address Spark collect review regressions
goutamadwant Nov 10, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

108 changes: 88 additions & 20 deletions datafusion/common/src/scalar/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1891,24 +1891,32 @@ impl ScalarValue {
)),

// List types
DataType::List(field) => {
let list =
ScalarValue::new_list(&[], field.data_type(), field.is_nullable());
Ok(ScalarValue::List(list))
}
DataType::FixedSizeList(field, _size) => {
let empty_arr = new_empty_array(field.data_type());
let values = Arc::new(
SingleRowListArrayBuilder::new(empty_arr)
.with_field(field)
.build_fixed_size_list_array(0),
);
Ok(ScalarValue::FixedSizeList(values))
}
DataType::LargeList(field) => {
let list = ScalarValue::new_large_list(&[], field.data_type());
Ok(ScalarValue::LargeList(list))
DataType::List(field) => Ok(SingleRowListArrayBuilder::new(new_empty_array(
field.data_type(),
))
.with_field(field)
.build_list_scalar()),
DataType::FixedSizeList(field, size) => {
let list_size = size.to_usize().ok_or_else(|| {
_internal_datafusion_err!("FixedSizeList size cannot be negative")
})?;
let values = ScalarValue::new_default(field.data_type())?
.to_array_of_size(list_size)?;
Ok(ScalarValue::FixedSizeList(Arc::new(
FixedSizeListArray::try_new_with_length(
Arc::clone(field),
*size,
values,
None,
1,
)?,
)))
}
DataType::LargeList(field) => Ok(ScalarValue::LargeList(Arc::new(
SingleRowListArrayBuilder::new(new_empty_array(field.data_type()))
.with_field(field)
.build_large_list_array(),
))),
DataType::ListView(field) => {
let empty_arr = new_empty_array(field.data_type());
let values = Arc::new(
Expand Down Expand Up @@ -1965,9 +1973,15 @@ impl ScalarValue {
}

// Map types
DataType::Map(field, _) => Ok(ScalarValue::Map(Arc::new(MapArray::from(
ArrayData::new_empty(field.data_type()),
)))),
DataType::Map(field, sorted) => {
Ok(ScalarValue::Map(Arc::new(MapArray::try_new(
Arc::clone(field),
arrow::buffer::OffsetBuffer::from_lengths([0]),
new_empty_array(field.data_type()).as_struct().clone(),
None,
*sorted,
)?)))
}

// Union types - return first variant with default value
DataType::Union(fields, mode) => {
Expand Down Expand Up @@ -11096,6 +11110,60 @@ mod tests {
}
}

#[test]
fn test_new_default_lists_preserve_item_field() -> Result<()> {
for nullable in [false, true] {
let field = Arc::new(
Field::new("custom_item", DataType::Int32, nullable).with_metadata(
HashMap::from([("tag".to_string(), "preserved".to_string())]),
),
);
for data_type in [
DataType::List(Arc::clone(&field)),
DataType::LargeList(Arc::clone(&field)),
] {
let value = ScalarValue::new_default(&data_type)?;
assert_eq!(value.data_type(), data_type);
let array = value.to_array()?;
assert_eq!(array.len(), 1);
assert!(array.is_valid(0));
array.to_data().validate_full()?;
let nested = DataType::Struct(Fields::from(vec![Field::new(
"items", data_type, false,
)]));
let value = ScalarValue::new_default(&nested)?;
assert_eq!(value.data_type(), nested);
value.to_array()?.to_data().validate_full()?;
}
}
Ok(())
}

#[test]
fn test_new_default_map_preserves_type_and_has_one_empty_value() -> Result<()> {
let entries = Arc::new(Field::new(
"entries",
DataType::Struct(Fields::from(vec![
Field::new("key", DataType::Int32, false),
Field::new("value", DataType::Int32, false),
])),
false,
));
for sorted in [false, true] {
let data_type = DataType::Map(Arc::clone(&entries), sorted);
let value = ScalarValue::new_default(&data_type)?;
assert_eq!(value.data_type(), data_type);
let ScalarValue::Map(map) = value else {
panic!("expected Map")
};
assert_eq!(map.len(), 1);
assert!(map.is_valid(0));
assert_eq!(map.value_length(0), 0);
map.to_data().validate_full()?;
}
Ok(())
}

#[test]
fn test_scalar_min() {
// Test integer types
Expand Down
30 changes: 29 additions & 1 deletion datafusion/common/src/utils/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1583,7 +1583,7 @@ mod tests {
use super::*;
use crate::ScalarValue::Null;
use arrow::{
array::{Float64Array, Int32Array, NullArray, StringArray},
array::{Float64Array, Int8Array, Int32Array, NullArray, StringArray},
datatypes::{Int8Type, Int32Type, UInt8Type},
};
#[cfg(feature = "sql")]
Expand Down Expand Up @@ -1678,6 +1678,34 @@ mod tests {
}
}

#[test]
fn single_row_list_builder_handles_conservatively_nullable_values() {
let keys = Int8Array::from(vec![0, 0]);
let dictionary_values = Arc::new(StringArray::from(vec![Some("a"), None]));
let values = Arc::new(
DictionaryArray::<Int8Type>::try_new(keys, dictionary_values).unwrap(),
) as ArrayRef;

assert_eq!(values.logical_null_count(), 0);
assert!(values.is_nullable());

let list = SingleRowListArrayBuilder::new(Arc::clone(&values))
.with_nullable(false)
.build_list_array();
list.to_data().validate_full().unwrap();

let value = ScalarValue::try_from_array(&list, 0).unwrap();
assert_eq!(value.data_type(), list.data_type().clone());

let list = SingleRowListArrayBuilder::new(values)
.with_nullable(false)
.build_large_list_array();
list.to_data().validate_full().unwrap();

let value = ScalarValue::try_from_array(&list, 0).unwrap();
assert_eq!(value.data_type(), list.data_type().clone());
}

#[test]
fn test_bisect_linear_left_and_right() -> Result<()> {
let arrays: Vec<ArrayRef> = vec![
Expand Down
1 change: 1 addition & 0 deletions datafusion/spark/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ name = "datafusion_spark"

[dependencies]
arrow = { workspace = true }
arrow-select = { workspace = true }
base64 = "0.23"
bigdecimal = { workspace = true }
chrono = { workspace = true }
Expand Down
Loading
Loading