diff --git a/native/shuffle/src/spark_unsafe/list.rs b/native/shuffle/src/spark_unsafe/list.rs index 3d8e89ebc56..32d08a4bd38 100644 --- a/native/shuffle/src/spark_unsafe/list.rs +++ b/native/shuffle/src/spark_unsafe/list.rs @@ -15,21 +15,30 @@ // specific language governing permissions and limitations // under the License. +use std::sync::Arc; + use crate::spark_unsafe::{ map::append_map_elements, row::{append_field, downcast_builder_ref, SparkUnsafeRow}, unsafe_object::{impl_primitive_accessors, SparkUnsafeObject}, }; -use arrow::array::{ - builder::{ - ArrayBuilder, BinaryBuilder, BooleanBuilder, Date32Builder, Decimal128Builder, - Float32Builder, Float64Builder, Int16Builder, Int32Builder, Int64Builder, Int8Builder, - ListBuilder, NullBuilder, StringBuilder, StructBuilder, Time64NanosecondBuilder, - TimestampMicrosecondBuilder, +use arrow::datatypes::{DataType, TimeUnit}; +use arrow::{ + array::{ + builder::{ + ArrayBuilder, BinaryBuilder, BooleanBuilder, Date32Builder, Decimal128Builder, + Float32Builder, Float64Builder, Int16Builder, Int32Builder, Int64Builder, Int8Builder, + ListBuilder, NullBuilder, StringBuilder, StructBuilder, Time64NanosecondBuilder, + TimestampMicrosecondBuilder, + }, + MapBuilder, PrimitiveArray, + }, + buffer::{BooleanBuffer, Buffer, NullBuffer, ScalarBuffer}, + datatypes::{ + Date32Type, Float32Type, Float64Type, Int16Type, Int32Type, Int64Type, Int8Type, + Time64NanosecondType, TimestampMicrosecondType, }, - MapBuilder, }; -use arrow::datatypes::{DataType, TimeUnit}; use datafusion_comet_jni_bridge::errors::CometError; /// Element count from which a nullable array with no null elements is appended with one copy @@ -46,39 +55,69 @@ const MIN_BULK_APPEND_ELEMENTS: usize = 8; /// - `null_bitset_ptr()` returns a pointer to `ceil(num_elements/64)` i64 words /// - These invariants are guaranteed by the SparkUnsafeArray layout from the JVM macro_rules! impl_append_to_builder { - ($method_name:ident, $builder_type:ty, $element_type:ty) => { - pub(crate) fn $method_name(&self, builder: &mut $builder_type) { + ($method_name:ident, $builder_type:ty, $element_type:ty, $arrow_type:ty + $(, $timezone:ident)?) => { + pub(crate) fn $method_name( + &self, + builder: &mut $builder_type, + $($timezone: Option>,)? + ) { let num_elements = self.num_elements; if num_elements == 0 { return; } + // Note: alignment is not guaranteed - that is why do this + // This runtime check is needed. Look at `unsafe_object.rs:49` for more info + let ptr = self.element_offset as *const $element_type; + let aligned = (ptr as usize).is_multiple_of(std::mem::align_of::<$element_type>()); // A nullable array without nulls takes the copy below too, once it is long enough. if NULLABLE && (num_elements < MIN_BULK_APPEND_ELEMENTS || self.has_null()) { - let mut ptr = self.element_offset as *const $element_type; let null_words = self.null_bitset_ptr(); - debug_assert!(!null_words.is_null(), "null_bitset_ptr is null"); - debug_assert!(!ptr.is_null(), "element_offset pointer is null"); - for idx in 0..num_elements { - // SAFETY: null_words has ceil(num_elements/64) words, idx < num_elements - let is_null = unsafe { Self::is_null_in_bitset(null_words, idx) }; - - if is_null { - builder.append_null(); - } else { - // SAFETY: ptr is within element data bounds - builder.append_value(unsafe { ptr.read_unaligned() }); + + // An array shorter than MIN_BULK_APPEND_ELEMENTS stays on the per-element loop, + // which costs less than building the validity and value buffers. + if aligned && num_elements >= MIN_BULK_APPEND_ELEMENTS { + // Raw values + let values = unsafe { std::slice::from_raw_parts(ptr, num_elements) }; + + // Note: in Spark bitmap is padded to 8 byte word-boundaries + // In Arrow we just use the needed number of whole bytes without padding + let null_mask_len = num_elements.div_ceil(8); + // Spark and Arrow both use little-endian bit ordering within each byte, + // so casting the i64 null words to *const u8 preserves correct bit indexing. + let null_mask = unsafe { + std::slice::from_raw_parts::(null_words as *const u8, null_mask_len) + }; + // We need to perform this flip due to the null bitmap Spark vs Arrow incompatibility + // In `Spark` we have 1 set in bitmap meaning that element IS NULL + // In `Arrow` we have 1 set in bitmap meaning that element IS VALID (non-null) + let flipped: Vec = null_mask.iter().map(|n| !n).collect(); + // Constructing null-buffer + let validity = + NullBuffer::new(BooleanBuffer::new(Buffer::from(flipped), 0, num_elements)); + + let arr = PrimitiveArray::<$arrow_type>::new( + ScalarBuffer::from(Buffer::from_slice_ref(values)), + Some(validity), + ); + // `append_array` requires the array's data type to match the builder's, + // timezone included. + $(let arr = arr.with_timezone_opt($timezone);)? + builder.append_array(&arr); + } else { + let mut ptr = ptr; + for idx in 0..num_elements { + if unsafe { Self::is_null_in_bitset(null_words, idx) } { + builder.append_null(); + } else { + builder.append_value(unsafe { ptr.read_unaligned() }); + } + ptr = unsafe { ptr.add(1) }; } - // SAFETY: ptr stays within bounds, iterating num_elements times - ptr = unsafe { ptr.add(1) }; } } else { - // SAFETY: element_offset points to contiguous data of length num_elements - debug_assert!(self.element_offset != 0, "element_offset is null"); - let ptr = self.element_offset as *const $element_type; - // Use bulk copy when data is properly aligned, fall back to - // per-element unaligned reads otherwise - if (ptr as usize).is_multiple_of(std::mem::align_of::<$element_type>()) { + if aligned { let slice = unsafe { std::slice::from_raw_parts(ptr, num_elements) }; builder.append_slice(slice); } else { @@ -196,19 +235,26 @@ impl SparkUnsafeArray { (null_words.add(word_idx).read_unaligned() & (1i64 << bit_idx)) != 0 } - impl_append_to_builder!(append_ints_to_builder, Int32Builder, i32); - impl_append_to_builder!(append_longs_to_builder, Int64Builder, i64); - impl_append_to_builder!(append_time64s_to_builder, Time64NanosecondBuilder, i64); - impl_append_to_builder!(append_shorts_to_builder, Int16Builder, i16); - impl_append_to_builder!(append_bytes_to_builder, Int8Builder, i8); - impl_append_to_builder!(append_floats_to_builder, Float32Builder, f32); - impl_append_to_builder!(append_doubles_to_builder, Float64Builder, f64); + impl_append_to_builder!(append_ints_to_builder, Int32Builder, i32, Int32Type); + impl_append_to_builder!(append_longs_to_builder, Int64Builder, i64, Int64Type); + impl_append_to_builder!( + append_time64s_to_builder, + Time64NanosecondBuilder, + i64, + Time64NanosecondType + ); + impl_append_to_builder!(append_shorts_to_builder, Int16Builder, i16, Int16Type); + impl_append_to_builder!(append_bytes_to_builder, Int8Builder, i8, Int8Type); + impl_append_to_builder!(append_floats_to_builder, Float32Builder, f32, Float32Type); + impl_append_to_builder!(append_doubles_to_builder, Float64Builder, f64, Float64Type); impl_append_to_builder!( append_timestamps_to_builder, TimestampMicrosecondBuilder, - i64 + i64, + TimestampMicrosecondType, + timezone ); - impl_append_to_builder!(append_dates_to_builder, Date32Builder, i32); + impl_append_to_builder!(append_dates_to_builder, Date32Builder, i32, Date32Type); /// Bulk append boolean values to builder. /// Booleans are stored as 1 byte each in SparkUnsafeArray, requiring special handling. @@ -220,37 +266,31 @@ impl SparkUnsafeArray { if num_elements == 0 { return; } - - let mut ptr = self.element_offset as *const u8; + // Bools have alignment == 1 + // We dont have to worry about the fallback. Hence, we do not care about it debug_assert!( - !ptr.is_null(), + self.element_offset != 0, "append_booleans: element_offset pointer is null" ); if NULLABLE { let null_words = self.null_bitset_ptr(); - debug_assert!( - !null_words.is_null(), - "append_booleans: null_bitset_ptr is null" - ); - for idx in 0..num_elements { - // SAFETY: null_words has ceil(num_elements/64) words, idx < num_elements - let is_null = unsafe { Self::is_null_in_bitset(null_words, idx) }; - - if is_null { + let slice = unsafe { + std::slice::from_raw_parts(self.element_offset as *const u8, num_elements) + }; + for (idx, &value) in slice.iter().enumerate() { + if unsafe { Self::is_null_in_bitset(null_words, idx) } { builder.append_null(); } else { - // SAFETY: ptr is within element data bounds - builder.append_value(unsafe { *ptr != 0 }); + builder.append_value(value != 0); } - // SAFETY: ptr stays within bounds, iterating num_elements times - ptr = unsafe { ptr.add(1) }; } } else { - for _ in 0..num_elements { - // SAFETY: ptr is within element data bounds - builder.append_value(unsafe { *ptr != 0 }); - ptr = unsafe { ptr.add(1) }; + let values = unsafe { + std::slice::from_raw_parts(self.element_offset as *const u8, num_elements) + }; + for &value in values { + builder.append_value(value != 0); } } } @@ -307,9 +347,9 @@ pub fn append_to_builder( let builder = downcast_builder_ref!(Float64Builder, builder); array.append_doubles_to_builder::(builder); } - DataType::Timestamp(TimeUnit::Microsecond, _) => { + DataType::Timestamp(TimeUnit::Microsecond, tz) => { let builder = downcast_builder_ref!(TimestampMicrosecondBuilder, builder); - array.append_timestamps_to_builder::(builder); + array.append_timestamps_to_builder::(builder, tz.clone()); } DataType::Date32 => { let builder = downcast_builder_ref!(Date32Builder, builder); @@ -415,6 +455,7 @@ pub fn append_list_element( mod test { use super::*; use arrow::array::builder::PrimitiveBuilder; + use arrow::array::Array; use arrow::datatypes::{ ArrowPrimitiveType, Date32Type, Int16Type, Int32Type, Int64Type, Int8Type, TimestampMicrosecondType, @@ -455,7 +496,7 @@ mod test { let base = unsafe { (words.as_mut_ptr() as *mut u8).add(shift) }; unsafe { std::ptr::copy_nonoverlapping(bytes.as_ptr(), base, bytes.len()) }; let array = SparkUnsafeArray::new(base as i64); - let mut builder = PrimitiveBuilder::::new(); + let mut builder = PrimitiveBuilder::::new().with_data_type(data_type.clone()); append_to_builder::(&data_type, &mut builder, &array).unwrap(); builder.finish().iter().map(|v| v.map(Into::into)).collect() } @@ -508,7 +549,98 @@ mod test { ), *values ); + // The builder carries the column's timezone, which `append_array` checks. + assert_eq!( + round_trip::( + DataType::Timestamp(TimeUnit::Microsecond, Some("America/Denver".into())), + values, + shift + ), + *values + ); } } } + + fn make_i32_array(num_elements: usize, null_indices: &[usize]) -> Vec { + let null_bitset_words = num_elements.div_ceil(64); + let header_size = 8 + null_bitset_words * 8; + let mut buffer = vec![0u8; header_size + num_elements * 4]; + + buffer[0..8].copy_from_slice(&(num_elements as i64).to_le_bytes()); + + for &i in null_indices { + let word_offset = 8 + (i / 64) * 8; + let word = i64::from_le_bytes(buffer[word_offset..word_offset + 8].try_into().unwrap()); + buffer[word_offset..word_offset + 8] + .copy_from_slice(&(word | (1i64 << (i % 64))).to_le_bytes()); + } + + for i in 0..num_elements { + let offset = header_size + i * 4; + buffer[offset..offset + 4].copy_from_slice(&(i as i32).to_le_bytes()); + } + buffer + } + + fn append_i32(buffer: &[u8]) -> arrow::array::Int32Array { + let array = SparkUnsafeArray::new(buffer.as_ptr() as i64); + let mut builder = Int32Builder::new(); + append_to_builder::(&DataType::Int32, &mut builder, &array).unwrap(); + builder.finish() + } + + #[test] + fn test_nullable_7_elements_null_in_last_byte() { + // 7 elements — partial last byte, null at index 6 + let buf = make_i32_array(7, &[6]); + let arr = append_i32(&buf); + assert_eq!(arr.len(), 7); + assert!(arr.is_null(6)); + for i in 0..6 { + assert!(arr.is_valid(i)); + assert_eq!(arr.value(i), i as i32); + } + } + + #[test] + fn test_nullable_9_elements_null_in_last_byte() { + let buf = make_i32_array(9, &[8]); + let arr = append_i32(&buf); + assert_eq!(arr.len(), 9); + assert!(arr.is_null(8)); + for i in 0..8 { + assert!(arr.is_valid(i)); + assert_eq!(arr.value(i), i as i32); + } + } + + #[test] + fn test_nullable_17_elements_null_in_last_byte() { + let buf = make_i32_array(17, &[16]); + let arr = append_i32(&buf); + assert_eq!(arr.len(), 17); + assert!(arr.is_null(16)); + for i in 0..16 { + assert!(arr.is_valid(i)); + assert_eq!(arr.value(i), i as i32); + } + } + + #[test] + fn test_all_null() { + let indices: Vec = (0..9).collect(); + let buf = make_i32_array(9, &indices); + let arr = append_i32(&buf); + assert_eq!(arr.len(), 9); + assert_eq!(arr.null_count(), 9); + } + + #[test] + fn test_all_valid() { + let buf = make_i32_array(9, &[]); + let arr = append_i32(&buf); + assert_eq!(arr.len(), 9); + assert_eq!(arr.null_count(), 0); + } } diff --git a/native/shuffle/src/spark_unsafe/row.rs b/native/shuffle/src/spark_unsafe/row.rs index 180673c37fb..61540227cbf 100644 --- a/native/shuffle/src/spark_unsafe/row.rs +++ b/native/shuffle/src/spark_unsafe/row.rs @@ -622,26 +622,43 @@ fn append_list_column_batch( // Helper macro for primitive element types - gets builder fresh each iteration // to avoid borrow conflicts with list_builder.append() macro_rules! process_primitive_lists { - ($builder_type:ty, $append_fn:ident) => {{ - for i in row_start..row_end { - read_row_at!(row, row_addresses_ptr, row_sizes_ptr, i); - - if row.is_null_at(column_idx) { - list_builder.append_null(); - } else { - let array = row.get_array(column_idx); - // Get values builder fresh each iteration to avoid borrow conflict - let values_builder = list_builder - .values() - .as_any_mut() - .downcast_mut::<$builder_type>() - .expect(stringify!($builder_type)); - array.$append_fn::(values_builder); - list_builder.append(true); - } + // No extra args + ($builder_type:ty, $append_fn:ident) => {{ + for i in row_start..row_end { + read_row_at!(row, row_addresses_ptr, row_sizes_ptr, i); + if row.is_null_at(column_idx) { + list_builder.append_null(); + } else { + let array = row.get_array(column_idx); + let values_builder = list_builder + .values() + .as_any_mut() + .downcast_mut::<$builder_type>() + .expect(stringify!($builder_type)); + array.$append_fn::(values_builder); + list_builder.append(true); } - }}; - } + } + }}; + // Extra args — for timestamps (timezone) and others + ($builder_type:ty, $append_fn:ident, $($extra:expr),*) => {{ + for i in row_start..row_end { + read_row_at!(row, row_addresses_ptr, row_sizes_ptr, i); + if row.is_null_at(column_idx) { + list_builder.append_null(); + } else { + let array = row.get_array(column_idx); + let values_builder = list_builder + .values() + .as_any_mut() + .downcast_mut::<$builder_type>() + .expect(stringify!($builder_type)); + array.$append_fn::(values_builder, $($extra),*); + list_builder.append(true); + } + } + }}; +} match element_type { DataType::Boolean => { @@ -668,8 +685,12 @@ fn append_list_column_batch( DataType::Date32 => { process_primitive_lists!(Date32Builder, append_dates_to_builder); } - DataType::Timestamp(TimeUnit::Microsecond, _) => { - process_primitive_lists!(TimestampMicrosecondBuilder, append_timestamps_to_builder); + DataType::Timestamp(TimeUnit::Microsecond, tz) => { + process_primitive_lists!( + TimestampMicrosecondBuilder, + append_timestamps_to_builder, + tz.clone() + ); } DataType::Time64(TimeUnit::Nanosecond) => { process_primitive_lists!(Time64NanosecondBuilder, append_time64s_to_builder);