From a2f260bb6a56e6133a233312fd438a9c25e7c125 Mon Sep 17 00:00:00 2001 From: Jefffrey Date: Thu, 13 Nov 2025 00:08:06 +0900 Subject: [PATCH 1/2] Refactor Spark bitshift signature --- datafusion/functions/src/macros.rs | 18 +- .../spark/src/function/bitwise/bit_shift.rs | 730 ++++-------------- datafusion/spark/src/function/bitwise/mod.rs | 18 +- .../test_files/spark/bitwise/shiftright.slt | 99 +++ .../spark/bitwise/shiftrightunsigned.slt | 87 +++ .../test_files/spark/math/shiftleft.slt | 84 ++ 6 files changed, 447 insertions(+), 589 deletions(-) diff --git a/datafusion/functions/src/macros.rs b/datafusion/functions/src/macros.rs index 9e195f2d52914..077690cd3f7bd 100644 --- a/datafusion/functions/src/macros.rs +++ b/datafusion/functions/src/macros.rs @@ -77,7 +77,8 @@ macro_rules! export_functions { } /// Creates a singleton `ScalarUDF` of the `$UDF` function and a function -/// named `$NAME` which returns that singleton. +/// named `$NAME` which returns that singleton. Optionally use a custom constructor +/// `$CTOR` which defaults to `$UDF::new()` if not specified. /// /// This is used to ensure creating the list of `ScalarUDF` only happens once. #[macro_export] @@ -97,6 +98,21 @@ macro_rules! make_udf_function { std::sync::Arc::clone(&INSTANCE) } }; + ($UDF:ty, $NAME:ident, $CTOR:path) => { + #[allow(rustdoc::redundant_explicit_links)] + #[doc = concat!("Return a [`ScalarUDF`](datafusion_expr::ScalarUDF) implementation of ", stringify!($NAME))] + pub fn $NAME() -> std::sync::Arc { + // Singleton instance of the function + static INSTANCE: std::sync::LazyLock< + std::sync::Arc, + > = std::sync::LazyLock::new(|| { + std::sync::Arc::new(datafusion_expr::ScalarUDF::new_from_impl( + $CTOR(), + )) + }); + std::sync::Arc::clone(&INSTANCE) + } + }; } /// Creates a singleton `ScalarUDF` of the `$UDF` function and a function diff --git a/datafusion/spark/src/function/bitwise/bit_shift.rs b/datafusion/spark/src/function/bitwise/bit_shift.rs index 68911b0492c56..cb0afd8c26aa4 100644 --- a/datafusion/spark/src/function/bitwise/bit_shift.rs +++ b/datafusion/spark/src/function/bitwise/bit_shift.rs @@ -18,36 +18,33 @@ use std::any::Any; use std::sync::Arc; -use arrow::array::{ArrayRef, ArrowPrimitiveType, AsArray, PrimitiveArray}; +use arrow::array::{ArrayRef, ArrowPrimitiveType, AsArray, Int32Array, PrimitiveArray}; use arrow::compute; use arrow::datatypes::{ ArrowNativeType, DataType, Int32Type, Int64Type, UInt32Type, UInt64Type, }; -use datafusion_common::{plan_err, Result}; +use datafusion_common::types::{ + logical_int16, logical_int32, logical_int64, logical_int8, logical_uint16, + logical_uint32, logical_uint64, logical_uint8, NativeType, +}; +use datafusion_common::utils::take_function_args; +use datafusion_common::{internal_err, Result}; use datafusion_expr::{ - ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility, + Coercion, ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, TypeSignature, + TypeSignatureClass, Volatility, }; use datafusion_functions::utils::make_scalar_function; -use crate::function::error_utils::{ - invalid_arg_count_exec_err, unsupported_data_type_exec_err, -}; - -/// Performs a bitwise left shift on each element of the `value` array by the corresponding amount in the `shift` array. -/// The shift amount is normalized to the bit width of the type, matching Spark/Java semantics for negative and large shifts. -/// -/// # Arguments -/// * `value` - The array of values to shift. -/// * `shift` - The array of shift amounts (must be Int32). -/// -/// # Returns -/// A new array with the shifted values. -fn shift_left( +/// Bitwise left shift on elements in `value` by corresponding `shift` amount. +/// The shift amount is normalized to the bit width of the type, matching Spark/Java +/// semantics for negative and large shifts. +fn shift_left( value: &PrimitiveArray, - shift: &PrimitiveArray, + shift: &Int32Array, ) -> Result> where - T::Native: ArrowNativeType + std::ops::Shl, + T: ArrowPrimitiveType, + T::Native: std::ops::Shl, { let bit_num = (T::Native::get_byte_width() * 8) as i32; let result = compute::binary::<_, Int32Type, _, _>( @@ -61,21 +58,16 @@ where Ok(result) } -/// Performs a bitwise right shift on each element of the `value` array by the corresponding amount in the `shift` array. -/// The shift amount is normalized to the bit width of the type, matching Spark/Java semantics for negative and large shifts. -/// -/// # Arguments -/// * `value` - The array of values to shift. -/// * `shift` - The array of shift amounts (must be Int32). -/// -/// # Returns -/// A new array with the shifted values. -fn shift_right( +/// Bitwise right shift on elements in `value` by corresponding `shift` amount. +/// The shift amount is normalized to the bit width of the type, matching Spark/Java +/// semantics for negative and large shifts. +fn shift_right( value: &PrimitiveArray, - shift: &PrimitiveArray, + shift: &Int32Array, ) -> Result> where - T::Native: ArrowNativeType + std::ops::Shr, + T: ArrowPrimitiveType, + T::Native: std::ops::Shr, { let bit_num = (T::Native::get_byte_width() * 8) as i32; let result = compute::binary::<_, Int32Type, _, _>( @@ -93,49 +85,44 @@ where /// This is used to mimic Java's `>>>` operator, which does not exist in Rust. /// For unsigned types, this is just the normal right shift. /// For signed types, this casts to the unsigned type, shifts, then casts back. -trait UShr { - fn ushr(self, rhs: Rhs) -> Self; +trait UShr { + fn ushr(self, rhs: i32) -> Self; } -impl UShr for u32 { +impl UShr for u32 { fn ushr(self, rhs: i32) -> Self { self >> rhs } } -impl UShr for u64 { +impl UShr for u64 { fn ushr(self, rhs: i32) -> Self { self >> rhs } } -impl UShr for i32 { +impl UShr for i32 { fn ushr(self, rhs: i32) -> Self { ((self as u32) >> rhs) as i32 } } -impl UShr for i64 { +impl UShr for i64 { fn ushr(self, rhs: i32) -> Self { ((self as u64) >> rhs) as i64 } } -/// Performs a bitwise unsigned right shift on each element of the `value` array by the corresponding amount in the `shift` array. -/// The shift amount is normalized to the bit width of the type, matching Spark/Java semantics for negative and large shifts. -/// -/// # Arguments -/// * `value` - The array of values to shift. -/// * `shift` - The array of shift amounts (must be Int32). -/// -/// # Returns -/// A new array with the shifted values. -fn shift_right_unsigned( +/// Bitwise unsigned right shift on elements in `value` by corresponding `shift` +/// amount. The shift amount is normalized to the bit width of the type, matching +/// Spark/Java semantics for negative and large shifts. +fn shift_right_unsigned( value: &PrimitiveArray, - shift: &PrimitiveArray, + shift: &Int32Array, ) -> Result> where - T::Native: ArrowNativeType + UShr, + T: ArrowPrimitiveType, + T::Native: UShr, { let bit_num = (T::Native::get_byte_width() * 8) as i32; let result = compute::binary::<_, Int32Type, _, _>( @@ -149,589 +136,162 @@ where Ok(result) } -trait BitShiftUDF: ScalarUDFImpl { - fn shift( - &self, - value: &PrimitiveArray, - shift: &PrimitiveArray, - ) -> Result> - where - T::Native: ArrowNativeType - + std::ops::Shl - + std::ops::Shr - + UShr; - - fn spark_shift(&self, arrays: &[ArrayRef]) -> Result { - let value_array = arrays[0].as_ref(); - let shift_array = arrays[1].as_ref(); - - // Ensure shift array is Int32 - let shift_array = if shift_array.data_type() != &DataType::Int32 { - return plan_err!("{} shift amount must be Int32", self.name()); - } else { - shift_array.as_primitive::() - }; - - match value_array.data_type() { - DataType::Int32 => { - let value_array = value_array.as_primitive::(); - Ok(Arc::new(self.shift(value_array, shift_array)?)) - } - DataType::Int64 => { - let value_array = value_array.as_primitive::(); - Ok(Arc::new(self.shift(value_array, shift_array)?)) - } - DataType::UInt32 => { - let value_array = value_array.as_primitive::(); - Ok(Arc::new(self.shift(value_array, shift_array)?)) - } - DataType::UInt64 => { - let value_array = value_array.as_primitive::(); - Ok(Arc::new(self.shift(value_array, shift_array)?)) - } - _ => { - plan_err!( - "{} function does not support data type: {}", - self.name(), - value_array.data_type() - ) - } - } - } -} +fn shift_inner( + arrays: &[ArrayRef], + name: &str, + bit_shift_type: BitShiftType, +) -> Result { + let [value_array, shift_array] = take_function_args(name, arrays)?; -fn bit_shift_coerce_types(arg_types: &[DataType], func: &str) -> Result> { - if arg_types.len() != 2 { - return Err(invalid_arg_count_exec_err(func, (2, 2), arg_types.len())); - } - if !arg_types[0].is_integer() && !arg_types[0].is_null() { - return Err(unsupported_data_type_exec_err( - func, - "Integer Type", - &arg_types[0], - )); - } - if !arg_types[1].is_integer() && !arg_types[1].is_null() { - return Err(unsupported_data_type_exec_err( - func, - "Integer Type", - &arg_types[1], - )); + if value_array.data_type().is_null() || shift_array.data_type().is_null() { + return Ok(Arc::new(Int32Array::new_null(value_array.len()))); } - // Coerce smaller integer types to Int32 - let coerced_first = match &arg_types[0] { - DataType::Int8 | DataType::Int16 | DataType::Null => DataType::Int32, - DataType::UInt8 | DataType::UInt16 => DataType::UInt32, - _ => arg_types[0].clone(), - }; + let shift_array = shift_array.as_primitive::(); - Ok(vec![coerced_first, DataType::Int32]) -} - -#[derive(Debug, Hash, Eq, PartialEq)] -pub struct SparkShiftLeft { - signature: Signature, -} - -impl Default for SparkShiftLeft { - fn default() -> Self { - Self::new() - } -} - -impl SparkShiftLeft { - pub fn new() -> Self { - Self { - signature: Signature::user_defined(Volatility::Immutable), - } - } -} - -impl BitShiftUDF for SparkShiftLeft { - fn shift( - &self, + fn shift( value: &PrimitiveArray, - shift: &PrimitiveArray, + shift: &Int32Array, + bit_shift_type: BitShiftType, ) -> Result> where - T::Native: ArrowNativeType - + std::ops::Shl + T: ArrowPrimitiveType, + T::Native: std::ops::Shl + std::ops::Shr - + UShr, + + UShr, { - shift_left(value, shift) - } -} - -impl ScalarUDFImpl for SparkShiftLeft { - fn as_any(&self) -> &dyn Any { - self - } - - fn name(&self) -> &str { - "shiftleft" - } - - fn signature(&self) -> &Signature { - &self.signature - } - - fn coerce_types(&self, arg_types: &[DataType]) -> Result> { - bit_shift_coerce_types(arg_types, "shiftleft") - } - - fn return_type(&self, arg_types: &[DataType]) -> Result { - if arg_types.len() != 2 { - return plan_err!("shiftleft expects exactly 2 arguments"); + match bit_shift_type { + BitShiftType::Left => shift_left(value, shift), + BitShiftType::Right => shift_right(value, shift), + BitShiftType::RightUnsigned => shift_right_unsigned(value, shift), } - // Return type is the same as the first argument (the value to shift) - Ok(arg_types[0].clone()) } - fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { - if args.args.len() != 2 { - return plan_err!("shiftleft expects exactly 2 arguments"); + match value_array.data_type() { + DataType::Int32 => { + let value_array = value_array.as_primitive::(); + Ok(Arc::new(shift(value_array, shift_array, bit_shift_type)?)) + } + DataType::Int64 => { + let value_array = value_array.as_primitive::(); + Ok(Arc::new(shift(value_array, shift_array, bit_shift_type)?)) + } + DataType::UInt32 => { + let value_array = value_array.as_primitive::(); + Ok(Arc::new(shift(value_array, shift_array, bit_shift_type)?)) + } + DataType::UInt64 => { + let value_array = value_array.as_primitive::(); + Ok(Arc::new(shift(value_array, shift_array, bit_shift_type)?)) + } + dt => { + internal_err!("{name} function does not support data type: {dt}") } - let inner = |arr: &[ArrayRef]| -> Result { self.spark_shift(arr) }; - make_scalar_function(inner, vec![])(&args.args) } } -#[derive(Debug, Hash, Eq, PartialEq)] -pub struct SparkShiftRightUnsigned { - signature: Signature, +#[derive(Debug, Hash, Copy, Clone, Eq, PartialEq)] +enum BitShiftType { + Left, + Right, + RightUnsigned, } -impl Default for SparkShiftRightUnsigned { - fn default() -> Self { - Self::new() - } +#[derive(Debug, Hash, Eq, PartialEq)] +pub struct SparkBitShift { + signature: Signature, + name: &'static str, + bit_shift_type: BitShiftType, } -impl SparkShiftRightUnsigned { - pub fn new() -> Self { +impl SparkBitShift { + fn new(name: &'static str, bit_shift_type: BitShiftType) -> Self { + let shift_amount = Coercion::new_implicit( + TypeSignatureClass::Native(logical_int32()), + vec![TypeSignatureClass::Integer], + NativeType::Int32, + ); Self { - signature: Signature::user_defined(Volatility::Immutable), + signature: Signature::one_of( + vec![ + // Upcast small ints to 32bit + TypeSignature::Coercible(vec![ + Coercion::new_implicit( + TypeSignatureClass::Native(logical_int32()), + vec![ + TypeSignatureClass::Native(logical_int8()), + TypeSignatureClass::Native(logical_int16()), + ], + NativeType::Int32, + ), + shift_amount.clone(), + ]), + TypeSignature::Coercible(vec![ + Coercion::new_implicit( + TypeSignatureClass::Native(logical_uint32()), + vec![ + TypeSignatureClass::Native(logical_uint8()), + TypeSignatureClass::Native(logical_uint16()), + ], + NativeType::UInt32, + ), + shift_amount.clone(), + ]), + // Otherwise accept direct 64 bit integers + TypeSignature::Coercible(vec![ + Coercion::new_exact(TypeSignatureClass::Native(logical_int64())), + shift_amount.clone(), + ]), + TypeSignature::Coercible(vec![ + Coercion::new_exact(TypeSignatureClass::Native(logical_uint64())), + shift_amount.clone(), + ]), + ], + Volatility::Immutable, + ), + name, + bit_shift_type, } } -} -impl BitShiftUDF for SparkShiftRightUnsigned { - fn shift( - &self, - value: &PrimitiveArray, - shift: &PrimitiveArray, - ) -> Result> - where - T::Native: ArrowNativeType - + std::ops::Shl - + std::ops::Shr - + UShr, - { - shift_right_unsigned(value, shift) + pub fn left() -> Self { + Self::new("shiftleft", BitShiftType::Left) } -} -impl ScalarUDFImpl for SparkShiftRightUnsigned { - fn as_any(&self) -> &dyn Any { - self + pub fn right() -> Self { + Self::new("shiftright", BitShiftType::Right) } - fn name(&self) -> &str { - "shiftrightunsigned" + pub fn right_unsigned() -> Self { + Self::new("shiftrightunsigned", BitShiftType::RightUnsigned) } - - fn signature(&self) -> &Signature { - &self.signature - } - - fn coerce_types(&self, arg_types: &[DataType]) -> Result> { - bit_shift_coerce_types(arg_types, "shiftrightunsigned") - } - - fn return_type(&self, arg_types: &[DataType]) -> Result { - if arg_types.len() != 2 { - return plan_err!("shiftrightunsigned expects exactly 2 arguments"); - } - // Return type is the same as the first argument (the value to shift) - Ok(arg_types[0].clone()) - } - - fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { - if args.args.len() != 2 { - return plan_err!("shiftrightunsigned expects exactly 2 arguments"); - } - let inner = |arr: &[ArrayRef]| -> Result { self.spark_shift(arr) }; - make_scalar_function(inner, vec![])(&args.args) - } -} - -#[derive(Debug, Hash, Eq, PartialEq)] -pub struct SparkShiftRight { - signature: Signature, } -impl Default for SparkShiftRight { - fn default() -> Self { - Self::new() - } -} - -impl SparkShiftRight { - pub fn new() -> Self { - Self { - signature: Signature::user_defined(Volatility::Immutable), - } - } -} - -impl BitShiftUDF for SparkShiftRight { - fn shift( - &self, - value: &PrimitiveArray, - shift: &PrimitiveArray, - ) -> Result> - where - T::Native: ArrowNativeType - + std::ops::Shl - + std::ops::Shr - + UShr, - { - shift_right(value, shift) - } -} - -impl ScalarUDFImpl for SparkShiftRight { +impl ScalarUDFImpl for SparkBitShift { fn as_any(&self) -> &dyn Any { self } fn name(&self) -> &str { - "shiftright" + self.name } fn signature(&self) -> &Signature { &self.signature } - fn coerce_types(&self, arg_types: &[DataType]) -> Result> { - bit_shift_coerce_types(arg_types, "shiftright") - } - fn return_type(&self, arg_types: &[DataType]) -> Result { - if arg_types.len() != 2 { - return plan_err!("shiftright expects exactly 2 arguments"); + if arg_types[0].is_null() { + Ok(DataType::Int32) + } else { + Ok(arg_types[0].clone()) } - // Return type is the same as the first argument (the value to shift) - Ok(arg_types[0].clone()) } fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { - if args.args.len() != 2 { - return plan_err!("shiftright expects exactly 2 arguments"); - } - let inner = |arr: &[ArrayRef]| -> Result { self.spark_shift(arr) }; + let inner = |arr: &[ArrayRef]| -> Result { + shift_inner(arr, self.name(), self.bit_shift_type) + }; make_scalar_function(inner, vec![])(&args.args) } } - -#[cfg(test)] -mod tests { - use super::*; - use arrow::array::{Array, Int32Array, Int64Array, UInt32Array, UInt64Array}; - - #[test] - fn test_shift_right_unsigned_int32() { - let value_array = Arc::new(Int32Array::from(vec![4, 8, 16, 32])); - let shift_array = Arc::new(Int32Array::from(vec![1, 2, 3, 4])); - let result = SparkShiftRightUnsigned::new() - .spark_shift(&[value_array, shift_array]) - .unwrap(); - let arr = result.as_primitive::(); - assert_eq!(arr.value(0), 2); // 4 >>> 1 = 2 - assert_eq!(arr.value(1), 2); // 8 >>> 2 = 2 - assert_eq!(arr.value(2), 2); // 16 >>> 3 = 2 - assert_eq!(arr.value(3), 2); // 32 >>> 4 = 2 - } - - #[test] - fn test_shift_right_unsigned_int64() { - let value_array = Arc::new(Int64Array::from(vec![4i64, 8, 16])); - let shift_array = Arc::new(Int32Array::from(vec![1, 2, 3])); - let result = SparkShiftRightUnsigned::new() - .spark_shift(&[value_array, shift_array]) - .unwrap(); - let arr = result.as_primitive::(); - assert_eq!(arr.value(0), 2); // 4 >>> 1 = 2 - assert_eq!(arr.value(1), 2); // 8 >>> 2 = 2 - assert_eq!(arr.value(2), 2); // 16 >>> 3 = 2 - } - - #[test] - fn test_shift_right_unsigned_uint32() { - let value_array = Arc::new(UInt32Array::from(vec![4u32, 8, 16])); - let shift_array = Arc::new(Int32Array::from(vec![1, 2, 3])); - let result = SparkShiftRightUnsigned::new() - .spark_shift(&[value_array, shift_array]) - .unwrap(); - let arr = result.as_primitive::(); - assert_eq!(arr.value(0), 2); // 4 >>> 1 = 2 - assert_eq!(arr.value(1), 2); // 8 >>> 2 = 2 - assert_eq!(arr.value(2), 2); // 16 >>> 3 = 2 - } - - #[test] - fn test_shift_right_unsigned_uint64() { - let value_array = Arc::new(UInt64Array::from(vec![4u64, 8, 16])); - let shift_array = Arc::new(Int32Array::from(vec![1, 2, 3])); - let result = SparkShiftRightUnsigned::new() - .spark_shift(&[value_array, shift_array]) - .unwrap(); - let arr = result.as_primitive::(); - assert_eq!(arr.value(0), 2); // 4 >>> 1 = 2 - assert_eq!(arr.value(1), 2); // 8 >>> 2 = 2 - assert_eq!(arr.value(2), 2); // 16 >>> 3 = 2 - } - - #[test] - fn test_shift_right_unsigned_nulls() { - let value_array = Arc::new(Int32Array::from(vec![Some(4), None, Some(8)])); - let shift_array = Arc::new(Int32Array::from(vec![Some(1), Some(2), None])); - let result = SparkShiftRightUnsigned::new() - .spark_shift(&[value_array, shift_array]) - .unwrap(); - let arr = result.as_primitive::(); - assert_eq!(arr.value(0), 2); // 4 >>> 1 = 2 - assert!(arr.is_null(1)); // null >>> 2 = null - assert!(arr.is_null(2)); // 8 >>> null = null - } - - #[test] - fn test_shift_right_unsigned_negative_shift() { - let value_array = Arc::new(Int32Array::from(vec![4, 8, 16])); - let shift_array = Arc::new(Int32Array::from(vec![-1, -2, -3])); - let result = SparkShiftRightUnsigned::new() - .spark_shift(&[value_array, shift_array]) - .unwrap(); - let arr = result.as_primitive::(); - assert_eq!(arr.value(0), 0); // 4 >>> -1 = 0 - assert_eq!(arr.value(1), 0); // 8 >>> -2 = 0 - assert_eq!(arr.value(2), 0); // 16 >>> -3 = 0 - } - - #[test] - fn test_shift_right_unsigned_negative_values() { - let value_array = Arc::new(Int32Array::from(vec![-4, -8, -16])); - let shift_array = Arc::new(Int32Array::from(vec![1, 2, 3])); - let result = SparkShiftRightUnsigned::new() - .spark_shift(&[value_array, shift_array]) - .unwrap(); - let arr = result.as_primitive::(); - // For unsigned right shift, negative values are treated as large positive values - // -4 as u32 = 4294967292, -4 >>> 1 = 2147483646 - assert_eq!(arr.value(0), 2147483646); - // -8 as u32 = 4294967288, -8 >>> 2 = 1073741822 - assert_eq!(arr.value(1), 1073741822); - // -16 as u32 = 4294967280, -16 >>> 3 = 536870910 - assert_eq!(arr.value(2), 536870910); - } - - #[test] - fn test_shift_right_int32() { - let value_array = Arc::new(Int32Array::from(vec![4, 8, 16, 32])); - let shift_array = Arc::new(Int32Array::from(vec![1, 2, 3, 4])); - let result = SparkShiftRight::new() - .spark_shift(&[value_array, shift_array]) - .unwrap(); - let arr = result.as_primitive::(); - assert_eq!(arr.value(0), 2); // 4 >> 1 = 2 - assert_eq!(arr.value(1), 2); // 8 >> 2 = 2 - assert_eq!(arr.value(2), 2); // 16 >> 3 = 2 - assert_eq!(arr.value(3), 2); // 32 >> 4 = 2 - } - - #[test] - fn test_shift_right_int64() { - let value_array = Arc::new(Int64Array::from(vec![4i64, 8, 16])); - let shift_array = Arc::new(Int32Array::from(vec![1, 2, 3])); - let result = SparkShiftRight::new() - .spark_shift(&[value_array, shift_array]) - .unwrap(); - let arr = result.as_primitive::(); - assert_eq!(arr.value(0), 2); // 4 >> 1 = 2 - assert_eq!(arr.value(1), 2); // 8 >> 2 = 2 - assert_eq!(arr.value(2), 2); // 16 >> 3 = 2 - } - - #[test] - fn test_shift_right_uint32() { - let value_array = Arc::new(UInt32Array::from(vec![4u32, 8, 16])); - let shift_array = Arc::new(Int32Array::from(vec![1, 2, 3])); - let result = SparkShiftRight::new() - .spark_shift(&[value_array, shift_array]) - .unwrap(); - let arr = result.as_primitive::(); - assert_eq!(arr.value(0), 2); // 4 >> 1 = 2 - assert_eq!(arr.value(1), 2); // 8 >> 2 = 2 - assert_eq!(arr.value(2), 2); // 16 >> 3 = 2 - } - - #[test] - fn test_shift_right_uint64() { - let value_array = Arc::new(UInt64Array::from(vec![4u64, 8, 16])); - let shift_array = Arc::new(Int32Array::from(vec![1, 2, 3])); - let result = SparkShiftRight::new() - .spark_shift(&[value_array, shift_array]) - .unwrap(); - let arr = result.as_primitive::(); - assert_eq!(arr.value(0), 2); // 4 >> 1 = 2 - assert_eq!(arr.value(1), 2); // 8 >> 2 = 2 - assert_eq!(arr.value(2), 2); // 16 >> 3 = 2 - } - - #[test] - fn test_shift_right_nulls() { - let value_array = Arc::new(Int32Array::from(vec![Some(4), None, Some(8)])); - let shift_array = Arc::new(Int32Array::from(vec![Some(1), Some(2), None])); - let result = SparkShiftRight::new() - .spark_shift(&[value_array, shift_array]) - .unwrap(); - let arr = result.as_primitive::(); - assert_eq!(arr.value(0), 2); // 4 >> 1 = 2 - assert!(arr.is_null(1)); // null >> 2 = null - assert!(arr.is_null(2)); // 8 >> null = null - } - - #[test] - fn test_shift_right_large_shift() { - let value_array = Arc::new(Int32Array::from(vec![1, 2, 3])); - let shift_array = Arc::new(Int32Array::from(vec![32, 33, 64])); - let result = SparkShiftRight::new() - .spark_shift(&[value_array, shift_array]) - .unwrap(); - let arr = result.as_primitive::(); - assert_eq!(arr.value(0), 1); // 1 >> 32 = 1 - assert_eq!(arr.value(1), 1); // 2 >> 33 = 1 - assert_eq!(arr.value(2), 3); // 3 >> 64 = 3 - } - - #[test] - fn test_shift_right_negative_shift() { - let value_array = Arc::new(Int32Array::from(vec![4, 8, 16])); - let shift_array = Arc::new(Int32Array::from(vec![-1, -2, -3])); - let result = SparkShiftRight::new() - .spark_shift(&[value_array, shift_array]) - .unwrap(); - let arr = result.as_primitive::(); - assert_eq!(arr.value(0), 0); // 4 >> -1 = 0 - assert_eq!(arr.value(1), 0); // 8 >> -2 = 0 - assert_eq!(arr.value(2), 0); // 16 >> -3 = 0 - } - - #[test] - fn test_shift_right_negative_values() { - let value_array = Arc::new(Int32Array::from(vec![-4, -8, -16])); - let shift_array = Arc::new(Int32Array::from(vec![1, 2, 3])); - let result = SparkShiftRight::new() - .spark_shift(&[value_array, shift_array]) - .unwrap(); - let arr = result.as_primitive::(); - // For signed integers, right shift preserves the sign bit - assert_eq!(arr.value(0), -2); // -4 >> 1 = -2 - assert_eq!(arr.value(1), -2); // -8 >> 2 = -2 - assert_eq!(arr.value(2), -2); // -16 >> 3 = -2 - } - - #[test] - fn test_shift_left_int32() { - let value_array = Arc::new(Int32Array::from(vec![1, 2, 3, 4])); - let shift_array = Arc::new(Int32Array::from(vec![1, 2, 3, 4])); - let result = SparkShiftLeft::new() - .spark_shift(&[value_array, shift_array]) - .unwrap(); - let arr = result.as_primitive::(); - assert_eq!(arr.value(0), 2); // 1 << 1 = 2 - assert_eq!(arr.value(1), 8); // 2 << 2 = 8 - assert_eq!(arr.value(2), 24); // 3 << 3 = 24 - assert_eq!(arr.value(3), 64); // 4 << 4 = 64 - } - - #[test] - fn test_shift_left_int64() { - let value_array = Arc::new(Int64Array::from(vec![1i64, 2, 3])); - let shift_array = Arc::new(Int32Array::from(vec![1, 2, 3])); - let result = SparkShiftLeft::new() - .spark_shift(&[value_array, shift_array]) - .unwrap(); - let arr = result.as_primitive::(); - assert_eq!(arr.value(0), 2); // 1 << 1 = 2 - assert_eq!(arr.value(1), 8); // 2 << 2 = 8 - assert_eq!(arr.value(2), 24); // 3 << 3 = 24 - } - - #[test] - fn test_shift_left_uint32() { - let value_array = Arc::new(UInt32Array::from(vec![1u32, 2, 3])); - let shift_array = Arc::new(Int32Array::from(vec![1, 2, 3])); - let result = SparkShiftLeft::new() - .spark_shift(&[value_array, shift_array]) - .unwrap(); - let arr = result.as_primitive::(); - assert_eq!(arr.value(0), 2); // 1 << 1 = 2 - assert_eq!(arr.value(1), 8); // 2 << 2 = 8 - assert_eq!(arr.value(2), 24); // 3 << 3 = 24 - } - - #[test] - fn test_shift_left_uint64() { - let value_array = Arc::new(UInt64Array::from(vec![1u64, 2, 3])); - let shift_array = Arc::new(Int32Array::from(vec![1, 2, 3])); - let result = SparkShiftLeft::new() - .spark_shift(&[value_array, shift_array]) - .unwrap(); - let arr = result.as_primitive::(); - assert_eq!(arr.value(0), 2); // 1 << 1 = 2 - assert_eq!(arr.value(1), 8); // 2 << 2 = 8 - assert_eq!(arr.value(2), 24); // 3 << 3 = 24 - } - - #[test] - fn test_shift_left_nulls() { - let value_array = Arc::new(Int32Array::from(vec![Some(2), None, Some(3)])); - let shift_array = Arc::new(Int32Array::from(vec![Some(1), Some(2), None])); - let result = SparkShiftLeft::new() - .spark_shift(&[value_array, shift_array]) - .unwrap(); - let arr = result.as_primitive::(); - assert_eq!(arr.value(0), 4); // 2 << 1 = 4 - assert!(arr.is_null(1)); // null << 2 = null - assert!(arr.is_null(2)); // 3 << null = null - } - - #[test] - fn test_shift_left_large_shift() { - let value_array = Arc::new(Int32Array::from(vec![1, 2, 3])); - let shift_array = Arc::new(Int32Array::from(vec![32, 33, 64])); - let result = SparkShiftLeft::new() - .spark_shift(&[value_array, shift_array]) - .unwrap(); - let arr = result.as_primitive::(); - assert_eq!(arr.value(0), 1); // 1 << 32 = 0 (overflow) - assert_eq!(arr.value(1), 4); // 2 << 33 = 0 (overflow) - assert_eq!(arr.value(2), 3); // 3 << 64 = 0 (overflow) - } - - #[test] - fn test_shift_left_negative_shift() { - let value_array = Arc::new(Int32Array::from(vec![4, 8, 16])); - let shift_array = Arc::new(Int32Array::from(vec![-1, -2, -3])); - let result = SparkShiftLeft::new() - .spark_shift(&[value_array, shift_array]) - .unwrap(); - let arr = result.as_primitive::(); - assert_eq!(arr.value(0), 0); // 4 << -1 = 0 - assert_eq!(arr.value(1), 0); // 8 << -2 = 0 - assert_eq!(arr.value(2), 0); // 16 << -3 = 0 - } -} diff --git a/datafusion/spark/src/function/bitwise/mod.rs b/datafusion/spark/src/function/bitwise/mod.rs index d729a3ddd09a1..769ecf5c2fef5 100644 --- a/datafusion/spark/src/function/bitwise/mod.rs +++ b/datafusion/spark/src/function/bitwise/mod.rs @@ -24,9 +24,21 @@ use datafusion_expr::ScalarUDF; use datafusion_functions::make_udf_function; use std::sync::Arc; -make_udf_function!(bit_shift::SparkShiftLeft, shiftleft); -make_udf_function!(bit_shift::SparkShiftRight, shiftright); -make_udf_function!(bit_shift::SparkShiftRightUnsigned, shiftrightunsigned); +make_udf_function!( + bit_shift::SparkBitShift, + shiftleft, + bit_shift::SparkBitShift::left +); +make_udf_function!( + bit_shift::SparkBitShift, + shiftright, + bit_shift::SparkBitShift::right +); +make_udf_function!( + bit_shift::SparkBitShift, + shiftrightunsigned, + bit_shift::SparkBitShift::right_unsigned +); make_udf_function!(bit_get::SparkBitGet, bit_get); make_udf_function!(bit_count::SparkBitCount, bit_count); make_udf_function!(bitwise_not::SparkBitwiseNot, bitwise_not); diff --git a/datafusion/sqllogictest/test_files/spark/bitwise/shiftright.slt b/datafusion/sqllogictest/test_files/spark/bitwise/shiftright.slt index 3587bcc7ca52b..04edf641e9137 100644 --- a/datafusion/sqllogictest/test_files/spark/bitwise/shiftright.slt +++ b/datafusion/sqllogictest/test_files/spark/bitwise/shiftright.slt @@ -145,3 +145,102 @@ query I select shiftright(3::int,-32); ---- 3 + +# i32 + nulls +query IT +SELECT + shiftright(arrow_cast(value, 'Int32'), shift), + arrow_typeof(shiftright(arrow_cast(value, 'Int32'), shift)) +FROM VALUES +(4, 1), +(8, 2), +(16, 3), +(32, 4), +(null, 2), +(8, null) +t(value, shift) +---- +2 Int32 +2 Int32 +2 Int32 +2 Int32 +NULL Int32 +NULL Int32 + +# big shifts +query IT +SELECT + shiftright(arrow_cast(value, 'Int32'), shift), + arrow_typeof(shiftright(arrow_cast(value, 'Int32'), shift)) +FROM VALUES +(1, 32), +(2, 33), +(3, 64) +t(value, shift) +---- +1 Int32 +1 Int32 +3 Int32 + +# negative shift +query IT +SELECT + shiftright(arrow_cast(value, 'Int32'), shift), + arrow_typeof(shiftright(arrow_cast(value, 'Int32'), shift)) +FROM VALUES +(4, -1), +(8, -2), +(16, -3) +t(value, shift) +---- +0 Int32 +0 Int32 +0 Int32 + +# For signed integers, right shift preserves sign bit +query IT +SELECT + shiftright(arrow_cast(value, 'Int32'), shift), + arrow_typeof(shiftright(arrow_cast(value, 'Int32'), shift)) +FROM VALUES +(-4, 1), +(-8, 2), +(-16, 3) +t(value, shift) +---- +-2 Int32 +-2 Int32 +-2 Int32 + +# i64 value +query IT +SELECT + shiftright(arrow_cast(value, 'Int64'), shift), + arrow_typeof(shiftright(arrow_cast(value, 'Int64'), shift)) +FROM VALUES (4, 1), (8, 2), (16, 3) t(value, shift) +---- +2 Int64 +2 Int64 +2 Int64 + +# u32 value +query IT +SELECT + shiftright(arrow_cast(value, 'UInt32'), shift), + arrow_typeof(shiftright(arrow_cast(value, 'UInt32'), shift)) +FROM VALUES (4, 1), (8, 2), (16, 3) t(value, shift) +---- +2 UInt32 +2 UInt32 +2 UInt32 + +# u64 value +query IT +SELECT + shiftright(arrow_cast(value, 'UInt64'), shift), + arrow_typeof(shiftright(arrow_cast(value, 'UInt64'), shift)) +FROM VALUES (4, 1), (8, 2), (16, 3) t(value, shift) +---- +2 UInt64 +2 UInt64 +2 UInt64 diff --git a/datafusion/sqllogictest/test_files/spark/bitwise/shiftrightunsigned.slt b/datafusion/sqllogictest/test_files/spark/bitwise/shiftrightunsigned.slt index b0d4cfaec7021..80307ed0ea84e 100644 --- a/datafusion/sqllogictest/test_files/spark/bitwise/shiftrightunsigned.slt +++ b/datafusion/sqllogictest/test_files/spark/bitwise/shiftrightunsigned.slt @@ -146,3 +146,90 @@ query I select shiftrightunsigned(3::int,-32); ---- 3 + +# i32 + nulls +query IT +SELECT + shiftrightunsigned(arrow_cast(value, 'Int32'), shift), + arrow_typeof(shiftrightunsigned(arrow_cast(value, 'Int32'), shift)) +FROM VALUES +(4, 1), +(8, 2), +(16, 3), +(32, 4), +(null, 2), +(8, null) +t(value, shift) +---- +2 Int32 +2 Int32 +2 Int32 +2 Int32 +NULL Int32 +NULL Int32 + +# negative shift +query IT +SELECT + shiftrightunsigned(arrow_cast(value, 'Int32'), shift), + arrow_typeof(shiftrightunsigned(arrow_cast(value, 'Int32'), shift)) +FROM VALUES +(4, -1), +(8, -2), +(16, -3) +t(value, shift) +---- +0 Int32 +0 Int32 +0 Int32 + +# Negative values are treated as large positive values +# -4 as u32 = 4294967292, 4294967292 >>> 1 = 2147483646 +# -8 as u32 = 4294967288, 4294967288 >>> 2 = 1073741822 +# -16 as u32 = 4294967280, 4294967280 >>> 3 = 536870910 +query IT +SELECT + shiftrightunsigned(arrow_cast(value, 'Int32'), shift), + arrow_typeof(shiftrightunsigned(arrow_cast(value, 'Int32'), shift)) +FROM VALUES +(-4, 1), +(-8, 2), +(-16, 3) +t(value, shift) +---- +2147483646 Int32 +1073741822 Int32 +536870910 Int32 + +# i64 value +query IT +SELECT + shiftrightunsigned(arrow_cast(value, 'Int64'), shift), + arrow_typeof(shiftrightunsigned(arrow_cast(value, 'Int64'), shift)) +FROM VALUES (4, 1), (8, 2), (16, 3) t(value, shift) +---- +2 Int64 +2 Int64 +2 Int64 + +# u32 value +query IT +SELECT + shiftrightunsigned(arrow_cast(value, 'UInt32'), shift), + arrow_typeof(shiftrightunsigned(arrow_cast(value, 'UInt32'), shift)) +FROM VALUES (4, 1), (8, 2), (16, 3) t(value, shift) +---- +2 UInt32 +2 UInt32 +2 UInt32 + +# u64 value +query IT +SELECT + shiftrightunsigned(arrow_cast(value, 'UInt64'), shift), + arrow_typeof(shiftrightunsigned(arrow_cast(value, 'UInt64'), shift)) +FROM VALUES (4, 1), (8, 2), (16, 3) t(value, shift) +---- +2 UInt64 +2 UInt64 +2 UInt64 diff --git a/datafusion/sqllogictest/test_files/spark/math/shiftleft.slt b/datafusion/sqllogictest/test_files/spark/math/shiftleft.slt index 3676e4c18153c..fc51a3a8cbd88 100644 --- a/datafusion/sqllogictest/test_files/spark/math/shiftleft.slt +++ b/datafusion/sqllogictest/test_files/spark/math/shiftleft.slt @@ -144,3 +144,87 @@ query I select shiftleft(3::int,-32); ---- 3 + +# i32 + nulls +query IT +SELECT + shiftleft(arrow_cast(value, 'Int32'), shift), + arrow_typeof(shiftleft(arrow_cast(value, 'Int32'), shift)) +FROM VALUES +(1, 1), +(2, 2), +(3, 3), +(4, 4), +(null, 2), +(8, null) +t(value, shift) +---- +2 Int32 +8 Int32 +24 Int32 +64 Int32 +NULL Int32 +NULL Int32 + +# big shifts +query IT +SELECT + shiftleft(arrow_cast(value, 'Int32'), shift), + arrow_typeof(shiftleft(arrow_cast(value, 'Int32'), shift)) +FROM VALUES +(1, 32), +(2, 33), +(3, 64) +t(value, shift) +---- +1 Int32 +4 Int32 +3 Int32 + +# negative shift +query IT +SELECT + shiftleft(arrow_cast(value, 'Int32'), shift), + arrow_typeof(shiftleft(arrow_cast(value, 'Int32'), shift)) +FROM VALUES +(4, -1), +(8, -2), +(16, -3) +t(value, shift) +---- +0 Int32 +0 Int32 +0 Int32 + +# i64 value +query IT +SELECT + shiftleft(arrow_cast(value, 'Int64'), shift), + arrow_typeof(shiftleft(arrow_cast(value, 'Int64'), shift)) +FROM VALUES (1, 1), (2, 2), (3, 3) t(value, shift) +---- +2 Int64 +8 Int64 +24 Int64 + +# u32 value +query IT +SELECT + shiftleft(arrow_cast(value, 'UInt32'), shift), + arrow_typeof(shiftleft(arrow_cast(value, 'UInt32'), shift)) +FROM VALUES (1, 1), (2, 2), (3, 3) t(value, shift) +---- +2 UInt32 +8 UInt32 +24 UInt32 + +# u64 value +query IT +SELECT + shiftleft(arrow_cast(value, 'UInt64'), shift), + arrow_typeof(shiftleft(arrow_cast(value, 'UInt64'), shift)) +FROM VALUES (1, 1), (2, 2), (3, 3) t(value, shift) +---- +2 UInt64 +8 UInt64 +24 UInt64 From 768a65759923b6754a653624cb5e912efdb15d2d Mon Sep 17 00:00:00 2001 From: Jefffrey Date: Thu, 13 Nov 2025 09:57:15 +0900 Subject: [PATCH 2/2] Fix null handling --- .../spark/src/function/bitwise/bit_shift.rs | 11 +---------- .../test_files/spark/bitwise/shiftright.slt | 16 ++++++++++++++++ .../spark/bitwise/shiftrightunsigned.slt | 16 ++++++++++++++++ .../test_files/spark/math/shiftleft.slt | 16 ++++++++++++++++ 4 files changed, 49 insertions(+), 10 deletions(-) diff --git a/datafusion/spark/src/function/bitwise/bit_shift.rs b/datafusion/spark/src/function/bitwise/bit_shift.rs index cb0afd8c26aa4..65df04858077f 100644 --- a/datafusion/spark/src/function/bitwise/bit_shift.rs +++ b/datafusion/spark/src/function/bitwise/bit_shift.rs @@ -142,11 +142,6 @@ fn shift_inner( bit_shift_type: BitShiftType, ) -> Result { let [value_array, shift_array] = take_function_args(name, arrays)?; - - if value_array.data_type().is_null() || shift_array.data_type().is_null() { - return Ok(Arc::new(Int32Array::new_null(value_array.len()))); - } - let shift_array = shift_array.as_primitive::(); fn shift( @@ -281,11 +276,7 @@ impl ScalarUDFImpl for SparkBitShift { } fn return_type(&self, arg_types: &[DataType]) -> Result { - if arg_types[0].is_null() { - Ok(DataType::Int32) - } else { - Ok(arg_types[0].clone()) - } + Ok(arg_types[0].clone()) } fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { diff --git a/datafusion/sqllogictest/test_files/spark/bitwise/shiftright.slt b/datafusion/sqllogictest/test_files/spark/bitwise/shiftright.slt index 04edf641e9137..3dd43509b1769 100644 --- a/datafusion/sqllogictest/test_files/spark/bitwise/shiftright.slt +++ b/datafusion/sqllogictest/test_files/spark/bitwise/shiftright.slt @@ -244,3 +244,19 @@ FROM VALUES (4, 1), (8, 2), (16, 3) t(value, shift) 2 UInt64 2 UInt64 2 UInt64 + +# pure null handling +query IT +SELECT shiftright(null, 1), arrow_typeof(shiftright(null, 1)); +---- +NULL Int32 + +query IT +SELECT shiftright(null, null), arrow_typeof(shiftright(null, null)); +---- +NULL Int32 + +query IT +SELECT shiftright(1::bigint, null), arrow_typeof(shiftright(1::bigint, null)); +---- +NULL Int64 diff --git a/datafusion/sqllogictest/test_files/spark/bitwise/shiftrightunsigned.slt b/datafusion/sqllogictest/test_files/spark/bitwise/shiftrightunsigned.slt index 80307ed0ea84e..b9ef530b36238 100644 --- a/datafusion/sqllogictest/test_files/spark/bitwise/shiftrightunsigned.slt +++ b/datafusion/sqllogictest/test_files/spark/bitwise/shiftrightunsigned.slt @@ -233,3 +233,19 @@ FROM VALUES (4, 1), (8, 2), (16, 3) t(value, shift) 2 UInt64 2 UInt64 2 UInt64 + +# pure null handling +query IT +SELECT shiftrightunsigned(null, 1), arrow_typeof(shiftrightunsigned(null, 1)); +---- +NULL Int32 + +query IT +SELECT shiftrightunsigned(null, null), arrow_typeof(shiftrightunsigned(null, null)); +---- +NULL Int32 + +query IT +SELECT shiftrightunsigned(1::bigint, null), arrow_typeof(shiftrightunsigned(1::bigint, null)); +---- +NULL Int64 diff --git a/datafusion/sqllogictest/test_files/spark/math/shiftleft.slt b/datafusion/sqllogictest/test_files/spark/math/shiftleft.slt index fc51a3a8cbd88..c8ddeb6740871 100644 --- a/datafusion/sqllogictest/test_files/spark/math/shiftleft.slt +++ b/datafusion/sqllogictest/test_files/spark/math/shiftleft.slt @@ -228,3 +228,19 @@ FROM VALUES (1, 1), (2, 2), (3, 3) t(value, shift) 2 UInt64 8 UInt64 24 UInt64 + +# pure null handling +query IT +SELECT shiftleft(null, 1), arrow_typeof(shiftleft(null, 1)); +---- +NULL Int32 + +query IT +SELECT shiftleft(null, null), arrow_typeof(shiftleft(null, null)); +---- +NULL Int32 + +query IT +SELECT shiftleft(1::bigint, null), arrow_typeof(shiftleft(1::bigint, null)); +---- +NULL Int64