From 725ac1e5f0fa83e1d691a032e2079d7082c9a4c6 Mon Sep 17 00:00:00 2001 From: sriram Date: Thu, 4 Dec 2025 15:58:49 +0530 Subject: [PATCH 1/3] cleaner commit --- .../src/function/datetime/make_dt_interval.rs | 57 +++++++++++-------- .../spark/datetime/make_dt_interval.slt | 6 +- 2 files changed, 36 insertions(+), 27 deletions(-) diff --git a/datafusion/spark/src/function/datetime/make_dt_interval.rs b/datafusion/spark/src/function/datetime/make_dt_interval.rs index bbfba44861344..31155a8d05943 100644 --- a/datafusion/spark/src/function/datetime/make_dt_interval.rs +++ b/datafusion/spark/src/function/datetime/make_dt_interval.rs @@ -23,11 +23,11 @@ use arrow::array::{ }; use arrow::datatypes::TimeUnit::Microsecond; use arrow::datatypes::{DataType, Float64Type, Int32Type}; -use datafusion_common::{ - exec_err, plan_datafusion_err, DataFusionError, Result, ScalarValue, -}; +use datafusion_common::types::{logical_float64, logical_int32, NativeType}; +use datafusion_common::{plan_datafusion_err, DataFusionError, Result, ScalarValue}; use datafusion_expr::{ - ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility, + Coercion, ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, TypeSignature, + TypeSignatureClass, Volatility, }; use datafusion_functions::utils::make_scalar_function; @@ -44,8 +44,36 @@ impl Default for SparkMakeDtInterval { impl SparkMakeDtInterval { pub fn new() -> Self { + let int32 = Coercion::new_implicit( + TypeSignatureClass::Native(logical_int32()), + vec![TypeSignatureClass::Integer], + NativeType::Int32, + ); + + let float64 = Coercion::new_implicit( + TypeSignatureClass::Native(logical_float64()), + vec![TypeSignatureClass::Numeric], + NativeType::Float64, + ); + + let variants = vec![ + // (days) + TypeSignature::Coercible(vec![int32.clone()]), + // (days, hours) + TypeSignature::Coercible(vec![int32.clone(), int32.clone()]), + // (days, hours, minutes) + TypeSignature::Coercible(vec![int32.clone(), int32.clone(), int32.clone()]), + // (days, hours, minutes, seconds) + TypeSignature::Coercible(vec![ + int32.clone(), + int32.clone(), + int32.clone(), + float64, + ]), + ]; + Self { - signature: Signature::user_defined(Volatility::Immutable), + signature: Signature::one_of(variants, Volatility::Immutable), } } } @@ -81,25 +109,6 @@ impl ScalarUDFImpl for SparkMakeDtInterval { } make_scalar_function(make_dt_interval_kernel, vec![])(&args.args) } - - fn coerce_types(&self, arg_types: &[DataType]) -> Result> { - if arg_types.len() > 4 { - return exec_err!( - "make_dt_interval expects between 0 and 4 arguments, got {}", - arg_types.len() - ); - } - - Ok((0..arg_types.len()) - .map(|i| { - if i == 3 { - DataType::Float64 - } else { - DataType::Int32 - } - }) - .collect()) - } } fn make_dt_interval_kernel(args: &[ArrayRef]) -> Result { diff --git a/datafusion/sqllogictest/test_files/spark/datetime/make_dt_interval.slt b/datafusion/sqllogictest/test_files/spark/datetime/make_dt_interval.slt index dc6c33caa9b4c..8b21b7af06b74 100644 --- a/datafusion/sqllogictest/test_files/spark/datetime/make_dt_interval.slt +++ b/datafusion/sqllogictest/test_files/spark/datetime/make_dt_interval.slt @@ -64,10 +64,10 @@ SELECT (make_dt_interval(0, 0, 0, null)) NULL # missing params -query ? +query error does not support zero arguments SELECT (make_dt_interval()) AS make_dt_interval ----- -0 days 0 hours 0 mins 0.000000 secs + + query ? SELECT (make_dt_interval(1)) AS make_dt_interval From 8991d8ba89275205aa7b7b49381eed163388b741 Mon Sep 17 00:00:00 2001 From: sriram Date: Thu, 4 Dec 2025 18:33:49 +0530 Subject: [PATCH 2/3] handle no args --- .../src/function/datetime/make_dt_interval.rs | 24 ++++++++++++++++++- .../spark/datetime/make_dt_interval.slt | 7 +++--- 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/datafusion/spark/src/function/datetime/make_dt_interval.rs b/datafusion/spark/src/function/datetime/make_dt_interval.rs index 31155a8d05943..dd67e72ba00d6 100644 --- a/datafusion/spark/src/function/datetime/make_dt_interval.rs +++ b/datafusion/spark/src/function/datetime/make_dt_interval.rs @@ -24,7 +24,9 @@ use arrow::array::{ use arrow::datatypes::TimeUnit::Microsecond; use arrow::datatypes::{DataType, Float64Type, Int32Type}; use datafusion_common::types::{logical_float64, logical_int32, NativeType}; -use datafusion_common::{plan_datafusion_err, DataFusionError, Result, ScalarValue}; +use datafusion_common::{ + exec_err, plan_datafusion_err, DataFusionError, Result, ScalarValue, +}; use datafusion_expr::{ Coercion, ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, TypeSignature, TypeSignatureClass, Volatility, @@ -57,6 +59,7 @@ impl SparkMakeDtInterval { ); let variants = vec![ + TypeSignature::Nullary, // (days) TypeSignature::Coercible(vec![int32.clone()]), // (days, hours) @@ -109,6 +112,25 @@ impl ScalarUDFImpl for SparkMakeDtInterval { } make_scalar_function(make_dt_interval_kernel, vec![])(&args.args) } + + fn coerce_types(&self, arg_types: &[DataType]) -> Result> { + if arg_types.len() > 4 { + return exec_err!( + "make_dt_interval expects between 0 and 4 arguments, got {}", + arg_types.len() + ); + } + + Ok((0..arg_types.len()) + .map(|i| { + if i == 3 { + DataType::Float64 + } else { + DataType::Int32 + } + }) + .collect()) + } } fn make_dt_interval_kernel(args: &[ArrayRef]) -> Result { diff --git a/datafusion/sqllogictest/test_files/spark/datetime/make_dt_interval.slt b/datafusion/sqllogictest/test_files/spark/datetime/make_dt_interval.slt index 8b21b7af06b74..1223b777d1d63 100644 --- a/datafusion/sqllogictest/test_files/spark/datetime/make_dt_interval.slt +++ b/datafusion/sqllogictest/test_files/spark/datetime/make_dt_interval.slt @@ -63,10 +63,11 @@ SELECT (make_dt_interval(0, 0, 0, null)) ---- NULL -# missing params -query error does not support zero arguments +# zero arguments - returns default zero duration +query ? SELECT (make_dt_interval()) AS make_dt_interval - +---- +0 days 0 hours 0 mins 0.000000 secs query ? From 97c7854b8461100dee1602164dd42de3eaf63d08 Mon Sep 17 00:00:00 2001 From: sriram Date: Thu, 4 Dec 2025 19:48:29 +0530 Subject: [PATCH 3/3] remove coerce_types fn and use invoke_wth_args for ut --- .../src/function/datetime/make_dt_interval.rs | 57 +++++++++---------- 1 file changed, 28 insertions(+), 29 deletions(-) diff --git a/datafusion/spark/src/function/datetime/make_dt_interval.rs b/datafusion/spark/src/function/datetime/make_dt_interval.rs index dd67e72ba00d6..cb8f6e1d792ba 100644 --- a/datafusion/spark/src/function/datetime/make_dt_interval.rs +++ b/datafusion/spark/src/function/datetime/make_dt_interval.rs @@ -24,9 +24,7 @@ use arrow::array::{ use arrow::datatypes::TimeUnit::Microsecond; use arrow::datatypes::{DataType, Float64Type, Int32Type}; use datafusion_common::types::{logical_float64, logical_int32, NativeType}; -use datafusion_common::{ - exec_err, plan_datafusion_err, DataFusionError, Result, ScalarValue, -}; +use datafusion_common::{plan_datafusion_err, DataFusionError, Result, ScalarValue}; use datafusion_expr::{ Coercion, ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, TypeSignature, TypeSignatureClass, Volatility, @@ -110,26 +108,13 @@ impl ScalarUDFImpl for SparkMakeDtInterval { Some(0), ))); } - make_scalar_function(make_dt_interval_kernel, vec![])(&args.args) - } - - fn coerce_types(&self, arg_types: &[DataType]) -> Result> { - if arg_types.len() > 4 { - return exec_err!( + if args.args.len() > 4 { + return Err(DataFusionError::Execution(format!( "make_dt_interval expects between 0 and 4 arguments, got {}", - arg_types.len() - ); + args.args.len() + ))); } - - Ok((0..arg_types.len()) - .map(|i| { - if i == 3 { - DataType::Float64 - } else { - DataType::Int32 - } - }) - .collect()) + make_scalar_function(make_dt_interval_kernel, vec![])(&args.args) } } @@ -496,19 +481,33 @@ mod tests { fn no_more_than_4_params() -> Result<()> { let udf = SparkMakeDtInterval::new(); - let arg_types = vec![ - DataType::Int32, - DataType::Int32, - DataType::Int32, - DataType::Float64, - DataType::Int32, + // Create args with 5 parameters (exceeds the limit of 4) + let args = vec![ + ColumnarValue::Scalar(ScalarValue::Int32(Some(1))), + ColumnarValue::Scalar(ScalarValue::Int32(Some(2))), + ColumnarValue::Scalar(ScalarValue::Int32(Some(3))), + ColumnarValue::Scalar(ScalarValue::Float64(Some(4.0))), + ColumnarValue::Scalar(ScalarValue::Int32(Some(5))), ]; - let res = udf.coerce_types(&arg_types); + let arg_fields = args + .iter() + .map(|arg| Field::new("a", arg.data_type(), true).into()) + .collect::>(); + + let func_args = ScalarFunctionArgs { + args, + arg_fields, + number_rows: 1, + return_field: Field::new("f", Duration(Microsecond), true).into(), + config_options: Arc::new(Default::default()), + }; + + let res = udf.invoke_with_args(func_args); assert!( matches!(res, Err(DataFusionError::Execution(_))), - "make_dt_interval should return execution error for too many arguments" + "make_dt_interval should return execution error for more than 4 arguments" ); Ok(())