diff --git a/datafusion/functions/src/datetime/to_local_time.rs b/datafusion/functions/src/datetime/to_local_time.rs index 6e0a150b0a35f..0495a4841fbf9 100644 --- a/datafusion/functions/src/datetime/to_local_time.rs +++ b/datafusion/functions/src/datetime/to_local_time.rs @@ -20,7 +20,7 @@ use std::ops::Add; use std::sync::Arc; use arrow::array::timezone::Tz; -use arrow::array::{Array, ArrayRef, PrimitiveBuilder}; +use arrow::array::{ArrayRef, PrimitiveBuilder}; use arrow::datatypes::DataType::Timestamp; use arrow::datatypes::TimeUnit::{Microsecond, Millisecond, Nanosecond, Second}; use arrow::datatypes::{ @@ -31,11 +31,12 @@ use chrono::{DateTime, MappedLocalTime, Offset, TimeDelta, TimeZone, Utc}; use datafusion_common::cast::as_primitive_array; use datafusion_common::{ - exec_err, internal_datafusion_err, plan_err, utils::take_function_args, Result, + exec_err, internal_datafusion_err, internal_err, utils::take_function_args, Result, ScalarValue, }; use datafusion_expr::{ - ColumnarValue, Documentation, ScalarUDFImpl, Signature, Volatility, + Coercion, ColumnarValue, Documentation, ScalarUDFImpl, Signature, TypeSignatureClass, + Volatility, }; use datafusion_macros::user_doc; @@ -111,133 +112,163 @@ impl Default for ToLocalTimeFunc { impl ToLocalTimeFunc { pub fn new() -> Self { Self { - signature: Signature::user_defined(Volatility::Immutable), + signature: Signature::coercible( + vec![Coercion::new_exact(TypeSignatureClass::Timestamp)], + Volatility::Immutable, + ), } } +} - fn to_local_time(&self, args: &[ColumnarValue]) -> Result { - let [time_value] = take_function_args(self.name(), args)?; +impl ScalarUDFImpl for ToLocalTimeFunc { + fn as_any(&self) -> &dyn Any { + self + } - let arg_type = time_value.data_type(); - match arg_type { - Timestamp(_, None) => { - // if no timezone specified, just return the input - Ok(time_value.clone()) - } - // If has timezone, adjust the underlying time value. The current time value - // is stored as i64 in UTC, even though the timezone may not be in UTC. Therefore, - // we need to adjust the time value to the local time. See [`adjust_to_local_time`] - // for more details. - // - // Then remove the timezone in return type, i.e. return None - Timestamp(_, Some(timezone)) => { - let tz: Tz = timezone.parse()?; - - match time_value { - ColumnarValue::Scalar(ScalarValue::TimestampNanosecond( - Some(ts), - Some(_), - )) => { - let adjusted_ts = - adjust_to_local_time::(*ts, tz)?; - Ok(ColumnarValue::Scalar(ScalarValue::TimestampNanosecond( - Some(adjusted_ts), - None, - ))) - } - ColumnarValue::Scalar(ScalarValue::TimestampMicrosecond( - Some(ts), - Some(_), - )) => { - let adjusted_ts = - adjust_to_local_time::(*ts, tz)?; - Ok(ColumnarValue::Scalar(ScalarValue::TimestampMicrosecond( - Some(adjusted_ts), - None, - ))) - } - ColumnarValue::Scalar(ScalarValue::TimestampMillisecond( - Some(ts), - Some(_), - )) => { - let adjusted_ts = - adjust_to_local_time::(*ts, tz)?; - Ok(ColumnarValue::Scalar(ScalarValue::TimestampMillisecond( - Some(adjusted_ts), - None, - ))) - } - ColumnarValue::Scalar(ScalarValue::TimestampSecond( - Some(ts), - Some(_), - )) => { - let adjusted_ts = - adjust_to_local_time::(*ts, tz)?; - Ok(ColumnarValue::Scalar(ScalarValue::TimestampSecond( - Some(adjusted_ts), - None, - ))) - } - ColumnarValue::Array(array) => { - fn transform_array( - array: &ArrayRef, - tz: Tz, - ) -> Result { - let mut builder = PrimitiveBuilder::::new(); - - let primitive_array = as_primitive_array::(array)?; - for ts_opt in primitive_array.iter() { - match ts_opt { - None => builder.append_null(), - Some(ts) => { - let adjusted_ts: i64 = - adjust_to_local_time::(ts, tz)?; - builder.append_value(adjusted_ts) - } - } - } - - Ok(ColumnarValue::Array(Arc::new(builder.finish()))) - } - - match array.data_type() { - Timestamp(_, None) => { - // if no timezone specified, just return the input - Ok(time_value.clone()) - } - Timestamp(Nanosecond, Some(_)) => { - transform_array::(array, tz) - } - Timestamp(Microsecond, Some(_)) => { - transform_array::(array, tz) - } - Timestamp(Millisecond, Some(_)) => { - transform_array::(array, tz) - } - Timestamp(Second, Some(_)) => { - transform_array::(array, tz) - } - _ => { - exec_err!("to_local_time function requires timestamp argument in array, got {:?}", array.data_type()) - } - } - } - _ => { - exec_err!( - "to_local_time function requires timestamp argument, got {:?}", - time_value.data_type() - ) - } - } - } - _ => { - exec_err!( - "to_local_time function requires timestamp argument, got {:?}", - arg_type - ) + fn name(&self) -> &str { + "to_local_time" + } + + fn signature(&self) -> &Signature { + &self.signature + } + + fn return_type(&self, arg_types: &[DataType]) -> Result { + match &arg_types[0] { + DataType::Null => Ok(Timestamp(Nanosecond, None)), + Timestamp(timeunit, _) => Ok(Timestamp(*timeunit, None)), + dt => internal_err!( + "The to_local_time function can only accept timestamp as the arg, got {dt}" + ), + } + } + + fn invoke_with_args( + &self, + args: datafusion_expr::ScalarFunctionArgs, + ) -> Result { + let [time_value] = take_function_args(self.name(), &args.args)?; + to_local_time(time_value) + } + + fn documentation(&self) -> Option<&Documentation> { + self.doc() + } +} + +fn transform_array( + array: &ArrayRef, + tz: Tz, +) -> Result { + let primitive_array = as_primitive_array::(array)?; + let mut builder = PrimitiveBuilder::::with_capacity(primitive_array.len()); + for ts_opt in primitive_array.iter() { + match ts_opt { + None => builder.append_null(), + Some(ts) => { + let adjusted_ts: i64 = adjust_to_local_time::(ts, tz)?; + builder.append_value(adjusted_ts) } } } + + Ok(ColumnarValue::Array(Arc::new(builder.finish()))) +} + +fn to_local_time(time_value: &ColumnarValue) -> Result { + let arg_type = time_value.data_type(); + + let tz: Tz = match &arg_type { + Timestamp(_, Some(timezone)) => timezone.parse()?, + Timestamp(_, None) => { + // if no timezone specified, just return the input + return Ok(time_value.clone()); + } + DataType::Null => { + return Ok(ColumnarValue::Scalar(ScalarValue::TimestampNanosecond( + None, None, + ))); + } + dt => { + return internal_err!( + "to_local_time function requires timestamp argument, got {dt}" + ) + } + }; + + // If has timezone, adjust the underlying time value. The current time value + // is stored as i64 in UTC, even though the timezone may not be in UTC. Therefore, + // we need to adjust the time value to the local time. See [`adjust_to_local_time`] + // for more details. + // + // Then remove the timezone in return type, i.e. return None + match time_value { + ColumnarValue::Scalar(ScalarValue::TimestampSecond(None, Some(_))) => Ok( + ColumnarValue::Scalar(ScalarValue::TimestampSecond(None, None)), + ), + ColumnarValue::Scalar(ScalarValue::TimestampMillisecond(None, Some(_))) => Ok( + ColumnarValue::Scalar(ScalarValue::TimestampMillisecond(None, None)), + ), + ColumnarValue::Scalar(ScalarValue::TimestampMicrosecond(None, Some(_))) => Ok( + ColumnarValue::Scalar(ScalarValue::TimestampMicrosecond(None, None)), + ), + ColumnarValue::Scalar(ScalarValue::TimestampNanosecond(None, Some(_))) => Ok( + ColumnarValue::Scalar(ScalarValue::TimestampNanosecond(None, None)), + ), + ColumnarValue::Scalar(ScalarValue::TimestampNanosecond(Some(ts), Some(_))) => { + let adjusted_ts = adjust_to_local_time::(*ts, tz)?; + Ok(ColumnarValue::Scalar(ScalarValue::TimestampNanosecond( + Some(adjusted_ts), + None, + ))) + } + ColumnarValue::Scalar(ScalarValue::TimestampMicrosecond(Some(ts), Some(_))) => { + let adjusted_ts = adjust_to_local_time::(*ts, tz)?; + Ok(ColumnarValue::Scalar(ScalarValue::TimestampMicrosecond( + Some(adjusted_ts), + None, + ))) + } + ColumnarValue::Scalar(ScalarValue::TimestampMillisecond(Some(ts), Some(_))) => { + let adjusted_ts = adjust_to_local_time::(*ts, tz)?; + Ok(ColumnarValue::Scalar(ScalarValue::TimestampMillisecond( + Some(adjusted_ts), + None, + ))) + } + ColumnarValue::Scalar(ScalarValue::TimestampSecond(Some(ts), Some(_))) => { + let adjusted_ts = adjust_to_local_time::(*ts, tz)?; + Ok(ColumnarValue::Scalar(ScalarValue::TimestampSecond( + Some(adjusted_ts), + None, + ))) + } + ColumnarValue::Array(array) + if matches!(array.data_type(), Timestamp(Nanosecond, Some(_))) => + { + transform_array::(array, tz) + } + ColumnarValue::Array(array) + if matches!(array.data_type(), Timestamp(Microsecond, Some(_))) => + { + transform_array::(array, tz) + } + ColumnarValue::Array(array) + if matches!(array.data_type(), Timestamp(Millisecond, Some(_))) => + { + transform_array::(array, tz) + } + ColumnarValue::Array(array) + if matches!(array.data_type(), Timestamp(Second, Some(_))) => + { + transform_array::(array, tz) + } + _ => { + internal_err!( + "to_local_time function requires timestamp argument, got {arg_type}" + ) + } + } } /// This function converts a timestamp with a timezone to a timestamp without a timezone. @@ -343,68 +374,6 @@ fn adjust_to_local_time(ts: i64, tz: Tz) -> Result { } } -impl ScalarUDFImpl for ToLocalTimeFunc { - fn as_any(&self) -> &dyn Any { - self - } - - fn name(&self) -> &str { - "to_local_time" - } - - fn signature(&self) -> &Signature { - &self.signature - } - - fn return_type(&self, arg_types: &[DataType]) -> Result { - let [time_value] = take_function_args(self.name(), arg_types)?; - - match time_value { - Timestamp(timeunit, _) => Ok(Timestamp(*timeunit, None)), - _ => exec_err!( - "The to_local_time function can only accept timestamp as the arg, got {:?}", time_value - ) - } - } - - fn invoke_with_args( - &self, - args: datafusion_expr::ScalarFunctionArgs, - ) -> Result { - let [time_value] = take_function_args(self.name(), args.args)?; - - self.to_local_time(std::slice::from_ref(&time_value)) - } - - fn coerce_types(&self, arg_types: &[DataType]) -> Result> { - if arg_types.len() != 1 { - return plan_err!( - "to_local_time function requires 1 argument, got {:?}", - arg_types.len() - ); - } - - let first_arg = arg_types[0].clone(); - match &first_arg { - DataType::Null => Ok(vec![Timestamp(Nanosecond, None)]), - Timestamp(Nanosecond, timezone) => { - Ok(vec![Timestamp(Nanosecond, timezone.clone())]) - } - Timestamp(Microsecond, timezone) => { - Ok(vec![Timestamp(Microsecond, timezone.clone())]) - } - Timestamp(Millisecond, timezone) => { - Ok(vec![Timestamp(Millisecond, timezone.clone())]) - } - Timestamp(Second, timezone) => Ok(vec![Timestamp(Second, timezone.clone())]), - _ => plan_err!("The to_local_time function can only accept Timestamp as the arg got {first_arg}"), - } - } - fn documentation(&self) -> Option<&Documentation> { - self.doc() - } -} - #[cfg(test)] mod tests { use std::sync::Arc; diff --git a/datafusion/sqllogictest/test_files/timestamps.slt b/datafusion/sqllogictest/test_files/timestamps.slt index 5c365b056d354..3a1a26257b171 100644 --- a/datafusion/sqllogictest/test_files/timestamps.slt +++ b/datafusion/sqllogictest/test_files/timestamps.slt @@ -2250,7 +2250,7 @@ SET TIME ZONE = '+05:00' statement ok CREATE TABLE foo (time TIMESTAMPTZ) AS VALUES - ('2020-01-01T00:00:00+05:00'), + ('2020-01-01T00:00:00+05:00'), ('2020-01-01T01:00:00+05:00'), ('2020-01-01T02:00:00+05:00'), ('2020-01-01T03:00:00+05:00') @@ -2335,17 +2335,17 @@ NULL 1970-01-01T00:00:00 2031-01-19T23:33:25 1970-01-01T00:00:01 1969-12-31T23:5 # verify timestamp syntax styles are consistent query BBBBBBBBBBBBB SELECT to_timestamp(null) is null as c1, - null::timestamp is null as c2, - cast(null as timestamp) is null as c3, - to_timestamp(0) = 0::timestamp as c4, - to_timestamp(1926632005) = 1926632005::timestamp as c5, - to_timestamp(1) = 1::timestamp as c6, - to_timestamp(-1) = -1::timestamp as c7, + null::timestamp is null as c2, + cast(null as timestamp) is null as c3, + to_timestamp(0) = 0::timestamp as c4, + to_timestamp(1926632005) = 1926632005::timestamp as c5, + to_timestamp(1) = 1::timestamp as c6, + to_timestamp(-1) = -1::timestamp as c7, to_timestamp(0-1) = (0-1)::timestamp as c8, - to_timestamp(0) = cast(0 as timestamp) as c9, - to_timestamp(1926632005) = cast(1926632005 as timestamp) as c10, - to_timestamp(1) = cast(1 as timestamp) as c11, - to_timestamp(-1) = cast(-1 as timestamp) as c12, + to_timestamp(0) = cast(0 as timestamp) as c9, + to_timestamp(1926632005) = cast(1926632005 as timestamp) as c10, + to_timestamp(1) = cast(1 as timestamp) as c11, + to_timestamp(-1) = cast(-1 as timestamp) as c12, to_timestamp(0-1) = cast(0-1 as timestamp) as c13 ---- true true true true true true true true true true true true true @@ -2358,10 +2358,10 @@ Timestamp(ns) Timestamp(ns) Timestamp(ns) # verify timestamp output types using timestamp literal syntax query BBBBBB -SELECT arrow_typeof(to_timestamp(1)) = arrow_typeof(1::timestamp) as c1, +SELECT arrow_typeof(to_timestamp(1)) = arrow_typeof(1::timestamp) as c1, arrow_typeof(to_timestamp(null)) = arrow_typeof(null::timestamp) as c2, arrow_typeof(to_timestamp('2023-01-10 12:34:56.000')) = arrow_typeof('2023-01-10 12:34:56.000'::timestamp) as c3, - arrow_typeof(to_timestamp(1)) = arrow_typeof(cast(1 as timestamp)) as c4, + arrow_typeof(to_timestamp(1)) = arrow_typeof(cast(1 as timestamp)) as c4, arrow_typeof(to_timestamp(null)) = arrow_typeof(cast(null as timestamp)) as c5, arrow_typeof(to_timestamp('2023-01-10 12:34:56.000')) = arrow_typeof(cast('2023-01-10 12:34:56.000' as timestamp)) as c6 ---- @@ -2605,13 +2605,13 @@ drop table table_a ########## statement ok -create table table_a (ts timestamp) as values - ('2020-09-08T11:42:29Z'::timestamp), +create table table_a (ts timestamp) as values + ('2020-09-08T11:42:29Z'::timestamp), ('2020-09-08T12:42:29Z'::timestamp), ('2020-09-08T13:42:29Z'::timestamp) statement ok -create table table_b (ts timestamp) as values +create table table_b (ts timestamp) as values ('2020-09-08T11:42:29.190Z'::timestamp), ('2020-09-08T13:42:29.190Z'::timestamp), ('2020-09-08T12:42:29.190Z'::timestamp) @@ -2733,8 +2733,8 @@ statement ok drop table t1 statement ok -create table table_a (val int, ts1 timestamp, ts2 timestamp) as values - (1, '2018-07-01T06:00:00'::timestamp, '2018-07-01T07:00:00'::timestamp), +create table table_a (val int, ts1 timestamp, ts2 timestamp) as values + (1, '2018-07-01T06:00:00'::timestamp, '2018-07-01T07:00:00'::timestamp), (2, '2018-07-01T07:00:00'::timestamp, '2018-07-01T08:00:00'::timestamp) query I? @@ -3038,7 +3038,7 @@ NULL query T SELECT to_char(date_column, '%Y-%m-%d') -FROM (VALUES +FROM (VALUES (DATE '2020-09-01'), (NULL) ) AS t(date_column); @@ -3048,7 +3048,7 @@ NULL query T SELECT to_char(date_column, '%Y-%m-%d') -FROM (VALUES +FROM (VALUES (NULL), (DATE '2020-09-01') ) AS t(date_column); @@ -3249,7 +3249,7 @@ statement error select to_local_time('2024-04-01T00:00:20Z'::timestamp, 'some string'); # invalid argument data type -statement error The to_local_time function can only accept Timestamp as the arg got Utf8 +statement error DataFusion error: Error during planning: Internal error: Expect TypeSignatureClass::Timestamp but received NativeType::String, DataType: Utf8 select to_local_time('2024-04-01T00:00:20Z'); # invalid timezone @@ -3277,6 +3277,13 @@ select to_local_time(NULL); ---- NULL +query PT +select + to_local_time(arrow_cast(null, 'Timestamp(s, "Asia/Tokyo")')), + arrow_typeof(to_local_time(arrow_cast(null, 'Timestamp(s, "Asia/Tokyo")'))); +---- +NULL Timestamp(s) + query PTPT select time,