Repository navigation
Refactor to_local_time() signature away from user_defined
#18707
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
7b11ef2
0f4677d
6bb8d25
ac119ba
5de389e
e547c5a
c36f5ea
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change | ||||||
|---|---|---|---|---|---|---|---|---|
|
|
@@ -20,7 +20,7 @@ use std::ops::Add; | |||||||
| use std::sync::Arc; | ||||||||
|
|
||||||||
| use arrow::array::timezone::Tz; | ||||||||
| use arrow::array::{Array, ArrayRef, PrimitiveBuilder}; | ||||||||
| use arrow::array::{ArrayRef, PrimitiveBuilder}; | ||||||||
| use arrow::datatypes::DataType::Timestamp; | ||||||||
| use arrow::datatypes::TimeUnit::{Microsecond, Millisecond, Nanosecond, Second}; | ||||||||
| use arrow::datatypes::{ | ||||||||
|
|
@@ -31,11 +31,12 @@ use chrono::{DateTime, MappedLocalTime, Offset, TimeDelta, TimeZone, Utc}; | |||||||
|
|
||||||||
| use datafusion_common::cast::as_primitive_array; | ||||||||
| use datafusion_common::{ | ||||||||
| exec_err, internal_datafusion_err, plan_err, utils::take_function_args, Result, | ||||||||
| exec_err, internal_datafusion_err, internal_err, utils::take_function_args, Result, | ||||||||
| ScalarValue, | ||||||||
| }; | ||||||||
| use datafusion_expr::{ | ||||||||
| ColumnarValue, Documentation, ScalarUDFImpl, Signature, Volatility, | ||||||||
| Coercion, ColumnarValue, Documentation, ScalarUDFImpl, Signature, TypeSignatureClass, | ||||||||
| Volatility, | ||||||||
| }; | ||||||||
| use datafusion_macros::user_doc; | ||||||||
|
|
||||||||
|
|
@@ -111,133 +112,163 @@ impl Default for ToLocalTimeFunc { | |||||||
| impl ToLocalTimeFunc { | ||||||||
| pub fn new() -> Self { | ||||||||
| Self { | ||||||||
| signature: Signature::user_defined(Volatility::Immutable), | ||||||||
| signature: Signature::coercible( | ||||||||
| vec![Coercion::new_exact(TypeSignatureClass::Timestamp)], | ||||||||
| Volatility::Immutable, | ||||||||
| ), | ||||||||
| } | ||||||||
| } | ||||||||
| } | ||||||||
|
|
||||||||
| fn to_local_time(&self, args: &[ColumnarValue]) -> Result<ColumnarValue> { | ||||||||
|
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I refactored |
||||||||
| let [time_value] = take_function_args(self.name(), args)?; | ||||||||
| impl ScalarUDFImpl for ToLocalTimeFunc { | ||||||||
| fn as_any(&self) -> &dyn Any { | ||||||||
| self | ||||||||
| } | ||||||||
|
|
||||||||
| let arg_type = time_value.data_type(); | ||||||||
| match arg_type { | ||||||||
| Timestamp(_, None) => { | ||||||||
| // if no timezone specified, just return the input | ||||||||
| Ok(time_value.clone()) | ||||||||
| } | ||||||||
| // If has timezone, adjust the underlying time value. The current time value | ||||||||
| // is stored as i64 in UTC, even though the timezone may not be in UTC. Therefore, | ||||||||
| // we need to adjust the time value to the local time. See [`adjust_to_local_time`] | ||||||||
| // for more details. | ||||||||
| // | ||||||||
| // Then remove the timezone in return type, i.e. return None | ||||||||
| Timestamp(_, Some(timezone)) => { | ||||||||
| let tz: Tz = timezone.parse()?; | ||||||||
|
|
||||||||
| match time_value { | ||||||||
| ColumnarValue::Scalar(ScalarValue::TimestampNanosecond( | ||||||||
| Some(ts), | ||||||||
| Some(_), | ||||||||
| )) => { | ||||||||
| let adjusted_ts = | ||||||||
| adjust_to_local_time::<TimestampNanosecondType>(*ts, tz)?; | ||||||||
| Ok(ColumnarValue::Scalar(ScalarValue::TimestampNanosecond( | ||||||||
| Some(adjusted_ts), | ||||||||
| None, | ||||||||
| ))) | ||||||||
| } | ||||||||
| ColumnarValue::Scalar(ScalarValue::TimestampMicrosecond( | ||||||||
| Some(ts), | ||||||||
| Some(_), | ||||||||
| )) => { | ||||||||
| let adjusted_ts = | ||||||||
| adjust_to_local_time::<TimestampMicrosecondType>(*ts, tz)?; | ||||||||
| Ok(ColumnarValue::Scalar(ScalarValue::TimestampMicrosecond( | ||||||||
| Some(adjusted_ts), | ||||||||
| None, | ||||||||
| ))) | ||||||||
| } | ||||||||
| ColumnarValue::Scalar(ScalarValue::TimestampMillisecond( | ||||||||
| Some(ts), | ||||||||
| Some(_), | ||||||||
| )) => { | ||||||||
| let adjusted_ts = | ||||||||
| adjust_to_local_time::<TimestampMillisecondType>(*ts, tz)?; | ||||||||
| Ok(ColumnarValue::Scalar(ScalarValue::TimestampMillisecond( | ||||||||
| Some(adjusted_ts), | ||||||||
| None, | ||||||||
| ))) | ||||||||
| } | ||||||||
| ColumnarValue::Scalar(ScalarValue::TimestampSecond( | ||||||||
| Some(ts), | ||||||||
| Some(_), | ||||||||
| )) => { | ||||||||
| let adjusted_ts = | ||||||||
| adjust_to_local_time::<TimestampSecondType>(*ts, tz)?; | ||||||||
| Ok(ColumnarValue::Scalar(ScalarValue::TimestampSecond( | ||||||||
| Some(adjusted_ts), | ||||||||
| None, | ||||||||
| ))) | ||||||||
| } | ||||||||
| ColumnarValue::Array(array) => { | ||||||||
| fn transform_array<T: ArrowTimestampType>( | ||||||||
| array: &ArrayRef, | ||||||||
| tz: Tz, | ||||||||
| ) -> Result<ColumnarValue> { | ||||||||
| let mut builder = PrimitiveBuilder::<T>::new(); | ||||||||
|
|
||||||||
| let primitive_array = as_primitive_array::<T>(array)?; | ||||||||
| for ts_opt in primitive_array.iter() { | ||||||||
| match ts_opt { | ||||||||
| None => builder.append_null(), | ||||||||
| Some(ts) => { | ||||||||
| let adjusted_ts: i64 = | ||||||||
| adjust_to_local_time::<T>(ts, tz)?; | ||||||||
| builder.append_value(adjusted_ts) | ||||||||
| } | ||||||||
| } | ||||||||
| } | ||||||||
|
|
||||||||
| Ok(ColumnarValue::Array(Arc::new(builder.finish()))) | ||||||||
| } | ||||||||
|
|
||||||||
| match array.data_type() { | ||||||||
| Timestamp(_, None) => { | ||||||||
| // if no timezone specified, just return the input | ||||||||
| Ok(time_value.clone()) | ||||||||
| } | ||||||||
| Timestamp(Nanosecond, Some(_)) => { | ||||||||
| transform_array::<TimestampNanosecondType>(array, tz) | ||||||||
| } | ||||||||
| Timestamp(Microsecond, Some(_)) => { | ||||||||
| transform_array::<TimestampMicrosecondType>(array, tz) | ||||||||
| } | ||||||||
| Timestamp(Millisecond, Some(_)) => { | ||||||||
| transform_array::<TimestampMillisecondType>(array, tz) | ||||||||
| } | ||||||||
| Timestamp(Second, Some(_)) => { | ||||||||
| transform_array::<TimestampSecondType>(array, tz) | ||||||||
| } | ||||||||
| _ => { | ||||||||
| exec_err!("to_local_time function requires timestamp argument in array, got {:?}", array.data_type()) | ||||||||
| } | ||||||||
| } | ||||||||
| } | ||||||||
| _ => { | ||||||||
| exec_err!( | ||||||||
| "to_local_time function requires timestamp argument, got {:?}", | ||||||||
| time_value.data_type() | ||||||||
| ) | ||||||||
| } | ||||||||
| } | ||||||||
| } | ||||||||
| _ => { | ||||||||
| exec_err!( | ||||||||
| "to_local_time function requires timestamp argument, got {:?}", | ||||||||
| arg_type | ||||||||
| ) | ||||||||
| fn name(&self) -> &str { | ||||||||
| "to_local_time" | ||||||||
| } | ||||||||
|
|
||||||||
| fn signature(&self) -> &Signature { | ||||||||
| &self.signature | ||||||||
| } | ||||||||
|
|
||||||||
| fn return_type(&self, arg_types: &[DataType]) -> Result<DataType> { | ||||||||
| match &arg_types[0] { | ||||||||
| DataType::Null => Ok(Timestamp(Nanosecond, None)), | ||||||||
|
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Matching the old behaviour wrt nulls here |
||||||||
| Timestamp(timeunit, _) => Ok(Timestamp(*timeunit, None)), | ||||||||
| dt => internal_err!( | ||||||||
| "The to_local_time function can only accept timestamp as the arg, got {dt}" | ||||||||
| ), | ||||||||
| } | ||||||||
| } | ||||||||
|
|
||||||||
| fn invoke_with_args( | ||||||||
| &self, | ||||||||
| args: datafusion_expr::ScalarFunctionArgs, | ||||||||
| ) -> Result<ColumnarValue> { | ||||||||
| let [time_value] = take_function_args(self.name(), &args.args)?; | ||||||||
| to_local_time(time_value) | ||||||||
| } | ||||||||
|
|
||||||||
| fn documentation(&self) -> Option<&Documentation> { | ||||||||
| self.doc() | ||||||||
| } | ||||||||
| } | ||||||||
|
|
||||||||
| fn transform_array<T: ArrowTimestampType>( | ||||||||
| array: &ArrayRef, | ||||||||
| tz: Tz, | ||||||||
| ) -> Result<ColumnarValue> { | ||||||||
| let primitive_array = as_primitive_array::<T>(array)?; | ||||||||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Suggested change
move the builder declaration here to have the primitive_array size.
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Thanks, added in e547c5a |
||||||||
| let mut builder = PrimitiveBuilder::<T>::with_capacity(primitive_array.len()); | ||||||||
| for ts_opt in primitive_array.iter() { | ||||||||
| match ts_opt { | ||||||||
| None => builder.append_null(), | ||||||||
| Some(ts) => { | ||||||||
| let adjusted_ts: i64 = adjust_to_local_time::<T>(ts, tz)?; | ||||||||
| builder.append_value(adjusted_ts) | ||||||||
| } | ||||||||
| } | ||||||||
| } | ||||||||
|
|
||||||||
| Ok(ColumnarValue::Array(Arc::new(builder.finish()))) | ||||||||
| } | ||||||||
|
|
||||||||
| fn to_local_time(time_value: &ColumnarValue) -> Result<ColumnarValue> { | ||||||||
|
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Refactored |
||||||||
| let arg_type = time_value.data_type(); | ||||||||
|
|
||||||||
| let tz: Tz = match &arg_type { | ||||||||
| Timestamp(_, Some(timezone)) => timezone.parse()?, | ||||||||
| Timestamp(_, None) => { | ||||||||
| // if no timezone specified, just return the input | ||||||||
| return Ok(time_value.clone()); | ||||||||
| } | ||||||||
| DataType::Null => { | ||||||||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Is this case possible ?
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Yeah this is a small quirk of the coercion API, where |
||||||||
| return Ok(ColumnarValue::Scalar(ScalarValue::TimestampNanosecond( | ||||||||
| None, None, | ||||||||
| ))); | ||||||||
| } | ||||||||
| dt => { | ||||||||
| return internal_err!( | ||||||||
| "to_local_time function requires timestamp argument, got {dt}" | ||||||||
| ) | ||||||||
| } | ||||||||
| }; | ||||||||
|
|
||||||||
| // If has timezone, adjust the underlying time value. The current time value | ||||||||
| // is stored as i64 in UTC, even though the timezone may not be in UTC. Therefore, | ||||||||
| // we need to adjust the time value to the local time. See [`adjust_to_local_time`] | ||||||||
| // for more details. | ||||||||
| // | ||||||||
| // Then remove the timezone in return type, i.e. return None | ||||||||
| match time_value { | ||||||||
| ColumnarValue::Scalar(ScalarValue::TimestampSecond(None, Some(_))) => Ok( | ||||||||
| ColumnarValue::Scalar(ScalarValue::TimestampSecond(None, None)), | ||||||||
| ), | ||||||||
| ColumnarValue::Scalar(ScalarValue::TimestampMillisecond(None, Some(_))) => Ok( | ||||||||
| ColumnarValue::Scalar(ScalarValue::TimestampMillisecond(None, None)), | ||||||||
| ), | ||||||||
| ColumnarValue::Scalar(ScalarValue::TimestampMicrosecond(None, Some(_))) => Ok( | ||||||||
| ColumnarValue::Scalar(ScalarValue::TimestampMicrosecond(None, None)), | ||||||||
| ), | ||||||||
| ColumnarValue::Scalar(ScalarValue::TimestampNanosecond(None, Some(_))) => Ok( | ||||||||
| ColumnarValue::Scalar(ScalarValue::TimestampNanosecond(None, None)), | ||||||||
| ), | ||||||||
| ColumnarValue::Scalar(ScalarValue::TimestampNanosecond(Some(ts), Some(_))) => { | ||||||||
| let adjusted_ts = adjust_to_local_time::<TimestampNanosecondType>(*ts, tz)?; | ||||||||
| Ok(ColumnarValue::Scalar(ScalarValue::TimestampNanosecond( | ||||||||
| Some(adjusted_ts), | ||||||||
| None, | ||||||||
| ))) | ||||||||
| } | ||||||||
| ColumnarValue::Scalar(ScalarValue::TimestampMicrosecond(Some(ts), Some(_))) => { | ||||||||
| let adjusted_ts = adjust_to_local_time::<TimestampMicrosecondType>(*ts, tz)?; | ||||||||
| Ok(ColumnarValue::Scalar(ScalarValue::TimestampMicrosecond( | ||||||||
| Some(adjusted_ts), | ||||||||
| None, | ||||||||
| ))) | ||||||||
| } | ||||||||
| ColumnarValue::Scalar(ScalarValue::TimestampMillisecond(Some(ts), Some(_))) => { | ||||||||
| let adjusted_ts = adjust_to_local_time::<TimestampMillisecondType>(*ts, tz)?; | ||||||||
| Ok(ColumnarValue::Scalar(ScalarValue::TimestampMillisecond( | ||||||||
| Some(adjusted_ts), | ||||||||
| None, | ||||||||
| ))) | ||||||||
| } | ||||||||
| ColumnarValue::Scalar(ScalarValue::TimestampSecond(Some(ts), Some(_))) => { | ||||||||
| let adjusted_ts = adjust_to_local_time::<TimestampSecondType>(*ts, tz)?; | ||||||||
| Ok(ColumnarValue::Scalar(ScalarValue::TimestampSecond( | ||||||||
| Some(adjusted_ts), | ||||||||
| None, | ||||||||
| ))) | ||||||||
| } | ||||||||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. What about as for the
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Good catch, added arms and a test: 5de389e |
||||||||
| ColumnarValue::Array(array) | ||||||||
| if matches!(array.data_type(), Timestamp(Nanosecond, Some(_))) => | ||||||||
| { | ||||||||
| transform_array::<TimestampNanosecondType>(array, tz) | ||||||||
| } | ||||||||
| ColumnarValue::Array(array) | ||||||||
| if matches!(array.data_type(), Timestamp(Microsecond, Some(_))) => | ||||||||
| { | ||||||||
| transform_array::<TimestampMicrosecondType>(array, tz) | ||||||||
| } | ||||||||
| ColumnarValue::Array(array) | ||||||||
| if matches!(array.data_type(), Timestamp(Millisecond, Some(_))) => | ||||||||
| { | ||||||||
| transform_array::<TimestampMillisecondType>(array, tz) | ||||||||
| } | ||||||||
| ColumnarValue::Array(array) | ||||||||
| if matches!(array.data_type(), Timestamp(Second, Some(_))) => | ||||||||
| { | ||||||||
| transform_array::<TimestampSecondType>(array, tz) | ||||||||
| } | ||||||||
| _ => { | ||||||||
| internal_err!( | ||||||||
| "to_local_time function requires timestamp argument, got {arg_type}" | ||||||||
| ) | ||||||||
| } | ||||||||
| } | ||||||||
| } | ||||||||
|
|
||||||||
| /// This function converts a timestamp with a timezone to a timestamp without a timezone. | ||||||||
|
|
@@ -343,68 +374,6 @@ fn adjust_to_local_time<T: ArrowTimestampType>(ts: i64, tz: Tz) -> Result<i64> { | |||||||
| } | ||||||||
| } | ||||||||
|
|
||||||||
| impl ScalarUDFImpl for ToLocalTimeFunc { | ||||||||
| fn as_any(&self) -> &dyn Any { | ||||||||
| self | ||||||||
| } | ||||||||
|
|
||||||||
| fn name(&self) -> &str { | ||||||||
| "to_local_time" | ||||||||
| } | ||||||||
|
|
||||||||
| fn signature(&self) -> &Signature { | ||||||||
| &self.signature | ||||||||
| } | ||||||||
|
|
||||||||
| fn return_type(&self, arg_types: &[DataType]) -> Result<DataType> { | ||||||||
| let [time_value] = take_function_args(self.name(), arg_types)?; | ||||||||
|
|
||||||||
| match time_value { | ||||||||
| Timestamp(timeunit, _) => Ok(Timestamp(*timeunit, None)), | ||||||||
| _ => exec_err!( | ||||||||
| "The to_local_time function can only accept timestamp as the arg, got {:?}", time_value | ||||||||
| ) | ||||||||
| } | ||||||||
| } | ||||||||
|
|
||||||||
| fn invoke_with_args( | ||||||||
| &self, | ||||||||
| args: datafusion_expr::ScalarFunctionArgs, | ||||||||
| ) -> Result<ColumnarValue> { | ||||||||
| let [time_value] = take_function_args(self.name(), args.args)?; | ||||||||
|
|
||||||||
| self.to_local_time(std::slice::from_ref(&time_value)) | ||||||||
| } | ||||||||
|
|
||||||||
| fn coerce_types(&self, arg_types: &[DataType]) -> Result<Vec<DataType>> { | ||||||||
| if arg_types.len() != 1 { | ||||||||
| return plan_err!( | ||||||||
| "to_local_time function requires 1 argument, got {:?}", | ||||||||
| arg_types.len() | ||||||||
| ); | ||||||||
| } | ||||||||
|
|
||||||||
| let first_arg = arg_types[0].clone(); | ||||||||
| match &first_arg { | ||||||||
| DataType::Null => Ok(vec![Timestamp(Nanosecond, None)]), | ||||||||
| Timestamp(Nanosecond, timezone) => { | ||||||||
| Ok(vec![Timestamp(Nanosecond, timezone.clone())]) | ||||||||
| } | ||||||||
| Timestamp(Microsecond, timezone) => { | ||||||||
| Ok(vec![Timestamp(Microsecond, timezone.clone())]) | ||||||||
| } | ||||||||
| Timestamp(Millisecond, timezone) => { | ||||||||
| Ok(vec![Timestamp(Millisecond, timezone.clone())]) | ||||||||
| } | ||||||||
| Timestamp(Second, timezone) => Ok(vec![Timestamp(Second, timezone.clone())]), | ||||||||
| _ => plan_err!("The to_local_time function can only accept Timestamp as the arg got {first_arg}"), | ||||||||
| } | ||||||||
| } | ||||||||
| fn documentation(&self) -> Option<&Documentation> { | ||||||||
| self.doc() | ||||||||
| } | ||||||||
| } | ||||||||
|
|
||||||||
| #[cfg(test)] | ||||||||
| mod tests { | ||||||||
| use std::sync::Arc; | ||||||||
|
|
||||||||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Signature change here