Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
339 changes: 154 additions & 185 deletions datafusion/functions/src/datetime/to_local_time.rs
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ use std::ops::Add;
use std::sync::Arc;

use arrow::array::timezone::Tz;
use arrow::array::{Array, ArrayRef, PrimitiveBuilder};
use arrow::array::{ArrayRef, PrimitiveBuilder};
use arrow::datatypes::DataType::Timestamp;
use arrow::datatypes::TimeUnit::{Microsecond, Millisecond, Nanosecond, Second};
use arrow::datatypes::{
Expand All @@ -31,11 +31,12 @@ use chrono::{DateTime, MappedLocalTime, Offset, TimeDelta, TimeZone, Utc};

use datafusion_common::cast::as_primitive_array;
use datafusion_common::{
exec_err, internal_datafusion_err, plan_err, utils::take_function_args, Result,
exec_err, internal_datafusion_err, internal_err, utils::take_function_args, Result,
ScalarValue,
};
use datafusion_expr::{
ColumnarValue, Documentation, ScalarUDFImpl, Signature, Volatility,
Coercion, ColumnarValue, Documentation, ScalarUDFImpl, Signature, TypeSignatureClass,
Volatility,
};
use datafusion_macros::user_doc;

Expand Down Expand Up @@ -111,133 +112,163 @@ impl Default for ToLocalTimeFunc {
impl ToLocalTimeFunc {
pub fn new() -> Self {
Self {
signature: Signature::user_defined(Volatility::Immutable),
signature: Signature::coercible(
vec![Coercion::new_exact(TypeSignatureClass::Timestamp)],
Volatility::Immutable,
),
Comment on lines +115 to +118

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Signature change here

}
}
}

fn to_local_time(&self, args: &[ColumnarValue]) -> Result<ColumnarValue> {

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I refactored to_local_time to move it outside of ToLocalTimeFunc struct since it didn't really need the self

let [time_value] = take_function_args(self.name(), args)?;
impl ScalarUDFImpl for ToLocalTimeFunc {
fn as_any(&self) -> &dyn Any {
self
}

let arg_type = time_value.data_type();
match arg_type {
Timestamp(_, None) => {
// if no timezone specified, just return the input
Ok(time_value.clone())
}
// If has timezone, adjust the underlying time value. The current time value
// is stored as i64 in UTC, even though the timezone may not be in UTC. Therefore,
// we need to adjust the time value to the local time. See [`adjust_to_local_time`]
// for more details.
//
// Then remove the timezone in return type, i.e. return None
Timestamp(_, Some(timezone)) => {
let tz: Tz = timezone.parse()?;

match time_value {
ColumnarValue::Scalar(ScalarValue::TimestampNanosecond(
Some(ts),
Some(_),
)) => {
let adjusted_ts =
adjust_to_local_time::<TimestampNanosecondType>(*ts, tz)?;
Ok(ColumnarValue::Scalar(ScalarValue::TimestampNanosecond(
Some(adjusted_ts),
None,
)))
}
ColumnarValue::Scalar(ScalarValue::TimestampMicrosecond(
Some(ts),
Some(_),
)) => {
let adjusted_ts =
adjust_to_local_time::<TimestampMicrosecondType>(*ts, tz)?;
Ok(ColumnarValue::Scalar(ScalarValue::TimestampMicrosecond(
Some(adjusted_ts),
None,
)))
}
ColumnarValue::Scalar(ScalarValue::TimestampMillisecond(
Some(ts),
Some(_),
)) => {
let adjusted_ts =
adjust_to_local_time::<TimestampMillisecondType>(*ts, tz)?;
Ok(ColumnarValue::Scalar(ScalarValue::TimestampMillisecond(
Some(adjusted_ts),
None,
)))
}
ColumnarValue::Scalar(ScalarValue::TimestampSecond(
Some(ts),
Some(_),
)) => {
let adjusted_ts =
adjust_to_local_time::<TimestampSecondType>(*ts, tz)?;
Ok(ColumnarValue::Scalar(ScalarValue::TimestampSecond(
Some(adjusted_ts),
None,
)))
}
ColumnarValue::Array(array) => {
fn transform_array<T: ArrowTimestampType>(
array: &ArrayRef,
tz: Tz,
) -> Result<ColumnarValue> {
let mut builder = PrimitiveBuilder::<T>::new();

let primitive_array = as_primitive_array::<T>(array)?;
for ts_opt in primitive_array.iter() {
match ts_opt {
None => builder.append_null(),
Some(ts) => {
let adjusted_ts: i64 =
adjust_to_local_time::<T>(ts, tz)?;
builder.append_value(adjusted_ts)
}
}
}

Ok(ColumnarValue::Array(Arc::new(builder.finish())))
}

match array.data_type() {
Timestamp(_, None) => {
// if no timezone specified, just return the input
Ok(time_value.clone())
}
Timestamp(Nanosecond, Some(_)) => {
transform_array::<TimestampNanosecondType>(array, tz)
}
Timestamp(Microsecond, Some(_)) => {
transform_array::<TimestampMicrosecondType>(array, tz)
}
Timestamp(Millisecond, Some(_)) => {
transform_array::<TimestampMillisecondType>(array, tz)
}
Timestamp(Second, Some(_)) => {
transform_array::<TimestampSecondType>(array, tz)
}
_ => {
exec_err!("to_local_time function requires timestamp argument in array, got {:?}", array.data_type())
}
}
}
_ => {
exec_err!(
"to_local_time function requires timestamp argument, got {:?}",
time_value.data_type()
)
}
}
}
_ => {
exec_err!(
"to_local_time function requires timestamp argument, got {:?}",
arg_type
)
fn name(&self) -> &str {
"to_local_time"
}

fn signature(&self) -> &Signature {
&self.signature
}

fn return_type(&self, arg_types: &[DataType]) -> Result<DataType> {
match &arg_types[0] {
DataType::Null => Ok(Timestamp(Nanosecond, None)),

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Matching the old behaviour wrt nulls here

Timestamp(timeunit, _) => Ok(Timestamp(*timeunit, None)),
dt => internal_err!(
"The to_local_time function can only accept timestamp as the arg, got {dt}"
),
}
}

fn invoke_with_args(
&self,
args: datafusion_expr::ScalarFunctionArgs,
) -> Result<ColumnarValue> {
let [time_value] = take_function_args(self.name(), &args.args)?;
to_local_time(time_value)
}

fn documentation(&self) -> Option<&Documentation> {
self.doc()
}
}

fn transform_array<T: ArrowTimestampType>(
array: &ArrayRef,
tz: Tz,
) -> Result<ColumnarValue> {
let primitive_array = as_primitive_array::<T>(array)?;

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
let primitive_array = as_primitive_array::<T>(array)?;
let primitive_array = as_primitive_array::<T>(array)?;
let mut builder = PrimitiveBuilder::<T>::with_capacity(primitive_array.len());

move the builder declaration here to have the primitive_array size.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks, added in e547c5a

let mut builder = PrimitiveBuilder::<T>::with_capacity(primitive_array.len());
for ts_opt in primitive_array.iter() {
match ts_opt {
None => builder.append_null(),
Some(ts) => {
let adjusted_ts: i64 = adjust_to_local_time::<T>(ts, tz)?;
builder.append_value(adjusted_ts)
}
}
}

Ok(ColumnarValue::Array(Arc::new(builder.finish())))
}

fn to_local_time(time_value: &ColumnarValue) -> Result<ColumnarValue> {

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Refactored to_local_time function here now; I also flattened the function since it was quite nested before

let arg_type = time_value.data_type();

let tz: Tz = match &arg_type {
Timestamp(_, Some(timezone)) => timezone.parse()?,
Timestamp(_, None) => {
// if no timezone specified, just return the input
return Ok(time_value.clone());
}
DataType::Null => {

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is this case possible ?
Coercion::new_exact(TypeSignatureClass::Timestamp) should make it impossible, no ?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah this is a small quirk of the coercion API, where Null types are passed through as is; I plan to look into it later to see if we can handle it better (need to check if there is an existing GitHub issue for this)

return Ok(ColumnarValue::Scalar(ScalarValue::TimestampNanosecond(
None, None,
)));
}
dt => {
return internal_err!(
"to_local_time function requires timestamp argument, got {dt}"
)
}
};

// If has timezone, adjust the underlying time value. The current time value
// is stored as i64 in UTC, even though the timezone may not be in UTC. Therefore,
// we need to adjust the time value to the local time. See [`adjust_to_local_time`]
// for more details.
//
// Then remove the timezone in return type, i.e. return None
match time_value {
ColumnarValue::Scalar(ScalarValue::TimestampSecond(None, Some(_))) => Ok(
ColumnarValue::Scalar(ScalarValue::TimestampSecond(None, None)),
),
ColumnarValue::Scalar(ScalarValue::TimestampMillisecond(None, Some(_))) => Ok(
ColumnarValue::Scalar(ScalarValue::TimestampMillisecond(None, None)),
),
ColumnarValue::Scalar(ScalarValue::TimestampMicrosecond(None, Some(_))) => Ok(
ColumnarValue::Scalar(ScalarValue::TimestampMicrosecond(None, None)),
),
ColumnarValue::Scalar(ScalarValue::TimestampNanosecond(None, Some(_))) => Ok(
ColumnarValue::Scalar(ScalarValue::TimestampNanosecond(None, None)),
),
ColumnarValue::Scalar(ScalarValue::TimestampNanosecond(Some(ts), Some(_))) => {
let adjusted_ts = adjust_to_local_time::<TimestampNanosecondType>(*ts, tz)?;
Ok(ColumnarValue::Scalar(ScalarValue::TimestampNanosecond(
Some(adjusted_ts),
None,
)))
}
ColumnarValue::Scalar(ScalarValue::TimestampMicrosecond(Some(ts), Some(_))) => {
let adjusted_ts = adjust_to_local_time::<TimestampMicrosecondType>(*ts, tz)?;
Ok(ColumnarValue::Scalar(ScalarValue::TimestampMicrosecond(
Some(adjusted_ts),
None,
)))
}
ColumnarValue::Scalar(ScalarValue::TimestampMillisecond(Some(ts), Some(_))) => {
let adjusted_ts = adjust_to_local_time::<TimestampMillisecondType>(*ts, tz)?;
Ok(ColumnarValue::Scalar(ScalarValue::TimestampMillisecond(
Some(adjusted_ts),
None,
)))
}
ColumnarValue::Scalar(ScalarValue::TimestampSecond(Some(ts), Some(_))) => {
let adjusted_ts = adjust_to_local_time::<TimestampSecondType>(*ts, tz)?;
Ok(ColumnarValue::Scalar(ScalarValue::TimestampSecond(
Some(adjusted_ts),
None,
)))
}

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What about ColumnarValue::Scalar(ScalarValue::TimestampXyz(None, Some(_))) => { ?
IMO it should return:

return Ok(ColumnarValue::Scalar(ScalarValue::TimestampNanosecond(
                None, None,
            )));

as for the DateType::Null case above

@Jefffrey Jefffrey Nov 19, 2025 •

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good catch, added arms and a test: 5de389e

ColumnarValue::Array(array)
if matches!(array.data_type(), Timestamp(Nanosecond, Some(_))) =>
{
transform_array::<TimestampNanosecondType>(array, tz)
}
ColumnarValue::Array(array)
if matches!(array.data_type(), Timestamp(Microsecond, Some(_))) =>
{
transform_array::<TimestampMicrosecondType>(array, tz)
}
ColumnarValue::Array(array)
if matches!(array.data_type(), Timestamp(Millisecond, Some(_))) =>
{
transform_array::<TimestampMillisecondType>(array, tz)
}
ColumnarValue::Array(array)
if matches!(array.data_type(), Timestamp(Second, Some(_))) =>
{
transform_array::<TimestampSecondType>(array, tz)
}
_ => {
internal_err!(
"to_local_time function requires timestamp argument, got {arg_type}"
)
}
}
}

/// This function converts a timestamp with a timezone to a timestamp without a timezone.
Expand Down Expand Up @@ -343,68 +374,6 @@ fn adjust_to_local_time<T: ArrowTimestampType>(ts: i64, tz: Tz) -> Result<i64> {
}
}

impl ScalarUDFImpl for ToLocalTimeFunc {
fn as_any(&self) -> &dyn Any {
self
}

fn name(&self) -> &str {
"to_local_time"
}

fn signature(&self) -> &Signature {
&self.signature
}

fn return_type(&self, arg_types: &[DataType]) -> Result<DataType> {
let [time_value] = take_function_args(self.name(), arg_types)?;

match time_value {
Timestamp(timeunit, _) => Ok(Timestamp(*timeunit, None)),
_ => exec_err!(
"The to_local_time function can only accept timestamp as the arg, got {:?}", time_value
)
}
}

fn invoke_with_args(
&self,
args: datafusion_expr::ScalarFunctionArgs,
) -> Result<ColumnarValue> {
let [time_value] = take_function_args(self.name(), args.args)?;

self.to_local_time(std::slice::from_ref(&time_value))
}

fn coerce_types(&self, arg_types: &[DataType]) -> Result<Vec<DataType>> {
if arg_types.len() != 1 {
return plan_err!(
"to_local_time function requires 1 argument, got {:?}",
arg_types.len()
);
}

let first_arg = arg_types[0].clone();
match &first_arg {
DataType::Null => Ok(vec![Timestamp(Nanosecond, None)]),
Timestamp(Nanosecond, timezone) => {
Ok(vec![Timestamp(Nanosecond, timezone.clone())])
}
Timestamp(Microsecond, timezone) => {
Ok(vec![Timestamp(Microsecond, timezone.clone())])
}
Timestamp(Millisecond, timezone) => {
Ok(vec![Timestamp(Millisecond, timezone.clone())])
}
Timestamp(Second, timezone) => Ok(vec![Timestamp(Second, timezone.clone())]),
_ => plan_err!("The to_local_time function can only accept Timestamp as the arg got {first_arg}"),
}
}
fn documentation(&self) -> Option<&Documentation> {
self.doc()
}
}

#[cfg(test)]
mod tests {
use std::sync::Arc;
Expand Down
Loading