diff --git a/native/core/src/execution/planner.rs b/native/core/src/execution/planner.rs index 80f8d890a34..728a0a63ff3 100644 --- a/native/core/src/execution/planner.rs +++ b/native/core/src/execution/planner.rs @@ -4743,7 +4743,10 @@ fn create_case_expr( if let Some(coerce_type) = get_coerce_type_for_case_expression(&then_types, else_type.as_ref()) { - let cast_options = SparkCastOptions::new_without_timezone(EvalMode::Legacy, false); + // The branches share a Spark type, so any difference is in the Arrow representation. For + // a timestamp that is the timezone label, and the cast only relabels it, but Comet's cast + // still needs a timezone. Every `TimestampType` value in a native plan is labelled UTC. + let cast_options = SparkCastOptions::new(EvalMode::Legacy, "UTC", false); let when_then_pairs = when_then_pairs .iter() @@ -7738,4 +7741,43 @@ mod tests { "unexpected error: {err}" ); } + + /// CASE branches that share a Spark timestamp type but carry different Arrow timezone labels + /// are reconciled by relabelling them, which used to panic because the cast had no timezone. + #[test] + fn case_reconciles_timestamp_timezone_labels() { + use arrow::array::{AsArray, BooleanArray, TimestampMicrosecondArray}; + use arrow::datatypes::{TimeUnit, TimestampMicrosecondType}; + + let utc = DataType::Timestamp(TimeUnit::Microsecond, Some("UTC".into())); + for then_label in [Some("Etc/UTC"), None] { + let then_type = DataType::Timestamp(TimeUnit::Microsecond, then_label.map(Into::into)); + let schema = Schema::new(vec![ + Field::new("b", DataType::Boolean, true), + Field::new("t", then_type, true), + Field::new("e", utc.clone(), true), + ]); + let batch = RecordBatch::try_new( + Arc::new(schema.clone()), + vec![ + Arc::new(BooleanArray::from(vec![true, false])), + Arc::new( + TimestampMicrosecondArray::from(vec![1, 2]).with_timezone_opt(then_label), + ), + Arc::new(TimestampMicrosecondArray::from(vec![10, 20]).with_timezone("UTC")), + ], + ) + .unwrap(); + let case = super::create_case_expr( + vec![(Arc::new(Column::new("b", 0)), Arc::new(Column::new("t", 1)))], + Some(Arc::new(Column::new("e", 2))), + &schema, + ) + .unwrap(); + let result = case.evaluate(&batch).unwrap().into_array(2).unwrap(); + assert_eq!(result.data_type(), &utc, "{then_label:?}"); + let values = result.as_primitive::(); + assert_eq!(values.values().to_vec(), vec![1, 20], "{then_label:?}"); + } + } } diff --git a/native/spark-expr/src/utils.rs b/native/spark-expr/src/utils.rs index 8007275ae3d..e393de172cf 100644 --- a/native/spark-expr/src/utils.rs +++ b/native/spark-expr/src/utils.rs @@ -71,7 +71,7 @@ pub fn array_with_timezone( ) -> Result { match array.data_type() { DataType::Timestamp(TimeUnit::Millisecond, None) => { - assert!(!timezone.is_empty()); + require_timezone(&timezone)?; match to_type { Some(DataType::Utf8) | Some(DataType::Date32) => Ok(array), Some(DataType::Timestamp(_, Some(target_tz))) => { @@ -90,7 +90,7 @@ pub fn array_with_timezone( } } DataType::Timestamp(TimeUnit::Microsecond, None) => { - assert!(!timezone.is_empty()); + require_timezone(&timezone)?; match to_type { Some(DataType::Utf8) | Some(DataType::Date32) => Ok(array), Some(DataType::Timestamp(_, Some(target_tz))) => { @@ -107,7 +107,7 @@ pub fn array_with_timezone( } } DataType::Timestamp(_, None) => { - assert!(!timezone.is_empty()); + require_timezone(&timezone)?; match to_type { Some(DataType::Utf8) | Some(DataType::Date32) => Ok(array), Some(DataType::Timestamp(_, Some(target_tz))) => { @@ -124,7 +124,7 @@ pub fn array_with_timezone( } } DataType::Timestamp(TimeUnit::Microsecond, Some(_)) => { - assert!(!timezone.is_empty()); + require_timezone(&timezone)?; let array = as_primitive_array::(&array); let array_with_timezone = array.clone().with_timezone(timezone.clone()); let array = Arc::new(array_with_timezone) as ArrayRef; @@ -136,7 +136,7 @@ pub fn array_with_timezone( } } DataType::Timestamp(TimeUnit::Millisecond, Some(_)) => { - assert!(!timezone.is_empty()); + require_timezone(&timezone)?; let array = as_primitive_array::(&array); let array_with_timezone = array.clone().with_timezone(timezone.clone()); let array = Arc::new(array_with_timezone) as ArrayRef; @@ -161,6 +161,17 @@ pub fn array_with_timezone( } } +/// Converting a timestamp needs the session timezone. An empty one means the caller built the +/// conversion without it, which is reported as an error rather than a panic. +fn require_timezone(timezone: &str) -> Result<(), ArrowError> { + if timezone.is_empty() { + return Err(ArrowError::InvalidArgumentError( + "Converting a timestamp requires a timezone, but none was given".to_string(), + )); + } + Ok(()) +} + fn datetime_cast_err(value: i64) -> ArrowError { ArrowError::CastError(format!( "Cannot convert TimestampMicrosecondType {value} to datetime. Comet only supports dates between Jan 1, 262145 BCE and Dec 31, 262143 CE", @@ -208,7 +219,6 @@ pub(crate) fn timestamp_ntz_to_timestamp( tz: &str, to_timezone: Option<&str>, ) -> Result { - assert!(!tz.is_empty()); match array.data_type() { DataType::Timestamp(TimeUnit::Microsecond, None) => { let array = as_primitive_array::(&array); @@ -261,7 +271,6 @@ pub(crate) fn cast_timestamp_to_ntz( array: ArrayRef, timezone: &str, ) -> Result { - assert!(!timezone.is_empty()); let tz: Tz = timezone.parse()?; match array.data_type() { DataType::Timestamp(TimeUnit::Microsecond, Some(_)) => { @@ -289,7 +298,6 @@ pub(crate) fn cast_timestamp_to_ntz( /// This takes for special pre-casting cases of Spark. E.g., Timestamp to String. fn pre_timestamp_cast(array: ArrayRef, timezone: String) -> Result { - assert!(!timezone.is_empty()); match array.data_type() { DataType::Timestamp(_, _) => { // Spark doesn't output timezone while casting timestamp to string, but arrow's cast @@ -426,6 +434,17 @@ mod tests { Arc::new(TimestampMicrosecondArray::from(vec![ts]).with_timezone(tz.to_string())) } + #[test] + fn test_array_with_timezone_requires_a_timezone() { + let ltz = ts_with_tz("2024-01-15 10:30:00", "UTC"); + let err = array_with_timezone(ltz, String::new(), Some(&DataType::Utf8)).unwrap_err(); + assert!(err.to_string().contains("requires a timezone"), "{err}"); + + let ntz = array_containing("2024-01-15 10:30:00"); + let utc = DataType::Timestamp(TimeUnit::Microsecond, Some("UTC".into())); + assert!(array_with_timezone(ntz, String::new(), Some(&utc)).is_err()); + } + #[test] fn test_cast_timestamp_to_ntz_utc() { // In UTC, local time == UTC time, so NTZ value == UTC epoch value