diff --git a/datafusion/common/src/nested_struct.rs b/datafusion/common/src/nested_struct.rs index 66e439e398a4e..468e1f64b9226 100644 --- a/datafusion/common/src/nested_struct.rs +++ b/datafusion/common/src/nested_struct.rs @@ -2,7 +2,11 @@ // SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements -use crate::error::{_plan_err, Result}; +use crate::{ + datatype::DataTypeExt, + error::{_plan_err, Result}, + format::DEFAULT_CAST_OPTIONS, +}; use arrow::{ array::{ Array, ArrayRef, AsArray, DictionaryArray, FixedSizeListArray, GenericListArray, @@ -21,6 +25,38 @@ use std::{ sync::Arc, }; +/// Options for DataFusion casts that combine Arrow's cast behavior with an +/// optional field-aware cast extension. +#[derive(Debug, Clone)] +pub struct DFCastOptions<'a> { + /// Options passed to Arrow casts and cast extensions. + pub arrow_options: CastOptions<'a>, + /// The top-level extension or extension registry used to resolve casts. + pub cast_extension: Option<&'a dyn CastExtension>, +} + +impl<'a> DFCastOptions<'a> { + /// Creates cast options without a cast extension. + pub fn new(arrow_options: CastOptions<'a>) -> Self { + Self { + arrow_options, + cast_extension: None, + } + } + + /// Sets the top-level cast extension or extension registry. + pub fn with_cast_extension(mut self, cast_extension: &'a dyn CastExtension) -> Self { + self.cast_extension = Some(cast_extension); + self + } +} + +impl Default for DFCastOptions<'static> { + fn default() -> Self { + Self::new(DEFAULT_CAST_OPTIONS.clone()) + } +} + /// Cast a struct column to match target struct fields, handling nested structs recursively. /// /// Structs can be cast to other structs as long as they share at least one field. The source column @@ -51,7 +87,7 @@ use std::{ fn cast_struct_column( source_col: &ArrayRef, target_fields: &[Arc], - cast_options: &CastOptions, + options: &DFCastOptions<'_>, ) -> Result { if source_col.data_type() == &DataType::Null { return Ok(new_null_array( @@ -68,6 +104,7 @@ fn cast_struct_column( source_fields, target_fields, Some(&mut mapped_source_fields), + options, )?; if !source_col.is_empty() && source_col.null_count() == source_col.len() { @@ -91,10 +128,11 @@ fn cast_struct_column( Some(source_child_col_idx) => { let source_child_col = source_struct.column(*source_child_col_idx); - let adapted_child = cast_column( + let adapted_child = cast_column_fields( source_child_col, - target_child_field.data_type(), - cast_options, + source_fields[*source_child_col_idx].as_ref(), + target_child_field.as_ref(), + options, ) .map_err(|e| { e.context(format!( @@ -141,13 +179,14 @@ fn cast_union_column( source_mode: &UnionMode, target_fields: &UnionFields, target_mode: &UnionMode, - cast_options: &CastOptions, + options: &DFCastOptions<'_>, ) -> Result { validate_union_schema_compatibility( source_fields, source_mode, target_fields, target_mode, + options, )?; let source_union = source_col @@ -163,17 +202,25 @@ fn cast_union_column( let mut children = Vec::with_capacity(target_fields.len()); for (target_type_id, target_field) in target_fields.iter() { + let (_, source_field) = source_fields + .iter() + .find(|(source_type_id, _)| *source_type_id == target_type_id) + .expect("union fields were validated"); let source_child = source_union.child(target_type_id); children.push( - cast_column(source_child, target_field.data_type(), cast_options).map_err( - |e| { - e.context(format!( - "While adapting Union child type ID {target_type_id} ('{}')", - target_field.name() - )) - }, - )?, + cast_column_fields( + source_child, + source_field.as_ref(), + target_field.as_ref(), + options, + ) + .map_err(|e| { + e.context(format!( + "While adapting Union child type ID {target_type_id} ('{}')", + target_field.name() + )) + })?, ); } @@ -244,52 +291,97 @@ pub fn cast_column( target_type: &DataType, cast_options: &CastOptions, ) -> Result { - match (source_col.data_type(), target_type) { + let options = DFCastOptions::new(cast_options.clone()); + cast_column_fields( + source_col, + &source_col.data_type().clone().into_nullable_field(), + &target_type.clone().into_nullable_field(), + &options, + ) +} + +/// Casts an array between fields, allowing custom casts that depend on field metadata. +/// +/// Unlike [`cast_column`], this function preserves source and target [`Field`] metadata +/// and recursively passes it through nested container types. It also passes the same +/// [`DFCastOptions`] through extension dispatch and every recursive child cast. +pub fn cast_column_fields( + source_col: &ArrayRef, + source_field: &Field, + target_field: &Field, + options: &DFCastOptions<'_>, +) -> Result { + if let Some(cast_extension) = options.cast_extension + && cast_extension.can_cast_fields(source_field, target_field, options)? + { + return cast_extension.cast_array_fields( + source_col, + source_field, + target_field, + options, + ); + } + + match (source_field.data_type(), target_field.data_type()) { (_, Struct(target_fields)) => { - cast_struct_column(source_col, target_fields, cast_options) + cast_struct_column(source_col, target_fields, options) } - (DataType::List(_), DataType::List(target_inner)) => { - cast_list_column::(source_col, target_inner, cast_options) + (DataType::List(source_inner), DataType::List(target_inner)) => { + cast_list_column::(source_col, source_inner, target_inner, options) } - (DataType::LargeList(_), DataType::LargeList(target_inner)) => { - cast_list_column::(source_col, target_inner, cast_options) + (DataType::LargeList(source_inner), DataType::LargeList(target_inner)) => { + cast_list_column::(source_col, source_inner, target_inner, options) } ( - DataType::FixedSizeList(_, source_list_size), + DataType::FixedSizeList(source_inner, source_list_size), DataType::FixedSizeList(target_inner, target_list_size), ) if source_list_size == target_list_size => cast_fixed_size_list_column( source_col, + source_inner, target_inner, *target_list_size, - cast_options, + options, ), - (DataType::ListView(_), DataType::ListView(target_inner)) => { - cast_list_view_column::(source_col, target_inner, cast_options) + (DataType::ListView(source_inner), DataType::ListView(target_inner)) => { + cast_list_view_column::(source_col, source_inner, target_inner, options) } - (DataType::LargeListView(_), DataType::LargeListView(target_inner)) => { - cast_list_view_column::(source_col, target_inner, cast_options) + ( + DataType::LargeListView(source_inner), + DataType::LargeListView(target_inner), + ) => { + cast_list_view_column::(source_col, source_inner, target_inner, options) } (DataType::Map(_, _), DataType::Map(target_entries, target_sorted)) => { - if requires_nested_struct_cast(source_col.data_type(), target_type) { + if options.cast_extension.is_some() + || requires_nested_struct_cast( + source_field.data_type(), + target_field.data_type(), + ) + { cast_map_column( source_col.as_map(), target_entries, *target_sorted, - cast_options, + options, ) } else { - Ok(cast_with_options(source_col, target_type, cast_options)?) + Ok(cast_with_options( + source_col, + target_field.data_type(), + &options.arrow_options, + )?) } } ( - DataType::Dictionary(source_key_type, _), + DataType::Dictionary(source_key_type, source_value_type), DataType::Dictionary(target_key_type, target_value_type), ) => cast_dictionary_column( source_col, source_key_type, + source_value_type, target_key_type, target_value_type, - cast_options, + options, ), ( DataType::Union(source_fields, source_mode), @@ -300,16 +392,21 @@ pub fn cast_column( source_mode, target_fields, target_mode, - cast_options, + options, ), - _ => Ok(cast_with_options(source_col, target_type, cast_options)?), + _ => Ok(cast_with_options( + source_col, + target_field.data_type(), + &options.arrow_options, + )?), } } fn cast_list_column( source_col: &ArrayRef, + source_inner_field: &FieldRef, target_inner_field: &FieldRef, - cast_options: &CastOptions, + options: &DFCastOptions<'_>, ) -> Result { let source_list = source_col.as_list::(); let offsets = source_list.value_offsets(); @@ -325,10 +422,11 @@ fn cast_list_column( }; let source_list = compacted_list.as_ref().unwrap_or(source_list); - let cast_values = cast_column( + let cast_values = cast_column_fields( source_list.values(), - target_inner_field.data_type(), - cast_options, + source_inner_field.as_ref(), + target_inner_field.as_ref(), + options, )?; let result = GenericListArray::::new( @@ -342,8 +440,9 @@ fn cast_list_column( fn cast_list_view_column( source_col: &ArrayRef, + source_inner_field: &FieldRef, target_inner_field: &FieldRef, - cast_options: &CastOptions, + options: &DFCastOptions<'_>, ) -> Result { let source_list = source_col.as_list_view::(); let compacted_values = compact_list_view_values(source_list)?; @@ -356,7 +455,12 @@ fn cast_list_view_column( ), }; - let cast_values = cast_column(values, target_inner_field.data_type(), cast_options)?; + let cast_values = cast_column_fields( + values, + source_inner_field.as_ref(), + target_inner_field.as_ref(), + options, + )?; let result = GenericListViewArray::::try_new( Arc::clone(target_inner_field), @@ -469,21 +573,26 @@ fn compact_list_view_values( fn cast_fixed_size_list_column( source_col: &ArrayRef, + source_inner_field: &FieldRef, target_inner_field: &FieldRef, target_list_size: i32, - cast_options: &CastOptions, + options: &DFCastOptions<'_>, ) -> Result { let source_list = source_col.as_fixed_size_list(); let source_values = source_list.values(); - let target_type = target_inner_field.data_type(); - - let cast_values = match cast_column(source_values, target_type, cast_options) { + let cast_values = match cast_column_fields( + source_values, + source_inner_field.as_ref(), + target_inner_field.as_ref(), + options, + ) { Ok(cast_values) => cast_values, Err(error) => match cast_fixed_size_list_values_with_parent_nulls( source_values, - target_type, - cast_options, + source_inner_field, + target_inner_field, + options, source_list.nulls(), target_list_size, ) { @@ -502,8 +611,9 @@ fn cast_fixed_size_list_column( fn cast_fixed_size_list_values_with_parent_nulls( source_values: &ArrayRef, - target_type: &DataType, - cast_options: &CastOptions, + source_field: &FieldRef, + target_field: &FieldRef, + options: &DFCastOptions<'_>, parent_nulls: Option<&NullBuffer>, list_size: i32, ) -> Option> { @@ -513,7 +623,14 @@ fn cast_fixed_size_list_values_with_parent_nulls( // values are semantically hidden, but recursive casts still inspect them. let hidden_child_nulls = parent_nulls.expand(list_size as usize); let masked_values = mask_array_values(source_values, &hidden_child_nulls); - Some(masked_values.and_then(|values| cast_column(&values, target_type, cast_options))) + Some(masked_values.and_then(|values| { + cast_column_fields( + &values, + source_field.as_ref(), + target_field.as_ref(), + options, + ) + })) } fn mask_array_values( @@ -559,7 +676,7 @@ fn cast_map_column( source_map: &MapArray, target_entries: &FieldRef, target_sorted: bool, - cast_options: &CastOptions, + options: &DFCastOptions<'_>, ) -> Result { let DataType::Map(source_entries, source_sorted) = source_map.data_type() else { unreachable!("MapArray data type must be Map") @@ -570,7 +687,9 @@ fn cast_map_column( target_entries, target_sorted, None, + options, )?; + let (source_key, source_value) = validate_map_entries_field(source_entries)?; let offsets = source_map.value_offsets(); let has_unreachable_entries = offsets[0] != 0 @@ -584,11 +703,20 @@ fn cast_map_column( }; let source_map = compacted_map.as_ref().unwrap_or(source_map); - let cast_keys = cast_column(source_map.keys(), target_key.data_type(), cast_options) - .map_err(|error| error.context("While casting Map keys"))?; - let cast_values = - cast_column(source_map.values(), target_value.data_type(), cast_options) - .map_err(|error| error.context("While casting Map values"))?; + let cast_keys = cast_column_fields( + source_map.keys(), + source_key.as_ref(), + target_key.as_ref(), + options, + ) + .map_err(|error| error.context("While casting Map keys"))?; + let cast_values = cast_column_fields( + source_map.values(), + source_value.as_ref(), + target_value.as_ref(), + options, + ) + .map_err(|error| error.context("While casting Map values"))?; let Struct(target_fields) = target_entries.data_type() else { unreachable!("validated Map entries must be Struct") }; @@ -619,9 +747,10 @@ fn compact_map_entries(map: &MapArray) -> Result { fn cast_dictionary_column( source_col: &ArrayRef, source_key_type: &DataType, + source_value_type: &DataType, target_key_type: &DataType, target_value_type: &DataType, - cast_options: &CastOptions, + options: &DFCastOptions<'_>, ) -> Result { // Dispatch on source key type to access keys/values, then recursively // cast values. Rebuild with the source key type first. @@ -631,8 +760,12 @@ fn cast_dictionary_column( .as_any() .downcast_ref::>() .expect("downcast must succeed"); - let cast_values = - cast_column(source_dict.values(), target_value_type, cast_options)?; + let cast_values = cast_column_fields( + source_dict.values(), + &source_value_type.clone().into_nullable_field(), + &target_value_type.clone().into_nullable_field(), + options, + )?; Ok(Arc::new(DictionaryArray::<$t>::new( source_dict.keys().clone(), cast_values, @@ -652,7 +785,11 @@ fn cast_dictionary_column( Box::new(target_key_type.clone()), Box::new(target_value_type.clone()), ); - Ok(cast_with_options(&result, &target_dict_type, cast_options)?) + Ok(cast_with_options( + &result, + &target_dict_type, + &options.arrow_options, + )?) } else { Ok(result) } @@ -700,7 +837,20 @@ pub fn validate_struct_compatibility( source_fields: &[FieldRef], target_fields: &[FieldRef], ) -> Result<()> { - map_struct_fields(source_fields, target_fields, None) + validate_struct_compatibility_with_options( + source_fields, + target_fields, + &DFCastOptions::default(), + ) +} + +/// Validates struct compatibility using the supplied DataFusion cast options. +pub fn validate_struct_compatibility_with_options( + source_fields: &[FieldRef], + target_fields: &[FieldRef], + options: &DFCastOptions<'_>, +) -> Result<()> { + map_struct_fields(source_fields, target_fields, None, options) } /// Performs the mapping of struct fields, checking compatibility and handling missing or extra fields. @@ -718,6 +868,7 @@ fn map_struct_fields( source_fields: &[FieldRef], target_fields: &[FieldRef], mut mapped_source_fields: Option<&mut Vec>>, + options: &DFCastOptions<'_>, ) -> Result<()> { if !source_fields.is_empty() { let has_overlap = has_one_of_more_common_fields(source_fields, target_fields); @@ -740,7 +891,7 @@ fn map_struct_fields( let source_field_idx = if let Some((source_field_idx, source_field)) = source_field_opt { - validate_field_compatibility(source_field, target_field)?; + validate_field_compatibility(source_field, target_field, options)?; Some(source_field_idx) } else { @@ -769,6 +920,7 @@ fn map_struct_fields( fn validate_field_compatibility( source_field: &Field, target_field: &Field, + options: &DFCastOptions<'_>, ) -> Result<()> { if source_field.data_type() == &DataType::Null { // Validate that target allows nulls before returning early. @@ -793,10 +945,17 @@ fn validate_field_compatibility( ); } - validate_data_type_compatibility( + if let Some(cast_extension) = options.cast_extension + && cast_extension.can_cast_fields(source_field, target_field, options)? + { + return Ok(()); + } + + validate_data_type_compatibility_with_options( target_field.name(), source_field.data_type(), target_field.data_type(), + options, ) } @@ -806,6 +965,7 @@ fn validate_map_compatibility<'a>( target_entries: &'a Field, target_sorted: bool, field_name: Option<&str>, + options: &DFCastOptions<'_>, ) -> Result<(&'a FieldRef, &'a FieldRef)> { if source_sorted != target_sorted { if let Some(field_name) = field_name { @@ -818,8 +978,8 @@ fn validate_map_compatibility<'a>( } let (source_key, source_value) = validate_map_entries_field(source_entries)?; let (target_key, target_value) = validate_map_entries_field(target_entries)?; - validate_map_key_compatibility(source_key, target_key, target_sorted)?; - validate_field_compatibility(source_value, target_value)?; + validate_map_key_compatibility(source_key, target_key, target_sorted, options)?; + validate_field_compatibility(source_value, target_value, options)?; Ok((target_key, target_value)) } @@ -827,23 +987,59 @@ fn validate_map_key_compatibility( source_key: &Field, target_key: &Field, sorted: bool, + options: &DFCastOptions<'_>, ) -> Result<()> { if sorted && source_key.data_type() != target_key.data_type() { return _plan_err!("Cannot evolve key type of a sorted Map"); } - validate_map_key_data_type(source_key.data_type(), target_key.data_type()) + validate_map_key_field(source_key, target_key, sorted, options) } -fn validate_map_key_data_type( - source_type: &DataType, - target_type: &DataType, +fn validate_map_key_field( + source_field: &Field, + target_field: &Field, + sorted: bool, + options: &DFCastOptions<'_>, ) -> Result<()> { - if source_type == target_type { + if let Some(cast_extension) = options.cast_extension + && cast_extension.can_cast_fields(source_field, target_field, options)? + { + if !is_injective_cast_fields(source_field, target_field, options)? { + return _plan_err!( + "Cannot safely evolve Map key type from {} to {}", + source_field.data_type(), + target_field.data_type() + ); + } + if sorted + && (!cast_extension.is_order_preserving_fields( + source_field, + target_field, + options, + )? || !cast_extension.preserves_nulls_fields( + source_field, + target_field, + options, + )?) + { + return _plan_err!( + "Cannot evolve a sorted Map key using a cast extension that does not preserve ordering and nulls" + ); + } + return Ok(()); + } + + let source_type = source_field.data_type(); + let target_type = target_field.data_type(); + if source_type == target_type && !matches!(source_type, Struct(_)) { return Ok(()); } match (source_type, target_type) { (Struct(source_fields), Struct(target_fields)) => { + if source_fields.is_empty() && target_fields.is_empty() { + return Ok(()); + } if !has_one_of_more_common_fields(source_fields, target_fields) { return _plan_err!( "Cannot cast Map key Struct because there is no field name overlap" @@ -873,10 +1069,7 @@ fn validate_map_key_data_type( source_field.name() ); } - validate_map_key_data_type( - source_field.data_type(), - target_field.data_type(), - )?; + validate_map_key_field(source_field, target_field, sorted, options)?; } for target_field in target_fields { if !source_names.contains(target_field.name().as_str()) @@ -937,6 +1130,28 @@ pub fn is_injective_cast(source_type: &DataType, target_type: &DataType) -> bool ) } +/// Returns whether the cast selected for `source_field` and `target_field` is +/// known to preserve value identity. +/// +/// If the configured cast extension handles the fields, its injectivity declaration takes +/// precedence because [`cast_column_fields`] also selects the extension before +/// Arrow's built-in cast. Otherwise, identical data types and casts recognized +/// by [`is_injective_cast`] are injective. +pub fn is_injective_cast_fields( + source_field: &Field, + target_field: &Field, + options: &DFCastOptions<'_>, +) -> Result { + if let Some(cast_extension) = options.cast_extension + && cast_extension.can_cast_fields(source_field, target_field, options)? + { + return cast_extension.is_injective_fields(source_field, target_field, options); + } + + Ok(source_field.data_type() == target_field.data_type() + || is_injective_cast(source_field.data_type(), target_field.data_type())) +} + fn validate_map_entries_field(entries: &Field) -> Result<(&FieldRef, &FieldRef)> { if entries.is_nullable() { return _plan_err!("Map entries field must be non-nullable"); @@ -977,6 +1192,7 @@ fn validate_union_schema_compatibility( source_mode: &UnionMode, target_fields: &UnionFields, target_mode: &UnionMode, + options: &DFCastOptions<'_>, ) -> Result<()> { if source_mode != target_mode { return _plan_err!( @@ -1004,7 +1220,15 @@ fn validate_union_schema_compatibility( ); }; - if !target_field.contains(source_field) { + if options.cast_extension.is_some() { + validate_field_compatibility(source_field, target_field, options).map_err( + |error| { + error.context(format!( + "Cannot adapt Union child with type ID {target_type_id}" + )) + }, + )?; + } else if !target_field.contains(source_field) { return _plan_err!( "Cannot adapt Union child with type ID {target_type_id}: source field {source_field} is not contained by target field {target_field}" ); @@ -1018,31 +1242,53 @@ pub fn validate_data_type_compatibility( field_name: &str, source_type: &DataType, target_type: &DataType, +) -> Result<()> { + validate_data_type_compatibility_with_options( + field_name, + source_type, + target_type, + &DFCastOptions::default(), + ) +} + +/// Validates type compatibility using the supplied DataFusion cast options. +pub fn validate_data_type_compatibility_with_options( + field_name: &str, + source_type: &DataType, + target_type: &DataType, + options: &DFCastOptions<'_>, ) -> Result<()> { match (source_type, target_type) { (Struct(source_nested), Struct(target_nested)) => { - validate_struct_compatibility(source_nested, target_nested)?; + validate_struct_compatibility_with_options( + source_nested, + target_nested, + options, + )?; } ( DataType::FixedSizeList(s, source_list_size), DataType::FixedSizeList(t, target_list_size), ) if source_list_size == target_list_size => { - validate_field_compatibility(s, t)?; + validate_field_compatibility(s, t, options)?; } (DataType::List(s), DataType::List(t)) | (DataType::LargeList(s), DataType::LargeList(t)) | (DataType::ListView(s), DataType::ListView(t)) | (DataType::LargeListView(s), DataType::LargeListView(t)) => { - validate_field_compatibility(s, t)?; + validate_field_compatibility(s, t, options)?; } (DataType::Map(s, source_sorted), DataType::Map(t, target_sorted)) => { - if requires_nested_struct_cast(source_type, target_type) { + if options.cast_extension.is_some() + || requires_nested_struct_cast(source_type, target_type) + { validate_map_compatibility( s, *source_sorted, t, *target_sorted, Some(field_name), + options, )?; } else if !can_cast_types(source_type, target_type) { return _plan_err!( @@ -1062,7 +1308,9 @@ pub fn validate_data_type_compatibility( field_name ); } - validate_data_type_compatibility(field_name, s_val, t_val)?; + validate_data_type_compatibility_with_options( + field_name, s_val, t_val, options, + )?; } ( DataType::Union(source_fields, source_mode), @@ -1073,6 +1321,7 @@ pub fn validate_data_type_compatibility( source_mode, target_fields, target_mode, + options, )?; } _ => { @@ -1174,15 +1423,90 @@ pub fn has_one_of_more_common_fields( .any(|field| source_names.contains(field.name().as_str())) } +/// Extends array casting with conversions selected using the source and target +/// [`Field`]s. +/// +/// Implementations may select conversions using field metadata, the fields' +/// bare [`DataType`]s, or both. Metadata is not required. Extensions are checked +/// before Arrow's built-in cast, including when recursively casting nested types. +/// +/// Implementations should return `true` from [`Self::can_cast_fields`] only for +/// conversions they can perform in [`Self::cast_array_fields`]. +/// +/// When a registry dispatches to an independently registered extension, `self` +/// is the selected extension while [`DFCastOptions::cast_extension`] remains the +/// top-level registry. Recursive casts should pass the same options back to +/// [`cast_column_fields`]. +pub trait CastExtension: std::fmt::Debug + Send + Sync { + /// Returns whether this extension can cast between the supplied fields. + /// + /// Implementations may match only [`Field::data_type`] when the conversion + /// does not depend on metadata. + fn can_cast_fields( + &self, + source_field: &Field, + target_field: &Field, + options: &DFCastOptions<'_>, + ) -> Result; + + /// Returns whether this extension's cast preserves value identity. + /// + /// This method is called only when [`Self::can_cast_fields`] returns `true`. + /// The conservative default prevents callers from relying on injectivity + /// unless the implementation explicitly guarantees it. + fn is_injective_fields( + &self, + _source_field: &Field, + _target_field: &Field, + _options: &DFCastOptions<'_>, + ) -> Result { + Ok(false) + } + + /// Returns whether this extension's cast preserves value ordering. + /// + /// This method is called only when [`Self::can_cast_fields`] returns `true`. + fn is_order_preserving_fields( + &self, + _source_field: &Field, + _target_field: &Field, + _options: &DFCastOptions<'_>, + ) -> Result { + Ok(false) + } + + /// Returns whether this extension maps null inputs to null outputs without + /// introducing nulls for non-null inputs. + /// + /// This method is called only when [`Self::can_cast_fields`] returns `true`. + fn preserves_nulls_fields( + &self, + _source_field: &Field, + _target_field: &Field, + _options: &DFCastOptions<'_>, + ) -> Result { + Ok(false) + } + + /// Casts `array` between the supplied fields. + fn cast_array_fields( + &self, + array: &ArrayRef, + source_field: &Field, + target_field: &Field, + options: &DFCastOptions<'_>, + ) -> Result; +} + #[cfg(test)] mod tests { use super::*; - use crate::{assert_contains, format::DEFAULT_CAST_OPTIONS}; + use crate::{assert_contains, error::_internal_err}; use arrow::{ array::{ - BinaryArray, FixedSizeListArray, Int32Array, Int32Builder, Int64Array, - Int64Builder, ListArray, ListViewArray, MapArray, MapBuilder, NullArray, - StringArray, StringBuilder, UInt32Array, + BinaryArray, FixedSizeListArray, Int8Array, Int32Array, Int32Builder, + Int64Array, Int64Builder, ListArray, ListViewArray, MapArray, MapBuilder, + NullArray, StringArray, StringBuilder, UInt32Array, }, buffer::{NullBuffer, OffsetBuffer, ScalarBuffer}, datatypes::{DataType, Field, FieldRef, Int32Type}, @@ -1225,6 +1549,149 @@ mod tests { Arc::new(struct_field(name, fields)) } + fn bool8_field(name: &str) -> Field { + Field::new(name, DataType::Int8, true).with_metadata(HashMap::from([( + "ARROW:extension:name".to_string(), + "arrow.bool8".to_string(), + )])) + } + + #[derive(Debug)] + struct Bool8ToStringCast; + + impl CastExtension for Bool8ToStringCast { + fn can_cast_fields( + &self, + source_field: &Field, + target_field: &Field, + _options: &DFCastOptions<'_>, + ) -> Result { + Ok(source_field + .metadata() + .get("ARROW:extension:name") + .map(String::as_str) + == Some("arrow.bool8") + && source_field.data_type() == &DataType::Int8 + && target_field.data_type() == &DataType::Utf8 + && target_field.extension_type_name().is_none()) + } + + fn cast_array_fields( + &self, + array: &ArrayRef, + source_field: &Field, + target_field: &Field, + options: &DFCastOptions<'_>, + ) -> Result { + if !self.can_cast_fields(source_field, target_field, options)? { + return _internal_err!("Bool8ToStringCast cannot cast supplied fields"); + } + let array = array + .as_any() + .downcast_ref::() + .expect("arrow.bool8 uses Int8 storage"); + Ok(Arc::new(StringArray::from_iter(array.iter().map( + |value| value.map(|value| if value == 0 { "false" } else { "true" }), + )))) + } + } + + #[derive(Debug)] + struct BareInt8ToStringCast; + + impl CastExtension for BareInt8ToStringCast { + fn can_cast_fields( + &self, + source_field: &Field, + target_field: &Field, + _options: &DFCastOptions<'_>, + ) -> Result { + Ok(source_field.data_type() == &DataType::Int8 + && target_field.data_type() == &DataType::Utf8) + } + + fn cast_array_fields( + &self, + array: &ArrayRef, + source_field: &Field, + target_field: &Field, + options: &DFCastOptions<'_>, + ) -> Result { + if !self.can_cast_fields(source_field, target_field, options)? { + return _internal_err!( + "BareInt8ToStringCast cannot cast supplied fields" + ); + } + let array = array + .as_any() + .downcast_ref::() + .expect("extension receives Int8 input"); + Ok(Arc::new(StringArray::from_iter(array.iter().map( + |value| value.map(|value| if value == 0 { "false" } else { "true" }), + )))) + } + } + + #[derive(Debug)] + struct Int8ToInt16Cast { + injective: bool, + order_preserving: bool, + preserves_nulls: bool, + } + + impl CastExtension for Int8ToInt16Cast { + fn can_cast_fields( + &self, + source_field: &Field, + target_field: &Field, + _options: &DFCastOptions<'_>, + ) -> Result { + Ok(source_field.data_type() == &DataType::Int8 + && target_field.data_type() == &DataType::Int16) + } + + fn is_injective_fields( + &self, + _source_field: &Field, + _target_field: &Field, + _options: &DFCastOptions<'_>, + ) -> Result { + Ok(self.injective) + } + + fn is_order_preserving_fields( + &self, + _source_field: &Field, + _target_field: &Field, + _options: &DFCastOptions<'_>, + ) -> Result { + Ok(self.order_preserving) + } + + fn preserves_nulls_fields( + &self, + _source_field: &Field, + _target_field: &Field, + _options: &DFCastOptions<'_>, + ) -> Result { + Ok(self.preserves_nulls) + } + + fn cast_array_fields( + &self, + array: &ArrayRef, + _source_field: &Field, + target_field: &Field, + options: &DFCastOptions<'_>, + ) -> Result { + Ok(cast_with_options( + array, + target_field.data_type(), + &options.arrow_options, + )?) + } + } + #[test] fn test_cast_simple_column() { let source = Arc::new(Int32Array::from(vec![1, 2, 3])) as ArrayRef; @@ -1239,6 +1706,260 @@ mod tests { assert_eq!(result.value(2), 3); } + #[test] + fn test_cast_extension_bare_data_types() { + let source_field = field("value", DataType::Int8); + let target_field = field("value", DataType::Utf8); + assert!(source_field.metadata().is_empty()); + assert!(target_field.metadata().is_empty()); + + let options = DFCastOptions::new(DEFAULT_CAST_OPTIONS.clone()) + .with_cast_extension(&BareInt8ToStringCast); + + let result = + cast_column_fields(&int8_values(), &source_field, &target_field, &options) + .unwrap(); + + assert_boolean_strings(&result); + } + + fn int8_values() -> ArrayRef { + Arc::new(Int8Array::from(vec![Some(0), Some(1), Some(-20), None])) + } + + fn cast_bool8_container( + source: ArrayRef, + source_field: Field, + target_field: Field, + ) -> ArrayRef { + let options = DFCastOptions::new(DEFAULT_CAST_OPTIONS.clone()) + .with_cast_extension(&Bool8ToStringCast); + validate_data_type_compatibility_with_options( + source_field.name(), + source_field.data_type(), + target_field.data_type(), + &options, + ) + .unwrap(); + cast_column_fields(&source, &source_field, &target_field, &options).unwrap() + } + + fn assert_boolean_strings(values: &ArrayRef) { + let values = values.as_string::(); + assert_eq!( + values.iter().collect::>(), + vec![Some("false"), Some("true"), Some("true"), None] + ); + } + + #[test] + fn test_cast_extension_nested_struct() { + let source_fields: Fields = vec![Arc::new(bool8_field("value"))].into(); + let source = Arc::new(StructArray::new( + source_fields.clone(), + vec![int8_values()], + None, + )); + let target_fields: Fields = vec![arc_field("value", DataType::Utf8)].into(); + let result = cast_bool8_container( + source, + field("root", Struct(source_fields)), + field("root", Struct(target_fields)), + ); + + assert_boolean_strings(result.as_struct().column(0)); + } + + #[test] + fn test_cast_extension_nested_list() { + let source_item = Arc::new(bool8_field("item")); + let source = Arc::new(ListArray::new( + Arc::clone(&source_item), + OffsetBuffer::new(ScalarBuffer::from(vec![0, 2, 4])), + int8_values(), + None, + )); + let target_item = arc_field("item", DataType::Utf8); + let result = cast_bool8_container( + source, + field("root", DataType::List(source_item)), + field("root", DataType::List(target_item)), + ); + + assert_boolean_strings(result.as_list::().values()); + } + + #[test] + fn test_cast_extension_nested_large_list() { + let source_item = Arc::new(bool8_field("item")); + let source = Arc::new(GenericListArray::::new( + Arc::clone(&source_item), + OffsetBuffer::new(ScalarBuffer::from(vec![0, 2, 4])), + int8_values(), + None, + )); + let target_item = arc_field("item", DataType::Utf8); + let result = cast_bool8_container( + source, + field("root", DataType::LargeList(source_item)), + field("root", DataType::LargeList(target_item)), + ); + + assert_boolean_strings(result.as_list::().values()); + } + + #[test] + fn test_cast_extension_nested_fixed_size_list() { + let source_item = Arc::new(bool8_field("item")); + let source = Arc::new(FixedSizeListArray::new( + Arc::clone(&source_item), + 2, + int8_values(), + None, + )); + let target_item = arc_field("item", DataType::Utf8); + let result = cast_bool8_container( + source, + field("root", DataType::FixedSizeList(source_item, 2)), + field("root", DataType::FixedSizeList(target_item, 2)), + ); + + assert_boolean_strings(result.as_fixed_size_list().values()); + } + + #[test] + fn test_cast_extension_nested_list_view() { + let source_item = Arc::new(bool8_field("item")); + let source = Arc::new(ListViewArray::new( + Arc::clone(&source_item), + ScalarBuffer::from(vec![0, 2]), + ScalarBuffer::from(vec![2, 2]), + int8_values(), + None, + )); + let target_item = arc_field("item", DataType::Utf8); + let result = cast_bool8_container( + source, + field("root", DataType::ListView(source_item)), + field("root", DataType::ListView(target_item)), + ); + + assert_boolean_strings(result.as_list_view::().values()); + } + + #[test] + fn test_cast_extension_nested_large_list_view() { + let source_item = Arc::new(bool8_field("item")); + let source = Arc::new(GenericListViewArray::::new( + Arc::clone(&source_item), + ScalarBuffer::from(vec![0, 2]), + ScalarBuffer::from(vec![2, 2]), + int8_values(), + None, + )); + let target_item = arc_field("item", DataType::Utf8); + let result = cast_bool8_container( + source, + field("root", DataType::LargeListView(source_item)), + field("root", DataType::LargeListView(target_item)), + ); + + assert_boolean_strings(result.as_list_view::().values()); + } + + #[test] + fn test_cast_extension_nested_map() { + let source_entries = StructArray::new( + vec![ + Arc::new(non_null_field("keys", DataType::Utf8)), + Arc::new(bool8_field("values")), + ] + .into(), + vec![ + Arc::new(StringArray::from(vec!["a", "b", "c", "d"])), + int8_values(), + ], + None, + ); + let source_entries_field = Arc::new(non_null_field( + "entries", + source_entries.data_type().clone(), + )); + let source = Arc::new(MapArray::new( + Arc::clone(&source_entries_field), + OffsetBuffer::new(ScalarBuffer::from(vec![0, 2, 4])), + source_entries, + None, + false, + )); + let target_entries_field = Arc::new(non_null_field( + "entries", + struct_type(vec![ + non_null_field("keys", DataType::Utf8), + field("values", DataType::Utf8), + ]), + )); + let result = cast_bool8_container( + source, + field("root", DataType::Map(source_entries_field, false)), + field("root", DataType::Map(target_entries_field, false)), + ); + + assert_boolean_strings(result.as_map().values()); + } + + #[test] + fn test_cast_extension_nested_dictionary() { + let source_values = Arc::new(StructArray::new( + vec![Arc::new(bool8_field("value"))].into(), + vec![int8_values()], + None, + )); + let source: ArrayRef = Arc::new(DictionaryArray::::new( + Int32Array::from(vec![0, 1, 2, 3]), + source_values, + )); + let target_type = DataType::Dictionary( + Box::new(DataType::Int32), + Box::new(struct_type(vec![field("value", DataType::Utf8)])), + ); + let result = cast_bool8_container( + Arc::clone(&source), + field("root", source.data_type().clone()), + field("root", target_type), + ); + let result = result + .as_any() + .downcast_ref::>() + .unwrap(); + + assert_boolean_strings(result.values().as_struct().column(0)); + } + + #[test] + fn test_cast_extension_nested_union() { + let source_fields = + UnionFields::try_new(vec![0], vec![bool8_field("value")]).unwrap(); + let source = Arc::new( + UnionArray::try_new( + source_fields.clone(), + ScalarBuffer::from(vec![0i8, 0, 0, 0]), + None, + vec![int8_values()], + ) + .unwrap(), + ); + let target_fields = + UnionFields::try_new(vec![0], vec![field("value", DataType::Utf8)]).unwrap(); + let result = cast_bool8_container( + source, + field("root", DataType::Union(source_fields, UnionMode::Sparse)), + field("root", DataType::Union(target_fields, UnionMode::Sparse)), + ); + + assert_boolean_strings(result.as_union().child(0)); + } + #[test] fn test_cast_column_with_options() { let source = Arc::new(Int64Array::from(vec![1, i64::MAX])) as ArrayRef; @@ -2281,6 +3002,72 @@ mod tests { } } + #[test] + fn test_is_injective_cast_fields_uses_extension_properties() { + let source = non_null_field("key", DataType::Int8); + let target = non_null_field("key", DataType::Int16); + assert!(is_injective_cast(source.data_type(), target.data_type())); + + let non_injective = Int8ToInt16Cast { + injective: false, + order_preserving: true, + preserves_nulls: true, + }; + let non_injective_options = + DFCastOptions::default().with_cast_extension(&non_injective); + assert!( + !is_injective_cast_fields(&source, &target, &non_injective_options).unwrap() + ); + + let injective = Int8ToInt16Cast { + injective: true, + order_preserving: true, + preserves_nulls: true, + }; + let injective_options = DFCastOptions::default().with_cast_extension(&injective); + assert!(is_injective_cast_fields(&source, &target, &injective_options).unwrap()); + assert!( + is_injective_cast_fields(&source, &target, &DFCastOptions::default()) + .unwrap() + ); + } + + #[test] + fn test_map_key_cast_uses_extension_properties() { + let source = map_type(DataType::Int8, DataType::Utf8); + let target = map_type(DataType::Int16, DataType::Utf8); + let non_injective = Int8ToInt16Cast { + injective: false, + order_preserving: true, + preserves_nulls: true, + }; + let non_injective_options = + DFCastOptions::default().with_cast_extension(&non_injective); + + let error = validate_data_type_compatibility_with_options( + "map_col", + &source, + &target, + &non_injective_options, + ) + .unwrap_err(); + assert_contains!(error.to_string(), "Cannot safely evolve Map key type"); + + let injective = Int8ToInt16Cast { + injective: true, + order_preserving: true, + preserves_nulls: true, + }; + let injective_options = DFCastOptions::default().with_cast_extension(&injective); + validate_data_type_compatibility_with_options( + "map_col", + &source, + &target, + &injective_options, + ) + .unwrap(); + } + #[test] fn test_map_entry_names_match_positionally_and_adapt_nested_structs() { let source_col = nested_struct_map_array("source_keys", "source_values", false);