From 976d21cd1c414e291f2b99931cfa8f9da58cae54 Mon Sep 17 00:00:00 2001 From: Connor Tsui Date: Mon, 9 Nov 2026 10:01:14 -0500 Subject: [PATCH] Compare nested extension types by storage in Vortex ID exports Signed-off-by: Connor Tsui --- vortex-arrow/src/session.rs | 107 ++++++++++++++++++++++++-------- vortex-arrow/tests/vortex_id.rs | 74 +++++++++++++++++++++- 2 files changed, 151 insertions(+), 30 deletions(-) diff --git a/vortex-arrow/src/session.rs b/vortex-arrow/src/session.rs index 651c72452bd..8320feb76f6 100644 --- a/vortex-arrow/src/session.rs +++ b/vortex-arrow/src/session.rs @@ -267,11 +267,13 @@ type ArrowImporterRegistry = ArcSwapMap>; /// schemas, so [`ExtDTypeRef::serialize_metadata`] must give equal bytes for equal types. /// /// A caller can also export an array to a field that names its Vortex ID, such as a field from -/// [`Self::to_arrow_storage_field`]. The export then checks that an import of the result gives back -/// the same type. The field must have the same metadata, and its Arrow type must convert to the -/// storage type (or to the type itself, such as an Arrow timestamp). Without these checks, values -/// could silently change meaning, such as milliseconds that read back as seconds. For the same -/// reason, an extension array cannot export to a field that names a different known Vortex ID. +/// [`Self::to_arrow_storage_field`]. The export then checks that the field describes the type of the +/// array. The field must have the same metadata, and its Arrow type must import as the storage type +/// (or as the type itself, such as an Arrow timestamp). Nested extension types count as their +/// storage types in this check, and the export checks each nested field when it reaches it. Without +/// these checks, values could silently change meaning, such as milliseconds that read back as +/// seconds. For the same reason, an extension array cannot export to a field that names a different +/// known Vortex ID. /// /// On import, the plugins run first. If none applies, a field that names a known Vortex ID imports /// as that type over its storage, and a field with any other name imports as plain storage. A @@ -529,11 +531,7 @@ impl ArrowSession { .as_extension_opt() .filter(|ext_dtype| ext_dtype.id() == ext_dtype_plugin.id()); if let Some(existing_ext_dtype) = existing_ext_dtype { - vortex_ensure!( - metadata == existing_ext_dtype.serialize_metadata()?, - "Arrow field {:?} must carry the metadata of {existing_ext_dtype}, got {metadata:?}", - field.name() - ); + ensure_ext_metadata(field, &metadata, existing_ext_dtype)?; return Ok(None); } @@ -969,10 +967,10 @@ impl ArrowSession { DataType::Map(entries_field, keys_sorted) => { let map = array.as_map(); let (entries, offsets) = trim_map_entries(map); + // `map_from_arrow_parts` needs the entries as a plain struct, so a Vortex ID on the + // entries field is ignored. Import plugins for its extension name still run, while + // `from_arrow_datatype` ignores the entries field completely. let entries_array: ArrowArrayRef = Arc::new(entries); - // Map entries must import as a plain struct, so a Vortex ID on their field is - // ignored, as `from_arrow_datatype` also does. Wrapping the entries in that - // extension type would only make `map_from_arrow_parts` fail. let entries = self.from_arrow_array_with_importers(entries_array, entries_field.as_ref())?; map_from_arrow_parts( @@ -1130,8 +1128,8 @@ impl ArrowExporter<'_> { } } - // The registry can hold an alias, so compare the array with the ID of the dtype plugin - // that the extension name resolves to. + // The extension name can be an alias, which the dtype registry resolves to the Vortex ID + // of its type, so the array can match either of them. let named_vortex_id = self .session .ext_dtypes @@ -1149,9 +1147,9 @@ impl ArrowExporter<'_> { return self.execute_arrow_ext_storage(current, &ext_dtype, target_field, ctx); } - // Only an extension array fails on a target that names another Vortex ID. A plain - // array still exports to it, because plugins export their storage to the target that - // they receive. + // An extension array of another type must not export to a field that names this Vortex + // ID. A plain array can, because a plugin such as `Vector` exports its storage to the + // same field that it received. if let (Some(ext_dtype), Some(named_vortex_id)) = (current.dtype().as_extension_opt(), named_vortex_id) { @@ -1182,25 +1180,21 @@ impl ArrowExporter<'_> { target: &Field, ctx: &mut ExecutionCtx, ) -> VortexResult { - let target_metadata = decode_arrow_ext_metadata(target)?; - vortex_ensure!( - target_metadata == ext_dtype.serialize_metadata()?, - "Arrow field {:?} must carry the metadata of {ext_dtype}, got {target_metadata:?}", - target.name() - ); + ensure_ext_metadata(target, &decode_arrow_ext_metadata(target)?, ext_dtype)?; let target_dtype = self .session .from_arrow_datatype(target.data_type(), ext_dtype.nullability())?; - // An Arrow type that already holds the extension type, such as an Arrow timestamp, exports - // the array itself, so that the executor of that type checks its own parameters. + // If the Arrow type of the target imports as the extension type itself, such as an Arrow + // timestamp, export the array to it. The executor of that Arrow type then checks the + // parameters of the type, such as the time unit of a timestamp. if target_dtype.eq_ignore_nullability(array.dtype()) { return execute_arrow_naive(array, Some(target.data_type()), self, ctx); } vortex_ensure!( - target_dtype.eq_ignore_nullability(ext_dtype.storage_dtype()), + eq_storage_layout(&target_dtype, ext_dtype.storage_dtype()), "Arrow field {:?} must have an Arrow type that imports as the storage of {ext_dtype}, got {}", target.name(), target.data_type() @@ -1215,6 +1209,65 @@ impl ArrowExporter<'_> { } } +/// Fails unless `metadata`, the extension metadata of `field`, is the metadata of `ext_dtype`. +fn ensure_ext_metadata( + field: &Field, + metadata: &[u8], + ext_dtype: &ExtDTypeRef, +) -> VortexResult<()> { + vortex_ensure!( + metadata == ext_dtype.serialize_metadata()?, + "Arrow field {:?} must carry the metadata of {ext_dtype}, got {metadata:?}", + field.name() + ); + + Ok(()) +} + +/// Returns whether `lhs` and `rhs` have the same storage layout, which means that they are equal, +/// ignoring nullability, after every nested extension type is replaced by its storage type. The +/// types at the top level are compared as they are. +/// +/// An export compares the type that its target imports as with the storage type this way, because +/// an import can lose the metadata of a nested extension type, such as the version of a UUID, or +/// not know its Vortex ID. The export of each nested field checks that field when it reaches it. +fn eq_storage_layout(lhs: &DType, rhs: &DType) -> bool { + match (lhs, rhs) { + (DType::List(lhs, _), DType::List(rhs, _)) => eq_nested_storage_layout(lhs, rhs), + (DType::FixedSizeList(lhs, lhs_size, _), DType::FixedSizeList(rhs, rhs_size, _)) => { + lhs_size == rhs_size && eq_nested_storage_layout(lhs, rhs) + } + (DType::Map(lhs, _), DType::Map(rhs, _)) => { + lhs.keys_sorted() == rhs.keys_sorted() + && eq_nested_storage_layout(&lhs.key_dtype(), &rhs.key_dtype()) + && eq_nested_storage_layout(&lhs.value_dtype(), &rhs.value_dtype()) + } + (DType::Struct(lhs, _), DType::Struct(rhs, _)) => { + lhs.names() == rhs.names() + && lhs + .fields() + .zip(rhs.fields()) + .all(|(lhs, rhs)| eq_nested_storage_layout(&lhs, &rhs)) + } + _ => lhs.eq_ignore_nullability(rhs), + } +} + +/// [`eq_storage_layout`] for two nested types, which replaces their own extension types by their +/// storage types too. +fn eq_nested_storage_layout(lhs: &DType, rhs: &DType) -> bool { + eq_storage_layout(innermost_storage_dtype(lhs), innermost_storage_dtype(rhs)) +} + +/// Returns `dtype` without its extension types, which can wrap each other. +fn innermost_storage_dtype(mut dtype: &DType) -> &DType { + while let Some(ext_dtype) = dtype.as_extension_opt() { + dtype = ext_dtype.storage_dtype(); + } + + dtype +} + /// Extension trait for accessing the [`ArrowSession`] on a Vortex session. pub trait ArrowSessionExt: SessionExt { /// Get the Arrow session. diff --git a/vortex-arrow/tests/vortex_id.rs b/vortex-arrow/tests/vortex_id.rs index 584feee6cbd..92d66d0148a 100644 --- a/vortex-arrow/tests/vortex_id.rs +++ b/vortex-arrow/tests/vortex_id.rs @@ -58,16 +58,24 @@ use vortex_error::vortex_bail; use vortex_session::VortexSession; use vortex_session::registry::CachedId; -/// A session that can look up [`Tagged`] by its Vortex ID. +/// A session that can look up [`Tagged`] by its Vortex ID and by [`TAGGED_ALIAS`]. static SESSION: LazyLock = LazyLock::new(|| { let session = VortexSession::empty(); session.dtypes().register(Tagged); + session + .dtypes() + .registry() + .insert(*TAGGED_ALIAS_ID, Arc::new(Tagged)); vortex_arrow::initialize(&session); session }); static TAGGED_ID: CachedId = CachedId::new("test.tagged"); +/// Another name that the dtype registry of [`SESSION`] resolves to [`Tagged`]. +const TAGGED_ALIAS: &str = "test.tagged.alias"; +static TAGGED_ALIAS_ID: CachedId = CachedId::new(TAGGED_ALIAS); + /// An extension type without an Arrow plugin. It accepts any storage, and its metadata is one tag /// byte. #[derive(Clone, Debug, PartialEq, Eq, Hash)] @@ -334,6 +342,66 @@ fn timestamp_round_trips_through_a_vortex_id_field( Ok(()) } +/// A field that names an alias of a Vortex ID exports and imports like a field that names the ID. +#[test] +fn alias_of_vortex_id_round_trips() -> VortexResult<()> { + let array = tagged_i64s(0xff)?; + let mut field = storage_field_as(Ok(array.clone()), DataType::Int64)?; + field + .metadata_mut() + .insert(EXTENSION_TYPE_NAME_KEY.to_owned(), TAGGED_ALIAS.to_owned()); + let mut ctx = SESSION.create_execution_ctx(); + + let exported = SESSION + .arrow() + .execute_arrow(array.clone(), Some(&field), &mut ctx)?; + let imported = SESSION.arrow().from_arrow_array(exported, &field)?; + assert_arrays_eq!(imported, array, &mut ctx); + + Ok(()) +} + +/// A plain array exports to a field that names a Vortex ID, because a plugin such as `Vector` +/// exports its storage to the same field that it received. +#[test] +fn plain_array_exports_to_a_vortex_id_field() -> VortexResult<()> { + let storage = PrimitiveArray::from_option_iter([Some(1i64), None, Some(3)]).into_array(); + let field = storage_field_as(tagged_i64s(0xff), DataType::Int64)?; + let mut ctx = SESSION.create_execution_ctx(); + + let exported = SESSION + .arrow() + .execute_arrow(storage.clone(), Some(&field), &mut ctx)?; + let plain_field = Field::new("t", DataType::Int64, true); + let imported = SESSION.arrow().from_arrow_array(exported, &plain_field)?; + assert_arrays_eq!(imported, storage, &mut ctx); + + Ok(()) +} + +/// The export to a field that names a Vortex ID compares nested extension types by their storage, +/// because the import of the field may not restore them. Here the session does not know the nested +/// Vortex ID, and an import plugin that loses metadata, such as the version of a UUID, has the same +/// effect. +#[test] +fn export_compares_nested_extension_types_by_storage() -> VortexResult<()> { + let array = tagged_array(0x01, struct_of_tagged()?)?; + let field = SESSION + .arrow() + .to_arrow_storage_field("t", array.dtype().as_extension())?; + let session = VortexSession::empty(); + vortex_arrow::initialize(&session); + let mut ctx = session.create_execution_ctx(); + + let exported = session + .arrow() + .execute_arrow(array.clone(), Some(&field), &mut ctx)?; + let imported = SESSION.arrow().from_arrow_array(exported, &field)?; + assert_arrays_eq!(imported, array, &mut ctx); + + Ok(()) +} + /// A session that does not know the Vortex ID imports the field as plain storage. #[test] fn unknown_vortex_id_imports_as_storage() -> VortexResult<()> { @@ -426,8 +494,8 @@ fn run_end_field_that_names_its_type_twice_wraps_it_once() -> VortexResult<()> { Ok(()) } -/// The entries field of a map is part of the map type, so a Vortex ID on it is ignored by both the -/// dtype import and the array import. +/// The entries field of a map is part of the map type, so the dtype import and the array import +/// both ignore a Vortex ID on it. #[test] fn map_entries_ignore_a_vortex_id() -> VortexResult<()> { let key_field = Arc::new(Field::new("key", DataType::Int64, false));