Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions encodings/alp/src/alp/plugin.rs
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,14 @@ impl ArrayPlugin for ALPPatchedPlugin {
ArrayVTable::id(&ALP)
}

fn read_ids(&self) -> Vec<ArrayId> {
vec![self.id()]
}

fn write_ids(&self) -> Vec<ArrayId> {
vec![self.id()]
}

fn serialize(
&self,
array: &ArrayRef,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,11 @@ impl ArrayPlugin for DecimalBytePartsPlugin {
VTable::id(&DecimalByteParts)
}

fn serialized_ids(&self) -> Vec<ArrayId> {
fn read_ids(&self) -> Vec<ArrayId> {
vec![decimal_byte_parts_v1_id(), decimal_byte_parts_v2_id()]
}

fn write_ids(&self) -> Vec<ArrayId> {
vec![decimal_byte_parts_v1_id(), decimal_byte_parts_v2_id()]
}

Expand Down
16 changes: 12 additions & 4 deletions encodings/fastlanes/src/bitpacking/plugin/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,11 @@ impl ArrayPlugin for BitPackedPlugin {
ArrayVTable::id(&BitPacked)
}

fn serialized_ids(&self) -> Vec<ArrayId> {
fn read_ids(&self) -> Vec<ArrayId> {
vec![bitpacked_v1_id(), bitpacked_v2_id()]
}

fn write_ids(&self) -> Vec<ArrayId> {
vec![bitpacked_v1_id(), bitpacked_v2_id()]
}

Expand Down Expand Up @@ -114,8 +118,12 @@ impl ArrayPlugin for BitPackedPatchedPlugin {
ArrayVTable::id(&BitPacked)
}

fn serialized_ids(&self) -> Vec<ArrayId> {
BitPackedPlugin.serialized_ids()
fn read_ids(&self) -> Vec<ArrayId> {
BitPackedPlugin.read_ids()
}

fn write_ids(&self) -> Vec<ArrayId> {
BitPackedPlugin.write_ids()
}

fn serialize(
Expand All @@ -133,7 +141,7 @@ impl ArrayPlugin for BitPackedPatchedPlugin {
session: &VortexSession,
) -> VortexResult<ArrayRef> {
vortex_ensure!(
self.serialized_ids().contains(&parts.serialized_id),
self.read_ids().contains(&parts.serialized_id),
"BitPacked plugin does not recognize serialized ID {}",
parts.serialized_id,
);
Expand Down
6 changes: 5 additions & 1 deletion encodings/fastlanes/src/for_/plugin/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,11 @@ impl ArrayPlugin for FoRPlugin {
VTable::id(&FoR)
}

fn serialized_ids(&self) -> Vec<ArrayId> {
fn read_ids(&self) -> Vec<ArrayId> {
vec![for_v1_id(), for_v2_id()]
}

fn write_ids(&self) -> Vec<ArrayId> {
vec![for_v1_id(), for_v2_id()]
}

Expand Down
49 changes: 30 additions & 19 deletions vortex-array/src/array/plugin.rs
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,8 @@ pub type ArrayPluginRef = Arc<dyn ArrayPlugin>;
/// The wire representation produced by an in-memory array's serializer.
///
/// A serializer may reuse the in-memory array's buffers and children with [`Self::from_array`],
/// or return different parts when an older wire representation requires a lossless structural
/// downgrade.
/// or return the wire representation of another encoding. The node must hold the logical values
/// of the array under the dtype supplied by the containing format.
#[derive(Clone, Debug)]
pub struct ArraySerialization {
/// The concrete array ID to write on the wire.
Expand Down Expand Up @@ -101,32 +101,35 @@ impl<'a> ArrayDeserialization<'a> {

/// Registry trait for serializing and deserializing an in-memory array representation.
///
/// A plugin has one [`id`](Self::id) for the in-memory representation and one or more
/// [`serialized_ids`](Self::serialized_ids) for wire representations. Its serializer chooses the
/// wire representation, and the serialization context validates that the chosen ID is permitted
/// before it is written.
/// A plugin writes one in-memory encoding, identified by [`id`](Self::id). It declares the wire
/// IDs that it reads and writes separately, so it can write another encoding's wire form without
/// replacing that encoding's reader. The serialization context validates that the chosen wire ID
/// is permitted before it is written.
///
/// Every serialized ID is also registered for deserialization. A current plugin may therefore
/// deserialize several historical IDs into the same in-memory representation. A reader that
/// predates a newer ID has no registration for it and reports it as unknown instead of silently
/// interpreting an unsupported representation.
/// A plugin can retain historical read IDs after it stops writing them. A reader that predates a
/// newer ID has no registration for it and reports it as unknown.
pub trait ArrayPlugin: 'static + Send + Sync {
/// Returns the ID of the in-memory array representation handled by this plugin.
/// Returns the ID of the in-memory encoding that this plugin writes.
fn id(&self) -> ArrayId;

/// Returns the serialized array IDs understood by this plugin, ordered oldest to newest.
/// Returns the wire IDs that [`deserialize`](Self::deserialize) accepts, oldest to newest.
///
/// The default uses the in-memory ID as the sole wire ID. Override this for an in-memory array
/// that has multiple serialized variants. IDs retained only for reading may also be included;
/// the single serializer need not select them.
fn serialized_ids(&self) -> Vec<ArrayId> {
vec![self.id()]
}
/// [`ArraySession::register`](crate::session::ArraySession::register) makes this plugin the
/// reader of each ID. An ID can stay in this list after the plugin stops writing it, so that
/// old files stay readable.
fn read_ids(&self) -> Vec<ArrayId>;

/// Returns the wire IDs that [`serialize`](Self::serialize) may return.
///
/// An ID in this list does not have to be in [`read_ids`](Self::read_ids). A plugin can write
/// its arrays in the wire form of another encoding. The plugin that owns that form reads it.
/// The node must hold the logical values of the array under the dtype of the array.
fn write_ids(&self) -> Vec<ArrayId>;

/// Serialize `array` to its wire representation.
///
/// This function is called only for arrays whose in-memory encoding matches [`id`](Self::id).
/// The returned ID must be declared by [`serialized_ids`](Self::serialized_ids). Return
/// The returned ID must be declared by [`write_ids`](Self::write_ids). Return
/// `Ok(None)` when the array cannot be serialized.
fn serialize(
&self,
Expand Down Expand Up @@ -167,6 +170,14 @@ impl<V: VTable> ArrayPlugin for V {
VTable::id(self)
}

fn read_ids(&self) -> Vec<ArrayId> {
vec![self.id()]
}

fn write_ids(&self) -> Vec<ArrayId> {
vec![self.id()]
}

fn serialize(
&self,
array: &ArrayRef,
Expand Down
8 changes: 8 additions & 0 deletions vortex-array/src/arrays/scalar_fn/plugin.rs
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,14 @@ impl<V: ScalarFnVTable + ScalarFnArrayVTable> ArrayPlugin for ScalarFnArrayPlugi
self.0.id()
}

fn read_ids(&self) -> Vec<ArrayId> {
vec![self.id()]
}

fn write_ids(&self) -> Vec<ArrayId> {
vec![self.id()]
}

fn serialize(
&self,
array: &ArrayRef,
Expand Down
14 changes: 11 additions & 3 deletions vortex-array/src/serde.rs
Original file line number Diff line number Diff line change
Expand Up @@ -818,7 +818,11 @@ mod tests {
old_primitive_id()
}

fn serialized_ids(&self) -> Vec<ArrayId> {
fn read_ids(&self) -> Vec<ArrayId> {
vec![old_primitive_id(), new_primitive_id()]
}

fn write_ids(&self) -> Vec<ArrayId> {
vec![old_primitive_id(), new_primitive_id()]
}

Expand Down Expand Up @@ -883,8 +887,12 @@ mod tests {
VersionedPrimitivePlugin.id()
}

fn serialized_ids(&self) -> Vec<ArrayId> {
VersionedPrimitivePlugin.serialized_ids()
fn read_ids(&self) -> Vec<ArrayId> {
VersionedPrimitivePlugin.read_ids()
}

fn write_ids(&self) -> Vec<ArrayId> {
VersionedPrimitivePlugin.write_ids()
}

fn serialize(
Expand Down
41 changes: 11 additions & 30 deletions vortex-array/src/session/mod.rs
Original file line number Diff line number Diff line change
@@ -1,6 +1,12 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright the Vortex contributors

//! Session registries for array readers and writers.
//!
//! Writers are keyed by in-memory encoding IDs, while readers are keyed by wire IDs. Keeping the
//! registries separate lets a writer reuse another encoding's wire form without replacing its
//! reader.

use std::any::Any;
use std::sync::Arc;

Expand Down Expand Up @@ -62,14 +68,14 @@ impl ArraySession {
&self.registry
}

/// Register an in-memory array plugin and all of its recognized serialized IDs.
/// Register an in-memory array writer and its declared wire readers.
///
/// This replaces any serializer with the same in-memory ID and any deserializer registered
/// under one of [`ArrayPlugin::serialized_ids`].
/// under one of [`ArrayPlugin::read_ids`].
pub fn register<P: ArrayPlugin>(&self, plugin: P) {
let plugin = Arc::new(plugin) as ArrayPluginRef;
self.serializers.insert(plugin.id(), Arc::clone(&plugin));
for serialized_id in plugin.serialized_ids() {
for serialized_id in plugin.read_ids() {
self.registry.insert(serialized_id, Arc::clone(&plugin));
}
}
Expand Down Expand Up @@ -144,9 +150,7 @@ pub trait ArraySessionExt: SessionExt {
return Ok(None);
};
vortex_ensure!(
plugin
.serialized_ids()
.contains(&serialization.serialized_id),
plugin.write_ids().contains(&serialization.serialized_id),
"array serializer {} produced undeclared serialized ID {}",
array.encoding_id(),
serialization.serialized_id,
Expand All @@ -158,27 +162,4 @@ pub trait ArraySessionExt: SessionExt {
impl<S: SessionExt> ArraySessionExt for S {}

#[cfg(test)]
mod tests {
use vortex_session::VortexSession;

use crate::ArrayVTable;
use crate::arrays::Bool;
use crate::session::ArraySession;
use crate::session::ArraySessionExt;

#[test]
fn array_session_default_registers_encodings() {
let session = VortexSession::empty().with::<ArraySession>();

assert!(session.arrays().registry().contains_key(&Bool.id()));
assert!(session.arrays().serializer(&Bool.id()).is_some());
}

#[test]
fn empty_array_session_registers_no_encodings() {
let session = VortexSession::empty().with_some(ArraySession::empty());

assert!(!session.arrays().registry().contains_key(&Bool.id()));
assert!(session.arrays().serializer(&Bool.id()).is_none());
}
}
mod tests;
Loading
Loading