Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1,091 changes: 1,091 additions & 0 deletions encodings/parquet-variant/src/extract.rs

Large diffs are not rendered by default.

419 changes: 419 additions & 0 deletions encodings/parquet-variant/src/kernel.rs

Large diffs are not rendered by default.

1 change: 1 addition & 0 deletions encodings/parquet-variant/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ mod array;
mod arrow;
mod builder;
mod compute;
mod extract;
#[cfg(test)]
mod json_to_variant_tests;
mod kernel;
Expand Down
57 changes: 57 additions & 0 deletions vortex-array/src/arrays/variant/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,10 @@ pub(crate) fn initialize(session: &vortex_session::VortexSession) {
}

use crate::ArrayRef;
use crate::ArraySlots;
use crate::Canonical;
use crate::ExecutionCtx;
use crate::IntoArray;
use crate::array::Array;
use crate::array::ArrayParts;
use crate::array::EmptyArrayData;
Expand Down Expand Up @@ -75,6 +79,59 @@ impl Array<Variant> {
}
}

/// Decodes the storage of an unshredded Variant `array`, so that every `variant_get` over it
/// reads the decoded children instead of decoding them again.
///
/// A Variant with shredded children is returned as it is: an extraction usually reads a shredded
/// child, and decoding the storage would be wasted.
pub fn decode_unshredded_storage(
array: ArrayRef,
ctx: &mut ExecutionCtx,
) -> VortexResult<ArrayRef> {
vortex_ensure!(
array.dtype().is_variant(),
"expected a Variant, found {}",
array.dtype()
);
let mut variant = array.execute::<VariantArray>(ctx)?;
// The core storage can itself be lazy, such as a filter over the stored encoding. Execute it
// until it is the encoding that holds the children, which executes to itself.
loop {
if variant.shredded().is_some() {
return Ok(variant.into_array());
}
let storage = variant
.core_storage()
.clone()
.execute::<VariantArray>(ctx)?;
if storage.core_storage().encoding_id() == variant.core_storage().encoding_id() {
break;
}
variant = storage;
}

let core_storage = variant.core_storage();
let slots = core_storage
.slots()
.iter()
.map(|slot| {
slot.as_ref()
.map(|child| {
// A Variant child is storage of its own, not a child to decode.
if child.dtype().is_variant() {
Ok(child.clone())
} else {
Ok(child.clone().execute::<Canonical>(ctx)?.into_array())
}
})
.transpose()
})
.collect::<VortexResult<ArraySlots>>()?;
// SAFETY: executing a slot to its canonical form keeps its logical values.
let core_storage = unsafe { core_storage.clone().with_slots(slots) }?;
Ok(VariantArray::try_new(core_storage, None)?.into_array())
}

#[cfg(test)]
mod tests {
use std::sync::LazyLock;
Expand Down
85 changes: 67 additions & 18 deletions vortex-array/src/arrays/variant/vtable/kernel.rs
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright the Vortex contributors

use vortex_buffer::Buffer;
use vortex_error::VortexError;
use vortex_error::VortexExpect;
use vortex_error::VortexResult;
use vortex_session::VortexSession;
Expand Down Expand Up @@ -68,25 +70,33 @@ impl ExecuteParentKernel<Variant> for VariantGetKernel {
.shredded()
.map(|shredded| {
typed_shredded_path(shredded, parent.options.path().elements(), ctx)?
.map(|typed| mask_with_validity(typed, &core_validity)?.optimize_ctx(ctx))
.map(|(typed, remaining)| {
let typed = mask_with_validity(typed, &core_validity)?.optimize_ctx(ctx)?;
Ok::<_, VortexError>((typed, remaining))
})
.transpose()
})
.transpose()?
.flatten();

let Some(typed) = typed else {
let Some((typed, remaining)) = typed else {
return make_fallback(ctx).map(Some);
};
// A shredded Variant child still needs VariantGet at the root to produce a
// concrete requested dtype.
// A shredded Variant child holds the rest of the path, in its own shredded tree or in
// its residual value. It also still needs VariantGet at its root to produce a concrete
// requested dtype.
if typed.dtype().is_variant()
&& parent
.options
.dtype()
.is_some_and(|dtype| !dtype.is_variant())
&& (!remaining.is_empty()
|| parent
.options
.dtype()
.is_some_and(|dtype| !dtype.is_variant()))
{
return execute_fallback_variant_get(
VariantGetOptions::new(VariantPath::root(), parent.options.dtype().cloned()),
VariantGetOptions::new(
VariantPath::new(remaining.iter().cloned()),
parent.options.dtype().cloned(),
),
typed,
ctx,
)
Expand Down Expand Up @@ -128,10 +138,41 @@ impl ExecuteParentKernel<Variant> for VariantGetKernel {

// Null typed rows are not necessarily missing from the logical variant value;
// fill those rows from core storage and keep valid typed rows unchanged.
// TODO: we are computing the entire fallback array here but only take indices where
// typed_mask is false, so we can narrow the core_storage to only the false
// indices and compute from there then zip
let fallback = make_fallback(ctx)?;
if typed_mask.all_false() {
return make_fallback(ctx).map(Some);
}
// Usually the null typed rows are missing from core storage as well, as for a field
// that only some objects have. Extract from just those rows first: when none of them
// holds a value, the typed child is the whole answer and the rest of core storage is
// never read.
let null_rows = !&typed_mask;
let null_rows_fallback = execute_fallback_variant_get(
parent.options.clone(),
array.core_storage().filter(null_rows)?,
ctx,
)?;
if null_rows_fallback
.validity()?
.execute_mask(null_rows_fallback.len(), ctx)?
.all_false()
{
return Ok(Some(typed));
}

// Scatter the extracted rows back to their positions: row `i` reads the extracted row
// that it is the n-th null typed row for. Valid typed rows read an arbitrary extracted
// row, which the zip discards.
let mut extracted = 0u64;
let positions = typed_mask
.to_bit_buffer()
.iter()
.map(|valid| {
let position = extracted;
extracted += u64::from(!valid);
position.min(null_rows_fallback.len() as u64 - 1)
})
.collect::<Buffer<u64>>();
let fallback = null_rows_fallback.take(positions.into_array())?;
typed_mask
.into_array()
.zip(typed, fallback)?
Expand All @@ -154,13 +195,21 @@ fn mask_with_validity(array: ArrayRef, validity: &Validity) -> VortexResult<Arra
}
}

fn typed_shredded_path(
/// Walks `path` through the shredded tree, returning the array it reaches and the elements of
/// `path` that are left.
///
/// The walk stops early at a shredded Variant child, which holds the rest of the path itself.
/// Returns `None` if the shredded tree does not represent the path.
fn typed_shredded_path<'a>(
shredded: &ArrayRef,
path: &[VariantPathElement],
path: &'a [VariantPathElement],
ctx: &mut ExecutionCtx,
) -> VortexResult<Option<ArrayRef>> {
) -> VortexResult<Option<(ArrayRef, &'a [VariantPathElement])>> {
let mut current = shredded.clone();
for element in path {
for (idx, element) in path.iter().enumerate() {
if current.dtype().is_variant() {
return Ok(Some((current, &path[idx..])));
}
let VariantPathElement::Field(name) = element else {
return Ok(None);
};
Expand All @@ -174,7 +223,7 @@ fn typed_shredded_path(
current = mask_with_validity(field.clone(), &current_struct.validity()?)?;
}

Ok(Some(current))
Ok(Some((current, &[])))
}

fn merge_typed_as_variant(
Expand Down
24 changes: 24 additions & 0 deletions vortex-array/src/expr/exprs.rs
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,8 @@ use crate::scalar_fn::fns::strpos::Strpos;
use crate::scalar_fn::fns::variant_get::VariantGet;
use crate::scalar_fn::fns::variant_get::VariantGetOptions;
use crate::scalar_fn::fns::variant_get::VariantPath;
use crate::scalar_fn::fns::variant_pack::VariantPack;
use crate::scalar_fn::fns::variant_pack::VariantPackOptions;
use crate::scalar_fn::fns::zip::Zip;

/// Creates an expression that references the root scope.
Expand Down Expand Up @@ -189,6 +191,28 @@ pub fn variant_get(
VariantGet.new_expr(VariantGetOptions::new(path.into(), dtype), vec![child])
}

/// Creates an expression that builds a Variant from its `core` storage and `shredded` child.
///
/// `complete` says that `shredded` holds every value it could hold; see
/// [`VariantPackOptions::complete`].
pub fn variant_pack(core: Expression, shredded: Expression, complete: bool) -> Expression {
VariantPack.new_expr(VariantPackOptions::new(complete), vec![core, shredded])
}

/// Creates an expression that builds a Variant from its `core` storage and a `shredded` struct
/// that is not nullable, whose validity is `validity`.
pub fn variant_pack_with_validity(
core: Expression,
shredded: Expression,
validity: Expression,
complete: bool,
) -> Expression {
VariantPack.new_expr(
VariantPackOptions::with_shredded_validity(complete),
vec![core, shredded, validity],
)
}

/// Creates a bound expression that extracts a path from a Variant expression.
pub fn bound_variant_get(
child: BoundExpression,
Expand Down
2 changes: 2 additions & 0 deletions vortex-array/src/expr/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,8 @@ pub use exprs::select_exclude;
pub use exprs::strpos;
pub use exprs::var;
pub use exprs::variant_get;
pub use exprs::variant_pack;
pub use exprs::variant_pack_with_validity;
pub use exprs::zip_expr;
pub use lambda::*;
pub use scope::*;
Expand Down
1 change: 1 addition & 0 deletions vortex-array/src/scalar_fn/fns/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -31,4 +31,5 @@ pub mod select;
pub mod stat;
pub mod strpos;
pub mod variant_get;
pub mod variant_pack;
pub mod zip;
Loading
Loading