Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
249 changes: 249 additions & 0 deletions vortex-array/src/arrays/extension/compute/between.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,249 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright the Vortex contributors

use vortex_error::VortexResult;

use crate::ArrayRef;
use crate::IntoArray;
use crate::array::ArrayView;
use crate::arrays::ConstantArray;
use crate::arrays::Extension;
use crate::arrays::extension::ExtensionArrayExt;
use crate::builtins::ArrayBuiltins;
use crate::dtype::extension::ExtId;
use crate::dtype::extension::ExtVTable;
use crate::extension::datetime::Date;
use crate::extension::datetime::Time;
use crate::extension::datetime::Timestamp;
use crate::scalar_fn::fns::between::BetweenOptions;
use crate::scalar_fn::fns::between::BetweenReduce;

impl BetweenReduce for Extension {
/// Evaluates between on the storage array, so a date or timestamp keeps its compressed
/// storage and reaches that encoding's between kernel instead of being decompressed.
///
/// Whether the pushdown is allowed is decided by the extension id: only the datetime ids
/// (`vortex.date`, `vortex.time`, `vortex.timestamp`) are known to order like their storage.
/// Any other extension type, such as JSON text, UUIDs or WKB geometry, may order differently
/// from its storage, so the rule declines it.
fn between(
array: ArrayView<'_, Extension>,
lower: &ArrayRef,
upper: &ArrayRef,
options: &BetweenOptions,
) -> VortexResult<Option<ArrayRef>> {
if !is_storage_ordered(array.ext_dtype().id()) {
return Ok(None);
}

// Storage values are only ordered alike when the bounds share the extension dtype
// (a timestamp in milliseconds must not be compared against raw seconds).
if !array.dtype().eq_ignore_nullability(lower.dtype())
|| !array.dtype().eq_ignore_nullability(upper.dtype())
{
return Ok(None);
}

let (Some(lower), Some(upper)) = (storage_of(lower), storage_of(upper)) else {
return Ok(None);
};

array
.storage_array()
.clone()
.between(lower, upper, options.clone())
.map(Some)
}
}

/// Whether values of the extension type with this id order the same as their storage values.
fn is_storage_ordered(id: ExtId) -> bool {
id == Date.id() || id == Time.id() || id == Timestamp.id()
}

/// The storage of a bound: a constant extension scalar becomes a constant storage scalar, and an
/// extension array its storage array.
fn storage_of(bound: &ArrayRef) -> Option<ArrayRef> {
if let Some(scalar) = bound.as_constant() {
let storage = scalar.as_extension().to_storage_scalar();
return Some(ConstantArray::new(storage, bound.len()).into_array());
}
bound
.as_opt::<Extension>()
.map(|ext| ext.storage_array().clone())
}

#[cfg(test)]
mod tests {
use std::sync::LazyLock;

use rstest::rstest;
use vortex_buffer::buffer;
use vortex_error::VortexResult;
use vortex_session::VortexSession;

use crate::ArrayRef;
use crate::Canonical;
use crate::IntoArray;
use crate::VortexSessionExecute;
use crate::arrays::ConstantArray;
use crate::arrays::Extension;
use crate::arrays::ExtensionArray;
use crate::arrays::PrimitiveArray;
use crate::assert_arrays_eq;
use crate::builtins::ArrayBuiltins;
use crate::dtype::Nullability;
use crate::extension::datetime::Date;
use crate::extension::datetime::TimeUnit;
use crate::extension::datetime::Timestamp;
use crate::extension::tests::divisible_int::DivisibleInt;
use crate::extension::tests::divisible_int::Divisor;
use crate::scalar::Scalar;
use crate::scalar_fn::fns::between::BetweenOptions;
use crate::scalar_fn::fns::between::BetweenReduce;
use crate::scalar_fn::fns::between::StrictComparison;
use crate::scalar_fn::fns::operators::Operator;

static SESSION: LazyLock<VortexSession> = LazyLock::new(crate::array_session);

fn dates(storage: ArrayRef) -> ArrayRef {
let nullability = storage.dtype().nullability();
ExtensionArray::new(Date::new(TimeUnit::Days, nullability).erased(), storage).into_array()
}

fn date(days: i32) -> Scalar {
Scalar::extension::<Date>(
TimeUnit::Days,
Scalar::primitive(days, Nullability::NonNullable),
)
}

/// The pushdown must agree with the two compares that between stands for.
fn assert_matches_compares(
array: ArrayRef,
lower: ArrayRef,
upper: ArrayRef,
options: BetweenOptions,
) -> VortexResult<()> {
let ctx = &mut SESSION.create_execution_ctx();
let expected = lower
.clone()
.binary(array.clone(), options.lower_strict.to_operator())?
.binary(
array
.clone()
.binary(upper.clone(), options.upper_strict.to_operator())?,
Operator::And,
)?
.execute::<Canonical>(ctx)?
.into_array();
let actual = array
.between(lower, upper, options)?
.execute::<Canonical>(ctx)?
.into_array();
assert_arrays_eq!(actual, expected, ctx);
Ok(())
}

#[rstest]
#[case(StrictComparison::NonStrict, StrictComparison::NonStrict)]
#[case(StrictComparison::NonStrict, StrictComparison::Strict)]
#[case(StrictComparison::Strict, StrictComparison::NonStrict)]
#[case(StrictComparison::Strict, StrictComparison::Strict)]
fn constant_bounds(
#[case] lower_strict: StrictComparison,
#[case] upper_strict: StrictComparison,
) -> VortexResult<()> {
let array = dates(buffer![1i32, 2, 3, 4, 5].into_array());
assert_matches_compares(
array,
ConstantArray::new(date(2), 5).into_array(),
ConstantArray::new(date(4), 5).into_array(),
BetweenOptions {
lower_strict,
upper_strict,
},
)
}

#[test]
fn nullable_storage() -> VortexResult<()> {
let array = dates(
PrimitiveArray::from_option_iter([Some(1i32), None, Some(3), Some(4), None])
.into_array(),
);
assert_matches_compares(
array,
ConstantArray::new(date(1), 5).into_array(),
ConstantArray::new(date(3), 5).into_array(),
BetweenOptions {
lower_strict: StrictComparison::NonStrict,
upper_strict: StrictComparison::NonStrict,
},
)
}

#[test]
fn array_bounds() -> VortexResult<()> {
let array = dates(buffer![1i32, 2, 3, 4, 5].into_array());
assert_matches_compares(
array,
dates(buffer![0i32, 3, 3, 5, 0].into_array()),
dates(buffer![2i32, 2, 9, 9, 4].into_array()),
BetweenOptions {
lower_strict: StrictComparison::NonStrict,
upper_strict: StrictComparison::Strict,
},
)
}

#[test]
fn timestamps() -> VortexResult<()> {
let dtype = Timestamp::new(TimeUnit::Microseconds, Nullability::NonNullable).erased();
let ts = |v: i64| {
Scalar::extension_ref(
dtype.clone(),
Scalar::primitive(v, Nullability::NonNullable),
)
};
let storage = buffer![10i64, 20, 30, 40].into_array();
let array = ExtensionArray::new(dtype.clone(), storage).into_array();
assert_matches_compares(
array,
ConstantArray::new(ts(15), 4).into_array(),
ConstantArray::new(ts(40), 4).into_array(),
BetweenOptions {
lower_strict: StrictComparison::Strict,
upper_strict: StrictComparison::NonStrict,
},
)
}

/// An extension type outside the datetime types is declined, even one whose storage happens
/// to order correctly, since the rule cannot know how an arbitrary type orders.
#[test]
fn declines_non_temporal_extension() -> VortexResult<()> {
let divisor = Divisor(2);
let value = |v: u64| {
Scalar::extension::<DivisibleInt>(
divisor,
Scalar::primitive(v, Nullability::NonNullable),
)
};
let dtype = value(0).dtype().as_extension().clone();
let array = ExtensionArray::new(dtype, buffer![2u64, 4, 6].into_array());

let reduced = <Extension as BetweenReduce>::between(
array.as_view(),
&ConstantArray::new(value(2), 3).into_array(),
&ConstantArray::new(value(4), 3).into_array(),
&BetweenOptions {
lower_strict: StrictComparison::NonStrict,
upper_strict: StrictComparison::NonStrict,
},
)?;

assert!(reduced.is_none());
Ok(())
}
}
1 change: 1 addition & 0 deletions vortex-array/src/arrays/extension/compute/mod.rs
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright the Vortex contributors

mod between;
mod cast;
mod compare;
mod filter;
Expand Down
2 changes: 2 additions & 0 deletions vortex-array/src/arrays/extension/compute/rules.rs
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ use crate::optimizer::rules::ArrayReduceRule;
use crate::optimizer::rules::ParentRuleSet;
use crate::optimizer::rules::ReduceRuleSet;
use crate::scalar::Scalar;
use crate::scalar_fn::fns::between::BetweenReduceAdaptor;
use crate::scalar_fn::fns::cast::CastReduceAdaptor;
use crate::scalar_fn::fns::mask::MaskReduceAdaptor;

Expand Down Expand Up @@ -47,6 +48,7 @@ impl ArrayReduceRule<Extension> for ExtensionConstantRule {

pub(crate) const PARENT_RULES: ParentRuleSet<Extension> = ParentRuleSet::new(&[
ParentRuleSet::lift(&ExtensionFilterPushDownRule),
ParentRuleSet::lift(&BetweenReduceAdaptor(Extension)),
ParentRuleSet::lift(&CastReduceAdaptor(Extension)),
ParentRuleSet::lift(&FilterReduceAdaptor(Extension)),
ParentRuleSet::lift(&MaskReduceAdaptor(Extension)),
Expand Down
2 changes: 1 addition & 1 deletion vortex-array/src/extension/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,4 +6,4 @@
pub mod datetime;

#[cfg(test)]
mod tests;
pub(crate) mod tests;
2 changes: 1 addition & 1 deletion vortex-array/src/extension/tests/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,4 +3,4 @@

//! Test extension types for exercising the [`ExtVTable`] contract.
mod divisible_int;
pub(crate) mod divisible_int;
Loading