Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
89 commits
Select commit Hold shift + click to select a range
768b3e9
impl map_from_entries
Dec 14, 2025
c68c342
Revert "impl map_from_entries"
Dec 16, 2025
d887555
Merge branch 'apache:main' into main
kazantsev-maksim Dec 16, 2025
231aa90
Merge branch 'apache:main' into main
kazantsev-maksim Dec 17, 2025
9500bbb
Merge branch 'apache:main' into main
kazantsev-maksim Dec 24, 2025
9577481
Merge branch 'apache:main' into main
kazantsev-maksim Dec 28, 2025
3791557
Merge branch 'apache:main' into main
kazantsev-maksim Jan 2, 2026
7c2f082
Merge branch 'apache:main' into main
kazantsev-maksim Jan 3, 2026
609a605
Merge branch 'apache:main' into main
kazantsev-maksim Jan 6, 2026
a151b2c
Merge branch 'apache:main' into main
kazantsev-maksim Jan 7, 2026
ad3e7f5
Merge branch 'apache:main' into main
kazantsev-maksim Jan 10, 2026
ea92e4b
Merge branch 'apache:main' into main
kazantsev-maksim Jan 14, 2026
8dfeca3
Merge branch 'apache:main' into main
kazantsev-maksim Jan 17, 2026
559741e
Merge branch 'apache:main' into main
kazantsev-maksim Jan 20, 2026
ebda14e
Merge branch 'apache:main' into main
kazantsev-maksim Jan 21, 2026
408152e
Merge branch 'apache:main' into main
kazantsev-maksim Jan 23, 2026
d7857b2
Merge branch 'apache:main' into main
kazantsev-maksim Jan 24, 2026
aef41be
Merge branch 'apache:main' into main
kazantsev-maksim Jan 29, 2026
5ac1c58
Merge branch 'apache:main' into main
kazantsev-maksim Jan 30, 2026
9ae8e23
Merge branch 'apache:main' into main
kazantsev-maksim Feb 1, 2026
5ca3888
Merge branch 'apache:main' into main
kazantsev-maksim Feb 4, 2026
160a817
Merge branch 'apache:main' into main
kazantsev-maksim Feb 5, 2026
88fc313
Merge branch 'apache:main' into main
kazantsev-maksim Feb 7, 2026
e14c180
Merge branch 'apache:main' into main
kazantsev-maksim Feb 13, 2026
610a885
Merge branch 'apache:main' into main
kazantsev-maksim Feb 20, 2026
f8acb2c
Merge branch 'apache:main' into main
kazantsev-maksim Feb 21, 2026
ec94897
Merge branch 'apache:main' into main
kazantsev-maksim Feb 26, 2026
43405e4
Merge branch 'apache:main' into main
kazantsev-maksim Feb 27, 2026
47b4915
Merge branch 'apache:main' into main
kazantsev-maksim Mar 1, 2026
26e2682
Merge branch 'apache:main' into main
kazantsev-maksim Mar 3, 2026
6cb5f07
Merge branch 'apache:main' into main
kazantsev-maksim Mar 4, 2026
ec194fb
Merge branch 'apache:main' into main
kazantsev-maksim Mar 31, 2026
256fccb
Merge branch 'apache:main' into main
kazantsev-maksim Apr 3, 2026
912c8f9
Merge branch 'apache:main' into main
kazantsev-maksim Apr 3, 2026
561a664
Merge branch 'apache:main' into main
kazantsev-maksim Apr 8, 2026
d926ef4
Merge branch 'apache:main' into main
kazantsev-maksim Apr 11, 2026
671412c
Merge branch 'apache:main' into main
kazantsev-maksim Apr 17, 2026
c9f52d1
Merge branch 'apache:main' into main
kazantsev-maksim Apr 22, 2026
67f72d9
Merge branch 'apache:main' into main
kazantsev-maksim Apr 23, 2026
314e594
Merge branch 'apache:main' into main
kazantsev-maksim Apr 24, 2026
ac8292f
Merge branch 'apache:main' into main
kazantsev-maksim May 1, 2026
c9c140e
Merge branch 'apache:main' into main
kazantsev-maksim May 7, 2026
decca58
Merge branch 'apache:main' into main
kazantsev-maksim May 13, 2026
0919b33
Merge branch 'apache:main' into main
kazantsev-maksim May 16, 2026
7495e21
Merge branch 'apache:main' into main
kazantsev-maksim May 19, 2026
0a37a60
Merge branch 'apache:main' into main
kazantsev-maksim May 21, 2026
abbba84
Merge branch 'apache:main' into main
kazantsev-maksim May 25, 2026
6020560
Merge branch 'apache:main' into main
kazantsev-maksim May 28, 2026
e2bdfb1
Merge branch 'apache:main' into main
kazantsev-maksim May 31, 2026
3edfc33
Merge branch 'apache:main' into main
kazantsev-maksim Jun 3, 2026
a39e860
Merge branch 'apache:main' into main
kazantsev-maksim Jun 4, 2026
e88dd7b
Merge branch 'apache:main' into main
kazantsev-maksim Jun 5, 2026
3e29d37
Merge branch 'apache:main' into main
kazantsev-maksim Jun 7, 2026
4068359
Merge branch 'apache:main' into main
kazantsev-maksim Jun 12, 2026
a3cb8de
Merge branch 'apache:main' into main
kazantsev-maksim Jun 13, 2026
b33726f
Merge branch 'apache:main' into main
kazantsev-maksim Jun 21, 2026
698f7a1
Merge branch 'apache:main' into main
kazantsev-maksim Jun 22, 2026
18162a6
Merge branch 'apache:main' into main
kazantsev-maksim Jun 23, 2026
6f6eb6f
Merge branch 'apache:main' into main
kazantsev-maksim Jul 1, 2026
c21a42e
Merge branch 'apache:main' into main
kazantsev-maksim Jul 2, 2026
618ae48
Merge branch 'apache:main' into main
kazantsev-maksim Jul 3, 2026
4d068e3
Merge branch 'apache:main' into main
kazantsev-maksim Jul 3, 2026
a2f519f
Merge branch 'apache:main' into main
kazantsev-maksim Jul 4, 2026
36e13d5
Merge branch 'apache:main' into main
kazantsev-maksim Jul 7, 2026
2c8ae52
Merge branch 'apache:main' into main
kazantsev-maksim Jul 8, 2026
593f7b6
Merge branch 'apache:main' into main
kazantsev-maksim Jul 11, 2026
b1d3a1a
Merge branch 'apache:main' into main
kazantsev-maksim Jul 14, 2026
e2de8c0
Merge branch 'apache:main' into main
kazantsev-maksim Jul 26, 2026
e6fd376
Merge branch 'apache:main' into main
kazantsev-maksim Aug 1, 2026
11528e3
Merge branch 'apache:main' into main
kazantsev-maksim Aug 1, 2026
e17397f
Merge branch 'apache:main' into main
kazantsev-maksim Aug 9, 2026
57d469c
work
Aug 9, 2026
3303a04
work
Aug 9, 2026
c1ec9a3
work
Aug 9, 2026
beda70b
fmt
Aug 9, 2026
ffc9e5f
clippy
Aug 10, 2026
b46a4f9
clippy
Aug 11, 2026
b79e747
Merge branch 'apache:main' into main
kazantsev-maksim Aug 12, 2026
3988b0f
Merge branch 'apache:main' into main
kazantsev-maksim Aug 19, 2026
f705c43
Merge branch 'apache:main' into main
kazantsev-maksim Aug 20, 2026
5e31ead
Merge remote-tracking branch 'origin/main' into perf_contains
Aug 20, 2026
2f8aef0
address comments
Aug 20, 2026
741747d
Merge branch 'apache:main' into main
kazantsev-maksim Aug 22, 2026
9495593
Merge branch 'apache:main' into main
kazantsev-maksim Aug 27, 2026
0ac77df
Merge branch 'apache:main' into main
kazantsev-maksim Aug 28, 2026
86f1707
Merge remote-tracking branch 'origin/main' into perf_contains
Aug 29, 2026
79efafd
address comments
Aug 29, 2026
37d1ddd
address comments
Sep 13, 2026
486b4fc
address comments
Sep 13, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
119 changes: 102 additions & 17 deletions native/spark-expr/benches/contains.rs
Original file line number Diff line number Diff line change
Expand Up @@ -28,9 +28,36 @@ use std::sync::Arc;
mod common;
use common::{string_array, NULL_RATIOS, ROW_COUNTS};

/// Scalar used as the haystack in the scalar/array shape. The matching needle
/// values below are chosen so the scalar/array shape performs real work.
const HAYSTACK_SCALAR: &str = "datafusion-comet";

/// Scalar used as the needle in the array/scalar shape.
const NEEDLE_SCALAR: &str = "comet";

fn build_args(
haystack: ColumnarValue,
needle: ColumnarValue,
number_rows: usize,
) -> ScalarFunctionArgs {
ScalarFunctionArgs {
args: vec![haystack, needle],
arg_fields: vec![],
number_rows,
return_field: Arc::new(Field::new("result", DataType::Boolean, true)),
config_options: Arc::new(ConfigOptions::default()),
}
}

fn criterion_benchmark(c: &mut Criterion) {
let udf = SparkContains::new();
let mut group = c.benchmark_group("spark_contains");

// ------------------------------------------------------------------
// Shape 1: array haystack vs scalar needle (`contains_array_scalar`).
// This path already used a scalar representation on `main`; included as a
// regression control since this PR touches it incidentally.
// ------------------------------------------------------------------
let mut group = c.benchmark_group("spark_contains/array_scalar");
for rows in ROW_COUNTS {
for (null_ratio, tag) in NULL_RATIOS {
let haystack = string_array(rows, null_ratio, |_| "datafusion-comet".to_string());
Expand All @@ -40,22 +67,80 @@ fn criterion_benchmark(c: &mut Criterion) {
|b, haystack| {
b.iter(|| {
black_box(
udf.invoke_with_args(ScalarFunctionArgs {
args: vec![
ColumnarValue::Array(Arc::clone(haystack)),
ColumnarValue::Scalar(ScalarValue::Utf8(Some(
"comet".to_string(),
))),
],
arg_fields: vec![],
number_rows: haystack.len(),
return_field: Arc::new(Field::new(
"result",
DataType::Boolean,
true,
)),
config_options: Arc::new(ConfigOptions::default()),
})
udf.invoke_with_args(build_args(
ColumnarValue::Array(Arc::clone(haystack)),
ColumnarValue::Scalar(ScalarValue::Utf8(Some(
NEEDLE_SCALAR.to_string(),
))),
haystack.len(),
))
.unwrap(),
)
})
},
);
}
}
group.finish();

// ------------------------------------------------------------------
// Shape 2: scalar haystack vs array needle (`contains_scalar_array`).
// This is the path the PR actually optimizes (it replaced
// `to_array_of_size(N)` with an O(1) broadcast), so it must be measured.
// The needle array is varied per row so the kernel does non-trivial work.
// ------------------------------------------------------------------
let mut group = c.benchmark_group("spark_contains/scalar_array");
for rows in ROW_COUNTS {
for (null_ratio, tag) in NULL_RATIOS {
let needle = string_array(rows, null_ratio, |i| {
if i % 2 == 0 {
"comet".to_string()
} else {
format!("comet-{i}")
}
});
group.bench_with_input(
BenchmarkId::from_parameter(format!("{rows}/{tag}")),
&needle,
|b, needle| {
b.iter(|| {
black_box(
udf.invoke_with_args(build_args(
ColumnarValue::Scalar(ScalarValue::Utf8(Some(
HAYSTACK_SCALAR.to_string(),
))),
ColumnarValue::Array(Arc::clone(needle)),
needle.len(),
))
.unwrap(),
)
})
},
);
}
}
group.finish();

// ------------------------------------------------------------------
// Shape 3: array haystack vs array needle (`arrow_contains` directly).
// Regression control for the straight-through path the PR does not touch.
// ------------------------------------------------------------------
let mut group = c.benchmark_group("spark_contains/array_array");
for rows in ROW_COUNTS {
for (null_ratio, tag) in NULL_RATIOS {
let haystack = string_array(rows, null_ratio, |_| "datafusion-comet".to_string());
let needle = string_array(rows, null_ratio, |_| "comet".to_string());
group.bench_with_input(
BenchmarkId::from_parameter(format!("{rows}/{tag}")),
&(haystack, needle),
|b, (haystack, needle)| {
b.iter(|| {
black_box(
udf.invoke_with_args(build_args(
ColumnarValue::Array(Arc::clone(haystack)),
ColumnarValue::Array(Arc::clone(needle)),
haystack.len(),
))
.unwrap(),
)
})
Expand Down
Loading
Loading