Skip to content

Commit 5f080f9

Browse files
authored
perf(hll): reduce union copying and defer dense estimator rebuilds (#317)
1 parent bb6ba57 commit 5f080f9

9 files changed

Lines changed: 468 additions & 83 deletions

File tree

‎CHANGELOG.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,10 +6,12 @@ All significant changes to this project will be documented in this file.
66

77
### New features
88

9+
* Add `HllUnion::serialize` to serialize union results directly.
910
* Add immutable `XorFilter` and `XorFilterBuilder` behind the `xor` feature, with 8-, 16-, and 32-bit fingerprints, consuming builder configuration with complete-input construction, bounded seeded retries and compact temporary storage, optional cardinality metadata for imported filters, standard serialization for all three widths (Java-compatible for 8/16-bit filters), and raw parts for application-defined storage.
1011

1112
### Improvements
1213

14+
* `HllUnion` is faster when adopting HLL8 inputs without downsampling or merging multiple dense inputs before querying or serializing the result.
1315
* `KllSketch::rank` is faster for repeated queries on compacted sketches.
1416
* Repeated scalar `TDigest::rank` queries are faster, including small digests and data with repeated values.
1517
* `TDigest` and `TDigestMut` CDF and PMF batches are faster for many split points; batches with sparse split points keep the original binary-search behavior.

‎benchmarks/hll/mod.rs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,4 +15,6 @@
1515
// specific language governing permissions and limitations
1616
// under the License.
1717

18+
mod serde;
19+
mod union;
1820
mod update;

‎benchmarks/hll/serde.rs‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
use datasketches::hll::HllType;
19+
use datasketches::hll::HllUnion;
20+
use divan::Bencher;
21+
use divan::black_box;
22+
23+
fn union(rows: u64) -> HllUnion {
24+
let mut union = HllUnion::new(18).unwrap();
25+
for value in 0..rows {
26+
union.update_value(value);
27+
}
28+
union
29+
}
30+
31+
#[divan::bench(args = [0, 4, 100, 80_000])]
32+
fn serialize_union(bencher: Bencher, rows: u64) {
33+
let union = union(rows);
34+
bencher.bench_local(|| black_box(&union).serialize());
35+
}
36+
37+
#[divan::bench(args = [0, 4, 100, 80_000])]
38+
fn serialize_union_via_sketch(bencher: Bencher, rows: u64) {
39+
let union = union(rows);
40+
bencher.bench_local(|| black_box(&union).to_sketch(HllType::Hll8).serialize());
41+
}

‎benchmarks/hll/union.rs‎

Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
use datasketches::hll::HllSketch;
19+
use datasketches::hll::HllType;
20+
use datasketches::hll::HllUnion;
21+
use divan::Bencher;
22+
use divan::black_box;
23+
24+
fn partials(lg_k: u8, count: usize, rows: usize) -> Vec<HllSketch> {
25+
(0..count)
26+
.map(|partial| {
27+
let mut sketch = HllSketch::new(lg_k, HllType::Hll8).unwrap();
28+
for value in partial * rows..(partial + 1) * rows {
29+
sketch.update(value as u64);
30+
}
31+
sketch
32+
})
33+
.collect()
34+
}
35+
36+
#[divan::bench(args = [12, 18])]
37+
fn empty_from_dense(bencher: Bencher, lg_k: u8) {
38+
let source = partials(lg_k, 1, 1 << lg_k).pop().unwrap();
39+
bencher.bench_local(|| {
40+
let mut union = HllUnion::new(lg_k).unwrap();
41+
union.update(black_box(&source));
42+
black_box(union)
43+
});
44+
}
45+
46+
#[divan::bench(args = [12, 18])]
47+
fn recreated_small_partials(bencher: Bencher, lg_k: u8) {
48+
let partials = partials(lg_k, 400, 200);
49+
bencher.bench_local(|| {
50+
let mut sketch = HllSketch::new(lg_k, HllType::Hll8).unwrap();
51+
for partial in black_box(&partials) {
52+
let mut union = HllUnion::new(lg_k).unwrap();
53+
union.update(&sketch);
54+
union.update(partial);
55+
sketch = union.to_sketch(HllType::Hll8);
56+
}
57+
black_box(sketch.estimate());
58+
black_box(sketch)
59+
});
60+
}
61+
62+
#[divan::bench(args = [12, 18])]
63+
fn persistent_small_partials(bencher: Bencher, lg_k: u8) {
64+
let partials = partials(lg_k, 400, 200);
65+
bencher.bench_local(|| {
66+
let mut union = HllUnion::new(lg_k).unwrap();
67+
for partial in black_box(&partials) {
68+
union.update(partial);
69+
}
70+
black_box(union.estimate());
71+
black_box(union.to_sketch(HllType::Hll8))
72+
});
73+
}
74+
75+
#[divan::bench(args = [12, 18])]
76+
fn dense_partials(bencher: Bencher, lg_k: u8) {
77+
let partials = partials(lg_k, 16, (1 << lg_k) / 4);
78+
bencher.bench_local(|| {
79+
let mut union = HllUnion::new(lg_k).unwrap();
80+
for partial in black_box(&partials) {
81+
union.update(partial);
82+
}
83+
black_box(union.estimate());
84+
black_box(union.to_sketch(HllType::Hll8))
85+
});
86+
}
87+
88+
#[divan::bench(args = [12, 18])]
89+
fn dense_partials_with_queries(bencher: Bencher, lg_k: u8) {
90+
let partials = partials(lg_k, 16, (1 << lg_k) / 4);
91+
bencher.bench_local(|| {
92+
let mut union = HllUnion::new(lg_k).unwrap();
93+
for partial in black_box(&partials) {
94+
union.update(partial);
95+
black_box(union.estimate());
96+
}
97+
black_box(union.to_sketch(HllType::Hll8))
98+
});
99+
}
100+
101+
#[divan::bench(args = [12, 18])]
102+
fn downsampled_dense_partials(bencher: Bencher, lg_k: u8) {
103+
let partials = partials(lg_k + 1, 16, (1 << lg_k) / 2);
104+
bencher.bench_local(|| {
105+
let mut union = HllUnion::new(lg_k).unwrap();
106+
for partial in black_box(&partials) {
107+
union.update(partial);
108+
}
109+
black_box(union.estimate());
110+
black_box(union.to_sketch(HllType::Hll8))
111+
});
112+
}

0 commit comments

Comments
 (0)