From ffa0694d4c4e779d8efd009c3df50f11ce551626 Mon Sep 17 00:00:00 2001 From: comphead Date: Sat, 14 Nov 2026 14:53:06 -0800 Subject: [PATCH] perf: stop computing an HLL estimate on every merge and size() call `SparkHllSketch` and `SparkHllUnion` inferred their heap size from `estimate()`, and `SparkHllUnion::merge` called `estimate()` on both sides before every update to keep a `dense` flag for that inference. `datasketches` 0.6.0 exposes the size directly: `estimated_size()` reads the coupon or register array length in O(1) and is exact. Use it, and drop the `layout` inference, both `dense` flags and the `preamble` module. On 0.6.0 this makes Final-mode `hll_sketch_agg` and `hll_union_agg` over many tiny groups about 20% faster (211 to 169 ns per partial row at lgConfigK 18). It matters more once Comet moves to a datasketches release with apache/datasketches-rust#317, which defers the register cache rebuild after a dense merge until the next query. The per-merge `estimate()` forced that rebuild on every merge. Built against #317, merging an already dense sketch drops from 266 us to 5.5 us, close to the 5.2 us a plain register-wise max takes. Part of #7341. --- native/spark-expr/src/agg_funcs/hll_sketch.rs | 150 +++--------------- 1 file changed, 24 insertions(+), 126 deletions(-) diff --git a/native/spark-expr/src/agg_funcs/hll_sketch.rs b/native/spark-expr/src/agg_funcs/hll_sketch.rs index 8624a5dc699..8295d2fe867 100644 --- a/native/spark-expr/src/agg_funcs/hll_sketch.rs +++ b/native/spark-expr/src/agg_funcs/hll_sketch.rs @@ -41,83 +41,6 @@ use datasketches::hll::{HllSketch, HllType, HllUnion}; #[derive(Debug)] pub struct SparkHllSketch { inner: HllSketch, - /// Whether `inner` is known to hold the dense register array; see `layout`. Once set it - /// stays set, because a union's result estimates from its registers, and near the promotion - /// point that estimate can fall back below `layout::max_coupons`. - dense: bool, -} - -/// The in-memory layout `datasketches` 0.3.0 gives an HLL_8 sketch, which the crate keeps -/// private (`HllSketch::mode` is `pub(super)`). Accumulator `size()` reports it: most sketches -/// stay small, and charging every group the dense register array up front makes grouped -/// high-`lgConfigK` queries exhaust the memory pool. -/// -/// A sketch starts in LIST mode, an 8-slot coupon array. When that fills it moves to SET mode, a -/// hash table that starts at 32 slots and doubles once more than 3/4 full. At `2^(lgConfigK - 3)` -/// slots it is promoted to the register array instead, one byte per register. Below lgConfigK 8 -/// a full LIST goes straight to the register array. Modes only move forward, and a union's -/// gadget follows the same path. -/// -/// The crate exposes only the estimate, which is enough. In LIST and SET mode `estimate()` is -/// `max(couponCount, interpolation)`, so it is never below the coupon count. Promotion seeds the -/// HIP accumulator with that estimate, so from the moment the register array exists the estimate -/// exceeds `max_coupons`. The one other way into the register array is a union with a sketch -/// already in it, which the caller tracks as `dense`. -/// -/// `heap_size_follows_the_crates_layout` checks this against the preamble the crate serializes, -/// so a `datasketches` bump that changes the layout fails that test. -mod layout { - const COUPON_BYTES: usize = 4; - const LIST_BYTES: usize = COUPON_BYTES << 3; - const MIN_SET_SLOTS: usize = 1 << 5; - /// Below this lgConfigK there is no SET mode. - const MIN_LG_K_WITH_SET: u8 = 8; - - /// The most coupons a sketch at `lg_config_k` holds before the register array replaces them. - pub fn max_coupons(lg_config_k: u8) -> usize { - if lg_config_k < MIN_LG_K_WITH_SET { - 7 - } else { - // 3/4 of the largest SET, 2^(lgConfigK - 3) slots. - 3 << (lg_config_k - 5) - } - } - - /// Whether a sketch at `lg_config_k` holds the register array, given its `estimate()` and - /// whether it is already known to. - pub fn is_dense(lg_config_k: u8, known_dense: bool, estimate: f64) -> bool { - known_dense || estimate > max_coupons(lg_config_k) as f64 - } - - /// Heap bytes held by an HLL_8 sketch or union gadget, never less than the crate allocated. - /// It is exact except just below a SET resize or the promotion, where the estimate, which - /// corrects for coupon collisions, runs slightly ahead of the coupon count and this charges - /// the next size up. - pub fn heap_bytes(lg_config_k: u8, known_dense: bool, estimate: f64) -> usize { - let registers = 1usize << lg_config_k; - if lg_config_k < MIN_LG_K_WITH_SET { - // The larger of a LIST and the register array, 128 bytes at most. - return registers.max(LIST_BYTES); - } - if is_dense(lg_config_k, known_dense, estimate) { - return registers; - } - let coupons = estimate as usize; - if coupons < 8 { - LIST_BYTES - } else { - let slots = (4 * coupons).div_ceil(3).next_power_of_two(); - COUPON_BYTES * slots.max(MIN_SET_SLOTS) - } - } -} - -/// Byte offsets into the DataSketches HLL preamble, and the bits we need from it. -mod preamble { - /// Mode byte. Low two bits are the current mode (0 LIST, 1 SET, 2 HLL). - pub const MODE: usize = 7; - pub const CUR_MODE_MASK: u8 = 0x3; - pub const CUR_MODE_HLL: u8 = 2; } impl SparkHllSketch { @@ -125,7 +48,6 @@ impl SparkHllSketch { pub fn new(lg_config_k: u8) -> Self { Self { inner: HllSketch::new(lg_config_k, HllType::Hll8).expect("lgConfigK is in [4, 21]"), - dense: false, } } @@ -157,9 +79,7 @@ impl SparkHllSketch { pub fn from_bytes(bytes: &[u8]) -> Result { let inner = HllSketch::deserialize(bytes) .map_err(|e| DataFusionError::Execution(format!("invalid HLL sketch bytes: {e}")))?; - // `deserialize` has validated the preamble, so the mode byte is there. - let dense = bytes[preamble::MODE] & preamble::CUR_MODE_MASK == preamble::CUR_MODE_HLL; - Ok(Self { inner, dense }) + Ok(Self { inner }) } /// The configured `lgConfigK`. @@ -172,13 +92,12 @@ impl SparkHllSketch { self.inner.estimate() } - fn is_dense(&self) -> bool { - layout::is_dense(self.lg_config_k(), self.dense, self.inner.estimate()) - } - - /// Heap bytes the sketch currently holds, for accumulator `size()`. + /// Heap bytes the sketch currently holds, for accumulator `size()`: the coupon array in LIST + /// and SET mode, the `2^lgConfigK`-byte register array once dense. Most groups stay small, and + /// charging every group the register array makes grouped high-`lgConfigK` queries exhaust the + /// memory pool. The crate reads this off its containers without computing an estimate. pub fn heap_size(&self) -> usize { - layout::heap_bytes(self.lg_config_k(), self.dense, self.inner.estimate()) + self.inner.estimated_size() } } @@ -186,9 +105,6 @@ impl SparkHllSketch { #[derive(Debug)] pub struct SparkHllUnion { inner: HllUnion, - /// Whether the union's gadget is known to hold the dense register array. Merging a sketch - /// that holds one always promotes the gadget. See `SparkHllSketch::dense`. - dense: bool, } impl SparkHllUnion { @@ -196,7 +112,6 @@ impl SparkHllUnion { pub fn new(lg_max_k: u8) -> Self { Self { inner: HllUnion::new(lg_max_k).expect("lgMaxK is in [4, 21]"), - dense: false, } } @@ -205,9 +120,9 @@ impl SparkHllUnion { self.inner.lg_max_k() } - /// Merge a sketch into the union. + /// Merge a sketch into the union without querying it. Querying after each dense merge would + /// rebuild the register cache, which apache/datasketches-rust#317 defers until it is needed. pub fn merge(&mut self, sketch: &SparkHllSketch) { - self.dense = self.is_dense() || sketch.is_dense(); self.inner.update(&sketch.inner); } @@ -216,13 +131,10 @@ impl SparkHllUnion { self.inner.to_sketch(HllType::Hll8).serialize() } - fn is_dense(&self) -> bool { - layout::is_dense(self.inner.lg_config_k(), self.dense, self.inner.estimate()) - } - - /// Heap bytes the union's gadget currently holds, for accumulator `size()`. + /// Heap bytes the union's gadget currently holds, for accumulator `size()`. See + /// `SparkHllSketch::heap_size`. pub fn heap_size(&self) -> usize { - layout::heap_bytes(self.inner.lg_config_k(), self.dense, self.inner.estimate()) + self.inner.estimated_size() } } @@ -389,45 +301,31 @@ mod tests { fn allocated(bytes: &[u8]) -> usize { const LG_K: usize = 3; const LG_ARR: usize = 4; - if bytes[preamble::MODE] & preamble::CUR_MODE_MASK == preamble::CUR_MODE_HLL { + // The low two bits of the mode byte are the current mode: 0 LIST, 1 SET, 2 HLL. + const MODE: usize = 7; + if bytes[MODE] & 0x3 == 2 { 1 << bytes[LG_K] } else { 4 << bytes[LG_ARR] } } - /// `heap_size` must never report less than the crate holds, and it should track the real - /// layout rather than the dense maximum. Walk each lgConfigK through every SET resize and the - /// promotion, with duplicates mixed in, and compare against what the crate serializes. + /// `heap_size` reports what the crate holds, not the dense maximum. Walk each lgConfigK + /// through every SET resize and the promotion, with duplicates mixed in, and compare against + /// what the crate serializes. #[test] fn heap_size_follows_the_crates_layout() { for lg_k in [4u8, 7, 8, 11, 14] { let mut sketch = SparkHllSketch::new(lg_k); - let steps = 2 * layout::max_coupons(lg_k) + 64; - let mut exact = 0; - for i in 0..steps as i64 { + // Past the promotion, which comes after 7 coupons below lgConfigK 8 and at 3/4 of + // 2^(lgConfigK - 3) coupons from there. + for i in 0..(1i64 << (lg_k - 3)) + 64 { sketch.update_i64(i); sketch.update_i64(i / 2); let actual = allocated(&sketch.to_sketch_bytes()); - let charged = sketch.heap_size(); - assert!( - charged >= actual, - "lgConfigK {lg_k}, value {i}: charged {charged} bytes, crate holds {actual}" - ); - // Off by at most one size step. Below lgConfigK 8 the charge is a flat 128 bytes - // at most. - assert!( - charged <= (2 * actual).max(128), - "lgConfigK {lg_k}, value {i}: charged {charged} bytes, crate holds {actual}" - ); - exact += usize::from(charged == actual); - } - if lg_k >= 8 { - assert!( - exact * 100 >= steps * 99, - "lgConfigK {lg_k}: exact for only {exact} of {steps} cardinalities" - ); + assert_eq!(sketch.heap_size(), actual, "lgConfigK {lg_k}, value {i}"); } + assert_eq!(sketch.heap_size(), 1 << lg_k, "lgConfigK {lg_k} ends dense"); } } @@ -452,8 +350,8 @@ mod tests { } /// A union's gadget stays small while it takes small sketches. Merging a sketch that holds the - /// register array promotes it however little that sketch has seen, so it is the preamble's - /// mode, not the estimate, that has to decide. + /// register array promotes it however little that sketch has seen, so the charge has to follow + /// the gadget's mode, not its estimate. #[test] fn union_heap_size_follows_the_crates_layout() { let sketch = |values: std::ops::Range| {