From dc0d2dc3a13cdf6fe88413eb112930218df805df Mon Sep 17 00:00:00 2001 From: jacktengg Date: Thu, 12 Nov 2026 16:01:34 +0800 Subject: [PATCH] [fix](be) Apply HAVING before the distinct streaming limit ### Background Distinct streaming aggregation emits new grouping keys as input arrives. BE can use this operator for GROUP BY without aggregate functions. A final aggregate can also contain HAVING and LIMIT. ### What problem does this PR solve? Issue Number: None Related PR: None PR Type: Bugfix Problem Summary: A final distinct streaming aggregate can discard valid groups when it has HAVING and LIMIT. For example, HAVING rejects group 1 and accepts group 2, but LIMIT 1 stops input after group 1. Apply the final limit after HAVING so the query returns group 2. ### Details - **Frequency:** Queries can lose rows when an early group fails HAVING and consumes the limit before filtering. - **Root cause:** push() truncated the aggregate block and set _reach_limit before pull() evaluated HAVING. - **Why existing tests missed it:** The existing operator tests covered LIMIT and local preaggregation without a final HAVING filter. - **Fix:** Skip early truncation when conjuncts are present. Reuse reached_limit() after the final HAVING filter. Preserve output accounting for local preaggregation and its handling of rows that bypass deduplication. - **Other paths and branches:** Ordinary aggregation and streaming aggregation already filter before applying the final limit. No release branch is changed. - **Behavior change:** LIMIT counts only groups that pass HAVING in the final distinct streaming aggregate. ### Testing - Before the fix, all four new BE tests failed. The new SQL suite also failed: it expected 2 and received no row. - Built ASAN BE with `DISABLE_BE_JAVA_EXTENSIONS=ON DISABLE_BE_CDC_CLIENT=ON ./build.sh --be -j24`. The build and its GLIBC_2.17 symbol check passed. - Ran `./run-be-ut.sh --run --filter='*DistinctStreamingAgg*' -j24`. All 15 tests passed. - Ran `./run-regression-test.sh --run -d query_p0/aggregate,query_p0/set_operations,correctness_p0 -s distinct_streaming_having_limit,distinct_streaming_repeated_keys,test_distinct_streaming_agg_local_shuffle`. All three suites passed on the local ASAN BE. - The new SQL suite uses two rows. It checks one final node for HAVING and LIMIT, then checks the result with distinct streaming on and off at batch sizes 1 and 1024. The expected output was generated by the regression script through the ordinary aggregate. - Direct SQL checks returned 2 for both batch sizes. Profiles confirmed the distinct streaming implementation. - clang-format v16, check-format, build structure checks, and clang-tidy passed for the changed scope. clang-tidy used the BE and UT compile commands with the compiler wrapper's toolchain paths. ### Release note Fix missing result rows for GROUP BY queries with HAVING and LIMIT when the final aggregate uses distinct streaming aggregation. ### Check List (For Author) - Behavior changed: Yes (see Details) - Does this need documentation: No - Pre-submission status: Ready ### AI Disclosure - Tool: OpenAI Codex - Generated: Code, tests, comments, and this description - Human review: Pending author review --- ...istinct_streaming_aggregation_operator.cpp | 12 +++- ...ct_streaming_aggregation_operator_test.cpp | 49 ++++++++++++++ .../distinct_streaming_having_limit.out | 13 ++++ .../distinct_streaming_having_limit.groovy | 67 +++++++++++++++++++ 4 files changed, 138 insertions(+), 3 deletions(-) create mode 100644 regression-test/data/query_p0/aggregate/distinct_streaming_having_limit.out create mode 100644 regression-test/suites/query_p0/aggregate/distinct_streaming_having_limit.groovy diff --git a/be/src/exec/operator/distinct_streaming_aggregation_operator.cpp b/be/src/exec/operator/distinct_streaming_aggregation_operator.cpp index 45ab5827fb7177..6c9c2f8f697f08 100644 --- a/be/src/exec/operator/distinct_streaming_aggregation_operator.cpp +++ b/be/src/exec/operator/distinct_streaming_aggregation_operator.cpp @@ -411,7 +411,9 @@ Status DistinctStreamingAggOperatorX::push(RuntimeState* state, Block* in_block, // Pass-through rows are not deduplicated; counting them against a pushed-down limit would let // duplicates consume the allowance and stop the child before enough distinct keys reached the // global stage (which applies the limit again). - if (_limit != -1 && !local_state._stop_emplace_flag && + // With HAVING, apply LIMIT in pull() after filtering. For example, if HAVING rejects + // group 1 and keeps group 2, LIMIT 1 must not stop input after group 1. + if (_limit != -1 && local_state._conjuncts.empty() && !local_state._stop_emplace_flag && (local_state._num_rows_returned + local_state._aggregated_block->rows()) >= _limit) { auto limit_rows = _limit - local_state._num_rows_returned; local_state._aggregated_block->set_num_rows(limit_rows); @@ -433,10 +435,14 @@ Status DistinctStreamingAggOperatorX::pull(RuntimeState* state, Block* block, bo local_state._make_nullable_output_key(block); if (!_is_streaming_preagg) { - // dispose the having clause, should not be execute in prestreaming agg + // Apply HAVING before LIMIT. Only groups that pass the filter count toward LIMIT. RETURN_IF_ERROR(local_state.filter_block(local_state._conjuncts, block)); + local_state.reached_limit(block, &local_state._reach_limit); + } else { + // Keep the input-stage count separate. Rows that bypass deduplication must not + // consume its pushed-down LIMIT; the final aggregate applies LIMIT to distinct groups. + local_state.add_num_rows_returned(block->rows()); } - local_state.add_num_rows_returned(block->rows()); // If the limit is not reached, it is important to ensure that _aggregated_block is empty // because it may still contain data. // However, if the limit is reached, there is no need to output data even if some exists. diff --git a/be/test/exec/operator/distinct_streaming_aggregation_operator_test.cpp b/be/test/exec/operator/distinct_streaming_aggregation_operator_test.cpp index f52a65ba186a33..fc7baa21b7b069 100644 --- a/be/test/exec/operator/distinct_streaming_aggregation_operator_test.cpp +++ b/be/test/exec/operator/distinct_streaming_aggregation_operator_test.cpp @@ -133,6 +133,55 @@ TEST_F(DistinctStreamingAggOperatorTest, test2) { } } +struct DistinctStreamingAggHavingLimitTest : public DistinctStreamingAggOperatorTest, + public ::testing::WithParamInterface { + void SetUp() override { + DistinctStreamingAggOperatorTest::SetUp(); + state->_batch_size = GetParam(); + op->_is_streaming_preagg = false; + op->_limit = 1; + auto type = std::make_shared(); + create_op({type}, {type}); + // Use the Boolean group key as HAVING: reject 0 and keep 1. + // These two groups are sufficient to test LIMIT after filtering. + local_state->_conjuncts = MockSlotRef::create_mock_contexts(0, type); + } +}; + +TEST_P(DistinctStreamingAggHavingLimitTest, filter_before_limit_in_one_block) { + // Both groups arrive in one block. LIMIT must not discard 1 before HAVING rejects 0. + mock_op->_outout_blocks.push_back(ColumnHelper::create_block({0, 1})); + Block block; + bool eos = false; + ASSERT_TRUE(op->get_block(state.get(), &block, &eos).ok()); + EXPECT_TRUE(ColumnHelper::block_equal(block, ColumnHelper::create_block({1}))); + EXPECT_TRUE(eos); + EXPECT_TRUE(local_state->_reach_limit); + EXPECT_EQ(local_state->_num_rows_returned, 1); +} + +TEST_P(DistinctStreamingAggHavingLimitTest, read_next_block_after_rejected_group) { + // A rejected group must not stop the child before the next block arrives. + mock_op->_outout_blocks.push_back(ColumnHelper::create_block({0})); + mock_op->_outout_blocks.push_back(ColumnHelper::create_block({1})); + Block block; + bool eos = false; + ASSERT_TRUE(op->get_block(state.get(), &block, &eos).ok()); + EXPECT_EQ(block.rows(), 0); + EXPECT_FALSE(eos); + EXPECT_FALSE(local_state->_reach_limit); + EXPECT_EQ(local_state->_num_rows_returned, 0); + + ASSERT_TRUE(op->get_block(state.get(), &block, &eos).ok()); + EXPECT_TRUE(ColumnHelper::block_equal(block, ColumnHelper::create_block({1}))); + EXPECT_TRUE(eos); + EXPECT_TRUE(local_state->_reach_limit); + EXPECT_EQ(local_state->_num_rows_returned, 1); +} + +INSTANTIATE_TEST_SUITE_P(BatchSizes, DistinctStreamingAggHavingLimitTest, + ::testing::Values(1, 1024)); + TEST_F(DistinctStreamingAggOperatorTest, test3) { // batch size = 10 op->_is_streaming_preagg = true; diff --git a/regression-test/data/query_p0/aggregate/distinct_streaming_having_limit.out b/regression-test/data/query_p0/aggregate/distinct_streaming_having_limit.out new file mode 100644 index 00000000000000..f78cffdf094b19 --- /dev/null +++ b/regression-test/data/query_p0/aggregate/distinct_streaming_having_limit.out @@ -0,0 +1,13 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !reference -- +2 + +-- !streaming -- +2 + +-- !reference -- +2 + +-- !streaming -- +2 + diff --git a/regression-test/suites/query_p0/aggregate/distinct_streaming_having_limit.groovy b/regression-test/suites/query_p0/aggregate/distinct_streaming_having_limit.groovy new file mode 100644 index 00000000000000..54d125b52bb869 --- /dev/null +++ b/regression-test/suites/query_p0/aggregate/distinct_streaming_having_limit.groovy @@ -0,0 +1,67 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("distinct_streaming_having_limit") { + sql "DROP TABLE IF EXISTS distinct_streaming_having_limit" + sql """ + CREATE TABLE distinct_streaming_having_limit (k INT NOT NULL) + DUPLICATE KEY(k) + DISTRIBUTED BY HASH(k) BUCKETS 1 + PROPERTIES ("replication_num" = "1") + """ + // Two groups are sufficient: HAVING rejects the first and keeps the second. + sql "INSERT INTO distinct_streaming_having_limit VALUES (1), (2)" + + // Keep one ordered input stream. Disable the other final aggregation paths. + // In particular, the final distinct streaming operator does not support spill. + sql "SET parallel_pipeline_task_num = 1" + sql "SET enable_bucketed_hash_agg = false" + sql "SET enable_spill = false" + // Execute each operator path instead of reusing the reference query's cached result. + sql "SET enable_sql_cache = false" + + // ORDER BY and the inner LIMIT make group 1 arrive first. + // RAND() keeps HAVING on the aggregate node. Its value is in [0, 1), so + // the predicate is always false for k = 1 and always true for k = 2. + // The final LIMIT must therefore return 2, regardless of the random value. + def query = """ + SELECT k + FROM (SELECT k FROM distinct_streaming_having_limit ORDER BY k LIMIT 2) s + GROUP BY k + HAVING RAND() < CAST(k - 1 AS DOUBLE) + LIMIT 1 + """ + + for (def batchSize : [1, 1024]) { + sql "SET batch_size = ${batchSize}" + // Use the ordinary aggregate as the reference for each batch size. + sql "SET enable_distinct_streaming_aggregation = false" + order_qt_reference query + + sql "SET enable_distinct_streaming_aggregation = true" + explain { + sql query + check { plan -> + // Require HAVING and LIMIT on the same final aggregate node. + // Each intervening node detail starts with a vertical bar. + (plan =~ /VAGGREGATE \([^\n]*finalize\)[^\n]*\n(?:\s*\|[^\n]*\n)*?\s*\|\s*having:[^\n]*\n(?:\s*\|[^\n]*\n)*?\s*\|\s*limit: 1\b/).find() + } + } + // Cover both one row per batch and the default batch size. + order_qt_streaming query + } +}