diff --git a/be/src/exec/operator/distinct_streaming_aggregation_operator.cpp b/be/src/exec/operator/distinct_streaming_aggregation_operator.cpp index 45ab5827fb7177..6c9c2f8f697f08 100644 --- a/be/src/exec/operator/distinct_streaming_aggregation_operator.cpp +++ b/be/src/exec/operator/distinct_streaming_aggregation_operator.cpp @@ -411,7 +411,9 @@ Status DistinctStreamingAggOperatorX::push(RuntimeState* state, Block* in_block, // Pass-through rows are not deduplicated; counting them against a pushed-down limit would let // duplicates consume the allowance and stop the child before enough distinct keys reached the // global stage (which applies the limit again). - if (_limit != -1 && !local_state._stop_emplace_flag && + // With HAVING, apply LIMIT in pull() after filtering. For example, if HAVING rejects + // group 1 and keeps group 2, LIMIT 1 must not stop input after group 1. + if (_limit != -1 && local_state._conjuncts.empty() && !local_state._stop_emplace_flag && (local_state._num_rows_returned + local_state._aggregated_block->rows()) >= _limit) { auto limit_rows = _limit - local_state._num_rows_returned; local_state._aggregated_block->set_num_rows(limit_rows); @@ -433,10 +435,14 @@ Status DistinctStreamingAggOperatorX::pull(RuntimeState* state, Block* block, bo local_state._make_nullable_output_key(block); if (!_is_streaming_preagg) { - // dispose the having clause, should not be execute in prestreaming agg + // Apply HAVING before LIMIT. Only groups that pass the filter count toward LIMIT. RETURN_IF_ERROR(local_state.filter_block(local_state._conjuncts, block)); + local_state.reached_limit(block, &local_state._reach_limit); + } else { + // Keep the input-stage count separate. Rows that bypass deduplication must not + // consume its pushed-down LIMIT; the final aggregate applies LIMIT to distinct groups. + local_state.add_num_rows_returned(block->rows()); } - local_state.add_num_rows_returned(block->rows()); // If the limit is not reached, it is important to ensure that _aggregated_block is empty // because it may still contain data. // However, if the limit is reached, there is no need to output data even if some exists. diff --git a/be/test/exec/operator/distinct_streaming_aggregation_operator_test.cpp b/be/test/exec/operator/distinct_streaming_aggregation_operator_test.cpp index f52a65ba186a33..fc7baa21b7b069 100644 --- a/be/test/exec/operator/distinct_streaming_aggregation_operator_test.cpp +++ b/be/test/exec/operator/distinct_streaming_aggregation_operator_test.cpp @@ -133,6 +133,55 @@ TEST_F(DistinctStreamingAggOperatorTest, test2) { } } +struct DistinctStreamingAggHavingLimitTest : public DistinctStreamingAggOperatorTest, + public ::testing::WithParamInterface { + void SetUp() override { + DistinctStreamingAggOperatorTest::SetUp(); + state->_batch_size = GetParam(); + op->_is_streaming_preagg = false; + op->_limit = 1; + auto type = std::make_shared(); + create_op({type}, {type}); + // Use the Boolean group key as HAVING: reject 0 and keep 1. + // These two groups are sufficient to test LIMIT after filtering. + local_state->_conjuncts = MockSlotRef::create_mock_contexts(0, type); + } +}; + +TEST_P(DistinctStreamingAggHavingLimitTest, filter_before_limit_in_one_block) { + // Both groups arrive in one block. LIMIT must not discard 1 before HAVING rejects 0. + mock_op->_outout_blocks.push_back(ColumnHelper::create_block({0, 1})); + Block block; + bool eos = false; + ASSERT_TRUE(op->get_block(state.get(), &block, &eos).ok()); + EXPECT_TRUE(ColumnHelper::block_equal(block, ColumnHelper::create_block({1}))); + EXPECT_TRUE(eos); + EXPECT_TRUE(local_state->_reach_limit); + EXPECT_EQ(local_state->_num_rows_returned, 1); +} + +TEST_P(DistinctStreamingAggHavingLimitTest, read_next_block_after_rejected_group) { + // A rejected group must not stop the child before the next block arrives. + mock_op->_outout_blocks.push_back(ColumnHelper::create_block({0})); + mock_op->_outout_blocks.push_back(ColumnHelper::create_block({1})); + Block block; + bool eos = false; + ASSERT_TRUE(op->get_block(state.get(), &block, &eos).ok()); + EXPECT_EQ(block.rows(), 0); + EXPECT_FALSE(eos); + EXPECT_FALSE(local_state->_reach_limit); + EXPECT_EQ(local_state->_num_rows_returned, 0); + + ASSERT_TRUE(op->get_block(state.get(), &block, &eos).ok()); + EXPECT_TRUE(ColumnHelper::block_equal(block, ColumnHelper::create_block({1}))); + EXPECT_TRUE(eos); + EXPECT_TRUE(local_state->_reach_limit); + EXPECT_EQ(local_state->_num_rows_returned, 1); +} + +INSTANTIATE_TEST_SUITE_P(BatchSizes, DistinctStreamingAggHavingLimitTest, + ::testing::Values(1, 1024)); + TEST_F(DistinctStreamingAggOperatorTest, test3) { // batch size = 10 op->_is_streaming_preagg = true; diff --git a/regression-test/data/query_p0/aggregate/distinct_streaming_having_limit.out b/regression-test/data/query_p0/aggregate/distinct_streaming_having_limit.out new file mode 100644 index 00000000000000..f78cffdf094b19 --- /dev/null +++ b/regression-test/data/query_p0/aggregate/distinct_streaming_having_limit.out @@ -0,0 +1,13 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !reference -- +2 + +-- !streaming -- +2 + +-- !reference -- +2 + +-- !streaming -- +2 + diff --git a/regression-test/suites/query_p0/aggregate/distinct_streaming_having_limit.groovy b/regression-test/suites/query_p0/aggregate/distinct_streaming_having_limit.groovy new file mode 100644 index 00000000000000..54d125b52bb869 --- /dev/null +++ b/regression-test/suites/query_p0/aggregate/distinct_streaming_having_limit.groovy @@ -0,0 +1,67 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("distinct_streaming_having_limit") { + sql "DROP TABLE IF EXISTS distinct_streaming_having_limit" + sql """ + CREATE TABLE distinct_streaming_having_limit (k INT NOT NULL) + DUPLICATE KEY(k) + DISTRIBUTED BY HASH(k) BUCKETS 1 + PROPERTIES ("replication_num" = "1") + """ + // Two groups are sufficient: HAVING rejects the first and keeps the second. + sql "INSERT INTO distinct_streaming_having_limit VALUES (1), (2)" + + // Keep one ordered input stream. Disable the other final aggregation paths. + // In particular, the final distinct streaming operator does not support spill. + sql "SET parallel_pipeline_task_num = 1" + sql "SET enable_bucketed_hash_agg = false" + sql "SET enable_spill = false" + // Execute each operator path instead of reusing the reference query's cached result. + sql "SET enable_sql_cache = false" + + // ORDER BY and the inner LIMIT make group 1 arrive first. + // RAND() keeps HAVING on the aggregate node. Its value is in [0, 1), so + // the predicate is always false for k = 1 and always true for k = 2. + // The final LIMIT must therefore return 2, regardless of the random value. + def query = """ + SELECT k + FROM (SELECT k FROM distinct_streaming_having_limit ORDER BY k LIMIT 2) s + GROUP BY k + HAVING RAND() < CAST(k - 1 AS DOUBLE) + LIMIT 1 + """ + + for (def batchSize : [1, 1024]) { + sql "SET batch_size = ${batchSize}" + // Use the ordinary aggregate as the reference for each batch size. + sql "SET enable_distinct_streaming_aggregation = false" + order_qt_reference query + + sql "SET enable_distinct_streaming_aggregation = true" + explain { + sql query + check { plan -> + // Require HAVING and LIMIT on the same final aggregate node. + // Each intervening node detail starts with a vertical bar. + (plan =~ /VAGGREGATE \([^\n]*finalize\)[^\n]*\n(?:\s*\|[^\n]*\n)*?\s*\|\s*having:[^\n]*\n(?:\s*\|[^\n]*\n)*?\s*\|\s*limit: 1\b/).find() + } + } + // Cover both one row per batch and the default batch size. + order_qt_streaming query + } +}