Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 9 additions & 3 deletions be/src/exec/operator/distinct_streaming_aggregation_operator.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -411,7 +411,9 @@ Status DistinctStreamingAggOperatorX::push(RuntimeState* state, Block* in_block,
// Pass-through rows are not deduplicated; counting them against a pushed-down limit would let
// duplicates consume the allowance and stop the child before enough distinct keys reached the
// global stage (which applies the limit again).
if (_limit != -1 && !local_state._stop_emplace_flag &&
// With HAVING, apply LIMIT in pull() after filtering. For example, if HAVING rejects
// group 1 and keeps group 2, LIMIT 1 must not stop input after group 1.
if (_limit != -1 && local_state._conjuncts.empty() && !local_state._stop_emplace_flag &&
(local_state._num_rows_returned + local_state._aggregated_block->rows()) >= _limit) {
auto limit_rows = _limit - local_state._num_rows_returned;
local_state._aggregated_block->set_num_rows(limit_rows);
Expand All @@ -433,10 +435,14 @@ Status DistinctStreamingAggOperatorX::pull(RuntimeState* state, Block* block, bo

local_state._make_nullable_output_key(block);
if (!_is_streaming_preagg) {
// dispose the having clause, should not be execute in prestreaming agg
// Apply HAVING before LIMIT. Only groups that pass the filter count toward LIMIT.
RETURN_IF_ERROR(local_state.filter_block(local_state._conjuncts, block));
local_state.reached_limit(block, &local_state._reach_limit);
} else {
// Keep the input-stage count separate. Rows that bypass deduplication must not
// consume its pushed-down LIMIT; the final aggregate applies LIMIT to distinct groups.
local_state.add_num_rows_returned(block->rows());
}
local_state.add_num_rows_returned(block->rows());
// If the limit is not reached, it is important to ensure that _aggregated_block is empty
// because it may still contain data.
// However, if the limit is reached, there is no need to output data even if some exists.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -133,6 +133,55 @@ TEST_F(DistinctStreamingAggOperatorTest, test2) {
}
}

struct DistinctStreamingAggHavingLimitTest : public DistinctStreamingAggOperatorTest,
public ::testing::WithParamInterface<int> {
void SetUp() override {
DistinctStreamingAggOperatorTest::SetUp();
state->_batch_size = GetParam();
op->_is_streaming_preagg = false;
op->_limit = 1;
auto type = std::make_shared<DataTypeUInt8>();
create_op({type}, {type});
// Use the Boolean group key as HAVING: reject 0 and keep 1.
// These two groups are sufficient to test LIMIT after filtering.
local_state->_conjuncts = MockSlotRef::create_mock_contexts(0, type);
}
};

TEST_P(DistinctStreamingAggHavingLimitTest, filter_before_limit_in_one_block) {
// Both groups arrive in one block. LIMIT must not discard 1 before HAVING rejects 0.
mock_op->_outout_blocks.push_back(ColumnHelper::create_block<DataTypeUInt8>({0, 1}));
Block block;
bool eos = false;
ASSERT_TRUE(op->get_block(state.get(), &block, &eos).ok());
EXPECT_TRUE(ColumnHelper::block_equal(block, ColumnHelper::create_block<DataTypeUInt8>({1})));
EXPECT_TRUE(eos);
EXPECT_TRUE(local_state->_reach_limit);
EXPECT_EQ(local_state->_num_rows_returned, 1);
}

TEST_P(DistinctStreamingAggHavingLimitTest, read_next_block_after_rejected_group) {
// A rejected group must not stop the child before the next block arrives.
mock_op->_outout_blocks.push_back(ColumnHelper::create_block<DataTypeUInt8>({0}));
mock_op->_outout_blocks.push_back(ColumnHelper::create_block<DataTypeUInt8>({1}));
Block block;
bool eos = false;
ASSERT_TRUE(op->get_block(state.get(), &block, &eos).ok());
EXPECT_EQ(block.rows(), 0);
EXPECT_FALSE(eos);
EXPECT_FALSE(local_state->_reach_limit);
EXPECT_EQ(local_state->_num_rows_returned, 0);

ASSERT_TRUE(op->get_block(state.get(), &block, &eos).ok());
EXPECT_TRUE(ColumnHelper::block_equal(block, ColumnHelper::create_block<DataTypeUInt8>({1})));
EXPECT_TRUE(eos);
EXPECT_TRUE(local_state->_reach_limit);
EXPECT_EQ(local_state->_num_rows_returned, 1);
}

INSTANTIATE_TEST_SUITE_P(BatchSizes, DistinctStreamingAggHavingLimitTest,
::testing::Values(1, 1024));

TEST_F(DistinctStreamingAggOperatorTest, test3) {
// batch size = 10
op->_is_streaming_preagg = true;
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
-- This file is automatically generated. You should know what you did if you want to edit this
-- !reference --
2

-- !streaming --
2

-- !reference --
2

-- !streaming --
2

Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

suite("distinct_streaming_having_limit") {
sql "DROP TABLE IF EXISTS distinct_streaming_having_limit"
sql """
CREATE TABLE distinct_streaming_having_limit (k INT NOT NULL)
DUPLICATE KEY(k)
DISTRIBUTED BY HASH(k) BUCKETS 1
PROPERTIES ("replication_num" = "1")
"""
// Two groups are sufficient: HAVING rejects the first and keeps the second.
sql "INSERT INTO distinct_streaming_having_limit VALUES (1), (2)"

// Keep one ordered input stream. Disable the other final aggregation paths.
// In particular, the final distinct streaming operator does not support spill.
sql "SET parallel_pipeline_task_num = 1"
sql "SET enable_bucketed_hash_agg = false"
sql "SET enable_spill = false"
// Execute each operator path instead of reusing the reference query's cached result.
sql "SET enable_sql_cache = false"

// ORDER BY and the inner LIMIT make group 1 arrive first.
// RAND() keeps HAVING on the aggregate node. Its value is in [0, 1), so
// the predicate is always false for k = 1 and always true for k = 2.
// The final LIMIT must therefore return 2, regardless of the random value.
def query = """
SELECT k
FROM (SELECT k FROM distinct_streaming_having_limit ORDER BY k LIMIT 2) s
GROUP BY k
HAVING RAND() < CAST(k - 1 AS DOUBLE)
LIMIT 1
"""

for (def batchSize : [1, 1024]) {
sql "SET batch_size = ${batchSize}"
// Use the ordinary aggregate as the reference for each batch size.
sql "SET enable_distinct_streaming_aggregation = false"
order_qt_reference query

sql "SET enable_distinct_streaming_aggregation = true"
explain {
sql query
check { plan ->
// Require HAVING and LIMIT on the same final aggregate node.
// Each intervening node detail starts with a vertical bar.
(plan =~ /VAGGREGATE \([^\n]*finalize\)[^\n]*\n(?:\s*\|[^\n]*\n)*?\s*\|\s*having:[^\n]*\n(?:\s*\|[^\n]*\n)*?\s*\|\s*limit: 1\b/).find()
}
}
// Cover both one row per batch and the default batch size.
order_qt_streaming query
}
}
Loading