Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions be/src/exec/sink/writer/iceberg/viceberg_partition_writer.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -217,6 +217,16 @@ Status VIcebergPartitionWriter::_build_iceberg_commit_data(TIcebergCommitData* c
commit_data->__set_file_size(_file_format_transformer->written_len());
commit_data->__set_file_content(TFileContent::DATA);
commit_data->__set_partition_values(_partition_values);
// Split offsets describe the physical file, even when the table disables all column metrics.
if (_file_format_type == TFileFormatType::FORMAT_PARQUET) {
std::vector<int64_t> offsets;
RETURN_IF_ERROR(static_cast<VIcebergParquetWriter*>(_file_format_transformer.get())
->collect_split_offsets_after_close(&offsets));
// Empty files have no splits; leave the optional field absent, as with an older BE.
if (!offsets.empty()) {
commit_data->__set_split_offsets(offsets);
}
}
// ORC collection reopens the file, so honor the FE policy before any footer work.
if (!_collect_column_stats) {
return Status::OK();
Expand Down
26 changes: 26 additions & 0 deletions be/src/format/transformer/viceberg_parquet_writer.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -115,6 +115,32 @@ std::unique_ptr<ArrowBlockConvertor> VIcebergParquetWriter::_create_arrow_block_
_iceberg_schema, &_iceberg_schema_json, timezone_name, timezone);
}

Status VIcebergParquetWriter::collect_split_offsets_after_close(std::vector<int64_t>* offsets) {
auto metadata = _file_metadata();
// Arrow publishes file metadata only after Close(), so offsets cannot be collected earlier.
if (metadata == nullptr) {
return Status::InternalError("File metadata is not available");
}
offsets->clear();
offsets->reserve(metadata->num_row_groups());
for (int i = 0; i < metadata->num_row_groups(); ++i) {
auto row_group = metadata->RowGroup(i);
// Empty files/groups have no page boundary to advertise as a scan split.
if (row_group->num_rows() == 0 || row_group->num_columns() == 0) {
continue;
}
auto first_column = row_group->ColumnChunk(0);
int64_t offset = first_column->data_page_offset();
// A split must include the dictionary page, not just the first data page that references it.
if (first_column->has_dictionary_page()) {
offset = std::min(offset, first_column->dictionary_page_offset());
}
offsets->push_back(offset);
}
std::sort(offsets->begin(), offsets->end());
return Status::OK();
}

Status VIcebergParquetWriter::collect_file_statistics_after_close(TIcebergColumnStats* stats) {
std::shared_ptr<::parquet::FileMetaData> file_metadata = _file_metadata();
if (file_metadata == nullptr) {
Expand Down
2 changes: 2 additions & 0 deletions be/src/format/transformer/viceberg_parquet_writer.h
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,8 @@ class VIcebergParquetWriter final : public VParquetWriter {

Status collect_file_statistics_after_close(TIcebergColumnStats* stats);

Status collect_split_offsets_after_close(std::vector<int64_t>* offsets);

protected:
std::unique_ptr<ArrowBlockConvertor> _create_arrow_block_convertor(
DataTypes types, std::vector<std::string> names, const std::string& timezone_name,
Expand Down
65 changes: 62 additions & 3 deletions be/test/exec/sink/writer/iceberg/iceberg_partition_writer_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -15,16 +15,24 @@
// specific language governing permissions and limitations
// under the License.

#include <arrow/memory_pool.h>
#include <gtest/gtest.h>

#include <optional>

#include "core/data_type/data_type_number.h"
#include "exec/sink/writer/iceberg/viceberg_partition_writer.h"
#include "exec/sink/writer/iceberg/viceberg_sort_writer.h"
#include "format/table/iceberg/schema_parser.h"
#include "format/transformer/viceberg_parquet_writer.h"
#include "io/fs/local_file_system.h"
#include "runtime/exec_env.h"
#include "testutil/mock/mock_descriptors.h"
#include "testutil/mock/mock_runtime_state.h"
#include "testutil/mock/mock_slot_ref.h"
#include "util/defer_op.h"
#include "util/timezone_utils.h"
#include "util/uid_util.h"

namespace doris {

Expand Down Expand Up @@ -73,13 +81,14 @@ class VIcebergPartitionWriterTest : public testing::Test {
static std::unique_ptr<VIcebergPartitionWriter> make_writer(
const TDataSink& sink, const VExprContextSPtrs& output_exprs,
const iceberg::Schema& schema, const std::string* schema_json,
const std::map<std::string, std::string>& hadoop_conf) {
const std::map<std::string, std::string>& hadoop_conf,
TFileFormatType::type format = TFileFormatType::FORMAT_ORC) {
IPartitionWriterBase::WriteInfo write_info;
write_info.file_type = TFileType::FILE_LOCAL;
return std::make_unique<VIcebergPartitionWriter>(
sink, std::vector<std::string> {}, output_exprs, schema, schema_json,
std::vector<std::string> {}, std::move(write_info), "data", 0,
TFileFormatType::FORMAT_ORC, TFileCompressType::ZLIB, hadoop_conf);
std::vector<std::string> {}, std::move(write_info), "data", 0, format,
TFileCompressType::ZLIB, hadoop_conf);
}

static void install_fake_transformer(VIcebergPartitionWriter* writer,
Expand All @@ -90,6 +99,11 @@ class VIcebergPartitionWriterTest : public testing::Test {
output_exprs, std::move(close_status), destroyed);
}

static void install_parquet_transformer(VIcebergPartitionWriter* writer,
std::unique_ptr<VIcebergParquetWriter> transformer) {
writer->_file_format_transformer = std::move(transformer);
}

static Status build_commit_data(VIcebergPartitionWriter* writer,
TIcebergCommitData* commit_data) {
return writer->_build_iceberg_commit_data(commit_data);
Expand All @@ -114,6 +128,51 @@ TEST_F(VIcebergPartitionWriterTest, OrcSkipsFooterCollectionWhenMetricsAreDisabl
EXPECT_FALSE(commit_data.__isset.column_stats);
}

TEST_F(VIcebergPartitionWriterTest, ParquetKeepsSplitOffsetsWhenColumnMetricsAreDisabled) {
TimezoneUtils::load_timezones_to_cache();
auto* env = ExecEnv::GetInstance();
auto* previous_pool = env->_arrow_memory_pool;
env->_arrow_memory_pool = arrow::default_memory_pool();
const auto path = "./iceberg_split_offsets_" + UniqueId::gen_uid().to_string() + ".parquet";
auto fs = io::global_local_filesystem();
Defer cleanup([&] {
env->_arrow_memory_pool = previous_pool;
static_cast<void>(fs->delete_file(path));
});
auto type = std::make_shared<DataTypeInt32>();
auto output_exprs = MockSlotRef::create_mock_contexts(DataTypes {type});
const std::string json =
R"({"type":"struct","fields":[{"id":1,"name":"value","required":true,"type":"int"}]})";
auto schema = iceberg::SchemaParser::from_json(json);
io::FileWriterPtr file_writer;
ASSERT_TRUE(fs->create_file(path, &file_writer).ok());
RuntimeState state;
state.set_timezone("UTC");
ParquetFileOptions options {.compression_type = TParquetCompressionType::SNAPPY,
.parquet_version = TParquetVersion::PARQUET_1_0,
.parquet_disable_dictionary = false,
.enable_int96_timestamps = false};
auto transformer = std::make_unique<VIcebergParquetWriter>(
&state, file_writer.get(), output_exprs, std::vector<std::string> {"value"}, false,
options, &json, *schema);
ASSERT_TRUE(transformer->open().ok());
auto column = type->create_column();
column->insert(Field::create_field<TYPE_INT>(7));
Block block;
block.insert({std::move(column), type, "value"});
ASSERT_TRUE(transformer->write(block).ok());
ASSERT_TRUE(transformer->close().ok());
auto writer = make_writer(make_table_sink(false), output_exprs, *schema, &json, {},
TFileFormatType::FORMAT_PARQUET);
install_parquet_transformer(writer.get(), std::move(transformer));
TIcebergCommitData commit;
ASSERT_TRUE(build_commit_data(writer.get(), &commit).ok());
EXPECT_FALSE(commit.__isset.column_stats);
ASSERT_TRUE(commit.__isset.split_offsets);
ASSERT_EQ(1, commit.split_offsets.size());
EXPECT_EQ(4, commit.split_offsets[0]);
}

TEST_F(VIcebergPartitionWriterTest, MissingPolicyKeepsCollectionEnabledForRollingUpgrade) {
VExprContextSPtrs output_exprs;
iceberg::Schema schema(std::vector<iceberg::NestedField> {});
Expand Down
66 changes: 66 additions & 0 deletions be/test/format/transformer/vparquet_writer_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
#include <optional>
#include <string_view>

#include "common/config.h"
#include "core/block/block.h"
#include "core/column/column_array.h"
#include "core/column/column_decimal.h"
Expand All @@ -50,6 +51,7 @@
#include "runtime/exec_env.h"
#include "runtime/runtime_state.h"
#include "testutil/mock/mock_slot_ref.h"
#include "util/defer_op.h"
#include "util/timezone_utils.h"
#include "util/uid_util.h"

Expand Down Expand Up @@ -128,11 +130,75 @@ class VParquetWriterTest : public testing::Test {
}
}

void check_split_offsets(int row_groups, bool dictionary) {
const auto previous_row_group_size = config::min_row_group_size;
config::min_row_group_size = 0;
Defer restore_row_group_size([&] { config::min_row_group_size = previous_row_group_size; });
auto type = std::make_shared<DataTypeInt32>();
auto output_exprs = MockSlotRef::create_mock_contexts(DataTypes {type});
const std::string json =
R"({"type":"struct","fields":[{"id":1,"name":"value","required":true,"type":"int"}]})";
auto schema = iceberg::SchemaParser::from_json(json);
io::FileWriterPtr file_writer;
ASSERT_TRUE(_fs->create_file(_file_path, &file_writer).ok());
RuntimeState state;
state.set_timezone("UTC");
ParquetFileOptions options {.compression_type = TParquetCompressionType::SNAPPY,
.parquet_version = TParquetVersion::PARQUET_1_0,
.parquet_disable_dictionary = !dictionary,
.enable_int96_timestamps = false};
VIcebergParquetWriter writer(&state, file_writer.get(), output_exprs, {"value"}, false,
options, &json, *schema);
ASSERT_TRUE(writer.open().ok());
for (int group = 0; group < row_groups; ++group) {
auto column = type->create_column();
for (int row = 0; row < 100; ++row) {
column->insert(Field::create_field<TYPE_INT>(row % 3));
}
Block block;
block.insert({std::move(column), type, "value"});
ASSERT_TRUE(writer.write(block).ok());
}
ASSERT_TRUE(writer.close().ok());
std::vector<int64_t> offsets {999999};
ASSERT_TRUE(writer.collect_split_offsets_after_close(&offsets).ok());
ASSERT_EQ(row_groups, offsets.size());
auto reader = ::parquet::ParquetFileReader::OpenFile(_file_path, false);
auto metadata = reader->metadata();
for (int i = 0; i < row_groups; ++i) {
auto row_group = metadata->RowGroup(i);
auto column = row_group->ColumnChunk(0);
ASSERT_EQ(dictionary, column->has_dictionary_page());
EXPECT_EQ(dictionary ? column->dictionary_page_offset() : column->data_page_offset(),
offsets[i]);
EXPECT_GT(offsets[i], i == 0 ? 0 : offsets[i - 1]);
}
std::vector<int64_t> repeated;
ASSERT_TRUE(writer.collect_split_offsets_after_close(&repeated).ok());
EXPECT_EQ(offsets, repeated);
}

arrow::MemoryPool* _previous_pool = nullptr;
std::string _file_path;
std::shared_ptr<io::FileSystem> _fs;
};

TEST_F(VParquetWriterTest, IcebergSplitOffsetsIncludeDictionaryPages) {
check_split_offsets(3, true);
}

TEST_F(VParquetWriterTest, IcebergSplitOffsetsWithoutDictionary) {
check_split_offsets(3, false);
}

TEST_F(VParquetWriterTest, IcebergSplitOffsetsSingleRowGroup) {
check_split_offsets(1, true);
}

TEST_F(VParquetWriterTest, IcebergSplitOffsetsEmptyFile) {
check_split_offsets(0, true);
}

TEST_F(VParquetWriterTest, WritesIcebergPrimitiveAndCollectsMetrics) {
auto type = make_nullable(std::make_shared<DataTypeInt32>());
auto output_exprs = MockSlotRef::create_mock_contexts(DataTypes {type});
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -980,7 +980,7 @@ private List<ConnectorScanRange> doPlanSystemTableScan(IcebergTableHandle handle
return doPlanPositionDeletesSystemTableScan(handle, metadataTable, columns, filter, session);
}
TableScan scan = buildScan(metadataTable, handle, filter, session);
// Project the metadata-table scan to ONLY the requested columns, in the SAME order BE lists them in
// Project the metadata-table scan to the requested columns first, in the SAME order BE lists them in
// required_fields (== the scan slot order: buildColumnHandles iterates desc.getSlots(), BE builds
// required_fields from the matching file_slot_descs). This mirrors legacy IcebergScanNode
// .getSystemTableProjectedSchema() + scan.project(...). Two reasons the projection is mandatory AND must
Expand Down Expand Up @@ -1009,6 +1009,13 @@ private List<ConnectorScanRange> doPlanSystemTableScan(IcebergTableHandle handle
projectedFields.add(field);
}
}
// Iceberg returns split_offsets only when the last offset is smaller than file_size_in_bytes.
// Without that dependency, even valid offsets such as [4, 2048] are hidden as null. Append it
// after the requested columns so BE's positional reads keep their original order.
if (seenFieldIds.contains(DataFile.SPLIT_OFFSETS.fieldId())
&& seenFieldIds.add(DataFile.FILE_SIZE.fieldId())) {
projectedFields.add(metadataSchema.findField(DataFile.FILE_SIZE.fieldId()));
}
scan = scan.project(new Schema(projectedFields));
}
List<ConnectorScanRange> ranges = new ArrayList<>();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -184,7 +184,7 @@ private static WriteResult convertToWriterResult(
partitionData = Optional.of(convertToPartitionData(partitionValues, spec, zone));
}
DataFile dataFile = genDataFile(fileFormat, location, spec, partitionData, recordCount, fileSize,
metrics, sortOrder);
metrics, sortOrder, commitData.getSplitOffsets());
dataFiles.add(dataFile);
}
return WriteResult.builder()
Expand All @@ -194,14 +194,18 @@ private static WriteResult convertToWriterResult(

private static DataFile genDataFile(FileFormat format, String location, PartitionSpec spec,
Optional<PartitionData> partitionData, long recordCount, long fileSize, Metrics metrics,
SortOrder sortOrder) {
SortOrder sortOrder, List<Long> splitOffsets) {
DataFiles.Builder builder = DataFiles.builder(spec)
.withPath(location)
.withFileSizeInBytes(fileSize)
.withRecordCount(recordCount)
.withMetrics(metrics)
.withSortOrder(sortOrder)
.withFormat(format);
// Older BE fragments omit this optional field; only Parquet data writers report these boundaries.
if (format == FileFormat.PARQUET && splitOffsets != null && !splitOffsets.isEmpty()) {
builder.withSplitOffsets(splitOffsets);
}
partitionData.ifPresent(builder::withPartition);
return builder.build();
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2613,6 +2613,52 @@ public void planSystemTableScanProjectsRequestedColumnsExcludingReadableMetrics(
+ "BufferUnderflows on complex/boolean bound columns when materialised unrequested");
}

@Test
public void planSystemTableScanKeepsSplitOffsetsWithTheirFileSizeDependency() throws Exception {
List<Long> offsets = Arrays.asList(4L, 2048L);
for (PartitionSpec spec : Arrays.asList(PartitionSpec.unpartitioned(),
PartitionSpec.builderFor(SCHEMA).identity("id").build())) {
Table table = createTable("t1", SCHEMA, spec);
table.newAppend().appendFile(dataFile(table.spec(), "data.parquet", 4096, offsets,
spec.isPartitioned() ? "id=1" : null)).commit();
IcebergScanPlanProvider provider = new IcebergScanPlanProvider(
IcebergCatalogProperties.of(Collections.emptyMap()), opsReturning(table));
for (String type : Arrays.asList("files", "data_files", "all_files", "all_data_files")) {
for (List<String> names : Arrays.asList(Collections.singletonList("split_offsets"),
Arrays.asList("content", "split_offsets"), Arrays.asList("split_offsets", "content"),
Arrays.asList("split_offsets", "file_size_in_bytes"),
Arrays.asList("file_size_in_bytes", "split_offsets"))) {
List<ConnectorColumnHandle> columns = names.stream()
.map(name -> (ConnectorColumnHandle) new IcebergColumnHandle(name, 1))
.collect(Collectors.toList());
List<ConnectorScanRange> ranges = provider.planScan(null, ConnectorScanRequest.builder(
IcebergTableHandle.forSystemTable("db1", "t1", type, -1L, null, -1L), columns).build());
int rowCount = 0;
for (ConnectorScanRange range : ranges) {
FileScanTask task = IcebergSystemTableSerialization.deserializeFromBase64(
((IcebergScanRange) range).getSerializedSplit());
try (CloseableIterable<StructLike> rows = task.asDataTask().rows()) {
for (StructLike row : rows) {
Assertions.assertEquals(offsets, row.get(names.indexOf("split_offsets"), List.class),
type + " " + names);
if (names.contains("content")) {
Assertions.assertEquals(0, row.get(names.indexOf("content"), Integer.class));
}
if (names.contains("file_size_in_bytes")) {
Assertions.assertEquals(4096L,
row.get(names.indexOf("file_size_in_bytes"), Long.class));
}
rowCount++;
}
}
Assertions.assertNull(task.schema().findField("readable_metrics"));
}
Assertions.assertEquals(1, rowCount);
}
}
}
}

@Test
public void planSystemTableScanProjectionPreservesRequestedColumnOrderForPositionalBeRead() throws Exception {
// WHY (regression — External Regression build 1000283): BE's IcebergSysTableJniScanner reads the
Expand Down
Loading
Loading