Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
75 changes: 28 additions & 47 deletions be/src/exec/sink/viceberg_delete_sink.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -147,14 +147,16 @@ Status VIcebergDeleteSink::init_properties(ObjectPool* pool) {
_compress_type = delete_sink.compress_type;
}

// Get output path and table location
if (delete_sink.__isset.output_path) {
_output_path = delete_sink.output_path;
}

if (delete_sink.__isset.table_location) {
_table_location = delete_sink.table_location;
// Every FE version sends both forms of the data location (see IcebergOutputLocation). Without
// the table's own form the commit could only record the address BE wrote the file at, e.g.
// s3://b/db/t/data/delete_pos_x.parquet for an oss:// table, which other engines resolve with
// the wrong file system.
if (delete_sink.output_path.empty() || delete_sink.table_location.empty()) {
return Status::InternalError(
"Iceberg delete sink must carry output_path and table_location, got '{}' and '{}'",
delete_sink.output_path, delete_sink.table_location);
}
_output_location.emplace(delete_sink.output_path, delete_sink.table_location);

// Get Hadoop configuration
if (delete_sink.__isset.hadoop_config) {
Expand Down Expand Up @@ -219,7 +221,7 @@ Status VIcebergDeleteSink::open(RuntimeState* state, RuntimeProfile* profile) {

LOG(INFO) << fmt::format(
"VIcebergDeleteSink opened: delete_type={}, output_path={}, format_version={}",
to_string(_delete_type), _output_path, _format_version);
to_string(_delete_type), _t_sink.iceberg_delete_sink.output_path, _format_version);

return Status::OK();
}
Expand Down Expand Up @@ -461,18 +463,16 @@ Status VIcebergDeleteSink::_collect_position_deletes(

Status VIcebergDeleteSink::_write_position_delete_files(
const std::map<std::string, IcebergFileDeletion>& file_deletions) {
DORIS_CHECK(_output_location.has_value());
constexpr size_t kBatchSize = 4096;
for (const auto& [data_file_path, deletion] : file_deletions) {
if (deletion.rows_to_delete.isEmpty()) {
continue;
}
// Generate unique delete file path
std::string delete_file_path = _generate_delete_file_path(data_file_path);

// Create delete file writer
auto writer = VIcebergDeleteFileWriterFactory::create_writer(
TFileContent::POSITION_DELETES, delete_file_path, _file_format_type,
_compress_type);
TFileContent::POSITION_DELETES, *_output_location,
_delete_file_name(data_file_path), _file_format_type, _compress_type);

// Build column names for position delete
std::vector<std::string> column_names = {"file_path", "pos"};
Expand All @@ -486,7 +486,7 @@ Status VIcebergDeleteSink::_write_position_delete_files(
writer->open(_state, _state->runtime_profile(), _position_delete_output_expr_ctxs,
column_names, _hadoop_conf, _file_type, _broker_addresses);
if (writer->file_system() != nullptr) {
_created_files.emplace_back(writer->file_system(), delete_file_path);
_created_files.emplace_back(writer->file_system(), writer->write_path());
}
RETURN_IF_ERROR(open_status);

Expand Down Expand Up @@ -523,7 +523,7 @@ Status VIcebergDeleteSink::_write_position_delete_files(
_delete_file_count++;

VLOG(1) << fmt::format("Written position delete file: path={}, rows={}, referenced_file={}",
delete_file_path, commit_data.row_count, data_file_path);
writer->write_path(), commit_data.row_count, data_file_path);
}

return Status::OK();
Expand Down Expand Up @@ -623,6 +623,7 @@ std::string VIcebergDeleteSink::_get_file_extension() const {

Status VIcebergDeleteSink::_write_deletion_vector_files(
const std::map<std::string, IcebergFileDeletion>& file_deletions) {
DORIS_CHECK(_output_location.has_value());
std::vector<DeletionVectorBlob> blobs;
for (const auto& [data_file_path, deletion] : file_deletions) {
if (deletion.rows_to_delete.isEmpty()) {
Expand Down Expand Up @@ -667,13 +668,15 @@ Status VIcebergDeleteSink::_write_deletion_vector_files(
return Status::OK();
}

std::string puffin_path = _generate_puffin_file_path();
const std::string puffin_file_name = _puffin_file_name();
int64_t puffin_file_size = 0;
RETURN_IF_ERROR(_write_puffin_file(puffin_path, &blobs, &puffin_file_size));
RETURN_IF_ERROR(_write_puffin_file(_output_location->write_path(puffin_file_name), &blobs,
&puffin_file_size));
const std::string commit_path = _output_location->commit_path(puffin_file_name);

for (const auto& blob : blobs) {
TIcebergCommitData commit_data;
commit_data.__set_file_path(puffin_path);
commit_data.__set_file_path(commit_path);
commit_data.__set_row_count(blob.merged_count);
commit_data.__set_affected_rows(blob.delete_count);
commit_data.__set_file_size(puffin_file_size);
Expand Down Expand Up @@ -783,37 +786,15 @@ std::string VIcebergDeleteSink::_build_puffin_footer_json(
return {buffer.GetString(), buffer.GetSize()};
}

std::string VIcebergDeleteSink::_generate_delete_file_path(
const std::string& referenced_data_file) {
// Generate unique delete file name using UUID
std::string uuid = generate_uuid_string();
std::string file_name;

std::string file_extension = _get_file_extension();
file_name =
fmt::format("delete_pos_{}_{}{}", uuid,
std::hash<std::string> {}(referenced_data_file) % 10000000, file_extension);

// Combine with output path or table location
std::string base_path = _output_path.empty() ? _table_location : _output_path;

// Ensure base path ends with /
if (!base_path.empty() && base_path.back() != '/') {
base_path += '/';
}

// Delete files are data files in Iceberg, write under data location
return fmt::format("{}{}", base_path, file_name);
std::string VIcebergDeleteSink::_delete_file_name(const std::string& referenced_data_file) const {
// Delete files live in the data location itself, next to the partition directories.
return fmt::format("delete_pos_{}_{}{}", generate_uuid_string(),
std::hash<std::string> {}(referenced_data_file) % 10000000,
_get_file_extension());
}

std::string VIcebergDeleteSink::_generate_puffin_file_path() {
std::string uuid = generate_uuid_string();
std::string file_name = fmt::format("delete_dv_{}.puffin", uuid);
std::string base_path = _output_path.empty() ? _table_location : _output_path;
if (!base_path.empty() && base_path.back() != '/') {
base_path += '/';
}
return fmt::format("{}{}", base_path, file_name);
std::string VIcebergDeleteSink::_puffin_file_name() const {
return fmt::format("delete_dv_{}.puffin", generate_uuid_string());
}

} // namespace doris
16 changes: 7 additions & 9 deletions be/src/exec/sink/viceberg_delete_sink.h
Original file line number Diff line number Diff line change
Expand Up @@ -24,12 +24,14 @@
#include <cstdint>
#include <map>
#include <memory>
#include <optional>
#include <string>
#include <vector>

#include "common/status.h"
#include "core/block/block.h"
#include "exec/sink/writer/async_result_writer.h"
#include "exec/sink/writer/iceberg/iceberg_output_location.h"
#include "exec/sink/writer/iceberg/viceberg_delete_file_writer.h"
#include "exprs/vexpr_fwd.h"
#include "roaring/roaring64map.hh"
Expand Down Expand Up @@ -114,12 +116,9 @@ class VIcebergDeleteSink final : public AsyncResultWriter {

std::string _build_puffin_footer_json(const std::vector<DeletionVectorBlob>& blobs);

std::string _generate_puffin_file_path();

/**
* Generate unique delete file path
*/
std::string _generate_delete_file_path(const std::string& referenced_data_file = "");
// Unique file names, relative to the data location (_output_location).
std::string _puffin_file_name() const;
std::string _delete_file_name(const std::string& referenced_data_file) const;

/**
* Get $row_id column index from block
Expand Down Expand Up @@ -162,9 +161,8 @@ class VIcebergDeleteSink final : public AsyncResultWriter {
TFileFormatType::type _file_format_type = TFileFormatType::FORMAT_PARQUET;
TFileCompressType::type _compress_type = TFileCompressType::SNAPPYBLOCK;

// Output directory for delete files
std::string _output_path;
std::string _table_location;
// The data location delete files are written to and committed under, set by init_properties().
std::optional<IcebergOutputLocation> _output_location;

TFileType::type _file_type = TFileType::FILE_HDFS;
std::vector<TNetworkAddress> _broker_addresses;
Expand Down
81 changes: 81 additions & 0 deletions be/src/exec/sink/writer/iceberg/iceberg_output_location.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

#pragma once

#include <string>
#include <string_view>
#include <utility>

#include "common/check.h"

namespace doris {

// The directory an Iceberg writer puts its files in, in both forms of the same location. BE
// creates, writes and deletes a file under write_dir, the form BE addresses the storage with.
// Iceberg metadata records the file under commit_dir, the table's own form, which every engine
// reading the table resolves. FE sends both: TIcebergTableSink output_path / original_output_path
// and TIcebergDeleteSink output_path / table_location. Example: an OSS table with data location
// oss://b/db/t/data reaches BE as write_dir s3://b/db/t/data and commit_dir oss://b/db/t/data, so
// delete_pos_1.parquet is written to s3://b/db/t/data/delete_pos_1.parquet and committed as
// oss://b/db/t/data/delete_pos_1.parquet.
//
// Every file a writer reports takes both of its paths from one IcebergOutputLocation, so it
// cannot commit the form it wrote: delete files used to be committed as s3://..., and other
// engines resolved them with a file system the table is not on.
class IcebergOutputLocation {
public:
IcebergOutputLocation(std::string write_dir, std::string commit_dir)
: _write_dir(std::move(write_dir)), _commit_dir(std::move(commit_dir)) {
DORIS_CHECK(!_write_dir.empty());
DORIS_CHECK(!_commit_dir.empty());
}

// The same subdirectory in both forms, e.g. the partition directory "p=1/q=2".
IcebergOutputLocation subdirectory(std::string_view relative_dir) const {
DORIS_CHECK(!relative_dir.empty());
return {_join(_write_dir, relative_dir), _join(_commit_dir, relative_dir)};
}

// The path BE writes file_name at.
std::string write_path(std::string_view file_name) const {
return _join(_write_dir, file_name);
}

// The path Iceberg metadata records file_name at.
std::string commit_path(std::string_view file_name) const {
return _join(_commit_dir, file_name);
}

private:
// FE takes write.data.path verbatim, so a directory may end with '/'. Both forms join with
// exactly one separator, as Iceberg's own location provider does: oss://b/t/data and
// oss://b/t/data/ both join "p=1" into oss://b/t/data/p=1.
static std::string _join(const std::string& dir, std::string_view name) {
std::string path = dir;
if (path.back() != '/') {
path.push_back('/');
}
path.append(name);
return path;
}

std::string _write_dir;
std::string _commit_dir;
};

} // namespace doris
17 changes: 10 additions & 7 deletions be/src/exec/sink/writer/iceberg/viceberg_delete_file_writer.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -43,11 +43,13 @@ std::unique_ptr<iceberg::Schema> build_position_delete_schema() {
}

VIcebergDeleteFileWriter::VIcebergDeleteFileWriter(TFileContent::type delete_type,
const std::string& output_path,
const IcebergOutputLocation& location,
const std::string& file_name,
TFileFormatType::type file_format,
TFileCompressType::type compress_type)
: _delete_type(delete_type),
_output_path(output_path),
_write_path(location.write_path(file_name)),
_commit_path(location.commit_path(file_name)),
_file_format(file_format),
_compress_type(compress_type) {}

Expand All @@ -73,7 +75,7 @@ Status VIcebergDeleteFileWriter::open(RuntimeState* state, RuntimeProfile* profi
fs_properties.broker_addresses = &broker_addresses;
}

io::FileDescription file_description = {.path = _output_path, .fs_name {}};
io::FileDescription file_description = {.path = _write_path, .fs_name {}};
_fs = DORIS_TRY(FileFactory::create_fs(fs_properties, file_description));

io::FileWriterOptions file_writer_options = {.used_by_s3_committer = false};
Expand Down Expand Up @@ -147,7 +149,7 @@ Status VIcebergDeleteFileWriter::close(TIcebergCommitData& commit_data) {
}

// Fill commit data (use __set_ to mark optional fields as present)
commit_data.__set_file_path(_output_path);
commit_data.__set_file_path(_commit_path);
commit_data.__set_row_count(_written_rows);
commit_data.__set_affected_rows(_written_rows);
commit_data.__set_file_size(_file_size);
Expand All @@ -168,9 +170,10 @@ Status VIcebergDeleteFileWriter::close(TIcebergCommitData& commit_data) {

// Factory method
std::unique_ptr<VIcebergDeleteFileWriter> VIcebergDeleteFileWriterFactory::create_writer(
TFileContent::type delete_type, const std::string& output_path,
TFileFormatType::type file_format, TFileCompressType::type compress_type) {
return std::make_unique<VIcebergDeleteFileWriter>(delete_type, output_path, file_format,
TFileContent::type delete_type, const IcebergOutputLocation& location,
const std::string& file_name, TFileFormatType::type file_format,
TFileCompressType::type compress_type) {
return std::make_unique<VIcebergDeleteFileWriter>(delete_type, location, file_name, file_format,
compress_type);
}

Expand Down
18 changes: 13 additions & 5 deletions be/src/exec/sink/writer/iceberg/viceberg_delete_file_writer.h
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@

#include "common/status.h"
#include "core/block/block.h"
#include "exec/sink/writer/iceberg/iceberg_output_location.h"
#include "exprs/vexpr_fwd.h"
#include "gen_cpp/DataSinks_types.h"
#include "io/fs/file_writer.h"
Expand Down Expand Up @@ -50,8 +51,10 @@ class VFileFormatTransformer;
*/
class VIcebergDeleteFileWriter {
public:
VIcebergDeleteFileWriter(TFileContent::type delete_type, const std::string& output_path,
TFileFormatType::type file_format,
// Writes file_name under location; the commit data reports the file under the table's own form
// of that directory, not the form it was written at.
VIcebergDeleteFileWriter(TFileContent::type delete_type, const IcebergOutputLocation& location,
const std::string& file_name, TFileFormatType::type file_format,
TFileCompressType::type compress_type);

~VIcebergDeleteFileWriter();
Expand Down Expand Up @@ -96,9 +99,13 @@ class VIcebergDeleteFileWriter {

std::shared_ptr<io::FileSystem> file_system() const { return _fs; }

// The path the file is written at, which is what cleanup must delete.
const std::string& write_path() const { return _write_path; }

private:
TFileContent::type _delete_type;
std::string _output_path;
std::string _write_path;
std::string _commit_path;
TFileFormatType::type _file_format;
TFileCompressType::type _compress_type = TFileCompressType::SNAPPYBLOCK;

Expand All @@ -122,8 +129,9 @@ class VIcebergDeleteFileWriter {
class VIcebergDeleteFileWriterFactory {
public:
static std::unique_ptr<VIcebergDeleteFileWriter> create_writer(
TFileContent::type delete_type, const std::string& output_path,
TFileFormatType::type file_format, TFileCompressType::type compress_type);
TFileContent::type delete_type, const IcebergOutputLocation& location,
const std::string& file_name, TFileFormatType::type file_format,
TFileCompressType::type compress_type);
};

} // namespace doris
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@ Status VIcebergPartitionWriter::open(RuntimeState* state, RuntimeProfile* profil
if (!_write_info.broker_addresses.empty()) {
fs_properties.broker_addresses = &(_write_info.broker_addresses);
}
_path = fmt::format("{}/{}", _write_info.write_path, _get_target_file_name());
_path = _write_info.location.write_path(_get_target_file_name());
io::FileDescription file_description = {.path = _path, .fs_name {}};
_fs = DORIS_TRY(FileFactory::create_fs(fs_properties, file_description));
io::FileWriterOptions file_writer_options = {.used_by_s3_committer = false};
Expand Down Expand Up @@ -165,8 +165,7 @@ Status VIcebergPartitionWriter::_build_iceberg_commit_data(TIcebergCommitData* c
DCHECK(commit_data != nullptr);
DCHECK(_file_format_transformer != nullptr);

commit_data->__set_file_path(
fmt::format("{}/{}", _write_info.original_write_path, _get_target_file_name()));
commit_data->__set_file_path(_write_info.location.commit_path(_get_target_file_name()));
commit_data->__set_row_count(_row_count);
commit_data->__set_affected_rows(_row_count);
commit_data->__set_file_size(_file_format_transformer->written_len());
Expand Down
Loading
Loading