Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 6 additions & 2 deletions be/src/format/table/iceberg_sys_table_jni_reader.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -58,8 +58,12 @@ IcebergSysTableJniReader::IcebergSysTableJniReader(
JniDataBridge::get_jni_type_with_different_string(desc->type()));
}
std::map<std::string, std::string> params;
params["serialized_split"] =
range.table_format_params.iceberg_params.serialized_split;
const auto& iceberg_params = range.table_format_params.iceberg_params;
params["serialized_split"] = iceberg_params.serialized_split;
if (iceberg_params.__isset.file_io_expiry_ms) {
params["file_io_expiry_ms"] =
std::to_string(iceberg_params.file_io_expiry_ms);
}
params["required_fields"] = join(required_fields, ",");
params["required_types"] = join(required_types, "#");
params["time_zone"] = state->timezone();
Expand Down
7 changes: 5 additions & 2 deletions be/src/format_v2/jni/iceberg_sys_table_reader.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -54,8 +54,11 @@ Status IcebergSysTableJniReader::build_scanner_params(
std::map<std::string, std::string>* params) const {
DORIS_CHECK(params != nullptr);
params->clear();
params->emplace("serialized_split",
_current_range.table_format_params.iceberg_params.serialized_split);
const auto& iceberg_params = _current_range.table_format_params.iceberg_params;
params->emplace("serialized_split", iceberg_params.serialized_split);
if (iceberg_params.__isset.file_io_expiry_ms) {
params->emplace("file_io_expiry_ms", std::to_string(iceberg_params.file_io_expiry_ms));
}

std::vector<std::string> required_types;
required_types.reserve(_projected_columns.size());
Expand Down
14 changes: 12 additions & 2 deletions be/src/io/file_factory.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -126,6 +126,10 @@ Result<io::FileSystemSPtr> FileFactory::create_fs(const io::FSPropertiesRef& fs_
*fs_properties.properties, io::FileSystem::TMP_FS_ID);
}
case TFileType::FILE_S3: {
// FILE_S3 is also the compatibility wire value for Azure. The
// provider-owned properties are consumed by S3ClientFactory, which
// dispatches Azure ranges to AzureObjStorageClient; no Hadoop reader
// is involved in this branch.
S3URI s3_uri(file_description.path);
RETURN_IF_ERROR_RESULT(s3_uri.parse());
S3Conf s3_conf;
Expand Down Expand Up @@ -195,10 +199,12 @@ Result<io::FileWriterPtr> FileFactory::create_file_writer(
S3Conf s3_conf;
RETURN_IF_ERROR_RESULT(
S3ClientFactory::convert_properties_to_s3_conf(properties, s3_uri, &s3_conf));
RETURN_IF_ERROR_RESULT(
s3_uri.parse(s3_conf.client_conf.provider == io::ObjStorageProvider::AZURE));
auto client = std::make_shared<io::ObjClientHolder>(std::move(s3_conf.client_conf));
RETURN_IF_ERROR_RESULT(client->init());
return std::make_unique<io::S3FileWriter>(std::move(client), std::move(s3_conf.bucket),
s3_uri.get_key(), &options);
s3_uri.get_key(), &options, path);
}
case TFileType::FILE_HDFS: {
THdfsParams hdfs_params = parse_properties(properties);
Expand Down Expand Up @@ -238,16 +244,20 @@ Result<io::FileReaderSPtr> FileFactory::_create_file_reader_internal(
return file_reader;
}
case TFileType::FILE_S3: {
// Azure uses this existing wire slot with provider=azure; the object
// client factory selects Azure's native Blob SDK below.
S3URI s3_uri(file_description.path);
RETURN_IF_ERROR_RESULT(s3_uri.parse());
S3Conf s3_conf;
RETURN_IF_ERROR_RESULT(S3ClientFactory::convert_properties_to_s3_conf(
system_properties.properties, s3_uri, &s3_conf));
RETURN_IF_ERROR_RESULT(
s3_uri.parse(s3_conf.client_conf.provider == io::ObjStorageProvider::AZURE));
auto client_holder = std::make_shared<io::ObjClientHolder>(s3_conf.client_conf);
RETURN_IF_ERROR_RESULT(client_holder->init());
auto options = with_storage_in_cache_key(reader_options, s3_conf.client_conf.endpoint);
return io::S3FileReader::create(std::move(client_holder), s3_conf.bucket, s3_uri.get_key(),
file_description.file_size, profile)
file_description.file_size, profile, file_description.path)
.and_then([&options](auto&& reader) {
return io::create_cached_file_reader(std::move(reader), options);
});
Expand Down
4 changes: 4 additions & 0 deletions be/src/io/file_factory.h
Original file line number Diff line number Diff line change
Expand Up @@ -123,6 +123,10 @@ class FileFactory {
case TStorageBackendType::S3:
return TFileType::FILE_S3;
case TStorageBackendType::AZURE:
// Keep the shipped FILE_S3 wire slot for compatibility. The
// provider field in the storage properties selects the native
// Azure client inside S3ClientFactory; this must never fall back
// to FILE_HDFS for an Azure URI.
return TFileType::FILE_S3;
case TStorageBackendType::BROKER:
return TFileType::FILE_BROKER;
Expand Down
20 changes: 16 additions & 4 deletions be/src/io/fs/s3_file_reader.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,15 @@

namespace doris::io {

namespace {
std::string sanitize_display_path(std::string path) {
if (const auto query = path.find('?'); query != std::string::npos) {
path.resize(query);
}
return path;
}
} // namespace

bvar::Adder<uint64_t> s3_file_reader_read_counter("s3_file_reader", "read_at");
bvar::Adder<uint64_t> s3_file_reader_total("s3_file_reader", "total_num");
bvar::Adder<uint64_t> s3_bytes_read_total("s3_file_reader", "bytes_read");
Expand All @@ -66,7 +75,8 @@ bvar::LatencyRecorder s3_file_reader_latency("s3_file_reader", "s3_latency");

Result<FileReaderSPtr> S3FileReader::create(std::shared_ptr<const ObjClientHolder> client,
std::string bucket, std::string key, int64_t file_size,
RuntimeProfile* profile) {
RuntimeProfile* profile, std::string display_path) {
RETURN_IF_ERROR_RESULT(client->validate_for_access());
if (file_size < 0) {
auto res = client->object_file_size(bucket, key);
if (!res.has_value()) {
Expand All @@ -77,12 +87,14 @@ Result<FileReaderSPtr> S3FileReader::create(std::shared_ptr<const ObjClientHolde
}

return std::make_shared<S3FileReader>(std::move(client), std::move(bucket), std::move(key),
file_size, profile);
file_size, profile, std::move(display_path));
}

S3FileReader::S3FileReader(std::shared_ptr<const ObjClientHolder> client, std::string bucket,
std::string key, size_t file_size, RuntimeProfile* profile)
: _path(fmt::format("s3://{}/{}", bucket, key)),
std::string key, size_t file_size, RuntimeProfile* profile,
std::string display_path)
: _path(display_path.empty() ? fmt::format("s3://{}/{}", bucket, key)
: sanitize_display_path(std::move(display_path))),
_file_size(file_size),
_bucket(std::move(bucket)),
_key(std::move(key)),
Expand Down
4 changes: 2 additions & 2 deletions be/src/io/fs/s3_file_reader.h
Original file line number Diff line number Diff line change
Expand Up @@ -38,10 +38,10 @@ class S3FileReader final : public FileReader {
public:
static Result<FileReaderSPtr> create(std::shared_ptr<const ObjClientHolder> client,
std::string bucket, std::string key, int64_t file_size,
RuntimeProfile* profile);
RuntimeProfile* profile, std::string display_path = {});

S3FileReader(std::shared_ptr<const ObjClientHolder> client, std::string bucket, std::string key,
size_t file_size, RuntimeProfile* profile);
size_t file_size, RuntimeProfile* profile, std::string display_path = {});

~S3FileReader() override;

Expand Down
72 changes: 47 additions & 25 deletions be/src/io/fs/s3_file_system.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -62,10 +62,14 @@ constexpr int LEN_OF_OSS_PRIVATE_SUFFIX = 9; // length of "-internal"
}
#endif

Result<std::string> get_key(const Path& full_path) {
// FIXME(plat1ko): Check bucket in full path and support relative path
Result<std::string> get_key(const Path& full_path, const S3ClientConf& conf) {
// Preserve the raw-object-key contract for internal callers. Full Azure
// URIs must belong to this holder even when a FileSystem is reused.
S3URI uri(full_path.native());
RETURN_IF_ERROR_RESULT(uri.parse());
RETURN_IF_ERROR_RESULT(uri.parse(conf.provider == ObjStorageProvider::AZURE));
if (conf.provider == ObjStorageProvider::AZURE) {
RETURN_IF_ERROR_RESULT(S3ClientFactory::validate_azure_uri(uri, conf));
}
return uri.get_key();
}

Expand All @@ -88,6 +92,7 @@ Status ObjClientHolder::reset(const S3ClientConf& conf) {
reset_conf.ak = conf.ak;
reset_conf.sk = conf.sk;
reset_conf.token = conf.token;
reset_conf.azure_credentials = conf.azure_credentials;
reset_conf.bucket = conf.bucket;
reset_conf.connect_timeout_ms = conf.connect_timeout_ms;
reset_conf.max_connections = conf.max_connections;
Expand Down Expand Up @@ -120,6 +125,12 @@ Status ObjClientHolder::reset(const S3ClientConf& conf) {
return Status::OK();
}

Status ObjClientHolder::validate_for_access() const {
TEST_SYNC_POINT("ObjClientHolder::validate_for_access:before_lock");
std::shared_lock lock(_mtx);
return S3ClientFactory::validate_credentials_for_access(_conf);
}

Result<int64_t> ObjClientHolder::object_file_size(const std::string& bucket,
const std::string& key) const {
auto client = get();
Expand All @@ -142,6 +153,7 @@ Result<int64_t> ObjClientHolder::object_file_size(const std::string& bucket,
}

std::string ObjClientHolder::full_s3_path(std::string_view bucket, std::string_view key) const {
std::shared_lock lock(_mtx);
return fmt::format("{}/{}/{}", _conf.endpoint, bucket, key);
}

Expand Down Expand Up @@ -183,18 +195,30 @@ S3FileSystem::~S3FileSystem() = default;

Status S3FileSystem::create_file_impl(const Path& file, FileWriterPtr* writer,
const FileWriterOptions* opts) {
RETURN_IF_ERROR(_client->validate_for_access());
const auto conf = _client->s3_client_conf();
auto client = _client->get();
CHECK_S3_CLIENT(client);
auto key = DORIS_TRY(get_key(file));
*writer = std::make_unique<S3FileWriter>(_client, _bucket, std::move(key), opts);
auto key = DORIS_TRY(get_key(file, conf));
std::string display_path;
if (file.native().find("://") != std::string::npos) {
display_path = file.native();
}
*writer = std::make_unique<S3FileWriter>(_client, _bucket, std::move(key), opts,
std::move(display_path));
return Status::OK();
}

Status S3FileSystem::open_file_internal(const Path& file, FileReaderSPtr* reader,
const FileReaderOptions& opts) {
TEST_SYNC_POINT_CALLBACK("S3FileSystem::open_file_internal", &file, &opts);
auto key = DORIS_TRY(get_key(file));
*reader = DORIS_TRY(S3FileReader::create(_client, _bucket, key, opts.file_size, nullptr));
auto key = DORIS_TRY(get_key(file, _client->s3_client_conf()));
std::string display_path;
if (file.native().find("://") != std::string::npos) {
display_path = file.native();
}
*reader = DORIS_TRY(S3FileReader::create(_client, _bucket, key, opts.file_size, nullptr,
std::move(display_path)));
return Status::OK();
}

Expand All @@ -206,7 +230,7 @@ Status S3FileSystem::delete_file_impl(const Path& file) {
auto client = _client->get();
CHECK_S3_CLIENT(client);

auto key = DORIS_TRY(get_key(file));
auto key = DORIS_TRY(get_key(file, _client->s3_client_conf()));

auto resp = client->delete_object({.bucket = _bucket, .key = key});

Expand All @@ -221,7 +245,7 @@ Status S3FileSystem::delete_directory_impl(const Path& dir) {
auto client = _client->get();
CHECK_S3_CLIENT(client);

auto prefix = DORIS_TRY(get_key(dir));
auto prefix = DORIS_TRY(get_key(dir, _client->s3_client_conf()));
if (!prefix.empty() && prefix.back() != '/') {
prefix.push_back('/');
}
Expand All @@ -248,7 +272,7 @@ Status S3FileSystem::batch_delete_impl(const std::vector<Path>& remote_files) {
auto path_begin = path_iter;
for (; path_iter != remote_files.end() && (path_iter - path_begin < max_delete_batch);
++path_iter) {
auto key = DORIS_TRY(get_key(*path_iter));
auto key = DORIS_TRY(get_key(*path_iter, _client->s3_client_conf()));
objects.emplace_back(std::move(key));
}
if (objects.empty()) {
Expand All @@ -267,7 +291,7 @@ Status S3FileSystem::batch_delete_impl(const std::vector<Path>& remote_files) {
Status S3FileSystem::exists_impl(const Path& path, bool* res) const {
auto client = _client->get();
CHECK_S3_CLIENT(client);
auto key = DORIS_TRY(get_key(path));
auto key = DORIS_TRY(get_key(path, _client->s3_client_conf()));

VLOG_DEBUG << "key:" << key << " path:" << path;

Expand All @@ -286,7 +310,7 @@ Status S3FileSystem::exists_impl(const Path& path, bool* res) const {
}

Status S3FileSystem::file_size_impl(const Path& file, int64_t* file_size) const {
auto key = DORIS_TRY(get_key(file));
auto key = DORIS_TRY(get_key(file, _client->s3_client_conf()));
*file_size = DORIS_TRY(_client->object_file_size(_bucket, key));
return Status::OK();
}
Expand All @@ -298,7 +322,7 @@ Status S3FileSystem::list_impl(const Path& dir, bool only_file, std::vector<File
*exists = true;
auto client = _client->get();
CHECK_S3_CLIENT(client);
auto prefix = DORIS_TRY(get_key(dir));
auto prefix = DORIS_TRY(get_key(dir, _client->s3_client_conf()));
if (!prefix.empty() && prefix.back() != '/') {
prefix.push_back('/');
}
Expand Down Expand Up @@ -328,10 +352,10 @@ Status S3FileSystem::upload_impl(const Path& local_file, const Path& remote_file
auto client = _client->get();
CHECK_S3_CLIENT(client);

auto key = DORIS_TRY(get_key(remote_file));
auto key = DORIS_TRY(get_key(remote_file, _client->s3_client_conf()));
auto start = std::chrono::steady_clock::now();
FileWriterPtr obj_writer;
RETURN_IF_ERROR(create_file_impl(key, &obj_writer, nullptr));
RETURN_IF_ERROR(create_file_impl(remote_file, &obj_writer, nullptr));
FileReaderSPtr local_reader;
RETURN_IF_ERROR(io::global_local_filesystem()->open_file(local_file, &local_reader));
size_t local_buffer_size = config::s3_file_system_local_upload_buffer_size;
Expand Down Expand Up @@ -372,9 +396,9 @@ Status S3FileSystem::batch_upload_impl(const std::vector<Path>& local_files,
const auto& local_file = local_files[idx];
const auto& remote_file = remote_files[idx];
auto& obj_writer = obj_writers[idx];
auto key = DORIS_TRY(get_key(remote_file));
auto key = DORIS_TRY(get_key(remote_file, _client->s3_client_conf()));
LOG(INFO) << "Start to upload " << local_file.native() << " to " << full_s3_path(key);
RETURN_IF_ERROR(create_file_impl(key, &obj_writer, nullptr));
RETURN_IF_ERROR(create_file_impl(remote_file, &obj_writer, nullptr));
FileReaderSPtr local_reader;
RETURN_IF_ERROR(io::global_local_filesystem()->open_file(local_file, &local_reader));
size_t local_buffer_size = config::s3_file_system_local_upload_buffer_size;
Expand Down Expand Up @@ -417,7 +441,7 @@ Status S3FileSystem::batch_upload_impl(const std::vector<Path>& local_files,
Status S3FileSystem::download_impl(const Path& remote_file, const Path& local_file) {
auto client = _client->get();
CHECK_S3_CLIENT(client);
auto key = DORIS_TRY(get_key(remote_file));
auto key = DORIS_TRY(get_key(remote_file, _client->s3_client_conf()));
int64_t size;
RETURN_IF_ERROR(file_size(remote_file, &size));
std::unique_ptr<char[]> buf = std::make_unique_for_overwrite<char[]>(size);
Expand Down Expand Up @@ -446,13 +470,11 @@ std::string S3FileSystem::generate_presigned_url(const Path& path, int64_t expir
bool is_public_endpoint) const {
std::string key = fmt::format("{}/{}", _prefix, path.native());
std::shared_ptr<ObjStorageClient> client;
if (is_public_endpoint &&
_client->s3_client_conf().endpoint.ends_with(OSS_PRIVATE_ENDPOINT_SUFFIX)) {
auto new_s3_conf = _client->s3_client_conf();
new_s3_conf.endpoint.erase(
_client->s3_client_conf().endpoint.size() - OSS_PRIVATE_ENDPOINT_SUFFIX.size(),
LEN_OF_OSS_PRIVATE_SUFFIX);
auto client_result = S3ClientFactory::instance().create(new_s3_conf);
auto conf = _client->s3_client_conf();
if (is_public_endpoint && conf.endpoint.ends_with(OSS_PRIVATE_ENDPOINT_SUFFIX)) {
conf.endpoint.erase(conf.endpoint.size() - OSS_PRIVATE_ENDPOINT_SUFFIX.size(),
LEN_OF_OSS_PRIVATE_SUFFIX);
auto client_result = S3ClientFactory::instance().create(conf);
if (!client_result) {
LOG(WARNING) << "failed to create S3 client for presigned URL: "
<< client_result.error();
Expand Down
11 changes: 10 additions & 1 deletion be/src/io/fs/s3_file_system.h
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,10 @@ class ObjClientHolder {
// Update s3 conf and reset client if `conf` is different. This method is threadsafe.
Status reset(const S3ClientConf& conf);

// Validate the current generation while holding the same lock that protects
// credential/client replacement. Existing readers still follow holder refreshes.
Status validate_for_access() const;

std::shared_ptr<ObjStorageClient> get() const {
std::shared_lock lock(_mtx);
return _client;
Expand All @@ -61,7 +65,12 @@ class ObjClientHolder {
// For error msg
std::string full_s3_path(std::string_view bucket, std::string_view key) const;

const S3ClientConf& s3_client_conf() { return _conf; }
// Credential refresh replaces _conf concurrently with file-system access.
// Return an owned snapshot so readers never retain a reference past the lock.
S3ClientConf s3_client_conf() const {
std::shared_lock lock(_mtx);
return _conf;
}

private:
mutable std::shared_mutex _mtx;
Expand Down
15 changes: 13 additions & 2 deletions be/src/io/fs/s3_file_writer.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,15 @@

namespace doris::io {

namespace {
std::string sanitize_display_path(std::string path) {
if (const auto query = path.find('?'); query != std::string::npos) {
path.resize(query);
}
return path;
}
} // namespace

bvar::Adder<uint64_t> s3_file_writer_total("s3_file_writer_total_num");
bvar::Adder<uint64_t> s3_bytes_written_total("s3_file_writer_bytes_written");
bvar::Adder<uint64_t> s3_file_created_total("s3_file_writer_file_created");
Expand All @@ -60,8 +69,10 @@ bvar::Window<bvar::IntRecorder> s3_file_writer_first_append_to_close_ms_window(
&s3_file_writer_first_append_to_close_ms_recorder, /*window_size=*/10);

S3FileWriter::S3FileWriter(std::shared_ptr<ObjClientHolder> client, std::string bucket,
std::string key, const FileWriterOptions* opts)
: _obj_storage_path_opts({.path = fmt::format("s3://{}/{}", bucket, key),
std::string key, const FileWriterOptions* opts, std::string display_path)
: _obj_storage_path_opts({.path = display_path.empty()
? fmt::format("s3://{}/{}", bucket, key)
: sanitize_display_path(std::move(display_path)),
.bucket = std::move(bucket),
.key = std::move(key)}),
_used_by_s3_committer(opts ? opts->used_by_s3_committer : false),
Expand Down
Loading
Loading