From e05defc1d91dc0acb8269ca7dde155098dd4e764 Mon Sep 17 00:00:00 2001 From: xylaaaaa <2392805527@qq.com> Date: Sun, 13 Sep 2026 20:01:21 +0800 Subject: [PATCH 01/10] [feature](be) Add native Azure authentication and client lifetime support ### What problem does this PR solve? Related PR: #67470 Problem Summary: Keep Azure credential types, authentication factories, URI/endpoint handling, native client caching and their direct unit tests together. This is a history-only regrouping of the native Azure PR. The final seven-commit series preserves the exact code tree of azure-native-before-squash-20260913550a886487e. ### Release note None ### Check List (For Author) - Test: Manual test / Unit Test - Final tree and PR diff are verified identical to the pre-squash snapshot - Existing final-snapshot verification: 711 selected FE tests passed - Tests were not rerun for this history-only rewrite - Behavior changed: No (relative to the pre-squash PR snapshot) - Does this need documentation: No --- be/src/util/s3_uri.cpp | 169 +++++- be/src/util/s3_uri.h | 35 +- be/src/util/s3_util.cpp | 559 ++++++++++++++++-- be/src/util/s3_util.h | 68 ++- be/test/cloud/cloud_compaction_test.cpp | 1 + be/test/io/azure_client_cache_test.cpp | 310 ++++++++++ ...zure_obj_storage_client_lifecycle_test.cpp | 185 ++++++ .../io/fs/azure_obj_storage_client_test.cpp | 352 ++++++++++- .../io/fs/s3_obj_storage_client_role_test.cpp | 1 + be/test/io/fs/s3_obj_storage_client_test.cpp | 1 + be/test/io/s3_client_factory_test.cpp | 391 ++++++++++++ be/test/storage/rowset/beta_rowset_test.cpp | 1 + be/test/storage/storage_resource_test.cpp | 1 + be/test/util/s3_uri_test.cpp | 149 +++++ .../obj-client/auth/azure_auth_factory.cpp | 221 ++++++- .../cpp/obj-client/auth/azure_auth_factory.h | 22 +- .../auth/azure_credential_options.h | 89 +++ .../obj-client/azure_obj_storage_client.cpp | 100 ++-- 18 files changed, 2471 insertions(+), 184 deletions(-) create mode 100644 be/test/io/azure_client_cache_test.cpp create mode 100644 be/test/io/fs/azure_obj_storage_client_lifecycle_test.cpp create mode 100644 common/cpp/obj-client/auth/azure_credential_options.h diff --git a/be/src/util/s3_uri.cpp b/be/src/util/s3_uri.cpp index 16c3bf3ffda47f..7c15bfadae3d12 100644 --- a/be/src/util/s3_uri.cpp +++ b/be/src/util/s3_uri.cpp @@ -18,13 +18,21 @@ #include "util/s3_uri.h" #include +#include #include +#include +#include +#include #include namespace doris { const std::string S3URI::_SCHEME_S3 = "s3"; +const std::string S3URI::_SCHEME_ABFS = "abfs"; +const std::string S3URI::_SCHEME_ABFSS = "abfss"; +const std::string S3URI::_SCHEME_WASB = "wasb"; +const std::string S3URI::_SCHEME_WASBS = "wasbs"; const std::string S3URI::_SCHEME_HTTP = "http"; const std::string S3URI::_SCHEME_HTTPS = "https"; const std::string S3URI::_SCHEME_DELIM = "://"; @@ -37,53 +45,152 @@ const std::string S3URI::_FRAGMENT_DELIM = "#"; /// _schema: s3 /// _bucket: bucket1 /// _key: path/to/file.txt -Status S3URI::parse() { +bool S3URI::is_azure_endpoint(std::string_view authority) { + auto host = absl::AsciiStrToLower(authority.substr(0, authority.find(':'))); + static constexpr std::string_view suffixes[] = { + ".blob.core.windows.net", ".dfs.core.windows.net", + ".blob.core.chinacloudapi.cn", ".dfs.core.chinacloudapi.cn", + ".blob.core.usgovcloudapi.net", ".dfs.core.usgovcloudapi.net", + ".blob.core.cloudapi.de", ".dfs.core.cloudapi.de"}; + return std::ranges::any_of(suffixes, [&host](std::string_view suffix) { + return host.ends_with(suffix) && host.size() > suffix.size(); + }); +} + +Status S3URI::_parsing_error(std::string_view message, bool azure_provider) const { + if (azure_provider || _is_azure) { + return Status::InvalidArgument("{}", message); + } + // The first HTTP parse may precede provider selection. Never echo a + // signed query, including for malformed or custom Azure endpoints. + const auto safe_location = std::string_view(_location).substr(0, _location.find_first_of("?#")); + return Status::InvalidArgument("{}: {}", message, safe_location); +} + +Status S3URI::_parse_authority(const std::string& scheme, const std::string& rest, + bool azure_provider) { + if (scheme == _SCHEME_S3) { + // has scheme, eg: s3://bucket1/path/to/file.txt + std::vector authority_split = + absl::StrSplit(rest, absl::MaxSplits(_PATH_DELIM, 1)); + if (authority_split.empty() || authority_split[0].empty()) { + return _parsing_error("Invalid S3 URI", azure_provider); + } + _bucket = authority_split[0]; + // support s3://bucket1 + _key = authority_split.size() == 1 ? "/" : authority_split[1]; + } else if (absl::EqualsIgnoreCase(scheme, _SCHEME_ABFS) || + absl::EqualsIgnoreCase(scheme, _SCHEME_ABFSS) || + absl::EqualsIgnoreCase(scheme, _SCHEME_WASB) || + absl::EqualsIgnoreCase(scheme, _SCHEME_WASBS)) { + // Azure Data Lake paths use container@account-host as the + // authority. Keep the account host so the native Azure client + // can derive its endpoint without consulting Hadoop settings. + _is_azure = true; + std::vector authority_split = + absl::StrSplit(rest, absl::MaxSplits(_PATH_DELIM, 1)); + if (authority_split.empty() || authority_split[0].empty()) { + return _parsing_error("Invalid Azure URI", azure_provider); + } + const auto at = authority_split[0].find('@'); + if (at == std::string::npos || at == 0 || at + 1 == authority_split[0].size() || + authority_split[0].find('@', at + 1) != std::string::npos) { + return _parsing_error("Invalid Azure URI authority", azure_provider); + } + _bucket = authority_split[0].substr(0, at); + _endpoint = authority_split[0].substr(at + 1); + if (_endpoint.empty()) { + return _parsing_error("Invalid Azure URI authority", azure_provider); + } + const auto dot = _endpoint.find('.'); + _account = dot == std::string::npos ? _endpoint : _endpoint.substr(0, dot); + _key = authority_split.size() == 1 ? "/" : authority_split[1]; + } else if (absl::EqualsIgnoreCase(scheme, _SCHEME_HTTP) || + absl::EqualsIgnoreCase(scheme, _SCHEME_HTTPS)) { + // has scheme, eg: http(s)://host/bucket1/path/to/file.txt + std::vector authority_split = + absl::StrSplit(rest, absl::MaxSplits(_PATH_DELIM, 2)); + _is_azure = is_azure_endpoint(authority_split[0]); + if (authority_split.size() != 3 || authority_split[0].empty() || + authority_split[1].empty()) { + return _parsing_error("Invalid S3 HTTP URI", azure_provider); + } + // authority_split[0] is host, authority_split[1] is bucket. + _endpoint = authority_split[0]; + const auto dot = _endpoint.find('.'); + _account = dot == std::string::npos ? _endpoint : _endpoint.substr(0, dot); + _bucket = authority_split[1]; + _key = authority_split[2]; + } else { + return _parsing_error("Invalid S3 URI", azure_provider); + } + return Status::OK(); +} + +Status S3URI::parse(bool azure_provider) { if (_location.empty()) { return Status::InvalidArgument("location is empty"); } - std::vector scheme_split = absl::StrSplit(_location, _SCHEME_DELIM); - std::string rest; + // S3URI instances are occasionally reused by callers. Do not retain a + // previous parse's Azure authority when parsing a second location. + _bucket.clear(); + _scheme.clear(); + _key.clear(); + _endpoint.clear(); + _account.clear(); + _is_azure = false; + + std::vector scheme_split = + absl::StrSplit(_location, absl::MaxSplits(_SCHEME_DELIM, 1)); if (scheme_split.size() == 2) { - if (scheme_split[0] == _SCHEME_S3) { - // has scheme, eg: s3://bucket1/path/to/file.txt - rest = scheme_split[1]; - std::vector authority_split = - absl::StrSplit(rest, absl::MaxSplits(_PATH_DELIM, 1)); - if (authority_split.size() < 1) { - return Status::InvalidArgument("Invalid S3 URI: {}", _location); - } - _bucket = authority_split[0]; - // support s3://bucket1 - _key = authority_split.size() == 1 ? "/" : authority_split[1]; - } else if (scheme_split[0] == _SCHEME_HTTP || scheme_split[0] == _SCHEME_HTTPS) { - // has scheme, eg: http(s)://host/bucket1/path/to/file.txt - rest = scheme_split[1]; - std::vector authority_split = - absl::StrSplit(rest, absl::MaxSplits(_PATH_DELIM, 2)); - if (authority_split.size() != 3) { - return Status::InvalidArgument("Invalid S3 HTTP URI: {}", _location); - } - // authority_split[1] is host - _bucket = authority_split[1]; - _key = authority_split[2]; - } else { - return Status::InvalidArgument("Invalid S3 URI: {}", _location); - } + const std::string& scheme = scheme_split[0]; + _scheme = absl::AsciiStrToLower(scheme); + RETURN_IF_ERROR(_parse_authority(scheme, scheme_split[1], azure_provider)); } else if (scheme_split.size() == 1) { // no scheme, eg: path/to/file.txt _bucket = ""; // unknown _key = _location; } else { - return Status::InvalidArgument("Invalid S3 URI: {}", _location); + return _parsing_error("Invalid S3 URI", azure_provider); + } + const bool azure_uri = + (_is_azure || azure_provider) && !_scheme.empty() && _scheme != _SCHEME_S3; + if (!azure_uri) { + absl::StripAsciiWhitespace(&_key); } - absl::StripAsciiWhitespace(&_key); if (_key.empty()) { - return Status::InvalidArgument("Invalid S3 key: {}", _location); + return _parsing_error("Invalid S3 key", azure_provider); } // Strip query and fragment if they exist std::vector _query_split = absl::StrSplit(_key, _QUERY_DELIM); std::vector _fragment_split = absl::StrSplit(_query_split[0], _FRAGMENT_DELIM); _key = _fragment_split[0]; + // ADLSFileIO's ADLSLocation passes ABFS/WASB names literally to the SDK: + // an object called p=a%2Fb must not become p=a/b. Only HTTP(S) locations + // represent URL-encoded names. The SDK encodes the resulting object name. + if (azure_uri && (_scheme == _SCHEME_HTTP || _scheme == _SCHEME_HTTPS)) { + std::string decoded; + decoded.reserve(_key.size()); + for (size_t i = 0; i < _key.size(); ++i) { + if (_key[i] != '%') { + // '+' is a literal object-name character, not form encoding. + decoded.push_back(_key[i]); + continue; + } + if (i + 2 >= _key.size()) { + return Status::InvalidArgument("Invalid percent encoding in Azure object path"); + } + unsigned int value = 0; + const auto* begin = _key.data() + i + 1; + const auto [end, error] = std::from_chars(begin, begin + 2, value, 16); + if (error != std::errc {} || end != begin + 2) { + return Status::InvalidArgument("Invalid percent encoding in Azure object path"); + } + decoded.push_back(static_cast(value)); + i += 2; + } + _key = std::move(decoded); + } return Status::OK(); } diff --git a/be/src/util/s3_uri.h b/be/src/util/s3_uri.h index 769053450bac36..51361c6ae53c4c 100644 --- a/be/src/util/s3_uri.h +++ b/be/src/util/s3_uri.h @@ -18,6 +18,7 @@ #pragma once #include +#include #include "common/status.h" #include "util/string_util.h" @@ -31,17 +32,45 @@ namespace doris { // 2. path/to/file.txt // bucket: "" // key: path/to/file.txt +// 3. abfss://container@account.dfs.core.windows.net/path/to/file.txt +// bucket: container +// key: path/to/file.txt +// endpoint: account.dfs.core.windows.net +// account: account +// +// Azure Data Lake locations intentionally retain their original URI form. The +// object-storage factory uses the parsed endpoint/account to construct the +// native Azure client; they are not routed through the Hadoop filesystem. class S3URI { public: S3URI(const std::string& location) : _location(location) {} - Status parse(); + // Azure SDK blob-name APIs encode their argument. Iceberg ADLSLocation passes + // ABFS/WASB names literally, so percent-decode only HTTP(S) Azure URL paths. + // Keep the old S3/raw-key contract. The provider flag also covers custom hosts. + Status parse(bool azure_provider = false); const std::string& get_bucket() const { return _bucket; } const std::string& get_key() const { return _key; } + // The authority host, when present. For Azure ABFS/WASB paths this is the + // account host (for example account.dfs.core.windows.net). + const std::string& get_endpoint() const { return _endpoint; } + // The storage account name parsed from an Azure authority. + const std::string& get_account() const { return _account; } + const std::string& get_scheme() const { return _scheme; } + bool is_azure() const { return _is_azure; } + static bool is_azure_endpoint(std::string_view authority); const std::string& get_location() const { return _location; } std::string to_string() const; private: + Status _parse_authority(const std::string& scheme, const std::string& rest, + bool azure_provider); + Status _parsing_error(std::string_view message, bool azure_provider) const; + static const std::string _SCHEME_S3; + static const std::string _SCHEME_ABFS; + static const std::string _SCHEME_ABFSS; + static const std::string _SCHEME_WASB; + static const std::string _SCHEME_WASBS; static const std::string _SCHEME_HTTP; static const std::string _SCHEME_HTTPS; static const std::string _SCHEME_DELIM; @@ -51,7 +80,11 @@ class S3URI { static const StringCaseSet _VALID_SCHEMES; std::string _location; + std::string _scheme; std::string _bucket; std::string _key; + std::string _endpoint; + std::string _account; + bool _is_azure = false; }; } // end namespace doris diff --git a/be/src/util/s3_util.cpp b/be/src/util/s3_util.cpp index cd5d019a18f2c5..2b79ee9b3ca4c1 100644 --- a/be/src/util/s3_util.cpp +++ b/be/src/util/s3_util.cpp @@ -33,12 +33,18 @@ #include #include #endif +#include +#include +#include #include #include #include #include +#include #include #include +#include +#include #include #include "common/config.h" @@ -64,24 +70,53 @@ namespace doris { namespace { +int64_t azure_client_time_millis() { + int64_t now = std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()) + .count(); + TEST_SYNC_POINT_CALLBACK("S3ClientFactory::azure_client_time", &now); + return now; +} + +Status validate_azure_credentials_for_access(const AzureCredentialOptions& credential, + int64_t now_ms, int64_t* effective_expiry_ms) { + *effective_expiry_ms = 0; + // Explicit expiry has no SDK dependency. Reject it before test creators as + // well, including builds without Azure support. + if (credential.type == AzureCredentialType::SAS) { + *effective_expiry_ms = credential.sas_expiration_time_ms; + if (*effective_expiry_ms > 0 && *effective_expiry_ms <= now_ms) { + return Status::InvalidArgument("Azure SAS credential is expired"); + } + } +#ifdef USE_AZURE + if (auto error = AzureAuthFactory::validate(credential, now_ms, effective_expiry_ms); + !error.empty()) { + return Status::InvalidArgument("{}", error); + } +#endif + return Status::OK(); +} + doris::Status is_s3_conf_valid(const S3ClientConf& conf) { if (conf.endpoint.empty()) { return Status::InvalidArgument("Invalid s3 conf, empty endpoint"); } + if (conf.provider == io::ObjStorageProvider::AZURE) { + if (auto error = conf.azure_credentials.validate(); !error.empty()) { + return Status::InvalidArgument("{}", error); + } + return Status::OK(); + } if (conf.region.empty()) { return Status::InvalidArgument("Invalid s3 conf, empty region"); } - if (conf.role_arn.empty()) { - // Allow anonymous access when both ak and sk are empty - bool hasAk = !conf.ak.empty(); - bool hasSk = !conf.sk.empty(); - - // Either both credentials are provided or both are empty (anonymous access) - if (hasAk && conf.sk.empty()) { + // Preserve the existing non-Azure anonymous/static credential contract. + if (!conf.ak.empty() && conf.sk.empty()) { return Status::InvalidArgument("Invalid s3 conf, empty sk"); } - if (hasSk && conf.ak.empty()) { + if (!conf.sk.empty() && conf.ak.empty()) { return Status::InvalidArgument("Invalid s3 conf, empty ak"); } } @@ -121,6 +156,11 @@ bool to_int(std::string_view str, int& res) { return ec == std::errc {}; } +bool to_int64(std::string_view str, int64_t& res) { + auto [ptr, ec] = std::from_chars(str.data(), str.data() + str.size(), res); + return ec == std::errc {} && ptr == str.data() + str.size(); +} + #ifdef USE_AZURE std::string env_or_empty(const char* env_name) { if (const char* value = std::getenv(env_name); value != nullptr) { @@ -129,6 +169,24 @@ std::string env_or_empty(const char* env_name) { return ""; } +std::string redact_azure_log_message(std::string message) { + // Azure SDK diagnostics may include a fully-qualified request URL. A SAS + // token is a bearer credential, so redact the query component before it + // reaches the Doris logger while retaining the host/path context. + size_t query = message.find('?'); + while (query != std::string::npos) { + size_t end = query + 1; + while (end < message.size() && message[end] != ' ' && message[end] != '\t' && + message[end] != '\r' && message[end] != '\n' && message[end] != '"' && + message[end] != '\'' && message[end] != ')' && message[end] != ']') { + ++end; + } + message.replace(query + 1, end - query - 1, ""); + query = message.find('?', query + std::string("?").size()); + } + return message; +} + std::string build_azure_tls_debug_context(const std::string& selected_ca_file) { bool selected_ca_exists = false; bool selected_ca_readable = false; @@ -165,8 +223,342 @@ constexpr char S3_NEED_OVERRIDE_ENDPOINT[] = "AWS_NEED_OVERRIDE_ENDPOINT"; constexpr char S3_ROLE_ARN[] = "AWS_ROLE_ARN"; constexpr char S3_EXTERNAL_ID[] = "AWS_EXTERNAL_ID"; constexpr char S3_CREDENTIALS_PROVIDER_TYPE[] = "AWS_CREDENTIALS_PROVIDER_TYPE"; + +// Native Azure binding keys. The AWS_* aliases above remain accepted for +// existing object-storage callers, but Azure scans use these provider-owned +// names so their meaning does not depend on the S3 adapter. +constexpr char AZURE_AUTH_TYPE[] = "AZURE_AUTH_TYPE"; +constexpr char AZURE_ENDPOINT[] = "AZURE_ENDPOINT"; +constexpr char AZURE_ACCOUNT_NAME[] = "AZURE_ACCOUNT_NAME"; +constexpr char AZURE_ACCOUNT_KEY[] = "AZURE_ACCOUNT_KEY"; +constexpr char AZURE_CONTAINER[] = "AZURE_CONTAINER"; +constexpr char AZURE_SAS_TOKEN[] = "AZURE_SAS_TOKEN"; +constexpr char AZURE_SAS_EXPIRY_MS[] = "AZURE_SAS_EXPIRY_MS"; +constexpr char AZURE_CLIENT_ID[] = "AZURE_CLIENT_ID"; +constexpr char AZURE_CLIENT_SECRET[] = "AZURE_CLIENT_SECRET"; +constexpr char AZURE_TENANT_ID[] = "AZURE_TENANT_ID"; +constexpr char AZURE_OAUTH_SERVER_URI[] = "AZURE_OAUTH_SERVER_URI"; + +const std::string* find_property(const StringCaseMap& properties, + std::initializer_list names) { + for (const auto* name : names) { + auto it = properties.find(name); + if (it != properties.end()) { + return &it->second; + } + } + return nullptr; +} + +bool has_property(const StringCaseMap& properties, + std::initializer_list names) { + return find_property(properties, names) != nullptr; +} + +std::string normalize_azure_endpoint(std::string endpoint) { + if (endpoint.empty()) { + return endpoint; + } + const bool has_scheme = endpoint.find("://") != std::string::npos; + if (!has_scheme) { + endpoint = "https://" + endpoint; + } + const auto scheme_end = endpoint.find("://"); + endpoint.replace(0, scheme_end, to_lower(endpoint.substr(0, scheme_end))); + const auto authority_begin = scheme_end == std::string::npos ? 0 : scheme_end + 3; + const auto authority_end = endpoint.find('/', authority_begin); + const auto authority_length = authority_end == std::string::npos + ? endpoint.size() - authority_begin + : authority_end - authority_begin; + const auto authority = endpoint.substr(authority_begin, authority_length); + if (authority.empty()) { + return endpoint; + } + + auto lower_authority = to_lower(authority); + endpoint.replace(authority_begin, authority_length, lower_authority); + // Match the host, not host:port, so explicit transport ports do not disable + // the official DFS-to-Blob conversion. Custom proxy hosts stay unchanged. + const auto host = lower_authority.substr(0, lower_authority.find(':')); + const auto dfs_pos = host.find(".dfs."); + const bool official_dfs = + dfs_pos != std::string::npos && (host.ends_with(".dfs.core.windows.net") || + host.ends_with(".dfs.core.chinacloudapi.cn") || + host.ends_with(".dfs.core.usgovcloudapi.net") || + host.ends_with(".dfs.core.cloudapi.de")); + if (official_dfs) { + endpoint.replace(authority_begin + dfs_pos, 5, ".blob."); + } else if (!has_scheme && authority.find('.') == std::string::npos && + authority.find(':') == std::string::npos) { + endpoint.insert(authority_begin + authority.size(), ".blob.core.windows.net"); + } + while (endpoint.ends_with('/')) { + endpoint.pop_back(); + } + return endpoint; +} + +std::string endpoint_authority(const std::string& endpoint) { + const auto begin = endpoint.find("://") + 3; + auto authority = endpoint.substr(begin, endpoint.find('/', begin) - begin); + if (endpoint.starts_with("https://") && authority.ends_with(":443")) { + authority.resize(authority.size() - 4); + } else if (endpoint.starts_with("http://") && authority.ends_with(":80")) { + authority.resize(authority.size() - 3); + } + return authority; +} + +// Only established SharedKey wire producers use AWS fields for Azure. Once +// translated here the native factory never inspects these fields again. +void import_legacy_azure_shared_key(S3ClientConf* conf) { + // Keep the established SharedKey endpoint normalization at the legacy + // boundary. Native endpoints and object keys retain their internal slashes. + conf->endpoint = normalize_http_uri(conf->endpoint); + conf->azure_credentials = {}; + conf->azure_credentials.type = AzureCredentialType::SHARED_KEY; + conf->azure_credentials.account_name = std::move(conf->ak); + conf->azure_credentials.account_key = std::move(conf->sk); + conf->ak.clear(); + conf->sk.clear(); + conf->token.clear(); + conf->region.clear(); + conf->role_arn.clear(); + conf->external_id.clear(); + conf->cred_provider_type = CredProviderType::Default; +} + +Status convert_legacy_azure_properties(const StringCaseMap& properties, + S3ClientConf* client_conf) { + auto& client = *client_conf; + // Compatibility is deliberately limited to the old SharedKey map. An + // incomplete native map must not be mistaken for that old protocol. + for (const auto& [key, value] : properties) { + const auto lower = to_lower(key); + if (lower.starts_with("azure") || lower.starts_with("fs.azure.") || + (lower == "aws_token" && !value.empty())) { + return Status::InvalidArgument("Azure native credentials require AZURE_AUTH_TYPE"); + } + } + if (!has_property(properties, {S3_ENDPOINT}) || !has_property(properties, {S3_AK}) || + !has_property(properties, {S3_SK})) { + return Status::InvalidArgument("Azure native credentials require AZURE_AUTH_TYPE"); + } + client.endpoint = *find_property(properties, {S3_ENDPOINT}); + client.ak = *find_property(properties, {S3_AK}); + client.sk = *find_property(properties, {S3_SK}); + import_legacy_azure_shared_key(&client); + for (const auto& [name, target] : + {std::pair {S3_MAX_CONN_SIZE, &client.max_connections}, + std::pair {S3_REQUEST_TIMEOUT_MS, &client.request_timeout_ms}, + std::pair {S3_CONN_TIMEOUT_MS, &client.connect_timeout_ms}}) { + if (const auto* value = find_property(properties, {name}); value != nullptr) { + if (!to_int(*value, *target)) { + return Status::InvalidArgument("invalid Azure connection option {}", name); + } + } + } + return Status::OK(); +} + +Status convert_native_azure_properties(const StringCaseMap& properties, + const S3URI& uri, const std::string& auth_type, + S3ClientConf* client_conf) { + auto& client = *client_conf; + for (const auto& [key, value] : properties) { + const auto lower = to_lower(key); + if (lower.starts_with("aws_") || lower.starts_with("azure.")) { + return Status::InvalidArgument( + "Azure native credentials cannot use AWS or catalog property aliases"); + } + } + // Older FE versions attach a Hadoop configuration view for OneLake to + // this map. Ignore those extra keys; they must never supply or override + // any native authentication field. FE routing separates the two views. + auto& credential = client.azure_credentials; + if (auth_type == "SHARED_KEY") { + credential.type = AzureCredentialType::SHARED_KEY; + } else if (auth_type == "SAS") { + credential.type = AzureCredentialType::SAS; + } else if (auth_type == "OAUTH2") { + credential.type = AzureCredentialType::OAUTH2; + } else { + return Status::InvalidArgument("unsupported AZURE_AUTH_TYPE in native credentials"); + } + auto set = [&](const char* key, std::string* target) { + if (const auto* value = find_property(properties, {key}); value != nullptr) { + *target = *value; + } + }; + set(AZURE_ENDPOINT, &client.endpoint); + set(AZURE_ACCOUNT_NAME, &credential.account_name); + set(AZURE_ACCOUNT_KEY, &credential.account_key); + set(AZURE_SAS_TOKEN, &credential.sas_token); + set(AZURE_CLIENT_ID, &credential.oauth_client_id); + set(AZURE_CLIENT_SECRET, &credential.oauth_client_secret); + set(AZURE_TENANT_ID, &credential.oauth_tenant_id); + set(AZURE_OAUTH_SERVER_URI, &credential.oauth_server_uri); + if (const auto* expiry = find_property(properties, {AZURE_SAS_EXPIRY_MS}); expiry != nullptr) { + if (!to_int64(*expiry, credential.sas_expiration_time_ms) || + credential.sas_expiration_time_ms <= 0) { + return Status::InvalidArgument("invalid Azure SAS expiry value"); + } + } + if (client.endpoint.empty() || credential.account_name.empty()) { + return Status::InvalidArgument( + "Azure native credentials require endpoint and account name"); + } + if (credential.type == AzureCredentialType::SHARED_KEY && credential.account_key.empty()) { + return Status::InvalidArgument("Azure native SharedKey requires an account key"); + } + // Existing Azure SharedKey catalogs also contain S3-spelled locations. + // This explicit compatibility case carries no account in its URI; + // SAS/OAuth2 native data locations must carry their Azure authority. + if (uri.get_scheme().empty() || + (uri.get_scheme() == "s3" && credential.type != AzureCredentialType::SHARED_KEY)) { + return Status::InvalidArgument("Azure native credentials require an Azure data URI"); + } + if (client.endpoint.find_first_of("?#@\r\n") != std::string::npos) { + return Status::InvalidArgument( + "Azure endpoint must not contain credentials, query or fragment"); + } + client.endpoint = normalize_azure_endpoint(client.endpoint); + if (!client.endpoint.starts_with("https://") && !client.endpoint.starts_with("http://")) { + return Status::InvalidArgument("Azure endpoint must use HTTP or HTTPS"); + } + // A container property may constrain legacy callers, but is never a + // fallback location. Every native data URI identifies its container. + if (const auto* container = find_property(properties, {AZURE_CONTAINER}); + container != nullptr && *container != uri.get_bucket()) { + return Status::InvalidArgument("Azure URI container conflicts with the storage binding"); + } + return Status::OK(); +} + +Status convert_azure_properties(const StringCaseMap& properties, const S3URI& uri, + S3Conf* conf) { + auto& client = conf->client_conf; + client.provider = io::ObjStorageProvider::AZURE; + const auto* auth_type = find_property(properties, {AZURE_AUTH_TYPE}); + if (auth_type == nullptr) { + RETURN_IF_ERROR(convert_legacy_azure_properties(properties, &client)); + } else { + RETURN_IF_ERROR(convert_native_azure_properties(properties, uri, *auth_type, &client)); + } + if (uri.get_bucket().empty()) { + return Status::InvalidArgument("Azure data URI requires a container"); + } + conf->bucket = uri.get_bucket(); + client.bucket = conf->bucket; + if (auth_type != nullptr) { + RETURN_IF_ERROR(S3ClientFactory::validate_azure_uri(uri, client)); + } + return is_s3_conf_valid(client); +} } // namespace +Status S3ClientFactory::validate_azure_uri(const S3URI& uri, const S3ClientConf& conf) { + if (uri.get_scheme().empty()) { + return Status::OK(); // Internal file-system callers may pass a raw object key. + } + if (uri.get_bucket() != conf.bucket) { + return Status::InvalidArgument("Azure URI container conflicts with the storage binding"); + } + if (uri.get_scheme() == "s3") { + if (conf.azure_credentials.type != AzureCredentialType::SHARED_KEY) { + return Status::InvalidArgument("Azure SAS/OAuth2 data access requires an Azure URI"); + } + return Status::OK(); // Old SharedKey file-system paths use the S3 wire spelling. + } + const auto uri_host = to_lower(uri.get_endpoint()); + if (uri_host.ends_with(".dfs.fabric.microsoft.com") || + uri_host.ends_with(".blob.fabric.microsoft.com")) { + return Status::NotSupported("OneLake data access requires its Hadoop storage binding"); + } + const auto endpoint = normalize_azure_endpoint(conf.endpoint); + const bool http_uri = uri.get_scheme() == "http" || uri.get_scheme() == "https"; + const auto uri_endpoint = normalize_azure_endpoint( + http_uri ? uri.get_scheme() + "://" + uri.get_endpoint() : uri.get_endpoint()); + // ABFS/WASB authorities identify the logical Azure account, while a configured custom + // endpoint may be a proxy or emulator that intentionally has a different HTTP authority. + // The native client uses conf.endpoint as the transport origin; retain account validation + // below, but do not reject this valid proxy form. HTTP(S) locations carry their transport + // origin directly and must still match exactly. + const bool custom_transport = !S3URI::is_azure_endpoint(endpoint_authority(endpoint)); + if (endpoint_authority(endpoint) != endpoint_authority(uri_endpoint) && + !(custom_transport && !http_uri)) { + return Status::InvalidArgument( + "Azure URI account host conflicts with the storage endpoint"); + } + if (http_uri && !endpoint.starts_with(uri.get_scheme() + "://")) { + return Status::InvalidArgument("Azure URI scheme conflicts with the storage endpoint"); + } + if (S3URI::is_azure_endpoint(uri.get_endpoint()) && + !iequal(conf.azure_credentials.account_name, uri.get_account())) { + return Status::InvalidArgument("Azure URI account conflicts with the credential account"); + } + return Status::OK(); +} + +bool S3ClientConf::operator==(const S3ClientConf& other) const { + if (std::tie(provider, endpoint, bucket, max_connections, request_timeout_ms, + connect_timeout_ms, is_internal_bucket) != + std::tie(other.provider, other.endpoint, other.bucket, other.max_connections, + other.request_timeout_ms, other.connect_timeout_ms, other.is_internal_bucket)) { + return false; + } + if (provider == io::ObjStorageProvider::AZURE) { + return azure_credentials == other.azure_credentials; + } + return std::tie(ak, sk, token, region, use_virtual_addressing, need_override_endpoint, + cred_provider_type, role_arn, external_id) == + std::tie(other.ak, other.sk, other.token, other.region, other.use_virtual_addressing, + other.need_override_endpoint, other.cred_provider_type, other.role_arn, + other.external_id); +} + +uint64_t S3ClientConf::get_hash() const { + uint64_t hash = crc32_hash(endpoint) ^ crc32_hash(bucket) ^ max_connections ^ + request_timeout_ms ^ connect_timeout_ms ^ static_cast(provider) ^ + is_internal_bucket; + if (provider == io::ObjStorageProvider::AZURE) { + const auto& credential = azure_credentials; + hash ^= static_cast(credential.type); + hash ^= crc32_hash(credential.account_name + credential.account_key); + hash ^= crc32_hash(credential.sas_token); + hash ^= credential.sas_expiration_time_ms; + hash ^= crc32_hash(credential.oauth_client_id); + hash ^= crc32_hash(credential.oauth_client_secret); + hash ^= crc32_hash(credential.oauth_tenant_id); + hash ^= crc32_hash(credential.oauth_server_uri); + return hash; + } + // Preserve the S3 hash and full equality contract, including collision handling. + return hash ^ crc32_hash(ak + sk) ^ crc32_hash(token) ^ crc32_hash(region) ^ + use_virtual_addressing ^ static_cast(cred_provider_type) ^ crc32_hash(role_arn) ^ + crc32_hash(external_id); +} + +std::string S3ClientConf::to_string() const { + if (provider == io::ObjStorageProvider::AZURE) { + // Endpoints have no credential query in the native protocol. Also redact + // legacy endpoints at this logging boundary, without rewriting requests. + return fmt::format( + "(provider=azure, auth_type={}, account={}, endpoint={}, bucket={}, " + "sas_expiration_time_ms={}, is_internal_bucket={})", + static_cast(azure_credentials.type), azure_credentials.account_name, + endpoint.substr(0, endpoint.find('?')), bucket, + azure_credentials.sas_expiration_time_ms, is_internal_bucket); + } + return fmt::format( + "(ak={}, token={}, endpoint={}, region={}, bucket={}, max_connections={}, " + "request_timeout_ms={}, connect_timeout_ms={}, use_virtual_addressing={}, " + "cred_provider_type={}, role_arn={}, external_id={}, is_internal_bucket={})", + hide_access_key(ak), token.empty() ? "" : "******", endpoint, region, bucket, + max_connections, request_timeout_ms, connect_timeout_ms, use_virtual_addressing, + cred_provider_type, role_arn, external_id, is_internal_bucket); +} + S3ClientFactory::S3ClientFactory() { _aws_options = Aws::SDKOptions {}; auto logLevel = static_cast(config::aws_log_level); @@ -183,22 +575,23 @@ S3ClientFactory::S3ClientFactory() { Azure::Core::Diagnostics::Logger::SetLevel(azureLogLevel); Azure::Core::Diagnostics::Logger::SetListener( [&](Azure::Core::Diagnostics::Logger::Level level, const std::string& message) { + const auto safe_message = redact_azure_log_message(message); switch (level) { case Azure::Core::Diagnostics::Logger::Level::Verbose: - LOG(INFO) << message; + LOG(INFO) << safe_message; break; case Azure::Core::Diagnostics::Logger::Level::Informational: - LOG(INFO) << message; + LOG(INFO) << safe_message; break; case Azure::Core::Diagnostics::Logger::Level::Warning: - LOG(WARNING) << message; + LOG(WARNING) << safe_message; break; case Azure::Core::Diagnostics::Logger::Level::Error: - LOG(ERROR) << message; + LOG(ERROR) << safe_message; break; default: LOG(WARNING) << "Unknown level: " << static_cast(level) - << ", message: " << message; + << ", message: " << safe_message; break; } }); @@ -216,6 +609,17 @@ S3ClientFactory& S3ClientFactory::instance() { Result> S3ClientFactory::create(const S3ClientConf& s3_conf) { RETURN_IF_ERROR_RESULT(is_s3_conf_valid(s3_conf)); + const bool is_azure = s3_conf.provider == io::ObjStorageProvider::AZURE; + int64_t azure_expiry_ms = 0; + if (is_azure) { + const auto now_ms = azure_client_time_millis(); + { + std::lock_guard l(_lock); + _prune_azure_clients(now_ms); + } + RETURN_IF_ERROR_RESULT(validate_azure_credentials_for_access(s3_conf.azure_credentials, + now_ms, &azure_expiry_ms)); + } #ifdef BE_TEST { @@ -228,18 +632,36 @@ Result> S3ClientFactory::create(const S3Cl { std::lock_guard l(_lock); - auto it = _cache.find(s3_conf); - if (it != _cache.end()) { - return it->second; + if (is_azure) { + if (azure_expiry_ms > 0 && azure_expiry_ms <= azure_client_time_millis()) { + _prune_azure_clients(azure_expiry_ms); + return ResultError(Status::InvalidArgument("Azure SAS credential is expired")); + } + auto it = _azure_cache.find(s3_conf); + if (it != _azure_cache.end()) { + it->second.last_access = ++_azure_cache_clock; + return it->second.client; + } + } else { + auto it = _cache.find(s3_conf); + if (it != _cache.end()) { + return it->second; + } } } + if (is_azure) { + TEST_SYNC_POINT("S3ClientFactory::azure_cache_miss"); + } auto client_result = (s3_conf.provider == io::ObjStorageProvider::AZURE) ? _create_azure_client(s3_conf) : _create_s3_client(s3_conf); if (!client_result.has_value()) { return ResultError(std::move(client_result).error()); } + if (is_azure) { + TEST_SYNC_POINT("S3ClientFactory::azure_client_built"); + } auto obj_client = std::move(client_result).value(); if (!config::is_cloud_mode() || s3_conf.is_internal_bucket) { obj_client = std::make_shared( @@ -248,11 +670,62 @@ Result> S3ClientFactory::create(const S3Cl { std::lock_guard l(_lock); + if (is_azure) { + return _publish_azure_client(s3_conf, obj_client, azure_expiry_ms); + } auto [it, _] = _cache.emplace(s3_conf, std::move(obj_client)); return it->second; } } +Result> S3ClientFactory::_publish_azure_client( + const S3ClientConf& s3_conf, std::shared_ptr& obj_client, + int64_t azure_expiry_ms) { + const auto now_ms = azure_client_time_millis(); + _prune_azure_clients(now_ms); + if (azure_expiry_ms > 0 && azure_expiry_ms <= now_ms) { + return ResultError(Status::InvalidArgument("Azure SAS credential is expired")); + } + // A concurrent creator may have published the same identity while + // SDK construction ran outside the lock. Reuse that client. + auto it = _azure_cache.find(s3_conf); + if (it != _azure_cache.end()) { + it->second.last_access = ++_azure_cache_clock; + return it->second.client; + } + if (_azure_cache.size() >= _azure_cache_capacity) { + auto oldest = + std::ranges::min_element(_azure_cache, [](const auto& left, const auto& right) { + return left.second.last_access < right.second.last_access; + }); + _azure_cache.erase(oldest); + } + auto [inserted, _] = + _azure_cache.emplace(s3_conf, AzureCachedClient {.client = std::move(obj_client), + .expiry_ms = azure_expiry_ms, + .last_access = ++_azure_cache_clock}); + return inserted->second.client; +} + +Status S3ClientFactory::validate_credentials_for_access(const S3ClientConf& conf) { + if (conf.provider != io::ObjStorageProvider::AZURE) { + return Status::OK(); + } + int64_t effective_expiry_ms = 0; + return validate_azure_credentials_for_access(conf.azure_credentials, azure_client_time_millis(), + &effective_expiry_ms); +} + +void S3ClientFactory::_prune_azure_clients(int64_t now_ms) { + for (auto it = _azure_cache.begin(); it != _azure_cache.end();) { + if (it->second.expiry_ms > 0 && it->second.expiry_ms <= now_ms) { + it = _azure_cache.erase(it); + } else { + ++it; + } + } +} + #ifdef BE_TEST void S3ClientFactory::set_client_creator_for_test( std::function(const S3ClientConf&)> creator) { @@ -269,11 +742,9 @@ void S3ClientFactory::clear_client_creator_for_test() { Result> S3ClientFactory::_create_azure_client( const S3ClientConf& s3_conf) { #ifdef USE_AZURE + const std::string endpoint = normalize_azure_endpoint(s3_conf.endpoint); const std::string container_name = s3_conf.bucket; - std::string uri = fmt::format("{}/{}", s3_conf.endpoint, container_name); - if (s3_conf.endpoint.find("://") == std::string::npos) { - uri = "https://" + uri; - } + std::string uri = fmt::format("{}/{}", endpoint, container_name); Azure::Storage::Blobs::BlobClientOptions options; options.Retry.StatusCodes.insert(Azure::Core::Http::HttpStatusCode::TooManyRequests); @@ -287,17 +758,12 @@ Result> S3ClientFactory::_create_azure_cli std::make_shared(std::move(curl_options)); } - std::string normalized_uri = normalize_http_uri(uri); - VLOG_DEBUG << "uri:" << uri << ", normalized_uri:" << normalized_uri; + const std::string normalized_uri = normalize_http_uri(uri); + VLOG_DEBUG << "azure container uri:" << normalized_uri.substr(0, normalized_uri.find('?')); std::string tls_debug_context = build_azure_tls_debug_context(ca_cert_file_path); - auto built = AzureAuthFactory::create(normalized_uri, - { - .type = AzureCredentialType::SHARED_KEY, - .account_name = s3_conf.ak, - .account_key = s3_conf.sk, - }, - std::move(options)); + auto built = + AzureAuthFactory::create(normalized_uri, s3_conf.azure_credentials, std::move(options)); if (!built) { return ResultError( Status::InvalidArgument("failed to create Azure client: {}", built.error)); @@ -306,9 +772,9 @@ Result> S3ClientFactory::_create_azure_cli return std::make_shared( std::move(built.container_client), ObjStorageEndpointInfo { - .endpoint = s3_conf.endpoint, - .ak = s3_conf.ak, - .sk = s3_conf.sk, + .endpoint = endpoint, + .ak = s3_conf.azure_credentials.account_name, + .sk = s3_conf.azure_credentials.account_key, .tls_debug_context = std::move(tls_debug_context), }, std::move(built.shared_key_credential)); @@ -407,6 +873,16 @@ Result> S3ClientFactory::_create_s3_client Status S3ClientFactory::convert_properties_to_s3_conf( const std::map& prop, const S3URI& s3_uri, S3Conf* s3_conf) { StringCaseMap properties(prop.begin(), prop.end()); + s3_conf->client_conf.provider = io::ObjStorageProvider::AWS; + s3_conf->client_conf.azure_credentials = {}; + if (const auto* provider = find_property(properties, {S3_PROVIDER}); + provider != nullptr && iequal(*provider, AZURE_PROVIDER_STRING)) { + *s3_conf = {}; + return convert_azure_properties(properties, s3_uri, s3_conf); + } + if (s3_uri.is_azure()) { + return Status::InvalidArgument("Azure data URI requires provider=azure"); + } if (auto it = properties.find(S3_AK); it != properties.end()) { s3_conf->client_conf.ak = it->second; } @@ -442,19 +918,12 @@ Status S3ClientFactory::convert_properties_to_s3_conf( it->second); } } - if (auto it = properties.find(S3_PROVIDER); it != properties.end()) { - // S3 Provider properties should be case insensitive. - if (0 == strcasecmp(it->second.c_str(), AZURE_PROVIDER_STRING)) { - s3_conf->client_conf.provider = io::ObjStorageProvider::AZURE; - } - } - if (s3_uri.get_bucket().empty()) { return Status::InvalidArgument("Invalid S3 URI {}, bucket is not specified", s3_uri.to_string()); + } else { + s3_conf->bucket = s3_uri.get_bucket(); } - s3_conf->bucket = s3_uri.get_bucket(); - // For azure's compatibility s3_conf->client_conf.bucket = s3_uri.get_bucket(); s3_conf->prefix = ""; @@ -520,6 +989,7 @@ S3Conf S3Conf::get_s3_conf(const cloud::ObjectStoreInfoPB& info) { .ak = info.ak(), .sk = info.sk(), .token = {}, + .azure_credentials = {}, .bucket = info.bucket(), .provider = io::ObjStorageProvider::AWS, .use_virtual_addressing = @@ -569,6 +1039,9 @@ S3Conf S3Conf::get_s3_conf(const cloud::ObjectStoreInfoPB& info) { LOG_FATAL("unknown provider type {}, info {}", info.provider(), ret.to_string()); } ret.client_conf.provider = type; + if (type == io::ObjStorageProvider::AZURE) { + import_legacy_azure_shared_key(&ret.client_conf); + } return ret; } @@ -582,6 +1055,7 @@ S3Conf S3Conf::get_s3_conf(const TS3StorageParam& param) { .ak = param.ak, .sk = param.sk, .token = param.token, + .azure_credentials = {}, .bucket = param.bucket, .provider = io::ObjStorageProvider::AWS, .max_connections = param.max_conn, @@ -635,6 +1109,9 @@ S3Conf S3Conf::get_s3_conf(const TS3StorageParam& param) { __builtin_unreachable(); } ret.client_conf.provider = type; + if (type == io::ObjStorageProvider::AZURE) { + import_legacy_azure_shared_key(&ret.client_conf); + } return ret; } diff --git a/be/src/util/s3_util.h b/be/src/util/s3_util.h index b2ca49d4751cb9..bf6dd6b1373dc3 100644 --- a/be/src/util/s3_util.h +++ b/be/src/util/s3_util.h @@ -28,6 +28,7 @@ #include #endif +#include #include #include #include @@ -39,6 +40,7 @@ #include "core/string_ref.h" #include "cpp/aws_common.h" #include "cpp/obj-client/auth/aws_credential_factory.h" +#include "cpp/obj-client/auth/azure_credential_options.h" #include "cpp/obj-client/obj_storage_client.h" namespace Aws::S3 { @@ -61,6 +63,7 @@ struct S3ClientConf { std::string ak; std::string sk; std::string token; + AzureCredentialOptions azure_credentials; // For azure we'd better support the bucket at the first time init azure blob container client std::string bucket; io::ObjStorageProvider provider = io::ObjStorageProvider::AWS; @@ -80,41 +83,11 @@ struct S3ClientConf { // returned bare in cloud mode. bool is_internal_bucket = false; - // Full-field identity. get_hash() is only good for picking an unordered_map - // bucket; distinct configurations can collide, so never treat hash equality as - // configuration equality. - bool operator==(const S3ClientConf&) const = default; - - uint64_t get_hash() const { - uint64_t hash_code = 0; - // Use crc32_hash(ak + sk) hash to prevent swapped AK/SK order from producing same result. - hash_code ^= crc32_hash(ak + sk); - hash_code ^= crc32_hash(token); - hash_code ^= crc32_hash(endpoint); - hash_code ^= crc32_hash(region); - hash_code ^= crc32_hash(bucket); - hash_code ^= max_connections; - hash_code ^= request_timeout_ms; - hash_code ^= connect_timeout_ms; - hash_code ^= use_virtual_addressing; - hash_code ^= static_cast(provider); - - hash_code ^= static_cast(cred_provider_type); - hash_code ^= crc32_hash(role_arn); - hash_code ^= crc32_hash(external_id); - hash_code ^= is_internal_bucket; - return hash_code; - } - - std::string to_string() const { - return fmt::format( - "(ak={}, token={}, endpoint={}, region={}, bucket={}, max_connections={}, " - "request_timeout_ms={}, connect_timeout_ms={}, use_virtual_addressing={}, " - "cred_provider_type={},role_arn={}, external_id={}, is_internal_bucket={}", - hide_access_key(ak), token.empty() ? "" : "******", endpoint, region, bucket, - max_connections, request_timeout_ms, connect_timeout_ms, use_virtual_addressing, - cred_provider_type, role_arn, external_id, is_internal_bucket); - } + // Compare provider-owned identity, never just a hash. Azure authentication + // must not borrow AWS fields or depend on an unrelated region/role setting. + bool operator==(const S3ClientConf&) const; + uint64_t get_hash() const; + std::string to_string() const; }; struct S3ClientConfHash { @@ -149,6 +122,12 @@ class S3ClientFactory { static Status convert_properties_to_s3_conf(const std::map& prop, const S3URI& s3_uri, S3Conf* s3_conf); + static Status validate_azure_uri(const S3URI& uri, const S3ClientConf& conf); + + // Reused file systems must validate credentials before opening a new reader + // or writer too. This does not expire or interrupt existing readers. + static Status validate_credentials_for_access(const S3ClientConf& conf); + static Aws::Client::ClientConfiguration& getClientConfiguration() { // The default constructor of ClientConfiguration will do some http call // such as Aws::Internal::GetEC2MetadataClient and other init operation, @@ -172,16 +151,35 @@ class S3ClientFactory { private: #ifdef BE_TEST FRIEND_TEST(S3ClientFactoryTest, RefreshCaCertForCredentialsProvider); + friend class AzureClientFactoryCacheTest; #endif Result> _create_s3_client(const S3ClientConf& s3_conf); Result> _create_azure_client(const S3ClientConf& s3_conf); std::string _get_ca_cert_file_path(); + // Caller holds _lock. Eviction releases only the cache's shared ownership; + // readers and writers keep their own client references. + void _prune_azure_clients(int64_t now_ms); + // Caller holds _lock. Keep a losing candidate owned by create so that its + // destruction happens after unlocking, as for a failed expiry recheck. + Result> _publish_azure_client( + const S3ClientConf& s3_conf, std::shared_ptr& obj_client, + int64_t azure_expiry_ms); S3ClientFactory(); Aws::SDKOptions _aws_options; std::mutex _lock; std::unordered_map, S3ClientConfHash> _cache; + struct AzureCachedClient { + std::shared_ptr client; + int64_t expiry_ms; + uint64_t last_access; + }; + // Azure's short-lived credentials rotate. Keep its bounded cache separate + // from the existing non-Azure cache, whose eviction semantics are unchanged. + std::unordered_map _azure_cache; + size_t _azure_cache_capacity = 256; + uint64_t _azure_cache_clock = 0; std::mutex _ca_cert_lock; std::string _ca_cert_file_path; #ifdef BE_TEST diff --git a/be/test/cloud/cloud_compaction_test.cpp b/be/test/cloud/cloud_compaction_test.cpp index db53f73fec9f7a..ceb5a61bb5cea2 100644 --- a/be/test/cloud/cloud_compaction_test.cpp +++ b/be/test/cloud/cloud_compaction_test.cpp @@ -1170,6 +1170,7 @@ TEST_F(CloudCompactionTest, test_set_storage_resource_from_input_rowsets) { .ak = "ak", .sk = "sk", .token = "", + .azure_credentials = {}, .bucket = "", .role_arn = "", .external_id = "", diff --git a/be/test/io/azure_client_cache_test.cpp b/be/test/io/azure_client_cache_test.cpp new file mode 100644 index 00000000000000..5bec7793ea7ab4 --- /dev/null +++ b/be/test/io/azure_client_cache_test.cpp @@ -0,0 +1,310 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "cpp/sync_point.h" +#include "io/fs/file_reader.h" +#include "io/fs/s3_file_system.h" +#include "util/defer_op.h" +#include "util/s3_util.h" + +#ifdef USE_AZURE + +namespace doris { + +// Exercise the real factory and cache. The legacy test creator deliberately +// bypasses caching, so it must not be used to prove cache lifetime or identity. +// SDK client construction is lazy and makes no storage request in these tests. +class AzureClientFactoryCacheTest : public testing::Test { +protected: + void SetUp() override { + auto& factory = S3ClientFactory::instance(); + { + std::lock_guard lock(factory._lock); + factory._azure_cache.clear(); + original_capacity = factory._azure_cache_capacity; + factory._azure_cache_capacity = 2; + factory._azure_cache_clock = 0; + } + auto* sync = SyncPoint::get_instance(); + was_enabled = sync->get_enable(); + sync->set_call_back( + "S3ClientFactory::azure_client_time", + [this](auto&& args) { *try_any_cast(args[0]) = now_ms.load(); }, + &clock_callback); + sync->enable_processing(); + } + + void TearDown() override { + auto* sync = SyncPoint::get_instance(); + clock_callback = SyncPoint::CallbackGuard {}; + if (!was_enabled) { + sync->disable_processing(); + } + auto& factory = S3ClientFactory::instance(); + std::lock_guard lock(factory._lock); + factory._azure_cache.clear(); + factory._azure_cache_capacity = original_capacity; + } + + size_t cache_size() { + auto& factory = S3ClientFactory::instance(); + std::lock_guard lock(factory._lock); + return factory._azure_cache.size(); + } + + S3ClientConf sas_conf(std::string signature) { + S3ClientConf conf; + conf.provider = io::ObjStorageProvider::AZURE; + conf.endpoint = "https://account.blob.core.windows.net"; + conf.bucket = "cache-test"; + conf.azure_credentials.type = AzureCredentialType::SAS; + conf.azure_credentials.account_name = "account"; + conf.azure_credentials.sas_token = "sv=2024-01-01&sr=c&sig=" + signature; + return conf; + } + + std::shared_ptr client(const S3ClientConf& conf) { + auto result = S3ClientFactory::instance().create(conf); + if (!result.has_value()) { + ADD_FAILURE() << result.error(); + return nullptr; + } + return std::move(result).value(); + } + + // 2100-01-01T00:00:00Z: future even for SDK construction's real clock. + static constexpr int64_t TOKEN_EXPIRY_MS = 4102444800000LL; + std::atomic now_ms {TOKEN_EXPIRY_MS - 60000}; + +private: + size_t original_capacity = 0; + bool was_enabled = false; + SyncPoint::CallbackGuard clock_callback; +}; + +TEST_F(AzureClientFactoryCacheTest, TokenExpiryRejectsHitAndReleasesOnlyCachedOwnership) { + auto conf = sas_conf("token-expiry"); + conf.azure_credentials.sas_token += "&se=2100-01-01T00%3A00%3A00Z"; + conf.azure_credentials.sas_expiration_time_ms = TOKEN_EXPIRY_MS + 60000; + auto active_reader_client = client(conf); + ASSERT_NE(active_reader_client, nullptr); + EXPECT_EQ(client(conf), active_reader_client); + std::weak_ptr reference = active_reader_client; + + now_ms = TOKEN_EXPIRY_MS; + auto expired = S3ClientFactory::instance().create(conf); + ASSERT_FALSE(expired.has_value()); + EXPECT_NE(expired.error().to_string().find("expired"), std::string::npos); + EXPECT_EQ(expired.error().to_string().find("token-expiry"), std::string::npos); + EXPECT_EQ(cache_size(), 0); + EXPECT_FALSE(reference.expired()); + active_reader_client.reset(); + EXPECT_TRUE(reference.expired()); +} + +TEST_F(AzureClientFactoryCacheTest, ExplicitEarlierExpiryControlsEviction) { + auto conf = sas_conf("explicit-expiry"); + conf.azure_credentials.sas_token += "&se=2100-01-01T00%3A00%3A00Z"; + conf.azure_credentials.sas_expiration_time_ms = TOKEN_EXPIRY_MS - 30000; + std::weak_ptr reference = client(conf); + EXPECT_FALSE(reference.expired()); + + now_ms = TOKEN_EXPIRY_MS - 30000; + auto expired = S3ClientFactory::instance().create(conf); + ASSERT_FALSE(expired.has_value()); + EXPECT_EQ(cache_size(), 0); + EXPECT_TRUE(reference.expired()); +} + +TEST_F(AzureClientFactoryCacheTest, ReusedFileSystemRejectsExpiredSasBeforeOpeningKnownSizeReader) { + auto conf = sas_conf("reused-reader"); + conf.azure_credentials.sas_token += "&se=2100-01-01T00%3A00%3A00Z"; + S3Conf fs_conf {.bucket = conf.bucket, .prefix = {}, .client_conf = conf}; + auto fs_result = io::S3FileSystem::create(std::move(fs_conf), "azure-access-expiry"); + ASSERT_TRUE(fs_result.has_value()); + auto fs = std::move(fs_result).value(); + io::FileReaderOptions options; + options.file_size = 1; // Skip HEAD: the expiry check must not depend on a service error. + io::FileReaderSPtr active_reader; + const std::string path = "abfss://cache-test@account.dfs.core.windows.net/object"; + ASSERT_TRUE(fs->open_file(path, &active_reader, &options).ok()); + + now_ms = TOKEN_EXPIRY_MS; + io::FileReaderSPtr rejected_reader; + auto status = fs->open_file(path, &rejected_reader, &options); + EXPECT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find("expired"), std::string::npos); + EXPECT_EQ(status.to_string().find("reused-reader"), std::string::npos); + EXPECT_EQ(rejected_reader, nullptr); + EXPECT_FALSE(active_reader->closed()); + EXPECT_EQ(active_reader->size(), 1); + + auto refreshed = sas_conf("refreshed-reader"); + refreshed.azure_credentials.sas_expiration_time_ms = TOKEN_EXPIRY_MS + 60000; + ASSERT_TRUE(fs->client_holder()->reset(refreshed).ok()); + io::FileReaderSPtr refreshed_reader; + EXPECT_TRUE(fs->open_file(path, &refreshed_reader, &options).ok()); + EXPECT_NE(refreshed_reader, nullptr); +} + +TEST_F(AzureClientFactoryCacheTest, ReaderAdmissionChecksCurrentCredentialAfterConcurrentReset) { + auto long_lived = sas_conf("admission-old"); + long_lived.azure_credentials.sas_expiration_time_ms = TOKEN_EXPIRY_MS + 60000; + S3Conf fs_conf {.bucket = long_lived.bucket, .prefix = {}, .client_conf = long_lived}; + auto fs_result = io::S3FileSystem::create(std::move(fs_conf), "azure-concurrent-admission"); + ASSERT_TRUE(fs_result.has_value()); + auto fs = std::move(fs_result).value(); + io::FileReaderOptions options; + options.file_size = 1; // Neither the successful nor the rejected path may issue HEAD. + io::FileReaderSPtr reader; + Status open_status; + std::mutex mutex; + std::condition_variable changed; + bool opener_paused = false; + bool release_opener = false; + SyncPoint::CallbackGuard admission_callback; + SyncPoint::get_instance()->set_call_back( + "ObjClientHolder::validate_for_access:before_lock", + [&](auto&&) { + std::unique_lock lock(mutex); + opener_paused = true; + changed.notify_all(); + EXPECT_TRUE(changed.wait_for(lock, std::chrono::seconds(10), + [&] { return release_opener; })); + }, + &admission_callback); + { + std::thread opener([&] { + open_status = fs->open_file("abfss://cache-test@account.dfs.core.windows.net/object", + &reader, &options); + }); + // Fatal assertions below must also release the callback and join before + // its captured state and the file system go out of scope. + Defer release_and_join = [&] { + { + std::lock_guard lock(mutex); + release_opener = true; + } + changed.notify_all(); + opener.join(); + }; + { + std::unique_lock lock(mutex); + ASSERT_TRUE(changed.wait_for(lock, std::chrono::seconds(10), + [&] { return opener_paused; })); + } + auto short_lived = sas_conf("admission-new"); + short_lived.azure_credentials.sas_expiration_time_ms = TOKEN_EXPIRY_MS; + ASSERT_TRUE(fs->client_holder()->reset(short_lived).ok()); + now_ms = TOKEN_EXPIRY_MS; + // The stale A snapshot would still pass; only the holder's current B + // credential can make this admission fail after the opener resumes. + EXPECT_TRUE(S3ClientFactory::validate_credentials_for_access(long_lived).ok()); + } + EXPECT_FALSE(open_status.ok()); + EXPECT_NE(open_status.to_string().find("expired"), std::string::npos); + EXPECT_EQ(open_status.to_string().find("admission-old"), std::string::npos); + EXPECT_EQ(open_status.to_string().find("admission-new"), std::string::npos); + EXPECT_EQ(reader, nullptr); +} + +TEST_F(AzureClientFactoryCacheTest, UnknownExpiryRotationsUseBoundedLru) { + auto first_conf = sas_conf("rotation-first"); + auto second_conf = sas_conf("rotation-second"); + auto third_conf = sas_conf("rotation-third"); + auto first = client(first_conf); + auto second = client(second_conf); + ASSERT_NE(first, nullptr); + ASSERT_NE(second, nullptr); + EXPECT_EQ(client(first_conf), first); // Refresh LRU order. + auto third = client(third_conf); + ASSERT_NE(third, nullptr); + EXPECT_EQ(cache_size(), 2); + EXPECT_EQ(client(first_conf), first); + // Eviction does not invalidate a reader holding the old identity's client. + auto recreated_second = client(second_conf); + ASSERT_NE(recreated_second, nullptr); + EXPECT_NE(recreated_second, second); + EXPECT_EQ(cache_size(), 2); +} + +TEST_F(AzureClientFactoryCacheTest, ExpiryDuringConstructionDoesNotPublishClient) { + auto conf = sas_conf("expires-during-construction"); + conf.azure_credentials.sas_expiration_time_ms = TOKEN_EXPIRY_MS; + int built_clients = 0; + SyncPoint::CallbackGuard built_callback; + SyncPoint::get_instance()->set_call_back( + "S3ClientFactory::azure_client_built", + [&](auto&&) { + ++built_clients; + now_ms = TOKEN_EXPIRY_MS; + }, + &built_callback); + + auto expired = S3ClientFactory::instance().create(conf); + ASSERT_FALSE(expired.has_value()); + EXPECT_EQ(built_clients, 1); + EXPECT_NE(expired.error().to_string().find("expired"), std::string::npos); + EXPECT_EQ(expired.error().to_string().find("expires-during-construction"), std::string::npos); + EXPECT_EQ(cache_size(), 0); +} + +TEST_F(AzureClientFactoryCacheTest, AzureRotationsDoNotEvictS3Clients) { + // Even an anonymous S3 client's SDK configuration can consult IMDS. Disable + // that lookup for this offline test, without changing the caller's environment. + std::optional original_metadata_disabled; + if (const char* value = std::getenv("AWS_EC2_METADATA_DISABLED")) { + original_metadata_disabled = value; + } + Defer restore_environment = [&] { + if (original_metadata_disabled.has_value()) { + EXPECT_EQ(setenv("AWS_EC2_METADATA_DISABLED", original_metadata_disabled->c_str(), 1), + 0); + } else { + EXPECT_EQ(unsetenv("AWS_EC2_METADATA_DISABLED"), 0); + } + }; + ASSERT_EQ(setenv("AWS_EC2_METADATA_DISABLED", "true", 1), 0); + S3ClientConf s3_conf; + s3_conf.endpoint = "azure-cache-s3-regression.example.com"; + s3_conf.region = "us-east-1"; + s3_conf.cred_provider_type = CredProviderType::Anonymous; + auto s3_client = client(s3_conf); + ASSERT_NE(s3_client, nullptr); + for (int index = 0; index < 8; ++index) { + ASSERT_NE(client(sas_conf("rotation-" + std::to_string(index))), nullptr); + } + EXPECT_EQ(cache_size(), 2); + EXPECT_EQ(client(s3_conf), s3_client); +} + +} // namespace doris + +#endif diff --git a/be/test/io/fs/azure_obj_storage_client_lifecycle_test.cpp b/be/test/io/fs/azure_obj_storage_client_lifecycle_test.cpp new file mode 100644 index 00000000000000..fa434261e030f6 --- /dev/null +++ b/be/test/io/fs/azure_obj_storage_client_lifecycle_test.cpp @@ -0,0 +1,185 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include + +#ifdef USE_AZURE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "cpp/obj-client/azure_obj_storage_client.h" + +namespace doris { +namespace { + +constexpr char AZURE_ENDPOINT[] = "https://account.blob.core.windows.net"; +constexpr char AZURE_CONTAINER_URL[] = "https://account.blob.core.windows.net/container"; + +class AzureLifecycleTransport final : public Azure::Core::Http::HttpTransport { +public: + std::unique_ptr Send(Azure::Core::Http::Request& request, + const Azure::Core::Context&) override { + methods.push_back(request.GetMethod().ToString()); + // The pre-fix abort issues DELETE. Answer it locally so the test records the mutation + // without relying on a real Azure account, its permissions, or its network availability. + return std::make_unique( + 1, 1, Azure::Core::Http::HttpStatusCode::Accepted, "Accepted"); + } + + std::vector methods; +}; + +class AzureLifecycleTokenCredential final : public Azure::Core::Credentials::TokenCredential { +public: + AzureLifecycleTokenCredential() : TokenCredential("AzureLifecycleTest") {} + + Azure::Core::Credentials::AccessToken GetToken( + const Azure::Core::Credentials::TokenRequestContext&, + const Azure::Core::Context&) const override { + ++token_requests; + return {.Token = "test-access-token", + .ExpiresOn = + Azure::DateTime(std::chrono::system_clock::now() + std::chrono::hours(1))}; + } + + mutable size_t token_requests = 0; +}; + +class AzureObjStorageClientLifecycleTest : public testing::Test { +protected: + void SetUp() override { + transport = std::make_shared(); + options.Transport.Transport = transport; + options.Retry.MaxRetries = 0; + } + + std::shared_ptr transport; + Azure::Storage::Blobs::BlobClientOptions options; +}; + +TEST_F(AzureObjStorageClientLifecycleTest, SasCannotPublishTheContainerCredential) { + auto container = std::make_shared( + std::string(AZURE_CONTAINER_URL) + "?sv=2024-01-01&sr=c&sp=rwdl&sig=private-sas", + options); + AzureObjStorageClient client(container, {.endpoint = AZURE_ENDPOINT}); + + EXPECT_TRUE(client.generate_presigned_url({.bucket = "container", .key = "directory/file"}, 60) + .empty()); + EXPECT_TRUE(transport->methods.empty()); +} + +TEST_F(AzureObjStorageClientLifecycleTest, OAuth2CannotPublishAnUnauthenticatedUrl) { + auto credential = std::make_shared(); + auto container = std::make_shared( + AZURE_CONTAINER_URL, credential, options); + AzureObjStorageClient client(container, {.endpoint = AZURE_ENDPOINT}); + + EXPECT_TRUE(client.generate_presigned_url({.bucket = "container", .key = "directory/file"}, 60) + .empty()); + EXPECT_EQ(credential->token_requests, 0); + EXPECT_TRUE(transport->methods.empty()); +} + +TEST_F(AzureObjStorageClientLifecycleTest, SharedKeyStillSignsOneReadOnlyBlobWithRequestedExpiry) { + auto credential = std::make_shared( + "account", "MDEyMzQ1Njc4OWFiY2RlZg=="); + auto container = std::make_shared( + AZURE_CONTAINER_URL, credential, options); + AzureObjStorageClient client(container, {.endpoint = AZURE_ENDPOINT}, credential); + constexpr int64_t expiry_seconds = 600; + const auto before = std::chrono::system_clock::now(); + const auto url = client.generate_presigned_url({.bucket = "container", .key = "directory/file"}, + expiry_seconds); + const auto after = std::chrono::system_clock::now(); + + ASSERT_TRUE(url.starts_with("https://account.blob.core.windows.net/container/directory/file?")); + const auto query = Azure::Core::Url(url).GetQueryParameters(); + EXPECT_EQ(query.at("sr"), "b"); + EXPECT_EQ(query.at("sp"), "r"); + EXPECT_EQ(query.at("spr"), "https"); + EXPECT_FALSE(query.at("sig").empty()); + const auto expiry = Azure::DateTime::Parse(Azure::Core::Url::Decode(query.at("se")), + Azure::DateTime::DateFormat::Rfc3339); + EXPECT_GE(expiry, Azure::DateTime(before + std::chrono::seconds(expiry_seconds - 1))); + EXPECT_LE(expiry, Azure::DateTime(after + std::chrono::seconds(expiry_seconds))); + EXPECT_TRUE(transport->methods.empty()); +} + +TEST_F(AzureObjStorageClientLifecycleTest, SharedKeyPresignEncodesRawObjectNamesExactlyOnce) { + auto credential = std::make_shared( + "account", "MDEyMzQ1Njc4OWFiY2RlZg=="); + auto container = std::make_shared( + AZURE_CONTAINER_URL, credential, options); + AzureObjStorageClient client(container, {.endpoint = AZURE_ENDPOINT}, credential); + for (const auto* key : + {"directory/100%.parquet", "directory/audit%2Fhistory", "directory/a b+c", + "directory/中文.parquet", "directory/a%252Fb", "directory/http://example/file"}) { + SCOPED_TRACE(key); + const auto signed_url = + client.generate_presigned_url({.bucket = "container", .key = key}, 600); + const Azure::Core::Url url(signed_url); + EXPECT_EQ(Azure::Core::Url::Decode(url.GetPath()), std::string("container/") + key); + const auto query = url.GetQueryParameters(); + Azure::Storage::Sas::BlobSasBuilder expected; + expected.BlobContainerName = "container"; + expected.BlobName = key; + expected.Resource = Azure::Storage::Sas::BlobSasResource::Blob; + expected.Protocol = Azure::Storage::Sas::SasProtocol::HttpsOnly; + expected.SetPermissions(Azure::Storage::Sas::BlobSasPermissions::Read); + expected.ExpiresOn = Azure::DateTime::Parse(Azure::Core::Url::Decode(query.at("se")), + Azure::DateTime::DateFormat::Rfc3339); + // Verify that the signature still names the raw blob, not its URL-encoded spelling. + EXPECT_EQ(signed_url.substr(signed_url.find('?')), expected.GenerateSasToken(*credential)); + } + EXPECT_TRUE(transport->methods.empty()); +} + +TEST_F(AzureObjStorageClientLifecycleTest, AbortDoesNotDeletePublishedBlobOrOtherWritersBlocks) { + auto container = std::make_shared( + AZURE_CONTAINER_URL, options); + AzureObjStorageClient client(container, {.endpoint = AZURE_ENDPOINT}); + const ObjStoragePath path {.bucket = "container", .key = "existing-blob"}; + + EXPECT_TRUE(client.abort_multipart_upload(path, "first-writer").ok()); + EXPECT_TRUE(client.abort_multipart_upload(path, "first-writer").ok()); + EXPECT_TRUE(client.abort_multipart_upload(path, "second-writer").ok()); + EXPECT_TRUE(transport->methods.empty()); +} + +TEST(AzureObjStorageClientLifecycleHelperTest, AbortDoesNotRequireAnSdkClient) { + AzureObjStorageClient client(std::shared_ptr {}, + {}); + + EXPECT_TRUE(client.abort_multipart_upload({.key = "uncommitted-blob"}, "upload-id").ok()); +} + +} // namespace +} // namespace doris + +#endif // USE_AZURE diff --git a/be/test/io/fs/azure_obj_storage_client_test.cpp b/be/test/io/fs/azure_obj_storage_client_test.cpp index ca04f5ea0ffb31..8a32204074b2ff 100644 --- a/be/test/io/fs/azure_obj_storage_client_test.cpp +++ b/be/test/io/fs/azure_obj_storage_client_test.cpp @@ -24,16 +24,24 @@ #include #include +#include #include +#include #include +#include +#include #include "cpp/obj-client/obj_storage_client.h" +#include "io/file_factory.h" +#include "io/fs/file_reader.h" #include "io/fs/file_system.h" +#include "util/s3_uri.h" #include "util/s3_util.h" #ifdef USE_AZURE #include +#include #include #include #include @@ -82,12 +90,324 @@ TEST(AzureObjStorageClientMultipartHelperTest, create_upload_is_provider_free) { using namespace Azure::Storage::Blobs; TEST(AzureAuthFactoryTest, AllowsEmptySharedKeyCredentials) { - auto result = AzureAuthFactory::create("https://account.blob.core.windows.net/container", - {.account_name = "", .account_key = ""}, {}); + auto result = + AzureAuthFactory::create("https://account.blob.core.windows.net/container", {}, {}); EXPECT_TRUE(result); } +TEST(AzureAuthFactoryTest, BuildsSasClientWithoutSharedKey) { + const auto expiry = std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()) + .count() + + 3600000; + AzureCredentialOptions credentials; + credentials.type = AzureCredentialType::SAS; + credentials.sas_token = "?sv=2024-01-01&sr=c&sig=temporary"; + credentials.sas_expiration_time_ms = expiry; + auto result = AzureAuthFactory::create("https://account.blob.core.windows.net/container", + credentials, {}); + + ASSERT_TRUE(result); + EXPECT_NE(result.container_client->GetUrl().find("sv=2024-01-01"), std::string::npos); + EXPECT_NE(result.container_client->GetUrl().find("sig=temporary"), std::string::npos); + EXPECT_EQ(result.shared_key_credential, nullptr); +} + +TEST(AzureAuthFactoryTest, RejectsExpiredOrMalformedSas) { + AzureCredentialOptions credentials; + credentials.type = AzureCredentialType::SAS; + credentials.sas_token = "sv=2024-01-01&sig=expired"; + credentials.sas_expiration_time_ms = 1; + auto expired = AzureAuthFactory::create("https://account.blob.core.windows.net/container", + credentials, {}); + EXPECT_FALSE(expired); + EXPECT_NE(expired.error.find("expired"), std::string::npos); + + credentials.sas_token.clear(); + credentials.sas_expiration_time_ms = 0; + auto empty = AzureAuthFactory::create("https://account.blob.core.windows.net/container", + credentials, {}); + EXPECT_FALSE(empty); + EXPECT_NE(empty.error.find("non-empty"), std::string::npos); + + credentials.sas_token = "sv=1\nsig=bad"; + auto newline = AzureAuthFactory::create("https://account.blob.core.windows.net/container", + credentials, {}); + EXPECT_FALSE(newline); + EXPECT_NE(newline.error.find("line break"), std::string::npos); + + credentials.sas_token = "sv=2024-01-01&se=2000-01-01T00%3A00%3A00Z&sig=expired"; + auto expired_in_token = AzureAuthFactory::create( + "https://account.blob.core.windows.net/container", credentials, {}); + EXPECT_FALSE(expired_in_token); + EXPECT_NE(expired_in_token.error.find("expired"), std::string::npos); +} + +TEST(AzureAuthFactoryTest, BuildsOAuth2ClientSecretCredential) { + AzureCredentialOptions credentials; + credentials.type = AzureCredentialType::OAUTH2; + credentials.oauth_client_id = "client-id"; + credentials.oauth_client_secret = "client-secret"; + credentials.oauth_tenant_id = "tenant-id"; + credentials.oauth_server_uri = "https://login.microsoftonline.com/tenant/oauth2/token"; + auto result = AzureAuthFactory::create("https://account.blob.core.windows.net/container", + credentials, {}); + EXPECT_TRUE(result) << result.error; + EXPECT_NE(result.container_client, nullptr); +} + +TEST(AzureAuthFactoryTest, DerivesOAuth2TenantFromServerUri) { + AzureCredentialOptions credentials; + credentials.type = AzureCredentialType::OAUTH2; + credentials.oauth_client_id = "client-id"; + credentials.oauth_client_secret = "client-secret"; + credentials.oauth_server_uri = "https://login.microsoftonline.com/tenant/oauth2/token"; + auto result = AzureAuthFactory::create("https://account.blob.core.windows.net/container", + credentials, {}); + EXPECT_TRUE(result) << result.error; +} + +TEST(AzureAuthFactoryTest, RejectsIncompleteOAuth2Credential) { + AzureCredentialOptions credentials; + credentials.type = AzureCredentialType::OAUTH2; + credentials.oauth_client_id = "client-id"; + credentials.oauth_server_uri = "https://login.microsoftonline.com/tenant/oauth2/token"; + auto result = AzureAuthFactory::create("https://account.blob.core.windows.net/container", + credentials, {}); + EXPECT_FALSE(result); + EXPECT_NE(result.error.find("client id"), std::string::npos); +} + +TEST(AzureAuthFactoryTest, RejectsConflictingAuthenticationGroups) { + AzureCredentialOptions shared_key; + shared_key.account_name = "account"; + shared_key.account_key = "key"; + shared_key.sas_token = "sig=secret"; + EXPECT_FALSE(AzureAuthFactory::create("https://account.blob.core.windows.net/container", + shared_key, {})); + AzureCredentialOptions oauth; + oauth.type = AzureCredentialType::OAUTH2; + oauth.sas_token = "sig=secret"; + oauth.oauth_client_id = "client-id"; + oauth.oauth_client_secret = "client-secret"; + oauth.oauth_server_uri = "https://login.microsoftonline.com/tenant/oauth2/token"; + auto result = + AzureAuthFactory::create("https://account.blob.core.windows.net/container", oauth, {}); + EXPECT_FALSE(result); + EXPECT_EQ(result.error.find("sig=secret"), std::string::npos); + EXPECT_EQ(result.error.find("client-secret"), std::string::npos); +} + +TEST(AzureAuthFactoryTest, RejectsDuplicateOrMissingSasSignatureAndExpiryFields) { + for (const auto* token : {"sv=1", "sig=", "sig=one&sig=two", + "se=2100-01-01T00:00:00Z&se=2099-01-01T00:00:00Z&sig=secret"}) { + AzureCredentialOptions credentials; + credentials.type = AzureCredentialType::SAS; + credentials.sas_token = token; + auto result = AzureAuthFactory::create("https://account.blob.core.windows.net/container", + credentials, {}); + EXPECT_FALSE(result); + EXPECT_EQ(result.error.find("sig=secret"), std::string::npos); + } +} + +namespace { + +// This transport exercises the real Azure SDK request/response and Doris +// native reader stack without credentials, DNS or a live storage account. +class AzureRangeTransport final : public Azure::Core::Http::HttpTransport { +public: + explicit AzureRangeTransport(std::string expected_key) + : _expected_key(std::move(expected_key)) {} + + std::unique_ptr Send(Azure::Core::Http::Request& request, + const Azure::Core::Context&) override { + using namespace Azure::Core::Http; + const bool head = request.GetMethod() == HttpMethod::Head; + const auto headers = request.GetHeaders(); + if (head) { + ++heads; + } else { + EXPECT_EQ(request.GetMethod(), HttpMethod::Get); + ++ranges; + EXPECT_EQ(headers.at(headers.contains("x-ms-range") ? "x-ms-range" : "range"), + "bytes=7-10"); + } + EXPECT_EQ(request.GetUrl().GetQueryParameters().at("sig"), "a%2Bb%3D"); + EXPECT_EQ(headers.find("Authorization"), headers.end()); + EXPECT_EQ(headers.find("authorization"), headers.end()); + const auto decoded = Azure::Core::Url::Decode(request.GetUrl().GetPath()); + EXPECT_TRUE(decoded.ends_with("container/" + _expected_key)); + auto response = std::make_unique( + 1, 1, head ? HttpStatusCode::Ok : HttpStatusCode::PartialContent, "OK"); + response->SetHeader("Content-Length", head ? "16" : "4"); + response->SetHeader("Content-Type", "application/octet-stream"); + response->SetHeader("Accept-Ranges", "bytes"); + response->SetHeader("ETag", "\"etag\""); + response->SetHeader("Last-Modified", "Wed, 01 Jan 2025 00:00:00 GMT"); + response->SetHeader("x-ms-creation-time", "Wed, 01 Jan 2025 00:00:00 GMT"); + response->SetHeader("x-ms-blob-type", "BlockBlob"); + response->SetHeader("x-ms-server-encrypted", "true"); + response->SetHeader("x-ms-lease-status", "unlocked"); + response->SetHeader("x-ms-lease-state", "available"); + // HEAD has no response bytes, but the SDK transport policy still consumes a body stream. + response->SetBodyStream(std::make_unique( + reinterpret_cast("data"), head ? 0 : 4)); + if (!head) { + response->SetHeader("Content-Range", "bytes 7-10/16"); + } + return response; + } + + int heads = 0; + int ranges = 0; + +private: + std::string _expected_key; +}; + +// Both SDK pipelines must use the configured transport. In production this is the CurlTransport +// carrying the custom CA/proxy settings. This fake also exercises the real GetToken request. +class AzureOAuthTransport final : public Azure::Core::Http::HttpTransport { +public: + std::unique_ptr Send(Azure::Core::Http::Request& request, + const Azure::Core::Context&) override { + using namespace Azure::Core::Http; + auto response = std::make_unique(1, 1, HttpStatusCode::Ok, "OK"); + if (request.GetUrl().GetHost() == "127.0.0.1") { + ++token_requests; + EXPECT_EQ(request.GetMethod(), HttpMethod::Post); + EXPECT_EQ(request.GetUrl().GetPath(), "tenant/oauth2/v2.0/token"); + static constexpr char body[] = + R"({"access_token":"test-access-token","expires_in":3600,"token_type":"Bearer"})"; + response->SetHeader("Content-Type", "application/json"); + response->SetHeader("Content-Length", std::to_string(sizeof(body) - 1)); + response->SetBodyStream(std::make_unique( + reinterpret_cast(body), sizeof(body) - 1)); + } else { + ++blob_requests; + EXPECT_EQ(request.GetUrl().GetHost(), "account.blob.core.windows.net"); + EXPECT_EQ(request.GetMethod(), HttpMethod::Head); + EXPECT_EQ(request.GetHeaders().at("Authorization"), "Bearer test-access-token"); + response->SetHeader("Content-Length", "16"); + response->SetHeader("ETag", "\"etag\""); + response->SetHeader("Last-Modified", "Wed, 01 Jan 2025 00:00:00 GMT"); + response->SetHeader("x-ms-creation-time", "Wed, 01 Jan 2025 00:00:00 GMT"); + response->SetHeader("x-ms-blob-type", "BlockBlob"); + response->SetHeader("x-ms-server-encrypted", "true"); + response->SetBodyStream(std::make_unique( + reinterpret_cast(""), 0)); + } + return response; + } + + int token_requests = 0; + int blob_requests = 0; +}; + +void assert_reader_range_contents(io::FileReader& reader) { + EXPECT_EQ(reader.size(), 16); + std::array buffer {}; + size_t bytes_read = 0; + ASSERT_TRUE(reader.read_at(7, Slice(buffer.data(), buffer.size()), &bytes_read).ok()); + EXPECT_EQ(bytes_read, 4); + EXPECT_EQ(std::string(buffer.data(), bytes_read), "data"); +} + +void assert_native_azure_client_conf(const S3ClientConf& received, const S3ClientConf& expected) { + EXPECT_EQ(received, expected); + EXPECT_EQ(received.provider, ObjStorageProvider::AZURE); +} + +void assert_native_sas_reader_range(const std::string& location, const std::string& expected_key) { + auto transport = std::make_shared(expected_key); + Azure::Storage::Blobs::BlobClientOptions options; + options.Transport.Transport = transport; + options.Retry.MaxRetries = 0; + io::FileSystemProperties properties { + .system_type = TFileType::FILE_S3, + .properties = {{"provider", "azure"}, + {"AZURE_AUTH_TYPE", "SAS"}, + {"AZURE_ACCOUNT_NAME", "account"}, + {"AZURE_ENDPOINT", "https://account.blob.core.windows.net"}, + {"AZURE_SAS_TOKEN", "sv=2024-01-01&sr=c&sig=a%2Bb%3D"}}, + .hdfs_params = {}, + .broker_addresses = {}}; + S3URI uri(location); + ASSERT_TRUE(uri.parse().ok()); + EXPECT_EQ(uri.get_key(), expected_key); + S3Conf conf; + ASSERT_TRUE( + S3ClientFactory::convert_properties_to_s3_conf(properties.properties, uri, &conf).ok()); + auto built = AzureAuthFactory::create(conf.client_conf.endpoint + "/" + conf.bucket, + conf.client_conf.azure_credentials, std::move(options)); + ASSERT_TRUE(built) << built.error; + auto native_client = std::make_shared(built.container_client, + ObjStorageEndpointInfo {}); + S3ClientFactory::instance().set_client_creator_for_test( + [native_client, + conf](const S3ClientConf& received) -> std::shared_ptr { + assert_native_azure_client_conf(received, conf.client_conf); + return native_client; + }); + auto result = FileFactory::create_file_reader(properties, {.path = location, .fs_name = {}}, {}, + nullptr); + S3ClientFactory::instance().clear_client_creator_for_test(); + ASSERT_TRUE(result.has_value()) << result.error(); + auto reader = std::move(result).value(); + ASSERT_NO_FATAL_FAILURE(assert_reader_range_contents(*reader)); + EXPECT_EQ(transport->heads, 1); + EXPECT_EQ(transport->ranges, 1); + EXPECT_TRUE(reader->close().ok()); +} + +} // namespace + +TEST(AzureAuthFactoryTest, OAuth2TokenRequestsInheritTheBlobTransport) { + auto transport = std::make_shared(); + Azure::Storage::Blobs::BlobClientOptions options; + options.Transport.Transport = transport; + options.Retry.MaxRetries = 0; + AzureCredentialOptions credentials; + credentials.type = AzureCredentialType::OAUTH2; + credentials.oauth_client_id = "client-id"; + credentials.oauth_client_secret = "client-secret"; + // A loopback authority prevents an external token request even if transport propagation + // regresses. The context deadline bounds that failure; the passing test uses no sockets. + credentials.oauth_server_uri = "https://127.0.0.1:1/tenant/oauth2/token"; + auto built = AzureAuthFactory::create("https://account.blob.core.windows.net/container", + credentials, std::move(options)); + ASSERT_TRUE(built) << built.error; + const auto context = Azure::Core::Context().WithDeadline(std::chrono::system_clock::now() + + std::chrono::seconds(2)); + const auto blob = built.container_client->GetBlobClient("file.parquet"); + + ASSERT_NO_THROW(EXPECT_EQ(blob.GetProperties({}, context).Value.BlobSize, 16)); + EXPECT_EQ(transport->token_requests, 1); + EXPECT_EQ(transport->blob_requests, 1); +} + +TEST(AzureAuthFactoryTest, NativeAdlsSasReaderPreservesLiteralNamesInHeadAndRangeRead) { + const std::string key = "path/p=a%2Fb/with%20space/a+%2520/http://example//100%file"; + for (const auto* prefix : {"abfs://container@account.dfs.core.windows.net/", + "abfss://container@account.dfs.core.windows.net/", + "wasb://container@account.blob.core.windows.net/", + "wasbs://container@account.blob.core.windows.net/"}) { + SCOPED_TRACE(prefix); + // ADLSLocation passes this raw name to the Java SDK. The C++ SDK must + // address that same object, not turn the literal %2F into a directory. + assert_native_sas_reader_range(std::string(prefix) + key, key); + } +} + +TEST(AzureAuthFactoryTest, NativeHttpSasReaderDecodesUrlNamesOnceInHeadAndRangeRead) { + assert_native_sas_reader_range( + "https://account.blob.core.windows.net/container/path/p=a%2Fb/with%20space/" + "a+%2520/http://example//file", + "path/p=a/b/with space/a+%20/http://example//file"); +} + TEST(AzureObjStorageClientTlsHelperTest, detects_tls_ca_error) { EXPECT_TRUE(io::is_azure_tls_ca_error_message( "Problem with the SSL CA cert (path? access rights?)")); @@ -121,28 +441,24 @@ class AzureObjStorageClientTest : public testing::Test { static std::shared_ptr obj_storage_client; static void SetUpTestSuite() { - if (!std::getenv("AZURE_ACCOUNT_NAME") || !std::getenv("AZURE_ACCOUNT_KEY") || - !std::getenv("AZURE_CONTAINER_NAME")) { + const auto* account_name = std::getenv("AZURE_ACCOUNT_NAME"); + const auto* account_key = std::getenv("AZURE_ACCOUNT_KEY"); + const auto* container_name = std::getenv("AZURE_CONTAINER_NAME"); + if (!account_name || !account_key || !container_name) { return; } - std::string accountName = std::getenv("AZURE_ACCOUNT_NAME"); - std::string accountKey = std::getenv("AZURE_ACCOUNT_KEY"); - std::string containerName = std::getenv("AZURE_CONTAINER_NAME"); - // Initialize Azure SDK [[maybe_unused]] auto& s3ClientFactory = S3ClientFactory::instance(); - auto client_result = S3ClientFactory::instance().create( - {.endpoint = fmt::format("https://{}.blob.core.windows.net", accountName), - .region = "dummy-region", - .ak = accountName, - .sk = accountKey, - .token = "", - .bucket = containerName, - .provider = ObjStorageProvider::AZURE, - .role_arn = "", - .external_id = ""}); + S3ClientConf conf; + conf.endpoint = fmt::format("https://{}.blob.core.windows.net", account_name); + conf.region = "dummy-region"; + conf.azure_credentials.account_name = account_name; + conf.azure_credentials.account_key = account_key; + conf.bucket = container_name; + conf.provider = ObjStorageProvider::AZURE; + auto client_result = S3ClientFactory::instance().create(conf); ASSERT_TRUE(client_result.has_value()) << client_result.error(); AzureObjStorageClientTest::obj_storage_client = std::move(client_result).value(); } diff --git a/be/test/io/fs/s3_obj_storage_client_role_test.cpp b/be/test/io/fs/s3_obj_storage_client_role_test.cpp index 86705c06730a18..86a0e68b5aa8f6 100644 --- a/be/test/io/fs/s3_obj_storage_client_role_test.cpp +++ b/be/test/io/fs/s3_obj_storage_client_role_test.cpp @@ -55,6 +55,7 @@ class S3ObjStorageClientRoleTest : public testing::Test { .ak = "", .sk = "", .token = "", + .azure_credentials = {}, .bucket = bucket, .provider = ObjStorageProvider::AWS, .use_virtual_addressing = false, diff --git a/be/test/io/fs/s3_obj_storage_client_test.cpp b/be/test/io/fs/s3_obj_storage_client_test.cpp index 4bafbc4dde71f7..c12fb4f0b7a0d7 100644 --- a/be/test/io/fs/s3_obj_storage_client_test.cpp +++ b/be/test/io/fs/s3_obj_storage_client_test.cpp @@ -48,6 +48,7 @@ class S3ObjStorageClientTest : public testing::Test { .ak = access_key, .sk = secret_key, .token = "", + .azure_credentials = {}, .bucket = bucket, .provider = ObjStorageProvider::AWS, .use_virtual_addressing = false, diff --git a/be/test/io/s3_client_factory_test.cpp b/be/test/io/s3_client_factory_test.cpp index 835b32bdaef7e5..3dd8cd68ef310b 100644 --- a/be/test/io/s3_client_factory_test.cpp +++ b/be/test/io/s3_client_factory_test.cpp @@ -24,12 +24,14 @@ #include #include +#include #include #include #include #include #include #include +#include #include #include @@ -549,6 +551,395 @@ TEST_F(S3ClientFactoryTest, ConvertPropertiesToS3ConfCredentialValidation) { } } +TEST_F(S3ClientFactoryTest, ConvertNativeAzureSasProperties) { + const auto expiry = std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()) + .count() + + 3600000; + std::map properties { + {"provider", "azure"}, + {"AZURE_AUTH_TYPE", "SAS"}, + {"AZURE_ENDPOINT", "account.dfs.core.windows.net"}, + {"AZURE_ACCOUNT_NAME", "account"}, + {"AZURE_CONTAINER", "container"}, + {"AZURE_SAS_TOKEN", "sv=2024-01-01&sr=c&sig=temporary"}, + {"AZURE_SAS_EXPIRY_MS", std::to_string(expiry)}, + }; + S3URI azure_uri("abfss://container@account.dfs.core.windows.net/path/file.parquet"); + ASSERT_TRUE(azure_uri.parse().ok()); + + S3Conf s3_conf; + ASSERT_TRUE( + S3ClientFactory::convert_properties_to_s3_conf(properties, azure_uri, &s3_conf).ok()); + EXPECT_EQ(s3_conf.client_conf.provider, ObjStorageProvider::AZURE); + EXPECT_EQ(s3_conf.client_conf.endpoint, "https://account.blob.core.windows.net"); + EXPECT_EQ(s3_conf.client_conf.azure_credentials.account_name, "account"); + EXPECT_TRUE(s3_conf.client_conf.ak.empty()); + EXPECT_TRUE(s3_conf.client_conf.sk.empty()); + EXPECT_EQ(s3_conf.client_conf.bucket, "container"); + EXPECT_EQ(s3_conf.bucket, "container"); + EXPECT_TRUE(s3_conf.client_conf.token.empty()); + EXPECT_TRUE(s3_conf.client_conf.region.empty()); + EXPECT_EQ(s3_conf.client_conf.azure_credentials.sas_token, "sv=2024-01-01&sr=c&sig=temporary"); + EXPECT_EQ(s3_conf.client_conf.azure_credentials.type, AzureCredentialType::SAS); + EXPECT_EQ(s3_conf.client_conf.azure_credentials.sas_expiration_time_ms, expiry); +} + +TEST_F(S3ClientFactoryTest, ConvertsNativeAzureOAuth2Properties) { + std::map properties { + {"provider", "azure"}, + {"AZURE_AUTH_TYPE", "OAUTH2"}, + {"AZURE_ENDPOINT", "account.blob.core.windows.net"}, + {"AZURE_ACCOUNT_NAME", "account"}, + {"AZURE_CONTAINER", "container"}, + {"AZURE_CLIENT_ID", "client-id"}, + {"AZURE_CLIENT_SECRET", "client-secret"}, + {"AZURE_TENANT_ID", "tenant-id"}, + {"AZURE_OAUTH_SERVER_URI", "https://login.microsoftonline.com/tenant/oauth2/token"}, + }; + S3URI azure_uri("abfss://container@account.dfs.core.windows.net/path/file.parquet"); + ASSERT_TRUE(azure_uri.parse().ok()); + + S3Conf s3_conf; + ASSERT_TRUE( + S3ClientFactory::convert_properties_to_s3_conf(properties, azure_uri, &s3_conf).ok()); + EXPECT_EQ(s3_conf.client_conf.azure_credentials.type, AzureCredentialType::OAUTH2); + EXPECT_EQ(s3_conf.client_conf.azure_credentials.oauth_client_id, "client-id"); + EXPECT_EQ(s3_conf.client_conf.azure_credentials.oauth_client_secret, "client-secret"); + EXPECT_EQ(s3_conf.client_conf.azure_credentials.oauth_tenant_id, "tenant-id"); + EXPECT_EQ(s3_conf.client_conf.azure_credentials.oauth_server_uri, + "https://login.microsoftonline.com/tenant/oauth2/token"); +} + +TEST_F(S3ClientFactoryTest, RejectsAzureSasAliasInference) { + std::map properties { + {"provider", "azure"}, + {"AWS_TOKEN", "sv=2024-01-01&sig=temporary"}, + }; + S3URI azure_uri("abfss://container@account.dfs.core.windows.net/path/file.parquet"); + ASSERT_TRUE(azure_uri.parse().ok()); + + S3Conf s3_conf; + ASSERT_FALSE( + S3ClientFactory::convert_properties_to_s3_conf(properties, azure_uri, &s3_conf).ok()); +} + +TEST_F(S3ClientFactoryTest, NonAzurePropertiesKeepLegacyCredentialContract) { + std::map properties { + {"AWS_ENDPOINT", "s3.us-west-2.amazonaws.com"}, + {"AWS_REGION", "us-west-2"}, + {"AWS_ACCESS_KEY", "ak"}, + {"AWS_SECRET_KEY", "sk"}, + // Azure-only expiry metadata must not change an ordinary S3 binding. + {"AZURE_SAS_EXPIRY_MS", "not-a-number"}, + }; + S3URI s3_uri("s3://test-bucket/path/file"); + ASSERT_TRUE(s3_uri.parse().ok()); + + S3Conf s3_conf; + ASSERT_TRUE(S3ClientFactory::convert_properties_to_s3_conf(properties, s3_uri, &s3_conf).ok()); + EXPECT_EQ(s3_conf.client_conf.provider, ObjStorageProvider::AWS); + EXPECT_EQ(s3_conf.client_conf.ak, "ak"); + EXPECT_EQ(s3_conf.client_conf.sk, "sk"); + EXPECT_TRUE(s3_conf.client_conf.azure_credentials.sas_token.empty()); +} + +namespace { + +std::map native_azure_shared_key_properties() { + return {{"provider", "azure"}, + {"AZURE_AUTH_TYPE", "SHARED_KEY"}, + {"AZURE_ENDPOINT", "https://account.blob.core.windows.net"}, + {"AZURE_ACCOUNT_NAME", "account"}, + {"AZURE_ACCOUNT_KEY", "c2hhcmVkLWtleQ=="}}; +} + +} // namespace + +TEST_F(S3ClientFactoryTest, ConvertsNativeAzureSharedKeyWithoutAwsFields) { + auto properties = native_azure_shared_key_properties(); + S3URI uri("abfss://container@account.dfs.core.windows.net/path/file"); + ASSERT_TRUE(uri.parse().ok()); + S3Conf conf; + ASSERT_TRUE(S3ClientFactory::convert_properties_to_s3_conf(properties, uri, &conf).ok()); + EXPECT_EQ(conf.client_conf.azure_credentials.type, AzureCredentialType::SHARED_KEY); + EXPECT_EQ(conf.client_conf.azure_credentials.account_name, "account"); + EXPECT_EQ(conf.client_conf.azure_credentials.account_key, "c2hhcmVkLWtleQ=="); + EXPECT_EQ(conf.bucket, "container"); + EXPECT_EQ(conf.client_conf.bucket, "container"); + EXPECT_TRUE(conf.client_conf.ak.empty()); + EXPECT_TRUE(conf.client_conf.sk.empty()); + EXPECT_TRUE(conf.client_conf.token.empty()); + EXPECT_TRUE(conf.client_conf.region.empty()); +#ifdef USE_AZURE + auto client = S3ClientFactory::instance().create(conf.client_conf); + ASSERT_TRUE(client.has_value()) << client.error(); +#endif +} + +TEST_F(S3ClientFactoryTest, LegacyAzureSharedKeyRetainsCustomEndpointAndS3Uri) { + std::map properties { + {"provider", "azure"}, + {"AWS_ENDPOINT", "https://custom.example.com:8443/base%2Fpath"}, + {"AWS_ACCESS_KEY", "original-account"}, + {"AWS_SECRET_KEY", "c2hhcmVkLWtleQ=="}, + {"AWS_REGION", "legacy-region"}, + {"AWS_TOKEN", ""}}; + S3URI uri("s3://container/path%20with+encoding"); + ASSERT_TRUE(uri.parse().ok()); + S3Conf conf; + ASSERT_TRUE(S3ClientFactory::convert_properties_to_s3_conf(properties, uri, &conf).ok()); + EXPECT_EQ(conf.client_conf.azure_credentials.account_name, "original-account"); + EXPECT_EQ(conf.client_conf.azure_credentials.account_key, "c2hhcmVkLWtleQ=="); + EXPECT_EQ(conf.client_conf.endpoint, properties.at("AWS_ENDPOINT")); + EXPECT_EQ(uri.get_key(), "path%20with+encoding"); + EXPECT_EQ(conf.bucket, "container"); + EXPECT_TRUE(conf.client_conf.ak.empty()); + EXPECT_TRUE(conf.client_conf.sk.empty()); + EXPECT_TRUE(conf.client_conf.region.empty()); +} + +TEST_F(S3ClientFactoryTest, RequiresExplicitAzureProviderAndAuthentication) { + S3URI uri("abfss://container@account.dfs.core.windows.net/path/file"); + ASSERT_TRUE(uri.parse().ok()); + for (const auto* missing : {"provider", "AZURE_AUTH_TYPE", "AZURE_ENDPOINT", + "AZURE_ACCOUNT_NAME", "AZURE_ACCOUNT_KEY"}) { + auto properties = native_azure_shared_key_properties(); + properties.erase(missing); + S3Conf conf; + EXPECT_FALSE(S3ClientFactory::convert_properties_to_s3_conf(properties, uri, &conf).ok()) + << missing; + } + for (const auto* type : {"", "SharedKey", "OAuth", "MANAGED_IDENTITY", "secret-as-type"}) { + auto properties = native_azure_shared_key_properties(); + properties["AZURE_AUTH_TYPE"] = type; + S3Conf conf; + auto status = S3ClientFactory::convert_properties_to_s3_conf(properties, uri, &conf); + EXPECT_FALSE(status.ok()); + EXPECT_EQ(status.to_string().find("secret-as-type"), std::string::npos); + } + auto properties = native_azure_shared_key_properties(); + properties["provider"] = "s3"; + S3Conf conf; + EXPECT_FALSE(S3ClientFactory::convert_properties_to_s3_conf(properties, uri, &conf).ok()); +} + +TEST_F(S3ClientFactoryTest, RejectsNativeAzureLocationConflicts) { + auto properties = native_azure_shared_key_properties(); + properties.erase("AZURE_ACCOUNT_KEY"); + properties["AZURE_AUTH_TYPE"] = "SAS"; + properties["AZURE_SAS_TOKEN"] = "sig=temporary"; + for (const auto* location : + {"s3://container/path/file", "abfss://container@other.dfs.core.windows.net/path/file", + "abfss://container@account.dfs.core.chinacloudapi.cn/path/file", + "http://account.blob.core.windows.net/container/path/file", + "https://unrelated.example.com/container/path/file"}) { + S3URI uri(location); + ASSERT_TRUE(uri.parse().ok()); + S3Conf conf; + EXPECT_FALSE(S3ClientFactory::convert_properties_to_s3_conf(properties, uri, &conf).ok()) + << location; + } + S3URI uri("abfss://container@account.dfs.core.windows.net/path/file"); + ASSERT_TRUE(uri.parse().ok()); + for (const auto& [key, value] : std::map { + {"AZURE_ACCOUNT_NAME", "another-account"}, + {"AZURE_CONTAINER", "another-container"}, + {"AZURE_ENDPOINT", "https://account.blob.core.windows.net?sig=do-not-log"}}) { + auto conflicting = properties; + conflicting[key] = value; + S3Conf conf; + auto status = S3ClientFactory::convert_properties_to_s3_conf(conflicting, uri, &conf); + EXPECT_FALSE(status.ok()); + EXPECT_EQ(status.to_string().find("do-not-log"), std::string::npos); + } +} + +TEST_F(S3ClientFactoryTest, AllowsAbfsUriWithCustomAzureTransportEndpoint) { + auto properties = native_azure_shared_key_properties(); + properties["AZURE_ENDPOINT"] = "https://proxy.example.test:8443/base"; + S3URI uri("abfss://container@account.dfs.core.windows.net/path/file"); + ASSERT_TRUE(uri.parse().ok()); + + S3Conf conf; + ASSERT_TRUE(S3ClientFactory::convert_properties_to_s3_conf(properties, uri, &conf).ok()); + EXPECT_TRUE(S3ClientFactory::validate_azure_uri(uri, conf.client_conf).ok()); +} + +TEST_F(S3ClientFactoryTest, PreservesSchemedAzureEndpointsWithoutDnsSuffix) { + for (const auto* endpoint : {"https://proxy", "http://localhost"}) { + auto properties = native_azure_shared_key_properties(); + properties["AZURE_ENDPOINT"] = endpoint; + S3URI uri("abfss://container@account.dfs.core.windows.net/path/file"); + ASSERT_TRUE(uri.parse().ok()); + + S3Conf conf; + ASSERT_TRUE(S3ClientFactory::convert_properties_to_s3_conf(properties, uri, &conf).ok()) + << endpoint; + EXPECT_EQ(conf.client_conf.endpoint, endpoint); + } +} + +TEST_F(S3ClientFactoryTest, PreservesCustomEndpointsContainingDfsLabels) { + for (const auto* endpoint : + {"https://proxy.dfs.internal", "http://proxy.dfs.internal:10000/base", + "https://account.dfs.core.windows.net.proxy.test:8443"}) { + auto properties = native_azure_shared_key_properties(); + properties["AZURE_ENDPOINT"] = endpoint; + S3URI uri("abfss://container@account.dfs.core.windows.net/path/file"); + ASSERT_TRUE(uri.parse().ok()); + S3Conf conf; + ASSERT_TRUE(S3ClientFactory::convert_properties_to_s3_conf(properties, uri, &conf).ok()) + << endpoint; + EXPECT_EQ(conf.client_conf.endpoint, endpoint); + } +} + +TEST_F(S3ClientFactoryTest, NormalizesOfficialDfsEndpointsWithExplicitPorts) { + for (const auto* suffix : {"core.windows.net", "core.chinacloudapi.cn", + "core.usgovcloudapi.net", "core.cloudapi.de"}) { + auto properties = native_azure_shared_key_properties(); + properties["AZURE_ENDPOINT"] = + fmt::format("https://account.dfs.{}:8443/base%2Fpath", suffix); + S3URI uri(fmt::format("https://account.dfs.{}:8443/container/path/file", suffix)); + ASSERT_TRUE(uri.parse().ok()); + S3Conf conf; + auto status = S3ClientFactory::convert_properties_to_s3_conf(properties, uri, &conf); + ASSERT_TRUE(status.ok()) << status; + EXPECT_EQ(conf.client_conf.endpoint, + fmt::format("https://account.blob.{}:8443/base%2Fpath", suffix)); + } +} + +TEST_F(S3ClientFactoryTest, NativeAzureRejectsMixedAndAliasedCredentials) { + S3URI uri("abfss://container@account.dfs.core.windows.net/path/file"); + ASSERT_TRUE(uri.parse().ok()); + for (const auto& [key, value] : + std::map {{"AZURE_SAS_TOKEN", "sig=do-not-log"}, + {"AZURE_CLIENT_SECRET", "do-not-log"}, + {"AWS_SECRET_KEY", "do-not-log"}, + {"azure.account_key", "do-not-log"}}) { + auto properties = native_azure_shared_key_properties(); + properties[key] = value; + S3Conf conf; + auto status = S3ClientFactory::convert_properties_to_s3_conf(properties, uri, &conf); + EXPECT_FALSE(status.ok()) << key; + EXPECT_EQ(status.to_string().find("do-not-log"), std::string::npos); + } + auto properties = native_azure_shared_key_properties(); + properties["AZURE_AUTH_TYPE"] = "OAUTH2"; + properties.erase("AZURE_ACCOUNT_KEY"); + properties["AZURE_CLIENT_ID"] = "client"; + properties["AZURE_OAUTH_SERVER_URI"] = "https://login.microsoftonline.com/tenant/oauth2/token"; + S3Conf conf; + EXPECT_FALSE(S3ClientFactory::convert_properties_to_s3_conf(properties, uri, &conf).ok()); +} + +TEST_F(S3ClientFactoryTest, AzureClientIdentityExcludesAwsCredentialsAndSigningSettings) { + S3ClientConf first; + first.endpoint = "https://account.blob.core.windows.net"; + first.bucket = "container"; + first.provider = ObjStorageProvider::AZURE; + first.azure_credentials.account_name = "account"; + first.azure_credentials.account_key = "key"; + auto second = first; + second.ak = "unrelated-aws-access-key"; + second.sk = "unrelated-aws-secret-key"; + second.token = "unrelated-aws-token"; + second.region = "unrelated-aws-region"; + second.role_arn = "unrelated-aws-role"; + second.use_virtual_addressing = false; + second.need_override_endpoint = false; + EXPECT_EQ(first, second); + EXPECT_EQ(first.get_hash(), second.get_hash()); + second.azure_credentials.account_key = "rotated-key"; + EXPECT_NE(first, second); + EXPECT_NE(first.get_hash(), second.get_hash()); + EXPECT_EQ(second.to_string().find("rotated-key"), std::string::npos); + first.azure_credentials = {}; + first.azure_credentials.type = AzureCredentialType::SAS; + first.azure_credentials.account_name = "account"; + first.azure_credentials.sas_token = "sig=first"; + second = first; + second.azure_credentials.sas_token = "sig=second"; + EXPECT_NE(first, second); + EXPECT_NE(first.get_hash(), second.get_hash()); + EXPECT_EQ(second.to_string().find("sig=second"), std::string::npos); + first.azure_credentials = {}; + first.azure_credentials.type = AzureCredentialType::OAUTH2; + first.azure_credentials.account_name = "account"; + first.azure_credentials.oauth_client_id = "client"; + first.azure_credentials.oauth_client_secret = "first-secret"; + first.azure_credentials.oauth_tenant_id = "tenant"; + first.azure_credentials.oauth_server_uri = + "https://login.microsoftonline.com/tenant/oauth2/token"; + second = first; + second.azure_credentials.oauth_client_secret = "second-secret"; + EXPECT_NE(first, second); + EXPECT_NE(first.get_hash(), second.get_hash()); + EXPECT_EQ(second.to_string().find("second-secret"), std::string::npos); +} + +TEST_F(S3ClientFactoryTest, NativeAzureRejectsExpiredSasBeforeClientCreation) { + S3ClientConf conf; + conf.provider = ObjStorageProvider::AZURE; + conf.endpoint = "https://account.blob.core.windows.net"; + conf.bucket = "container"; + conf.azure_credentials.type = AzureCredentialType::SAS; + conf.azure_credentials.account_name = "account"; + conf.azure_credentials.sas_token = "sig=do-not-log"; + conf.azure_credentials.sas_expiration_time_ms = 1; + int creates = 0; + S3ClientFactory::instance().set_client_creator_for_test( + [&](const S3ClientConf&) -> std::shared_ptr { + ++creates; + return {}; + }); + auto result = S3ClientFactory::instance().create(conf); + ASSERT_FALSE(result.has_value()); + EXPECT_EQ(creates, 0); + EXPECT_NE(result.error().to_string().find("expired"), std::string::npos); + EXPECT_EQ(result.error().to_string().find("do-not-log"), std::string::npos); +#ifdef USE_AZURE + conf.azure_credentials.sas_expiration_time_ms = 0; + conf.azure_credentials.sas_token = "se=2000-01-01T00:00:00Z&sig=do-not-log"; + result = S3ClientFactory::instance().create(conf); + ASSERT_FALSE(result.has_value()); + EXPECT_EQ(creates, 0); + EXPECT_NE(result.error().to_string().find("expired"), std::string::npos); + EXPECT_EQ(result.error().to_string().find("do-not-log"), std::string::npos); +#endif +} + +TEST_F(S3ClientFactoryTest, ObjClientHolderResetReplacesAzureCredentialGroup) { + S3ClientConf conf; + conf.endpoint = "https://account.blob.core.windows.net"; + conf.bucket = "container"; + conf.provider = ObjStorageProvider::AZURE; + conf.azure_credentials.account_name = "account"; + conf.azure_credentials.account_key = "key"; + int create_count = 0; + S3ClientFactory::instance().set_client_creator_for_test( + [&](const S3ClientConf&) -> std::shared_ptr { + ++create_count; + return std::make_shared( + std::shared_ptr {}); + }); + io::ObjClientHolder holder(conf); + ASSERT_TRUE(holder.init().ok()); + auto first_client = holder.get(); + conf.azure_credentials = {}; + conf.azure_credentials.type = AzureCredentialType::SAS; + conf.azure_credentials.account_name = "account"; + conf.azure_credentials.sas_token = "sig=temporary"; + ASSERT_TRUE(holder.reset(conf).ok()); + EXPECT_EQ(create_count, 2); + EXPECT_NE(holder.get(), first_client); + EXPECT_EQ(holder.s3_client_conf().azure_credentials, conf.azure_credentials); + EXPECT_TRUE(holder.s3_client_conf().azure_credentials.account_key.empty()); + EXPECT_TRUE(holder.s3_client_conf().sk.empty()); +} + TEST_F(S3ClientFactoryTest, AwsCredentialsProviderV2ProviderTypeWithoutRoleArn) { S3ClientFactory& factory = S3ClientFactory::instance(); config::aws_credentials_provider_version = "v2"; diff --git a/be/test/storage/rowset/beta_rowset_test.cpp b/be/test/storage/rowset/beta_rowset_test.cpp index 52ca6c1be89d9d..d093e74eefd3f1 100644 --- a/be/test/storage/rowset/beta_rowset_test.cpp +++ b/be/test/storage/rowset/beta_rowset_test.cpp @@ -310,6 +310,7 @@ TEST_F(BetaRowsetTest, ReadTest) { .ak = "ak", .sk = "sk", .token = "", + .azure_credentials = {}, .bucket = "", .role_arn = "", .external_id = "", diff --git a/be/test/storage/storage_resource_test.cpp b/be/test/storage/storage_resource_test.cpp index e0ebdc7d75d314..f0c427bc91fd96 100644 --- a/be/test/storage/storage_resource_test.cpp +++ b/be/test/storage/storage_resource_test.cpp @@ -34,6 +34,7 @@ TEST(StorageResourceTest, RemotePath) { .ak = "ak", .sk = "sk", .token = "", + .azure_credentials = {}, .bucket = "", .role_arn = "", .external_id = "", diff --git a/be/test/util/s3_uri_test.cpp b/be/test/util/s3_uri_test.cpp index 93aa2c72eb809d..724c0455b971f8 100644 --- a/be/test/util/s3_uri_test.cpp +++ b/be/test/util/s3_uri_test.cpp @@ -70,6 +70,155 @@ TEST_F(S3URITest, HttpURI) { EXPECT_EQ("path/to/file", uri2.get_key()); } +TEST_F(S3URITest, AzureDataLakeURI) { + S3URI uri("abfss://container@account.dfs.core.windows.net/path/to/file.parquet"); + ASSERT_TRUE(uri.parse().ok()); + EXPECT_EQ("container", uri.get_bucket()); + EXPECT_EQ("path/to/file.parquet", uri.get_key()); + EXPECT_EQ("account.dfs.core.windows.net", uri.get_endpoint()); + EXPECT_EQ("account", uri.get_account()); + EXPECT_TRUE(uri.is_azure()); + + S3URI uppercase("ABFSS://container@account.dfs.core.windows.net/path/file.parquet"); + ASSERT_TRUE(uppercase.parse().ok()); + EXPECT_EQ("container", uppercase.get_bucket()); + EXPECT_EQ("path/file.parquet", uppercase.get_key()); + EXPECT_TRUE(uppercase.is_azure()); + + S3URI wasbs("wasbs://container@account.blob.core.windows.net/path/to/file.parquet"); + ASSERT_TRUE(wasbs.parse().ok()); + EXPECT_EQ("container", wasbs.get_bucket()); + EXPECT_EQ("path/to/file.parquet", wasbs.get_key()); + EXPECT_EQ("account.blob.core.windows.net", wasbs.get_endpoint()); + EXPECT_EQ("account", wasbs.get_account()); + EXPECT_TRUE(wasbs.is_azure()); + + S3URI https("https://account.blob.core.windows.net/container/path/to/file.parquet"); + ASSERT_TRUE(https.parse().ok()); + EXPECT_EQ("container", https.get_bucket()); + EXPECT_EQ("path/to/file.parquet", https.get_key()); + EXPECT_EQ("account.blob.core.windows.net", https.get_endpoint()); + EXPECT_EQ("account", https.get_account()); + EXPECT_TRUE(https.is_azure()); +} + +TEST_F(S3URITest, InvalidAzureDataLakeAuthority) { + S3URI missing_container("abfss://@account.dfs.core.windows.net/path/file.parquet"); + EXPECT_FALSE(missing_container.parse().ok()); + + S3URI missing_account("abfss://container@/path/file.parquet"); + EXPECT_FALSE(missing_account.parse().ok()); + + S3URI duplicate_separator( + "abfss://container@account@other.dfs.core.windows.net/path/file.parquet"); + EXPECT_FALSE(duplicate_separator.parse().ok()); +} + +TEST_F(S3URITest, AzureParsingErrorsDoNotEchoLocations) { + for (const auto* location : + {"abfss:///hidden-object?sig=sentinel", + "abfss://@account.dfs.core.windows.net/hidden-object?sig=sentinel", + "abfss://container@/hidden-object?sig=sentinel", + "WASBS://container@account@other.blob.core.windows.net/hidden-object?sig=sentinel", + "https://account.blob.core.windows.net//hidden-object?sig=sentinel"}) { + S3URI uri(location); + auto status = uri.parse(); + EXPECT_FALSE(status.ok()); + EXPECT_EQ(status.to_string().find("sentinel"), std::string::npos); + EXPECT_EQ(status.to_string().find("hidden-object"), std::string::npos); + EXPECT_EQ(status.to_string().find("://"), std::string::npos); + } +} + +TEST_F(S3URITest, ParsingErrorsRedactQueriesBeforeProviderSelection) { + for (const auto* location : {"https://custom.example.com//hidden-object?sig=sentinel", + "s3:///hidden-object?sig=sentinel#fragment-sentinel", + "unknown://host/hidden-object#fragment-sentinel"}) { + S3URI uri(location); + auto status = uri.parse(); + EXPECT_FALSE(status.ok()); + EXPECT_EQ(status.to_string().find("sentinel"), std::string::npos); + EXPECT_NE(status.to_string().find("hidden-object"), std::string::npos); + } + + S3URI custom("https://custom.example.com//hidden-object?sig=sentinel"); + auto status = custom.parse(true); + EXPECT_FALSE(status.ok()); + EXPECT_EQ(status.to_string().find("sentinel"), std::string::npos); + EXPECT_EQ(status.to_string().find("hidden-object"), std::string::npos); +} + +TEST_F(S3URITest, SchemeSplitPreservesSeparatorsInObjectNames) { + for (const auto* location : + {"s3://container/path/http://example//file", + "abfss://container@account.dfs.core.windows.net/path/http://example//file", + "https://account.blob.core.windows.net/container/path/http://example//file"}) { + S3URI uri(location); + ASSERT_TRUE(uri.parse().ok()); + EXPECT_EQ(uri.get_key(), "path/http://example//file"); + EXPECT_EQ(uri.get_bucket(), "container"); + } +} + +TEST_F(S3URITest, IcebergAdlsPathsPreserveLiteralPercentSequences) { + for (const auto* scheme : {"abfs", "abfss", "wasb", "wasbs"}) { + for (const auto* key : + {"data/p=a%2Fb/file.parquet", "path/a%20b+%2520/http://example//file", + "path/100%/%2/%GG", "path/a b "}) { + const std::string service = + std::string_view(scheme).starts_with("wasb") ? "blob" : "dfs"; + S3URI uri(std::string(scheme) + "://container@account." + service + + ".core.windows.net/" + key); + ASSERT_TRUE(uri.parse().ok()); + EXPECT_EQ(uri.get_key(), key); + ASSERT_TRUE(uri.parse(true).ok()); + EXPECT_EQ(uri.get_key(), key); + } + } +} + +TEST_F(S3URITest, AzureHttpPercentDecodingOccursExactlyOnceAndPreservesPlus) { + S3URI uri("https://account.blob.core.windows.net/container/a%20b+%2520//c%3Ad%2Fe%3Ff%23g"); + ASSERT_TRUE(uri.parse().ok()); + EXPECT_EQ(uri.get_key(), "a b+%20//c:d/e?f#g"); + ASSERT_TRUE(uri.parse(true).ok()); + EXPECT_EQ(uri.get_key(), "a b+%20//c:d/e?f#g"); + + S3URI custom("https://custom.example.com/container/a%20b+%2520"); + ASSERT_TRUE(custom.parse().ok()); + EXPECT_FALSE(custom.is_azure()); + EXPECT_EQ(custom.get_key(), "a%20b+%2520"); + ASSERT_TRUE(custom.parse(true).ok()); + EXPECT_EQ(custom.get_key(), "a b+%20"); + + S3URI raw("a%20b+%2520"); + ASSERT_TRUE(raw.parse(true).ok()); + EXPECT_EQ(raw.get_key(), "a%20b+%2520"); + S3URI legacy("s3://container/a%20b+%2520"); + ASSERT_TRUE(legacy.parse(true).ok()); + EXPECT_EQ(legacy.get_key(), "a%20b+%2520"); +} + +TEST_F(S3URITest, RejectsInvalidAzureHttpPercentEncodingWithoutEchoingInput) { + for (const auto* key : {"path%", "path%1", "path%1g", "path%gg"}) { + S3URI uri(std::string("https://account.blob.core.windows.net/container/") + key + + "?sig=secret"); + auto status = uri.parse(); + EXPECT_FALSE(status.ok()); + EXPECT_EQ(status.to_string().find("sig=secret"), std::string::npos); + } +} + +TEST_F(S3URITest, AzureHostRecognitionDoesNotClaimUnrelatedHttpDomains) { + EXPECT_TRUE(S3URI::is_azure_endpoint("ACCOUNT.blob.core.windows.net:443")); + EXPECT_TRUE(S3URI::is_azure_endpoint("account.dfs.core.chinacloudapi.cn")); + EXPECT_TRUE(S3URI::is_azure_endpoint("account.blob.core.usgovcloudapi.net")); + EXPECT_TRUE(S3URI::is_azure_endpoint("account.blob.core.cloudapi.de")); + EXPECT_FALSE(S3URI::is_azure_endpoint("service.blob.example.com")); + EXPECT_FALSE(S3URI::is_azure_endpoint("account.blob.core.windows.net.example.com")); + EXPECT_FALSE(S3URI::is_azure_endpoint("onelake.dfs.fabric.microsoft.com")); +} + TEST_F(S3URITest, InvalidSchema) { std::string p1 = "xxx://a.b.com/bucket/path/to/file"; S3URI uri1(p1); diff --git a/common/cpp/obj-client/auth/azure_auth_factory.cpp b/common/cpp/obj-client/auth/azure_auth_factory.cpp index 3e30546a5529db..040ae538f6adde 100644 --- a/common/cpp/obj-client/auth/azure_auth_factory.cpp +++ b/common/cpp/obj-client/auth/azure_auth_factory.cpp @@ -17,25 +17,230 @@ #include "azure_auth_factory.h" +#include +#include +#include #include +#include +#include +#include +#include namespace doris { +namespace { + +std::string normalize_sas_token(std::string token) { + while (!token.empty() && (token.front() == '?' || token.front() == '&')) { + token.erase(token.begin()); + } + return token; +} + +bool contains_line_break(std::string_view value) { + return value.find('\r') != std::string_view::npos || value.find('\n') != std::string_view::npos; +} + +int64_t unix_millis_now() { + return std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()) + .count(); +} + +std::optional sas_expiry_from_token(std::string_view token, std::string* error) { + std::optional result; + bool has_signature = false; + size_t begin = 0; + while (begin <= token.size()) { + const auto end = token.find('&', begin); + const auto field = token.substr( + begin, end == std::string_view::npos ? token.size() - begin : end - begin); + const auto separator = field.find('='); + if (separator == std::string_view::npos || separator == 0) { + *error = "Azure SAS credential has an invalid query field"; + return std::nullopt; + } + if (field.starts_with("sig=")) { + if (has_signature || field.size() == 4) { + *error = "Azure SAS credential has an invalid signature field"; + return std::nullopt; + } + has_signature = true; + } else if (field.starts_with("se=")) { + if (result.has_value()) { + *error = "Azure SAS credential has duplicate expiry fields"; + return std::nullopt; + } + const auto encoded_expiry = field.substr(3); + if (encoded_expiry.empty()) { + *error = "Azure SAS credential has an empty expiry"; + return std::nullopt; + } + try { + const auto expiry = Azure::DateTime::Parse( + Azure::Core::Url::Decode(std::string(encoded_expiry)), + Azure::DateTime::DateFormat::Rfc3339); + const auto system_time = static_cast(expiry); + result = std::chrono::duration_cast( + system_time.time_since_epoch()) + .count(); + } catch (const std::exception&) { + *error = "Azure SAS credential has an invalid expiry"; + return std::nullopt; + } + } + if (end == std::string_view::npos) { + break; + } + begin = end + 1; + } + if (!has_signature) { + *error = "Azure SAS credential requires a signature field"; + } + return result; +} + +std::string oauth_tenant_from_uri(std::string_view oauth_server_uri) { + Azure::Core::Url url {std::string(oauth_server_uri)}; + const auto path = Azure::Core::Url::Decode(url.GetPath()); + size_t begin = 0; + while (begin < path.size() && path[begin] == '/') { + ++begin; + } + while (begin < path.size()) { + const auto end = path.find('/', begin); + const auto part = + path.substr(begin, end == std::string::npos ? path.size() - begin : end - begin); + if (!part.empty() && part != "oauth2" && part != "v2.0" && part != "token") { + return part; + } + if (end == std::string::npos) { + break; + } + begin = end + 1; + } + return {}; +} + +std::string oauth_authority_from_uri(std::string_view oauth_server_uri) { + Azure::Core::Url url {std::string(oauth_server_uri)}; + if (url.GetScheme() != "https" || url.GetHost().empty()) { + return {}; + } + std::string authority = url.GetScheme() + "://" + url.GetHost(); + if (url.GetPort() != 0) { + authority += ":" + std::to_string(url.GetPort()); + } + return authority; +} + +} // namespace + +std::string AzureAuthFactory::validate(const AzureCredentialOptions& credential) { + int64_t effective_expiry_ms = 0; + return validate(credential, unix_millis_now(), &effective_expiry_ms); +} + +std::string AzureAuthFactory::validate(const AzureCredentialOptions& credential, int64_t now_ms, + int64_t* effective_expiry_ms) { + *effective_expiry_ms = 0; + if (auto error = credential.validate(); !error.empty()) { + return error; + } + if (credential.type != AzureCredentialType::SAS) { + return {}; + } + const auto token = normalize_sas_token(credential.sas_token); + if (token.empty()) { + return "Azure SAS credential requires a non-empty token"; + } + if (contains_line_break(token)) { + return "Azure SAS token contains a line break"; + } + std::string error; + const auto token_expiry = sas_expiry_from_token(token, &error); + if (!error.empty()) { + return error; + } + *effective_expiry_ms = credential.sas_expiration_time_ms; + if (token_expiry.has_value() && + (*effective_expiry_ms == 0 || *token_expiry < *effective_expiry_ms)) { + *effective_expiry_ms = *token_expiry; + } + if ((token_expiry.has_value() && *token_expiry <= now_ms) || + (*effective_expiry_ms > 0 && *effective_expiry_ms <= now_ms)) { + return "Azure SAS credential is expired"; + } + return {}; +} + AzureClientBuildResult AzureAuthFactory::create( std::string_view container_url, const AzureCredentialOptions& credential, Azure::Storage::Blobs::BlobClientOptions client_options) { + if (auto error = validate(credential); !error.empty()) { + return {.error = std::move(error)}; + } + if (credential.type == AzureCredentialType::OAUTH2) { + try { + std::string tenant_id = credential.oauth_tenant_id; + if (tenant_id.empty()) { + tenant_id = oauth_tenant_from_uri(credential.oauth_server_uri); + } + if (tenant_id.empty()) { + return {.error = "Azure OAuth2 credential requires a tenant id"}; + } + Azure::Identity::ClientSecretCredentialOptions identity_options; + // Identity owns a separate HTTP pipeline. Reuse the configured transport so token + // requests honor the same custom CA/proxy settings as Blob requests; do not copy + // storage-specific policies into the token pipeline. + identity_options.Transport = client_options.Transport; + identity_options.AuthorityHost = oauth_authority_from_uri(credential.oauth_server_uri); + if (identity_options.AuthorityHost.empty()) { + return {.error = "Azure OAuth2 credential has an invalid OAuth server URI"}; + } + auto token_credential = std::make_shared( + std::move(tenant_id), credential.oauth_client_id, + credential.oauth_client_secret, std::move(identity_options)); + std::shared_ptr credential_view = + std::move(token_credential); + auto client = std::make_shared( + std::string(container_url), std::move(credential_view), + std::move(client_options)); + return {.container_client = std::move(client)}; + } catch (const std::exception&) { + return {.error = "failed to create Azure OAuth2 credential"}; + } + } + + if (credential.type == AzureCredentialType::SAS) { + auto token = normalize_sas_token(credential.sas_token); + std::string sas_url(container_url); + sas_url += sas_url.find('?') == std::string::npos ? '?' : '&'; + sas_url += token; + try { + auto client = std::make_shared( + std::move(sas_url), std::move(client_options)); + return {.container_client = std::move(client)}; + } catch (const std::exception&) { + // SDK URL parse exceptions may contain the complete signed URL. + return {.error = "failed to create Azure SAS client"}; + } + } + if (credential.type != AzureCredentialType::SHARED_KEY) { return {.error = "unsupported Azure credential type"}; } - auto shared_key = std::make_shared( - credential.account_name, credential.account_key); - auto client = std::make_shared( - std::string(container_url), shared_key, std::move(client_options)); - return { - .container_client = std::move(client), - .shared_key_credential = std::move(shared_key), - }; + try { + auto shared_key = std::make_shared( + credential.account_name, credential.account_key); + auto client = std::make_shared( + std::string(container_url), shared_key, std::move(client_options)); + return {.container_client = std::move(client), + .shared_key_credential = std::move(shared_key)}; + } catch (const std::exception&) { + return {.error = "failed to create Azure SharedKey client"}; + } } } // namespace doris diff --git a/common/cpp/obj-client/auth/azure_auth_factory.h b/common/cpp/obj-client/auth/azure_auth_factory.h index f41e6b8aefc198..44fdcb3cce11f9 100644 --- a/common/cpp/obj-client/auth/azure_auth_factory.h +++ b/common/cpp/obj-client/auth/azure_auth_factory.h @@ -22,22 +22,14 @@ #include #include +#include "azure_credential_options.h" + namespace Azure::Storage { class StorageSharedKeyCredential; } namespace doris { -enum class AzureCredentialType { - SHARED_KEY, -}; - -struct AzureCredentialOptions { - AzureCredentialType type = AzureCredentialType::SHARED_KEY; - std::string account_name; - std::string account_key; -}; - struct AzureClientBuildResult { std::shared_ptr container_client {}; std::shared_ptr shared_key_credential {}; @@ -48,6 +40,16 @@ struct AzureClientBuildResult { class AzureAuthFactory { public: + // Called before cache lookup as well as SDK construction. Does not access + // the network or refresh a SAS credential. + static std::string validate(const AzureCredentialOptions& credential); + + // Returns the earliest known SAS expiry (zero when unknown) without changing + // the token or inventing a lifetime. The caller supplies its access timestamp + // so cache lookup and SDK construction use the same expiry rule. + static std::string validate(const AzureCredentialOptions& credential, int64_t now_ms, + int64_t* effective_expiry_ms); + static AzureClientBuildResult create(std::string_view container_url, const AzureCredentialOptions& credential, Azure::Storage::Blobs::BlobClientOptions client_options); diff --git a/common/cpp/obj-client/auth/azure_credential_options.h b/common/cpp/obj-client/auth/azure_credential_options.h new file mode 100644 index 00000000000000..c3daff713ef9c2 --- /dev/null +++ b/common/cpp/obj-client/auth/azure_credential_options.h @@ -0,0 +1,89 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +namespace doris { + +// This protocol model must remain independent of the Azure SDK: S3 client +// configuration is also used in builds where BUILD_AZURE is disabled. +enum class AzureCredentialType { + SHARED_KEY, + SAS, + OAUTH2, +}; + +struct AzureCredentialOptions { + AzureCredentialType type = AzureCredentialType::SHARED_KEY; + std::string account_name; + std::string account_key; + std::string sas_token; + int64_t sas_expiration_time_ms = 0; + std::string oauth_client_id; + std::string oauth_client_secret; + std::string oauth_tenant_id; + std::string oauth_server_uri; + + bool operator==(const AzureCredentialOptions&) const = default; + + // Validate the selected credential group, never infer or fall back to + // another authentication mode. SAS expiry is checked at client access. + std::string validate() const { + const bool has_oauth = !oauth_client_id.empty() || !oauth_client_secret.empty() || + !oauth_tenant_id.empty() || !oauth_server_uri.empty(); + switch (type) { + case AzureCredentialType::SHARED_KEY: + if (!sas_token.empty() || sas_expiration_time_ms != 0 || has_oauth) { + return "Azure SharedKey cannot be combined with SAS or OAuth2 credentials"; + } + // Retain the legacy empty-pair behavior for storage vault callers; + // the canonical native protocol separately requires both fields. + if (account_name.empty() != account_key.empty()) { + return "Azure SharedKey requires both account name and account key"; + } + break; + case AzureCredentialType::SAS: + if (sas_token.empty()) { + return "Azure SAS credential requires a non-empty token"; + } + if (!account_key.empty() || has_oauth) { + return "Azure SAS cannot be combined with SharedKey or OAuth2 credentials"; + } + if (sas_expiration_time_ms < 0) { + return "Azure SAS credential has an invalid expiry"; + } + break; + case AzureCredentialType::OAUTH2: + if (!account_key.empty() || !sas_token.empty() || sas_expiration_time_ms != 0) { + return "Azure OAuth2 cannot be combined with SharedKey or SAS credentials"; + } + if (oauth_client_id.empty() || oauth_client_secret.empty() || + oauth_server_uri.empty()) { + return "Azure OAuth2 requires client id, client secret, and OAuth server URI"; + } + break; + default: + return "unsupported Azure credential type"; + } + return {}; + } +}; + +} // namespace doris diff --git a/common/cpp/obj-client/azure_obj_storage_client.cpp b/common/cpp/obj-client/azure_obj_storage_client.cpp index 7e2f02f243bd3a..22c346fc209c9e 100644 --- a/common/cpp/obj-client/azure_obj_storage_client.cpp +++ b/common/cpp/obj-client/azure_obj_storage_client.cpp @@ -19,6 +19,7 @@ #include +#include #include #include #include @@ -30,8 +31,12 @@ using namespace Azure::Storage::Blobs; namespace { std::string wrap_object_storage_path_msg(const doris::ObjStoragePath& opts) { + std::string path = opts.path.native(); + if (const auto query = path.find('?'); query != std::string::npos) { + path.resize(query); + } return fmt::format("bucket {}, key {}, prefix {}, path {}", opts.bucket, opts.key, opts.prefix, - opts.path.native()); + path); } doris::ObjStoragePath with_object_key(const doris::ObjStoragePath& opts, std::string_view key) { @@ -42,11 +47,19 @@ doris::ObjStoragePath with_object_key(const doris::ObjStoragePath& opts, std::st std::string to_lower_ascii(std::string_view input) { std::string lowered(input); - std::transform(lowered.begin(), lowered.end(), lowered.begin(), - [](unsigned char ch) { return static_cast(std::tolower(ch)); }); + std::ranges::transform(lowered, lowered.begin(), + [](unsigned char ch) { return static_cast(std::tolower(ch)); }); return lowered; } +std::string redact_azure_url(std::string_view url) { + const auto query = url.find('?'); + if (query == std::string_view::npos) { + return std::string(url); + } + return fmt::format("{}?", url.substr(0, query)); +} + std::string encode_azure_block_id(std::string_view upload_id, int part_num) { // Azure has no multipart upload namespace. Include the writer UUID in every block ID so // concurrent writers for the same key cannot stage interchangeable blocks. @@ -59,7 +72,6 @@ std::string encode_azure_block_id(std::string_view upload_id, int part_num) { return Aws::Utils::HashingUtils::Base64Encode(bytes); } -constexpr char SAS_TOKEN_URL_TEMPLATE[] = "{}/{}/{}{}"; constexpr char BlobNotFound[] = "BlobNotFound"; } // namespace @@ -99,8 +111,8 @@ std::string build_azure_tls_debug_suffix(std::string_view error_message, static ObjStorageResponse make_azure_std_exception_response(const std::exception& e, const ObjStoragePath& opts, std::string_view tls_debug_context) { - auto msg = fmt::format("Azure request failed because {}, path msg {}{}", e.what(), - wrap_object_storage_path_msg(opts), + auto msg = fmt::format("Azure request failed because {}, path msg {}{}", + redact_azure_url(e.what()), wrap_object_storage_path_msg(opts), build_azure_tls_debug_suffix(e.what(), tls_debug_context)); LOG(WARNING) << msg; return {.status = ObjStorageStatus {ObjStorageStatus::INTERNAL_ERROR, std::move(msg)}, @@ -117,8 +129,8 @@ ObjStorageResponse do_azure_client_call(Func f, const ObjStoragePath& opts, doris::record_object_request_failed(static_cast(e.StatusCode)); auto msg = fmt::format( "Azure request failed because {}, error msg {}, http code {}, path msg {}{}", - e.what(), e.Message, static_cast(e.StatusCode), - wrap_object_storage_path_msg(opts), + redact_azure_url(e.what()), redact_azure_url(e.Message), + static_cast(e.StatusCode), wrap_object_storage_path_msg(opts), build_azure_tls_debug_suffix(fmt::format("{} {}", e.what(), e.Message), tls_debug_context)); LOG(WARNING) << msg; @@ -179,7 +191,8 @@ struct AzureBatchDeleter { auto msg = fmt::format( "Azure request failed because {}, error msg {}, http code {}, path msg " "{}{}", - e.what(), e.Message, static_cast(e.StatusCode), + redact_azure_url(e.what()), redact_azure_url(e.Message), + static_cast(e.StatusCode), wrap_object_storage_path_msg(with_object_key(_opts, deferred.key)), build_azure_tls_debug_suffix(fmt::format("{} {}", e.what(), e.Message), _tls_debug_context)); @@ -250,8 +263,9 @@ ObjStorageUploadResult AzureObjStorageClient::upload_part(const ObjStoragePath& fmt::format("{} {}", e.what(), e.Message), _config.tls_debug_context); auto msg = fmt::format( "Azure request failed because {}, error msg {}, http code {}, path msg {}{}", - e.what(), e.Message, static_cast(e.StatusCode), - wrap_object_storage_path_msg(opts), tls_debug_suffix); + redact_azure_url(e.what()), redact_azure_url(e.Message), + static_cast(e.StatusCode), wrap_object_storage_path_msg(opts), + tls_debug_suffix); LOG(WARNING) << msg; // clang-format off return { @@ -309,8 +323,9 @@ ObjStorageHeadResult AzureObjStorageClient::head_object(const ObjStoragePath& op fmt::format("{} {}", e.what(), e.Message), _config.tls_debug_context); auto msg = fmt::format( "Azure request failed because {}, error msg {}, http code {}, path msg {}{}", - e.what(), e.Message, static_cast(e.StatusCode), - wrap_object_storage_path_msg(opts), tls_debug_suffix); + redact_azure_url(e.what()), redact_azure_url(e.Message), + static_cast(e.StatusCode), wrap_object_storage_path_msg(opts), + tls_debug_suffix); LOG(WARNING) << msg << ", request_id=" << e.RequestId; return ObjStorageHeadResult { .resp = {.status = obj_storage_status_from_http_code(static_cast(e.StatusCode), @@ -398,19 +413,21 @@ ObjStorageListPageResult AzureObjStorageClient::list_objects_page( auto tls_debug_suffix = build_azure_tls_debug_suffix( fmt::format("{} {}", e.what(), e.Message), _config.tls_debug_context); LOG(WARNING) << fmt::format("Azure request failed because {}, url: {}, prefix: {}{}", - e.what(), _client->GetUrl(), request.Prefix.Value(), - tls_debug_suffix); + redact_azure_url(e.what()), redact_azure_url(_client->GetUrl()), + request.Prefix.Value(), tls_debug_suffix); return { - .resp = {.status = obj_storage_status_from_http_code(static_cast(e.StatusCode), - e.Message + tls_debug_suffix), + .resp = {.status = obj_storage_status_from_http_code( + static_cast(e.StatusCode), + redact_azure_url(e.Message) + tls_debug_suffix), .http_code = static_cast(e.StatusCode), .request_id = std::move(e.RequestId)}, }; } catch (std::exception& e) { LOG(WARNING) << fmt::format("Azure request failed because {}, url: {}, prefix: {}", - e.what(), _client->GetUrl(), request.Prefix.Value()); + redact_azure_url(e.what()), redact_azure_url(_client->GetUrl()), + request.Prefix.Value()); return { - .resp = {.status = {ObjStorageStatus::INTERNAL_ERROR, e.what()}, + .resp = {.status = {ObjStorageStatus::INTERNAL_ERROR, redact_azure_url(e.what())}, .http_code = 0, .request_id = ""}, }; @@ -472,8 +489,9 @@ ObjStorageResponse AzureObjStorageClient::delete_object(const ObjStoragePath& op fmt::format("{} {}", e.what(), e.Message), _config.tls_debug_context); auto msg = fmt::format( "Azure request failed because {}, error msg {}, http code {}, path msg {}{}", - e.what(), e.Message, static_cast(e.StatusCode), - wrap_object_storage_path_msg(opts), tls_debug_suffix); + redact_azure_url(e.what()), redact_azure_url(e.Message), + static_cast(e.StatusCode), wrap_object_storage_path_msg(opts), + tls_debug_suffix); LOG(WARNING) << msg; return { .status = obj_storage_status_from_http_code(static_cast(e.StatusCode), @@ -482,8 +500,8 @@ ObjStorageResponse AzureObjStorageClient::delete_object(const ObjStoragePath& op .request_id = std::move(e.RequestId), }; } catch (std::exception& e) { - auto msg = fmt::format("Azure request failed because {}, path msg {}{}", e.what(), - wrap_object_storage_path_msg(opts), + auto msg = fmt::format("Azure request failed because {}, path msg {}{}", + redact_azure_url(e.what()), wrap_object_storage_path_msg(opts), build_azure_tls_debug_suffix(e.what(), _config.tls_debug_context)); LOG(WARNING) << msg; return { @@ -496,6 +514,14 @@ ObjStorageResponse AzureObjStorageClient::delete_object(const ObjStoragePath& op std::string AzureObjStorageClient::generate_presigned_url(const ObjStoragePath& opts, int64_t expiration_secs) { + // A vended SAS may authorize an entire container. Returning the SDK URL would expose that + // credential without reducing its scope or lifetime. OAuth2 user-delegation SAS signing is + // not implemented. Preserve the string API's failure signal, which callers must not publish. + if (_credential == nullptr) { + LOG(WARNING) << "Azure presigned URL generation requires a SharedKey credential; " + "signing is not supported for SAS or OAuth2 credentials"; + return {}; + } Azure::Storage::Sas::BlobSasBuilder sas_builder; sas_builder.ExpiresOn = std::chrono::system_clock::now() + std::chrono::seconds(expiration_secs); @@ -505,19 +531,11 @@ std::string AzureObjStorageClient::generate_presigned_url(const ObjStoragePath& sas_builder.Protocol = Azure::Storage::Sas::SasProtocol::HttpsOnly; sas_builder.SetPermissions(Azure::Storage::Sas::BlobSasPermissions::Read); - auto credential = _credential; - if (credential == nullptr) { - credential = std::make_shared(_config.ak, - _config.sk); - } - std::string sasToken = sas_builder.GenerateSasToken(*credential); + std::string sasToken = sas_builder.GenerateSasToken(*_credential); - auto sasURL = - fmt::format(SAS_TOKEN_URL_TEMPLATE, _config.endpoint, opts.bucket, opts.key, sasToken); - if (sasURL.find("://") == std::string::npos) { - sasURL = "https://" + sasURL; - } - return sasURL; + // The SDK encodes the raw name exactly as it does for open/HEAD/range reads. Concatenating + // opts.key directly would reinterpret a literal %2F as a slash after signing a different blob. + return _client->GetBlobClient(opts.key).GetUrl() + sasToken; } ObjStorageResponse AzureObjStorageClient::get_lifecycle(const std::string& /*bucket*/, @@ -532,10 +550,12 @@ ObjStorageResponse AzureObjStorageClient::check_versioning(const std::string& /* return ObjStorageResponse::OK(); } -ObjStorageResponse AzureObjStorageClient::abort_multipart_upload(const ObjStoragePath& opts, - const std::string& upload_id) { - // delete uncommitted blobs - // https://learn.microsoft.com/en-us/rest/api/storageservices/delete-blob?tabs=microsoft-entra-id#remarks - return delete_object(opts); +ObjStorageResponse AzureObjStorageClient::abort_multipart_upload(const ObjStoragePath& /*opts*/, + const std::string& /*upload_id*/) { + // Azure has no server-side multipart session to abort. Blocks are isolated by upload UUID; + // deleting their blob would also remove any previously committed data or another writer's + // blocks. Leave uncommitted blocks to service garbage collection instead. This is idempotent + // abandonment, not a claim that the service has immediately reclaimed the staged blocks. + return ObjStorageResponse::OK(); } } // namespace doris From 2d55dadce9b398beb47a9a559c063a0f9ecebbe7 Mon Sep 17 00:00:00 2001 From: xylaaaaa <2392805527@qq.com> Date: Sun, 13 Sep 2026 20:01:21 +0800 Subject: [PATCH 02/10] [feature](be) Route file reads and writes through native Azure clients ### What problem does this PR solve? Related PR: #67470 Problem Summary: Keep FILE_S3 compatibility routing, S3 file reader/writer and file-factory integration together with the affected BE fixtures. This is a history-only regrouping of the native Azure PR. The final seven-commit series preserves the exact code tree of azure-native-before-squash-20260913550a886487e. ### Release note None ### Check List (For Author) - Test: Manual test / Unit Test - Final tree and PR diff are verified identical to the pre-squash snapshot - Existing final-snapshot verification: 711 selected FE tests passed - Tests were not rerun for this history-only rewrite - Behavior changed: No (relative to the pre-squash PR snapshot) - Does this need documentation: No --- be/src/io/file_factory.cpp | 14 +++- be/src/io/file_factory.h | 4 ++ be/src/io/fs/s3_file_reader.cpp | 20 ++++-- be/src/io/fs/s3_file_reader.h | 4 +- be/src/io/fs/s3_file_system.cpp | 72 ++++++++++++------- be/src/io/fs/s3_file_system.h | 11 ++- be/src/io/fs/s3_file_writer.cpp | 15 +++- be/src/io/fs/s3_file_writer.h | 2 +- be/src/runtime/runtime_state.cpp | 7 ++ ...iceberg_delete_file_reader_helper_test.cpp | 31 ++++++++ be/test/io/client/s3_file_system_test.cpp | 7 ++ 11 files changed, 150 insertions(+), 37 deletions(-) diff --git a/be/src/io/file_factory.cpp b/be/src/io/file_factory.cpp index c17742d9e2eb27..cdccdaccff7c8e 100644 --- a/be/src/io/file_factory.cpp +++ b/be/src/io/file_factory.cpp @@ -126,6 +126,10 @@ Result FileFactory::create_fs(const io::FSPropertiesRef& fs_ *fs_properties.properties, io::FileSystem::TMP_FS_ID); } case TFileType::FILE_S3: { + // FILE_S3 is also the compatibility wire value for Azure. The + // provider-owned properties are consumed by S3ClientFactory, which + // dispatches Azure ranges to AzureObjStorageClient; no Hadoop reader + // is involved in this branch. S3URI s3_uri(file_description.path); RETURN_IF_ERROR_RESULT(s3_uri.parse()); S3Conf s3_conf; @@ -195,10 +199,12 @@ Result FileFactory::create_file_writer( S3Conf s3_conf; RETURN_IF_ERROR_RESULT( S3ClientFactory::convert_properties_to_s3_conf(properties, s3_uri, &s3_conf)); + RETURN_IF_ERROR_RESULT( + s3_uri.parse(s3_conf.client_conf.provider == io::ObjStorageProvider::AZURE)); auto client = std::make_shared(std::move(s3_conf.client_conf)); RETURN_IF_ERROR_RESULT(client->init()); return std::make_unique(std::move(client), std::move(s3_conf.bucket), - s3_uri.get_key(), &options); + s3_uri.get_key(), &options, path); } case TFileType::FILE_HDFS: { THdfsParams hdfs_params = parse_properties(properties); @@ -238,16 +244,20 @@ Result FileFactory::_create_file_reader_internal( return file_reader; } case TFileType::FILE_S3: { + // Azure uses this existing wire slot with provider=azure; the object + // client factory selects Azure's native Blob SDK below. S3URI s3_uri(file_description.path); RETURN_IF_ERROR_RESULT(s3_uri.parse()); S3Conf s3_conf; RETURN_IF_ERROR_RESULT(S3ClientFactory::convert_properties_to_s3_conf( system_properties.properties, s3_uri, &s3_conf)); + RETURN_IF_ERROR_RESULT( + s3_uri.parse(s3_conf.client_conf.provider == io::ObjStorageProvider::AZURE)); auto client_holder = std::make_shared(s3_conf.client_conf); RETURN_IF_ERROR_RESULT(client_holder->init()); auto options = with_storage_in_cache_key(reader_options, s3_conf.client_conf.endpoint); return io::S3FileReader::create(std::move(client_holder), s3_conf.bucket, s3_uri.get_key(), - file_description.file_size, profile) + file_description.file_size, profile, file_description.path) .and_then([&options](auto&& reader) { return io::create_cached_file_reader(std::move(reader), options); }); diff --git a/be/src/io/file_factory.h b/be/src/io/file_factory.h index 3ff7f3decaf84c..06f0eb898f3be2 100644 --- a/be/src/io/file_factory.h +++ b/be/src/io/file_factory.h @@ -123,6 +123,10 @@ class FileFactory { case TStorageBackendType::S3: return TFileType::FILE_S3; case TStorageBackendType::AZURE: + // Keep the shipped FILE_S3 wire slot for compatibility. The + // provider field in the storage properties selects the native + // Azure client inside S3ClientFactory; this must never fall back + // to FILE_HDFS for an Azure URI. return TFileType::FILE_S3; case TStorageBackendType::BROKER: return TFileType::FILE_BROKER; diff --git a/be/src/io/fs/s3_file_reader.cpp b/be/src/io/fs/s3_file_reader.cpp index 5151d53b93909f..b30490bced65d0 100644 --- a/be/src/io/fs/s3_file_reader.cpp +++ b/be/src/io/fs/s3_file_reader.cpp @@ -50,6 +50,15 @@ namespace doris::io { +namespace { +std::string sanitize_display_path(std::string path) { + if (const auto query = path.find('?'); query != std::string::npos) { + path.resize(query); + } + return path; +} +} // namespace + bvar::Adder s3_file_reader_read_counter("s3_file_reader", "read_at"); bvar::Adder s3_file_reader_total("s3_file_reader", "total_num"); bvar::Adder s3_bytes_read_total("s3_file_reader", "bytes_read"); @@ -66,7 +75,8 @@ bvar::LatencyRecorder s3_file_reader_latency("s3_file_reader", "s3_latency"); Result S3FileReader::create(std::shared_ptr client, std::string bucket, std::string key, int64_t file_size, - RuntimeProfile* profile) { + RuntimeProfile* profile, std::string display_path) { + RETURN_IF_ERROR_RESULT(client->validate_for_access()); if (file_size < 0) { auto res = client->object_file_size(bucket, key); if (!res.has_value()) { @@ -77,12 +87,14 @@ Result S3FileReader::create(std::shared_ptr(std::move(client), std::move(bucket), std::move(key), - file_size, profile); + file_size, profile, std::move(display_path)); } S3FileReader::S3FileReader(std::shared_ptr client, std::string bucket, - std::string key, size_t file_size, RuntimeProfile* profile) - : _path(fmt::format("s3://{}/{}", bucket, key)), + std::string key, size_t file_size, RuntimeProfile* profile, + std::string display_path) + : _path(display_path.empty() ? fmt::format("s3://{}/{}", bucket, key) + : sanitize_display_path(std::move(display_path))), _file_size(file_size), _bucket(std::move(bucket)), _key(std::move(key)), diff --git a/be/src/io/fs/s3_file_reader.h b/be/src/io/fs/s3_file_reader.h index 40e3ac61d3aca6..376907f5cb0381 100644 --- a/be/src/io/fs/s3_file_reader.h +++ b/be/src/io/fs/s3_file_reader.h @@ -38,10 +38,10 @@ class S3FileReader final : public FileReader { public: static Result create(std::shared_ptr client, std::string bucket, std::string key, int64_t file_size, - RuntimeProfile* profile); + RuntimeProfile* profile, std::string display_path = {}); S3FileReader(std::shared_ptr client, std::string bucket, std::string key, - size_t file_size, RuntimeProfile* profile); + size_t file_size, RuntimeProfile* profile, std::string display_path = {}); ~S3FileReader() override; diff --git a/be/src/io/fs/s3_file_system.cpp b/be/src/io/fs/s3_file_system.cpp index d307335d15a517..7131a17719de82 100644 --- a/be/src/io/fs/s3_file_system.cpp +++ b/be/src/io/fs/s3_file_system.cpp @@ -62,10 +62,14 @@ constexpr int LEN_OF_OSS_PRIVATE_SUFFIX = 9; // length of "-internal" } #endif -Result get_key(const Path& full_path) { - // FIXME(plat1ko): Check bucket in full path and support relative path +Result get_key(const Path& full_path, const S3ClientConf& conf) { + // Preserve the raw-object-key contract for internal callers. Full Azure + // URIs must belong to this holder even when a FileSystem is reused. S3URI uri(full_path.native()); - RETURN_IF_ERROR_RESULT(uri.parse()); + RETURN_IF_ERROR_RESULT(uri.parse(conf.provider == ObjStorageProvider::AZURE)); + if (conf.provider == ObjStorageProvider::AZURE) { + RETURN_IF_ERROR_RESULT(S3ClientFactory::validate_azure_uri(uri, conf)); + } return uri.get_key(); } @@ -88,6 +92,7 @@ Status ObjClientHolder::reset(const S3ClientConf& conf) { reset_conf.ak = conf.ak; reset_conf.sk = conf.sk; reset_conf.token = conf.token; + reset_conf.azure_credentials = conf.azure_credentials; reset_conf.bucket = conf.bucket; reset_conf.connect_timeout_ms = conf.connect_timeout_ms; reset_conf.max_connections = conf.max_connections; @@ -120,6 +125,12 @@ Status ObjClientHolder::reset(const S3ClientConf& conf) { return Status::OK(); } +Status ObjClientHolder::validate_for_access() const { + TEST_SYNC_POINT("ObjClientHolder::validate_for_access:before_lock"); + std::shared_lock lock(_mtx); + return S3ClientFactory::validate_credentials_for_access(_conf); +} + Result ObjClientHolder::object_file_size(const std::string& bucket, const std::string& key) const { auto client = get(); @@ -142,6 +153,7 @@ Result ObjClientHolder::object_file_size(const std::string& bucket, } std::string ObjClientHolder::full_s3_path(std::string_view bucket, std::string_view key) const { + std::shared_lock lock(_mtx); return fmt::format("{}/{}/{}", _conf.endpoint, bucket, key); } @@ -183,18 +195,30 @@ S3FileSystem::~S3FileSystem() = default; Status S3FileSystem::create_file_impl(const Path& file, FileWriterPtr* writer, const FileWriterOptions* opts) { + RETURN_IF_ERROR(_client->validate_for_access()); + const auto conf = _client->s3_client_conf(); auto client = _client->get(); CHECK_S3_CLIENT(client); - auto key = DORIS_TRY(get_key(file)); - *writer = std::make_unique(_client, _bucket, std::move(key), opts); + auto key = DORIS_TRY(get_key(file, conf)); + std::string display_path; + if (file.native().find("://") != std::string::npos) { + display_path = file.native(); + } + *writer = std::make_unique(_client, _bucket, std::move(key), opts, + std::move(display_path)); return Status::OK(); } Status S3FileSystem::open_file_internal(const Path& file, FileReaderSPtr* reader, const FileReaderOptions& opts) { TEST_SYNC_POINT_CALLBACK("S3FileSystem::open_file_internal", &file, &opts); - auto key = DORIS_TRY(get_key(file)); - *reader = DORIS_TRY(S3FileReader::create(_client, _bucket, key, opts.file_size, nullptr)); + auto key = DORIS_TRY(get_key(file, _client->s3_client_conf())); + std::string display_path; + if (file.native().find("://") != std::string::npos) { + display_path = file.native(); + } + *reader = DORIS_TRY(S3FileReader::create(_client, _bucket, key, opts.file_size, nullptr, + std::move(display_path))); return Status::OK(); } @@ -206,7 +230,7 @@ Status S3FileSystem::delete_file_impl(const Path& file) { auto client = _client->get(); CHECK_S3_CLIENT(client); - auto key = DORIS_TRY(get_key(file)); + auto key = DORIS_TRY(get_key(file, _client->s3_client_conf())); auto resp = client->delete_object({.bucket = _bucket, .key = key}); @@ -221,7 +245,7 @@ Status S3FileSystem::delete_directory_impl(const Path& dir) { auto client = _client->get(); CHECK_S3_CLIENT(client); - auto prefix = DORIS_TRY(get_key(dir)); + auto prefix = DORIS_TRY(get_key(dir, _client->s3_client_conf())); if (!prefix.empty() && prefix.back() != '/') { prefix.push_back('/'); } @@ -248,7 +272,7 @@ Status S3FileSystem::batch_delete_impl(const std::vector& remote_files) { auto path_begin = path_iter; for (; path_iter != remote_files.end() && (path_iter - path_begin < max_delete_batch); ++path_iter) { - auto key = DORIS_TRY(get_key(*path_iter)); + auto key = DORIS_TRY(get_key(*path_iter, _client->s3_client_conf())); objects.emplace_back(std::move(key)); } if (objects.empty()) { @@ -267,7 +291,7 @@ Status S3FileSystem::batch_delete_impl(const std::vector& remote_files) { Status S3FileSystem::exists_impl(const Path& path, bool* res) const { auto client = _client->get(); CHECK_S3_CLIENT(client); - auto key = DORIS_TRY(get_key(path)); + auto key = DORIS_TRY(get_key(path, _client->s3_client_conf())); VLOG_DEBUG << "key:" << key << " path:" << path; @@ -286,7 +310,7 @@ Status S3FileSystem::exists_impl(const Path& path, bool* res) const { } Status S3FileSystem::file_size_impl(const Path& file, int64_t* file_size) const { - auto key = DORIS_TRY(get_key(file)); + auto key = DORIS_TRY(get_key(file, _client->s3_client_conf())); *file_size = DORIS_TRY(_client->object_file_size(_bucket, key)); return Status::OK(); } @@ -298,7 +322,7 @@ Status S3FileSystem::list_impl(const Path& dir, bool only_file, std::vectorget(); CHECK_S3_CLIENT(client); - auto prefix = DORIS_TRY(get_key(dir)); + auto prefix = DORIS_TRY(get_key(dir, _client->s3_client_conf())); if (!prefix.empty() && prefix.back() != '/') { prefix.push_back('/'); } @@ -328,10 +352,10 @@ Status S3FileSystem::upload_impl(const Path& local_file, const Path& remote_file auto client = _client->get(); CHECK_S3_CLIENT(client); - auto key = DORIS_TRY(get_key(remote_file)); + auto key = DORIS_TRY(get_key(remote_file, _client->s3_client_conf())); auto start = std::chrono::steady_clock::now(); FileWriterPtr obj_writer; - RETURN_IF_ERROR(create_file_impl(key, &obj_writer, nullptr)); + RETURN_IF_ERROR(create_file_impl(remote_file, &obj_writer, nullptr)); FileReaderSPtr local_reader; RETURN_IF_ERROR(io::global_local_filesystem()->open_file(local_file, &local_reader)); size_t local_buffer_size = config::s3_file_system_local_upload_buffer_size; @@ -372,9 +396,9 @@ Status S3FileSystem::batch_upload_impl(const std::vector& local_files, const auto& local_file = local_files[idx]; const auto& remote_file = remote_files[idx]; auto& obj_writer = obj_writers[idx]; - auto key = DORIS_TRY(get_key(remote_file)); + auto key = DORIS_TRY(get_key(remote_file, _client->s3_client_conf())); LOG(INFO) << "Start to upload " << local_file.native() << " to " << full_s3_path(key); - RETURN_IF_ERROR(create_file_impl(key, &obj_writer, nullptr)); + RETURN_IF_ERROR(create_file_impl(remote_file, &obj_writer, nullptr)); FileReaderSPtr local_reader; RETURN_IF_ERROR(io::global_local_filesystem()->open_file(local_file, &local_reader)); size_t local_buffer_size = config::s3_file_system_local_upload_buffer_size; @@ -417,7 +441,7 @@ Status S3FileSystem::batch_upload_impl(const std::vector& local_files, Status S3FileSystem::download_impl(const Path& remote_file, const Path& local_file) { auto client = _client->get(); CHECK_S3_CLIENT(client); - auto key = DORIS_TRY(get_key(remote_file)); + auto key = DORIS_TRY(get_key(remote_file, _client->s3_client_conf())); int64_t size; RETURN_IF_ERROR(file_size(remote_file, &size)); std::unique_ptr buf = std::make_unique_for_overwrite(size); @@ -446,13 +470,11 @@ std::string S3FileSystem::generate_presigned_url(const Path& path, int64_t expir bool is_public_endpoint) const { std::string key = fmt::format("{}/{}", _prefix, path.native()); std::shared_ptr client; - if (is_public_endpoint && - _client->s3_client_conf().endpoint.ends_with(OSS_PRIVATE_ENDPOINT_SUFFIX)) { - auto new_s3_conf = _client->s3_client_conf(); - new_s3_conf.endpoint.erase( - _client->s3_client_conf().endpoint.size() - OSS_PRIVATE_ENDPOINT_SUFFIX.size(), - LEN_OF_OSS_PRIVATE_SUFFIX); - auto client_result = S3ClientFactory::instance().create(new_s3_conf); + auto conf = _client->s3_client_conf(); + if (is_public_endpoint && conf.endpoint.ends_with(OSS_PRIVATE_ENDPOINT_SUFFIX)) { + conf.endpoint.erase(conf.endpoint.size() - OSS_PRIVATE_ENDPOINT_SUFFIX.size(), + LEN_OF_OSS_PRIVATE_SUFFIX); + auto client_result = S3ClientFactory::instance().create(conf); if (!client_result) { LOG(WARNING) << "failed to create S3 client for presigned URL: " << client_result.error(); diff --git a/be/src/io/fs/s3_file_system.h b/be/src/io/fs/s3_file_system.h index d1b969eedeca95..cd581f7239406e 100644 --- a/be/src/io/fs/s3_file_system.h +++ b/be/src/io/fs/s3_file_system.h @@ -51,6 +51,10 @@ class ObjClientHolder { // Update s3 conf and reset client if `conf` is different. This method is threadsafe. Status reset(const S3ClientConf& conf); + // Validate the current generation while holding the same lock that protects + // credential/client replacement. Existing readers still follow holder refreshes. + Status validate_for_access() const; + std::shared_ptr get() const { std::shared_lock lock(_mtx); return _client; @@ -61,7 +65,12 @@ class ObjClientHolder { // For error msg std::string full_s3_path(std::string_view bucket, std::string_view key) const; - const S3ClientConf& s3_client_conf() { return _conf; } + // Credential refresh replaces _conf concurrently with file-system access. + // Return an owned snapshot so readers never retain a reference past the lock. + S3ClientConf s3_client_conf() const { + std::shared_lock lock(_mtx); + return _conf; + } private: mutable std::shared_mutex _mtx; diff --git a/be/src/io/fs/s3_file_writer.cpp b/be/src/io/fs/s3_file_writer.cpp index 67355e640a0ccc..743005567eb0c5 100644 --- a/be/src/io/fs/s3_file_writer.cpp +++ b/be/src/io/fs/s3_file_writer.cpp @@ -47,6 +47,15 @@ namespace doris::io { +namespace { +std::string sanitize_display_path(std::string path) { + if (const auto query = path.find('?'); query != std::string::npos) { + path.resize(query); + } + return path; +} +} // namespace + bvar::Adder s3_file_writer_total("s3_file_writer_total_num"); bvar::Adder s3_bytes_written_total("s3_file_writer_bytes_written"); bvar::Adder s3_file_created_total("s3_file_writer_file_created"); @@ -60,8 +69,10 @@ bvar::Window s3_file_writer_first_append_to_close_ms_window( &s3_file_writer_first_append_to_close_ms_recorder, /*window_size=*/10); S3FileWriter::S3FileWriter(std::shared_ptr client, std::string bucket, - std::string key, const FileWriterOptions* opts) - : _obj_storage_path_opts({.path = fmt::format("s3://{}/{}", bucket, key), + std::string key, const FileWriterOptions* opts, std::string display_path) + : _obj_storage_path_opts({.path = display_path.empty() + ? fmt::format("s3://{}/{}", bucket, key) + : sanitize_display_path(std::move(display_path)), .bucket = std::move(bucket), .key = std::move(key)}), _used_by_s3_committer(opts ? opts->used_by_s3_committer : false), diff --git a/be/src/io/fs/s3_file_writer.h b/be/src/io/fs/s3_file_writer.h index 5ae786cec3ebbd..99aff4eeae69b0 100644 --- a/be/src/io/fs/s3_file_writer.h +++ b/be/src/io/fs/s3_file_writer.h @@ -51,7 +51,7 @@ class ObjClientHolder; class S3FileWriter final : public FileWriter { public: S3FileWriter(std::shared_ptr client, std::string bucket, std::string key, - const FileWriterOptions* opts); + const FileWriterOptions* opts, std::string display_path = {}); ~S3FileWriter() override; Status appendv(const Slice* data, size_t data_cnt) override; diff --git a/be/src/runtime/runtime_state.cpp b/be/src/runtime/runtime_state.cpp index 049d6b165c9012..699bc9fc364b1e 100644 --- a/be/src/runtime/runtime_state.cpp +++ b/be/src/runtime/runtime_state.cpp @@ -556,6 +556,13 @@ std::string RuntimeState::get_error_log_file_path() { auto presigned_url = s3_error_fs->generate_presigned_url(remote_error_log_file_path, EXPIRATION_SECONDS, config::use_public_endpoint_for_error_log); + if (presigned_url.empty()) { + // Upload success does not imply that this credential can sign a user-facing URL. Keep + // the local error log available when the object store cannot provide a signed URL. + LOG(WARNING) << "Fail to generate a presigned URL for the uploaded error log; " + "keeping the local error log file path"; + return local_error_log_file_path; + } std::lock_guard load_lock(_load_error_log_lock); _error_log_file_path = std::move(presigned_url); return _error_log_file_path; diff --git a/be/test/format/table/iceberg/iceberg_delete_file_reader_helper_test.cpp b/be/test/format/table/iceberg/iceberg_delete_file_reader_helper_test.cpp index 72b519e2191a47..09b8bdd20fb7d1 100644 --- a/be/test/format/table/iceberg/iceberg_delete_file_reader_helper_test.cpp +++ b/be/test/format/table/iceberg/iceberg_delete_file_reader_helper_test.cpp @@ -28,6 +28,7 @@ #include #include #include +#include #include #include #include @@ -224,6 +225,36 @@ TEST(IcebergDeleteFileReaderHelperTest, BuildDeleteFileRange) { EXPECT_EQ(range2.file_size, 1024); } +TEST(IcebergDeleteFileReaderHelperTest, NativeAzureDeleteParamsDoNotBuildHdfsParams) { + // Native Azure credentials travel through the generic object-storage properties map while + // FILE_HDFS is reserved for genuine Hadoop callers. Keeping this distinction is what prevents + // UPDATE/MERGE deletion-vector reads from re-entering the Hadoop ABFS builder. + const std::map native_azure_properties { + {"AZURE_AUTH_TYPE", "SAS"}, + {"AZURE_ENDPOINT", "https://account.blob.core.windows.net"}, + {"AZURE_ACCOUNT_NAME", "account"}, + {"AZURE_CONTAINER", "container"}, + {"AZURE_SAS_TOKEN", "sv=2024-01-01&sig=redacted"}, + {"AZURE_SAS_EXPIRY_MS", "4102444800000"}, + }; + + const auto native_params = + build_iceberg_delete_scan_range_params(native_azure_properties, TFileType::FILE_S3, {}); + ASSERT_TRUE(native_params.__isset.file_type); + EXPECT_EQ(TFileType::FILE_S3, native_params.file_type); + ASSERT_TRUE(native_params.__isset.properties); + EXPECT_EQ(native_azure_properties, native_params.properties); + EXPECT_FALSE(native_params.__isset.hdfs_params); + + // Preserve the existing Hadoop contract for non-Azure callers. + const auto hdfs_params = build_iceberg_delete_scan_range_params(native_azure_properties, + TFileType::FILE_HDFS, {}); + ASSERT_TRUE(hdfs_params.__isset.file_type); + EXPECT_EQ(TFileType::FILE_HDFS, hdfs_params.file_type); + ASSERT_TRUE(hdfs_params.__isset.hdfs_params); + ASSERT_TRUE(hdfs_params.hdfs_params.__isset.hdfs_conf); +} + TEST(IcebergDeleteFileReaderHelperTest, IsDeletionVector) { TIcebergDeleteFileDesc delete_file; delete_file.__set_content(3); diff --git a/be/test/io/client/s3_file_system_test.cpp b/be/test/io/client/s3_file_system_test.cpp index b7ba5966843ea3..384b09cb9454f0 100644 --- a/be/test/io/client/s3_file_system_test.cpp +++ b/be/test/io/client/s3_file_system_test.cpp @@ -195,6 +195,13 @@ class S3FileSystemTest : public ::testing::Test { s3_conf.client_conf.bucket = config_->get_bucket(); s3_conf.client_conf.region = config_->get_region(); s3_conf.client_conf.provider = convert_provider(config_->get_provider()); + if (s3_conf.client_conf.provider == io::ObjStorageProvider::AZURE) { + s3_conf.client_conf.azure_credentials.account_name = std::move(s3_conf.client_conf.ak); + s3_conf.client_conf.azure_credentials.account_key = std::move(s3_conf.client_conf.sk); + s3_conf.client_conf.ak.clear(); + s3_conf.client_conf.sk.clear(); + s3_conf.client_conf.region.clear(); + } if (config_->use_instance_profile()) { s3_conf.client_conf.cred_provider_type = CredProviderType::InstanceProfile; s3_conf.client_conf.role_arn = config_->get_role_arn(); From 671ee01fdaf1c62a1a5c0af1ec5ef86a1ab62c73 Mon Sep 17 00:00:00 2001 From: xylaaaaa <2392805527@qq.com> Date: Sun, 13 Sep 2026 20:01:21 +0800 Subject: [PATCH 03/10] [feature](fe) Add Azure filesystem bindings and provider-owned properties ### What problem does this PR solve? Related PR: #67470 Problem Summary: Keep the filesystem property interfaces, Azure provider, endpoint and vended-credential binding, plugin wiring and focused tests together. This is a history-only regrouping of the native Azure PR. The final seven-commit series preserves the exact code tree of azure-native-before-squash-20260913550a886487e. ### Release note None ### Check List (For Author) - Test: Manual test / Unit Test - Final tree and PR diff are verified identical to the pre-squash snapshot - Existing final-snapshot verification: 711 selected FE tests passed - Tests were not rerun for this history-only rewrite - Behavior changed: No (relative to the pre-squash PR snapshot) - Does this need documentation: No --- fe/fe-filesystem/README.md | 2 +- .../properties/BackendStorageKind.java | 6 +- .../properties/FileSystemProperties.java | 33 +- .../properties/StorageProperties.java | 82 +- .../StoragePropertiesInterfaceTest.java | 77 ++ fe/fe-filesystem/fe-filesystem-azure/pom.xml | 8 + .../filesystem/azure/AzureAccountHost.java | 146 +++ .../doris/filesystem/azure/AzureAuthType.java | 51 + .../azure/AzureFileIOSharedKey.java | 81 ++ .../filesystem/azure/AzureFileSystem.java | 100 +- .../azure/AzureFileSystemProperties.java | 937 ++++++++++++++++-- .../azure/AzureFileSystemProvider.java | 61 +- .../filesystem/azure/AzureObjStorage.java | 42 +- .../doris/filesystem/azure/AzureSasToken.java | 144 +++ .../doris/filesystem/azure/AzureUri.java | 119 ++- .../filesystem/azure/AzureVendedSas.java | 132 +++ .../azure/AzureAccountHostTest.java | 108 ++ .../azure/AzureAdlsLocationContractTest.java | 72 ++ .../filesystem/azure/AzureAuthTypeTest.java | 72 ++ .../azure/AzureBackendViewTest.java | 315 ++++++ .../azure/AzureCredentialScopeTest.java | 235 +++++ .../azure/AzureFileSystemPropertiesTest.java | 542 +++++++++- .../azure/AzureFileSystemProviderTest.java | 52 + .../filesystem/azure/AzureFileSystemTest.java | 153 +++ .../AzureIcebergFileIOPropertiesTest.java | 268 +++++ .../azure/AzureObjStorageExtensionTest.java | 93 +- .../filesystem/azure/AzureSasTokenTest.java | 117 +++ .../doris/filesystem/azure/AzureUriTest.java | 168 +++- .../azure/AzureVendedCredentialsTest.java | 362 +++++++ .../hdfs/HdfsFileSystemProvider.java | 13 +- .../hdfs/properties/HdfsProperties.java | 13 + .../hdfs/HdfsFileSystemProviderTest.java | 57 ++ .../filesystem/spi/FileSystemProvider.java | 27 + .../spi/FileSystemPluginSurfaceTest.java | 18 + ...FileSystemProviderTypedPropertiesTest.java | 10 + .../resources/filesystem-plugin-surface.txt | 38 + fe/fe-filesystem/pom.xml | 2 +- 37 files changed, 4455 insertions(+), 301 deletions(-) create mode 100644 fe/fe-filesystem/fe-filesystem-azure/src/main/java/org/apache/doris/filesystem/azure/AzureAccountHost.java create mode 100644 fe/fe-filesystem/fe-filesystem-azure/src/main/java/org/apache/doris/filesystem/azure/AzureAuthType.java create mode 100644 fe/fe-filesystem/fe-filesystem-azure/src/main/java/org/apache/doris/filesystem/azure/AzureFileIOSharedKey.java create mode 100644 fe/fe-filesystem/fe-filesystem-azure/src/main/java/org/apache/doris/filesystem/azure/AzureSasToken.java create mode 100644 fe/fe-filesystem/fe-filesystem-azure/src/main/java/org/apache/doris/filesystem/azure/AzureVendedSas.java create mode 100644 fe/fe-filesystem/fe-filesystem-azure/src/test/java/org/apache/doris/filesystem/azure/AzureAccountHostTest.java create mode 100644 fe/fe-filesystem/fe-filesystem-azure/src/test/java/org/apache/doris/filesystem/azure/AzureAdlsLocationContractTest.java create mode 100644 fe/fe-filesystem/fe-filesystem-azure/src/test/java/org/apache/doris/filesystem/azure/AzureAuthTypeTest.java create mode 100644 fe/fe-filesystem/fe-filesystem-azure/src/test/java/org/apache/doris/filesystem/azure/AzureBackendViewTest.java create mode 100644 fe/fe-filesystem/fe-filesystem-azure/src/test/java/org/apache/doris/filesystem/azure/AzureCredentialScopeTest.java create mode 100644 fe/fe-filesystem/fe-filesystem-azure/src/test/java/org/apache/doris/filesystem/azure/AzureIcebergFileIOPropertiesTest.java create mode 100644 fe/fe-filesystem/fe-filesystem-azure/src/test/java/org/apache/doris/filesystem/azure/AzureSasTokenTest.java create mode 100644 fe/fe-filesystem/fe-filesystem-azure/src/test/java/org/apache/doris/filesystem/azure/AzureVendedCredentialsTest.java diff --git a/fe/fe-filesystem/README.md b/fe/fe-filesystem/README.md index eb8e34ff06631a..2e53c9589fd736 100644 --- a/fe/fe-filesystem/README.md +++ b/fe/fe-filesystem/README.md @@ -105,7 +105,7 @@ request. For example: | S3 | `AWS_ACCESS_KEY` + (`AWS_ENDPOINT` or `AWS_REGION`) | | OSS | Endpoint contains `aliyuncs.com` or `_STORAGE_TYPE_` = `"OSS"` | | HDFS | `_STORAGE_TYPE_` = `"HDFS"` or URI scheme is `hdfs`/`viewfs`/`ofs`/`jfs`/`oss` | -| Azure | `AZURE_ACCOUNT_NAME` or endpoint contains `blob.core.windows.net` | +| Azure | Azure account/SAS properties, or a parsed host matching an Azure Blob/DFS suffix (case-insensitive) | | Local | URI starts with `file://` or `local://` | | Broker | `_STORAGE_TYPE_` = `"BROKER"` and `BROKER_HOST` present | diff --git a/fe/fe-filesystem/fe-filesystem-api/src/main/java/org/apache/doris/filesystem/properties/BackendStorageKind.java b/fe/fe-filesystem/fe-filesystem-api/src/main/java/org/apache/doris/filesystem/properties/BackendStorageKind.java index 8e6bf326580b4c..d2f29e81dc5f03 100644 --- a/fe/fe-filesystem/fe-filesystem-api/src/main/java/org/apache/doris/filesystem/properties/BackendStorageKind.java +++ b/fe/fe-filesystem/fe-filesystem-api/src/main/java/org/apache/doris/filesystem/properties/BackendStorageKind.java @@ -21,8 +21,8 @@ * Backend-facing storage classification used to select FE-to-BE adapters. * *

This is intentionally more specific than {@link StorageKind}. For example, - * Azure Blob is object storage at the filesystem layer, but it may be sent to BE - * either through the S3-compatible adapter or through its native AZURE storage type. + * Azure Blob is object storage at the filesystem layer, but uses native Azure credentials + * and client selection even while sharing the existing FILE_S3 transport type. */ public enum BackendStorageKind { /** @@ -31,7 +31,7 @@ public enum BackendStorageKind { S3_COMPATIBLE, /** - * Storage that should keep its own backend storage type, such as AZURE. + * Storage with a provider-specific backend credential and client contract, such as Azure. */ NATIVE, diff --git a/fe/fe-filesystem/fe-filesystem-api/src/main/java/org/apache/doris/filesystem/properties/FileSystemProperties.java b/fe/fe-filesystem/fe-filesystem-api/src/main/java/org/apache/doris/filesystem/properties/FileSystemProperties.java index fa1d84b80bf797..b1fb0fac11a988 100644 --- a/fe/fe-filesystem/fe-filesystem-api/src/main/java/org/apache/doris/filesystem/properties/FileSystemProperties.java +++ b/fe/fe-filesystem/fe-filesystem-api/src/main/java/org/apache/doris/filesystem/properties/FileSystemProperties.java @@ -117,6 +117,21 @@ default String validateAndNormalizeUri(String uri) { return uri; } + /** + * Whether this already-bound provider claims a URI with a generic scheme such as HTTP(S). + * The default does not claim any URI; provider-specific schemes continue to route through + * {@link #getSupportedSchemes()}. + * + *

This is an identity probe, not validation or credential access. Callers must first select + * the claiming binding, then call {@link #validateAndNormalizeUri(String)} and propagate its + * errors. In particular, a malformed object path on a recognized endpoint must not fall back + * to another provider. A provider must not advertise every HTTP(S) URI in its supported schemes + * merely because it accepts its own endpoint's HTTP(S) URLs.

+ */ + default boolean claimsUri(String uri) { + return false; + } + /** * Extracts the storage URI from the given load properties and validates it via * {@link #validateAndNormalizeUri(String)}. @@ -132,9 +147,10 @@ default String validateAndGetUri(Map loadProperties) { /** * Stable fingerprint of this storage identity, published under - * {@code doris.fs.cache.key.} in every BE-bound / Hadoop-bound property map derived - * from it, so that the Doris-patched {@code org.apache.hadoop.fs.FileSystem} never shares one + * {@code doris.fs.cache.key.} in Hadoop-compatible backend / Hadoop configuration maps, + * so that the Doris-patched {@code org.apache.hadoop.fs.FileSystem} never shares one * cached instance between different credential sets. See {@link FsCacheKeys}. + * Native-only backend views do not use these Hadoop cache keys. * *

The default hashes the concrete class name together with * {@link FsCacheKeys#identityProperties}, i.e. the user-supplied keys this binding consumed @@ -158,6 +174,19 @@ default Optional toBackendProperties() { return Optional.empty(); } + /** + * Selects the backend view for a URI already normalized by this binding. The returned kind + * and map must describe the same access path, for example a native view or a Hadoop view, + * without mixing their configuration vocabularies. + * + *

Selection performs no I/O or credential refresh. Time-dependent access validation still + * applies when the selected view's map is emitted. Providers with a single backend view keep + * their existing behavior through this default.

+ */ + default Optional resolveBackendProperties(String normalizedUri) { + return toBackendProperties(); + } + /** * Converts to Hadoop configuration properties if this provider supports Hadoop access. */ diff --git a/fe/fe-filesystem/fe-filesystem-api/src/main/java/org/apache/doris/filesystem/properties/StorageProperties.java b/fe/fe-filesystem/fe-filesystem-api/src/main/java/org/apache/doris/filesystem/properties/StorageProperties.java index ebeb1e6d78be04..1e293068fba673 100644 --- a/fe/fe-filesystem/fe-filesystem-api/src/main/java/org/apache/doris/filesystem/properties/StorageProperties.java +++ b/fe/fe-filesystem/fe-filesystem-api/src/main/java/org/apache/doris/filesystem/properties/StorageProperties.java @@ -18,7 +18,9 @@ package org.apache.doris.filesystem.properties; import org.apache.doris.filesystem.FileSystemType; +import org.apache.doris.filesystem.Location; +import java.util.Collections; import java.util.Map; import java.util.Optional; @@ -47,11 +49,43 @@ public interface StorageProperties { FileSystemType type(); /** - * Validates the already-bound property model. + * Returns whether the registry added this binding as a fallback without matching the user's + * storage configuration. The origin is fixed at binding creation, not inferred from raw keys. + * Normal provider bindings, including heuristic matches, return false. + */ + default boolean isSyntheticDefault() { + return false; + } + + /** + * Validates the format and required fields of the already-bound property model. + * This may run during metadata replay, so time-dependent checks belong in + * {@link #validateForAccess()} instead. */ default void validate() { } + /** + * Checks whether the bound credentials can be used now, for example against a known expiry. + * Callers must invoke this on each credential access, including when returning a cached + * configuration map. The default preserves providers without time-dependent validation. + * + *

This is local validation only: implementations must not perform I/O, refresh credentials, + * or mutate the binding. Invalid credentials must fail explicitly without exposing secrets.

+ */ + default void validateForAccess() { + } + + /** + * Returns whether a location is equal to or below a credential's location prefix. + * Providers own storage identity and object-name interpretation; equivalent URI spellings + * may require provider-specific parsing. This is local validation only, without credential + * access, I/O or path normalization. A directory caller must supply its child-path prefix. + */ + default boolean matchesLocationPrefix(String rawLocation, String rawPrefix) { + return Location.of(rawLocation).startsWith(Location.of(rawPrefix)); + } + /** * Returns the original raw properties passed to the provider. */ @@ -75,4 +109,50 @@ default Optional toBackendProperties() { default Optional toHadoopProperties() { return Optional.empty(); } + + /** + * Hadoop configuration needed by Iceberg's default REST FileIO selection. Unlike a + * filesystem-backed catalog or an explicitly selected Hadoop/custom FileIO, that path + * may use a native FileIO whose credentials must be selected from the load response first. + * Providers can omit an unused Hadoop view without accessing an obsolete credential. + * The default preserves every other provider's existing Hadoop configuration. + * + *

This does not affect native data readers, ordinary Hadoop consumers or credential + * validity checks when the selected FileIO configuration is emitted.

+ */ + default Optional toIcebergHadoopProperties() { + return toHadoopProperties(); + } + + /** + * Connection-only properties for the official Iceberg FileIO, without authentication or + * FileIO selection. A response may replace an obsolete credential while retaining the + * provider's endpoint, so this view must not access credentials or validate their expiry. + * Implementations must still validate connection fields and reject embedded credentials. + * + *

This performs no I/O and returns an immutable map. The default contributes nothing; + * credential emission and access validation remain in {@link #toIcebergFileIOProperties()}.

+ */ + default Map toIcebergFileIOConnectionProperties() { + return Collections.emptyMap(); + } + + /** + * Storage authentication and connection properties for the official Iceberg FileIO. + * This is a separate consumer dialect from the native backend and Hadoop maps. The + * provider owns translation; no Iceberg or SDK objects cross the plugin boundary. + * + *

Emitting credentials is an access point, so implementations must validate their + * validity here. The default contributes nothing and preserves existing FileIO selection. + * Returned maps must be immutable and must not be logged.

+ * + *

A provider may include {@code io-impl} when its authentication requires a specific + * default FileIO. Consumers must reconcile this requirement with explicit configuration, + * REST server overrides and vended credentials before constructing FileIO; they must not + * blindly overwrite those choices or silently substitute another identity. Hadoop + * authentication still comes from {@link #toHadoopProperties()}, not this map.

+ */ + default Map toIcebergFileIOProperties() { + return Collections.emptyMap(); + } } diff --git a/fe/fe-filesystem/fe-filesystem-api/src/test/java/org/apache/doris/filesystem/properties/StoragePropertiesInterfaceTest.java b/fe/fe-filesystem/fe-filesystem-api/src/test/java/org/apache/doris/filesystem/properties/StoragePropertiesInterfaceTest.java index 73cbcae0b25d63..092c3b9114f98b 100644 --- a/fe/fe-filesystem/fe-filesystem-api/src/test/java/org/apache/doris/filesystem/properties/StoragePropertiesInterfaceTest.java +++ b/fe/fe-filesystem/fe-filesystem-api/src/test/java/org/apache/doris/filesystem/properties/StoragePropertiesInterfaceTest.java @@ -24,6 +24,7 @@ import java.util.Collections; import java.util.Map; +import java.util.Optional; class StoragePropertiesInterfaceTest { @@ -34,6 +35,82 @@ void fileSystemPropertiesIsStorageProperties() { Assertions.assertTrue(properties instanceof StorageProperties); } + @Test + void normalBindingsAreNotSyntheticByDefault() { + StorageProperties properties = new TestProperties(); + + Assertions.assertFalse(properties.isSyntheticDefault()); + } + + @Test + void syntheticOriginIsVisibleThroughStorageProperties() { + StorageProperties properties = new TestProperties() { + @Override + public boolean isSyntheticDefault() { + return true; + } + }; + + Assertions.assertTrue(properties.isSyntheticDefault()); + } + + @Test + void accessValidationDefaultsToNoOp() { + StorageProperties properties = new TestProperties(); + + Assertions.assertDoesNotThrow(properties::validateForAccess); + } + + @Test + void fileIOPropertiesDefaultDoesNotChangeOtherProvidersSelectionOrCredentials() { + StorageProperties properties = new TestProperties(); + + Map output = properties.toIcebergFileIOProperties(); + + Assertions.assertEquals(Collections.emptyMap(), output); + Assertions.assertThrows(UnsupportedOperationException.class, () -> output.put("io-impl", "other")); + } + + @Test + void fileIOConnectionPropertiesDefaultIsEmptyAndImmutable() { + StorageProperties properties = new TestProperties(); + + Map output = properties.toIcebergFileIOConnectionProperties(); + + Assertions.assertEquals(Collections.emptyMap(), output); + Assertions.assertThrows(UnsupportedOperationException.class, + () -> output.put("adls.connection-string.account", "https://example.org")); + } + + @Test + void defaultIcebergHadoopViewPreservesOtherProvidersConfigurationAndIdentity() { + HadoopStorageProperties hadoop = () -> Map.of("fs.defaultFS", "hdfs://namenode:8020"); + StorageProperties properties = new TestProperties() { + @Override + public Optional toHadoopProperties() { + return Optional.of(hadoop); + } + }; + + Assertions.assertSame(hadoop, properties.toIcebergHadoopProperties().orElseThrow()); + Assertions.assertEquals(Map.of("fs.defaultFS", "hdfs://namenode:8020"), + properties.toIcebergHadoopProperties().orElseThrow().toHadoopConfigurationMap()); + Assertions.assertTrue(new TestProperties().toIcebergHadoopProperties().isEmpty()); + } + + @Test + void bindingValidationDoesNotPerformAccessValidation() { + StorageProperties properties = new TestProperties() { + @Override + public void validateForAccess() { + throw new IllegalArgumentException("Test credential is expired"); + } + }; + + Assertions.assertDoesNotThrow(properties::validate); + Assertions.assertThrows(IllegalArgumentException.class, properties::validateForAccess); + } + private static class TestProperties implements FileSystemProperties { @Override public String providerName() { diff --git a/fe/fe-filesystem/fe-filesystem-azure/pom.xml b/fe/fe-filesystem/fe-filesystem-azure/pom.xml index cb76d8c2f09e8b..83ba98ee84c854 100644 --- a/fe/fe-filesystem/fe-filesystem-azure/pom.xml +++ b/fe/fe-filesystem/fe-filesystem-azure/pom.xml @@ -120,6 +120,14 @@ under the License. mockito-inline test + + + org.apache.iceberg + iceberg-azure + ${iceberg.version} + test + diff --git a/fe/fe-filesystem/fe-filesystem-azure/src/main/java/org/apache/doris/filesystem/azure/AzureAccountHost.java b/fe/fe-filesystem/fe-filesystem-azure/src/main/java/org/apache/doris/filesystem/azure/AzureAccountHost.java new file mode 100644 index 00000000000000..0e48fd6b57d650 --- /dev/null +++ b/fe/fe-filesystem/fe-filesystem-azure/src/main/java/org/apache/doris/filesystem/azure/AzureAccountHost.java @@ -0,0 +1,146 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.filesystem.azure; + +import org.apache.doris.foundation.property.StoragePropertiesException; + +import java.net.URI; +import java.net.URISyntaxException; +import java.util.Locale; +import java.util.Set; + +/** Parsed Azure account authority and endpoint shared by provider configuration renderers. */ +public final class AzureAccountHost { + + private static final String DEFAULT_CLOUD_SUFFIX = "core.windows.net"; + private static final String DFS_MARKER = ".dfs."; + private static final String BLOB_MARKER = ".blob."; + private static final Set AZURE_CLOUD_SUFFIXES = Set.of( + "core.windows.net", "core.chinacloudapi.cn", "core.usgovcloudapi.net", + "core.cloudapi.de", "fabric.microsoft.com"); + private static final Set AZURE_BLOB_CLOUD_SUFFIXES = Set.of( + "core.windows.net", "core.chinacloudapi.cn", "core.usgovcloudapi.net", + "core.cloudapi.de"); + + private final String accountName; + private final String cloudSuffix; + private final boolean dfsHost; + private final URI endpoint; + + private AzureAccountHost(String accountName, String cloudSuffix, boolean dfsHost, URI endpoint) { + this.accountName = accountName; + this.cloudSuffix = cloudSuffix; + this.dfsHost = dfsHost; + this.endpoint = endpoint; + } + + /** + * Parses an account host or endpoint such as {@code account.dfs.core.windows.net}. + * A URI scheme is optional. Explicit transport settings are retained when rendering a Blob + * endpoint; only official Azure DFS endpoints are rewritten. + */ + public static AzureAccountHost parse(String hostOrEndpoint) { + if (hostOrEndpoint == null || hostOrEndpoint.isBlank()) { + throw new StoragePropertiesException("Azure account host must not be empty"); + } + String value = hostOrEndpoint.trim(); + String uriValue = value.contains("://") ? value : "https://" + value; + final URI endpoint; + try { + endpoint = new URI(uriValue); + } catch (URISyntaxException e) { + // URISyntaxException includes the input, which can contain a credential-bearing query. + throw new StoragePropertiesException("Invalid Azure account host"); + } + String host = endpoint.getHost(); + if (host == null || host.isBlank()) { + throw new StoragePropertiesException("Invalid Azure account host"); + } + + String lowerHost = host.toLowerCase(Locale.ROOT); + validateEndpoint(endpoint); + int dfsIndex = lowerHost.indexOf(DFS_MARKER); + if (dfsIndex > 0) { + return new AzureAccountHost(host.substring(0, dfsIndex), + host.substring(dfsIndex + DFS_MARKER.length()), true, endpoint); + } + int blobIndex = lowerHost.indexOf(BLOB_MARKER); + if (blobIndex > 0) { + return new AzureAccountHost(host.substring(0, blobIndex), + host.substring(blobIndex + BLOB_MARKER.length()), false, endpoint); + } + + int dot = host.indexOf('.'); + String accountName = dot > 0 ? host.substring(0, dot) : host; + return new AzureAccountHost(accountName, "", false, endpoint); + } + + private static void validateEndpoint(URI endpoint) { + if (!("http".equalsIgnoreCase(endpoint.getScheme()) + || "https".equalsIgnoreCase(endpoint.getScheme())) + || endpoint.getRawUserInfo() != null || endpoint.getRawQuery() != null + || endpoint.getRawFragment() != null) { + throw new StoragePropertiesException( + "Azure endpoint must use HTTP(S) and must not contain credentials, query or fragment"); + } + } + + /** Creates a public-cloud account host when only the account name is configured. */ + public static AzureAccountHost fromAccountName(String accountName) { + if (accountName == null || accountName.isBlank()) { + throw new StoragePropertiesException("Azure account name must not be empty"); + } + return parse(accountName.trim() + BLOB_MARKER + DEFAULT_CLOUD_SUFFIX); + } + + public String accountName() { + return accountName; + } + + public String cloudSuffix() { + return cloudSuffix; + } + + public String dfsHost() { + return cloudSuffix.isEmpty() ? endpoint.getHost() : accountName + DFS_MARKER + cloudSuffix; + } + + public String blobHost() { + return cloudSuffix.isEmpty() ? endpoint.getHost() : accountName + BLOB_MARKER + cloudSuffix; + } + + public String blobEndpoint() { + if (!dfsHost || !AZURE_BLOB_CLOUD_SUFFIXES.contains(cloudSuffix.toLowerCase(Locale.ROOT))) { + return endpoint.toString(); + } + String value = endpoint.toString(); + int hostStart = value.indexOf("://") + 3; + // Replace only the authority host: URI reconstruction would decode/re-encode paths. + return value.substring(0, hostStart) + blobHost() + + value.substring(hostStart + endpoint.getHost().length()); + } + + public boolean isDfsHost() { + return dfsHost; + } + + /** Whether this authority is an official Azure account host, rather than a custom transport. */ + public boolean isAzureCloudHost() { + return AZURE_CLOUD_SUFFIXES.contains(cloudSuffix.toLowerCase(Locale.ROOT)); + } +} diff --git a/fe/fe-filesystem/fe-filesystem-azure/src/main/java/org/apache/doris/filesystem/azure/AzureAuthType.java b/fe/fe-filesystem/fe-filesystem-azure/src/main/java/org/apache/doris/filesystem/azure/AzureAuthType.java new file mode 100644 index 00000000000000..a9aed1f43e4aea --- /dev/null +++ b/fe/fe-filesystem/fe-filesystem-azure/src/main/java/org/apache/doris/filesystem/azure/AzureAuthType.java @@ -0,0 +1,51 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.filesystem.azure; + +import org.apache.doris.foundation.property.StoragePropertiesException; + +import org.apache.commons.lang3.StringUtils; + +/** Azure authentication modes. Enum names are the canonical native backend values. */ +public enum AzureAuthType { + SHARED_KEY("SharedKey"), + SAS("SAS"), + OAUTH2("OAuth2"); + + private final String propertyValue; + + AzureAuthType(String propertyValue) { + this.propertyValue = propertyValue; + } + + public String propertyValue() { + return propertyValue; + } + + public static AzureAuthType parse(String value) { + String normalized = StringUtils.trim(value); + for (AzureAuthType authType : values()) { + if (authType.name().equalsIgnoreCase(normalized) + || authType.propertyValue.equalsIgnoreCase(normalized)) { + return authType; + } + } + throw new StoragePropertiesException( + "Unsupported Azure auth_type. Expected SharedKey/SHARED_KEY, SAS or OAuth2/OAUTH2."); + } +} diff --git a/fe/fe-filesystem/fe-filesystem-azure/src/main/java/org/apache/doris/filesystem/azure/AzureFileIOSharedKey.java b/fe/fe-filesystem/fe-filesystem-azure/src/main/java/org/apache/doris/filesystem/azure/AzureFileIOSharedKey.java new file mode 100644 index 00000000000000..791dd414663a5a --- /dev/null +++ b/fe/fe-filesystem/fe-filesystem-azure/src/main/java/org/apache/doris/filesystem/azure/AzureFileIOSharedKey.java @@ -0,0 +1,81 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.filesystem.azure; + +import org.apache.doris.foundation.property.StoragePropertiesException; + +import java.util.Map; +import java.util.Optional; + +/** The complete SharedKey identity selected by the official Iceberg FileIO. */ +final class AzureFileIOSharedKey { + private static final String ACCOUNT_NAME = "adls.auth.shared-key.account.name"; + private static final String ACCOUNT_KEY = "adls.auth.shared-key.account.key"; + + private final String accountName; + private final String accountKey; + + private AzureFileIOSharedKey(String accountName, String accountKey) { + this.accountName = accountName; + this.accountKey = accountKey; + } + + static Optional parse(Map credentials) { + String accountName = null; + String accountKey = null; + for (Map.Entry entry : credentials.entrySet()) { + boolean isName = ACCOUNT_NAME.equalsIgnoreCase(entry.getKey()); + boolean isKey = ACCOUNT_KEY.equalsIgnoreCase(entry.getKey()); + if (!isName && !isKey) { + continue; + } + String value = entry.getValue(); + if (value == null || value.isBlank()) { + throw new StoragePropertiesException("Azure FileIO SharedKey account name and key must not be empty"); + } + String previous = isName ? accountName : accountKey; + if (previous != null && !previous.equals(value)) { + throw new StoragePropertiesException("Conflicting Azure FileIO SharedKey credential fields"); + } + if (isName) { + accountName = value; + } else { + accountKey = value; + } + } + if (accountName == null && accountKey == null) { + return Optional.empty(); + } + if (accountName == null || accountKey == null) { + throw new StoragePropertiesException("Azure FileIO SharedKey requires both account name and key"); + } + return Optional.of(new AzureFileIOSharedKey(accountName, accountKey)); + } + + String accountName() { + return accountName; + } + + String accountKey() { + return accountKey; + } + + Map properties() { + return Map.of(ACCOUNT_NAME, accountName, ACCOUNT_KEY, accountKey); + } +} diff --git a/fe/fe-filesystem/fe-filesystem-azure/src/main/java/org/apache/doris/filesystem/azure/AzureFileSystem.java b/fe/fe-filesystem/fe-filesystem-azure/src/main/java/org/apache/doris/filesystem/azure/AzureFileSystem.java index 02cb3479ef03fd..85f9e23a6f6eaa 100644 --- a/fe/fe-filesystem/fe-filesystem-azure/src/main/java/org/apache/doris/filesystem/azure/AzureFileSystem.java +++ b/fe/fe-filesystem/fe-filesystem-azure/src/main/java/org/apache/doris/filesystem/azure/AzureFileSystem.java @@ -77,9 +77,7 @@ protected boolean isNotFoundError(IOException e) { @Override public void mkdirs(Location location) throws IOException { // Azure is a flat namespace; create a zero-byte directory marker for compatibility. - String path = location.uri().endsWith(DIR_MARKER_SUFFIX) - ? location.uri() - : location.uri() + DIR_MARKER_SUFFIX; + String path = directoryPrefix(location.uri()); // Idempotent: if a marker blob already exists at /, don't re-upload it // (putObject uses overwrite=true and would truncate any pre-existing blob with // exactly the same key). Only proceed when the marker is genuinely missing. @@ -110,8 +108,7 @@ public boolean exists(Location location) throws IOException { throw e; } } - String prefix = location.uri().endsWith(DIR_MARKER_SUFFIX) - ? location.uri() : location.uri() + DIR_MARKER_SUFFIX; + String prefix = directoryPrefix(location.uri()); return hasChildUnder(prefix); } @@ -132,8 +129,7 @@ public boolean exists(Location location) throws IOException { */ @Override public void delete(Location location, boolean recursive) throws IOException { - String prefix = location.uri().endsWith(DIR_MARKER_SUFFIX) - ? location.uri() : location.uri() + DIR_MARKER_SUFFIX; + String prefix = directoryPrefix(location.uri()); if (recursive) { deleteRecursive(prefix); return; @@ -158,26 +154,39 @@ public void delete(Location location, boolean recursive) throws IOException { */ private boolean hasChildUnder(String prefix) throws IOException { String prefixKey = AzureUri.parse(prefix).key(); - RemoteObjects firstPage = objStorage.listObjects(prefix, null); - for (RemoteObject obj : firstPage.getObjectList()) { - if (obj.getKey().length() > prefixKey.length()) { - return true; + String continuationToken = null; + do { + RemoteObjects page = objStorage.listObjects(prefix, continuationToken); + for (RemoteObject obj : page.getObjectList()) { + if (obj.getKey().length() > prefixKey.length()) { + return true; + } } - } + continuationToken = page.isTruncated() ? page.getContinuationToken() : null; + } while (continuationToken != null); return false; } private void deleteRecursive(String prefix) throws IOException { + AzureUri prefixUri = AzureUri.parse(prefix); String continuationToken = null; do { RemoteObjects batch = objStorage.listObjects(prefix, continuationToken); for (RemoteObject obj : batch.getObjectList()) { - objStorage.deleteObject(rebuildUri(prefix, obj.getKey())); + objStorage.deleteObject(prefixUri.withKey(obj.getKey()).toString()); } continuationToken = batch.isTruncated() ? batch.getContinuationToken() : null; } while (continuationToken != null); } + private static String directoryPrefix(String location) throws IOException { + AzureUri uri = AzureUri.parse(location); + String key = uri.key(); + // Append to the object name, not after a URL query/fragment where the slash would + // be discarded by AzureUri.parse and accidentally include same-prefix siblings. + return uri.withKey(key.isEmpty() || key.endsWith(DIR_MARKER_SUFFIX) ? key : key + DIR_MARKER_SUFFIX).toString(); + } + /** * Azure Blob Storage does not support atomic directory renames. * Single-blob renames are supported via copy-then-delete. @@ -199,7 +208,7 @@ public void rename(Location src, Location dst) throws IOException { } // Refuse rename when src is a virtual directory (children exist under /); // a copy-then-delete on the literal blob name would silently leave children behind. - if (hasChildUnder(src.uri() + DIR_MARKER_SUFFIX)) { + if (hasChildUnder(directoryPrefix(src.uri()))) { throw new IOException( "Renaming directories is not supported in Azure Blob Storage: " + src); } @@ -243,8 +252,7 @@ public void rename(Location src, Location dst) throws IOException { @Override public void renameDirectory(Location src, Location dst, Runnable whenSrcNotExists) throws IOException { - String prefix = src.uri().endsWith(DIR_MARKER_SUFFIX) - ? src.uri() : src.uri() + DIR_MARKER_SUFFIX; + String prefix = directoryPrefix(src.uri()); if (hasChildUnder(prefix)) { throw new UnsupportedOperationException( "renameDirectory is not supported by AzureFileSystem; " @@ -311,11 +319,14 @@ public List listFiles(Location location) throws IOException { */ private List listFilesSingleLevelGlob(Location location) throws IOException { String uri = location.uri(); - int lastSlash = uri.lastIndexOf('/'); - String parentPrefix = uri.substring(0, lastSlash + 1); - String basenameGlob = uri.substring(lastSlash + 1); + AzureUri parsed = AzureUri.parse(uri); + String keyPattern = parsed.key(); + int lastSlash = keyPattern.lastIndexOf('/'); + String parentKey = lastSlash >= 0 ? keyPattern.substring(0, lastSlash + 1) : ""; + String basenameGlob = keyPattern.substring(lastSlash + 1); Pattern matcher = Pattern.compile(globToRegex(basenameGlob)); - String parentKey = AzureUri.parse(parentPrefix).key(); + AzureUri parentUri = parsed.withKey(parentKey); + String parentPrefix = parentUri.toString(); List result = new ArrayList<>(); String continuation = null; @@ -337,7 +348,7 @@ private List listFilesSingleLevelGlob(Location location) throws IOExc continue; } result.add(new FileEntry( - Location.of(parentPrefix + relative), + Location.of(parentUri.withKey(key).toString()), obj.getSize(), false, obj.getModificationTime(), List.of())); } continuation = page.isTruncated() ? page.getContinuationToken() : null; @@ -352,11 +363,7 @@ public FileIterator list(Location location) throws IOException { // because they share the same string prefix. The FileSystem.list // contract specifies a directory, so enforce a trailing '/' to // constrain the prefix to a true directory boundary. - String uri = location.uri(); - if (!uri.endsWith(DIR_MARKER_SUFFIX)) { - uri = uri + DIR_MARKER_SUFFIX; - } - return new AzureFileIterator(uri); + return new AzureFileIterator(directoryPrefix(location.uri())); } @Override @@ -369,24 +376,6 @@ public DorisOutputFile newOutputFile(Location location) throws IOException { return new AzureOutputFile(location); } - private static String rebuildUri(String prefix, String key) { - // prefix is like "wasbs://container@account.blob.core.windows.net/path/", - // key is the full blob name; reconstruct the full URI for the key. - int schemeEnd = prefix.indexOf("://"); - if (schemeEnd < 0) { - return prefix + key; - } - String scheme = prefix.substring(0, schemeEnd); - String withoutScheme = prefix.substring(schemeEnd + 3); - // For wasb/abfs: "container@account.host/path/" - int firstSlash = withoutScheme.indexOf('/'); - if (firstSlash < 0) { - return prefix + key; - } - String authority = withoutScheme.substring(0, firstSlash); - return scheme + "://" + authority + "/" + key; - } - @Override public GlobListing globListWithLimit(Location path, String startAfter, long maxBytes, long maxFiles) throws IOException { @@ -394,12 +383,9 @@ public GlobListing globListWithLimit(Location path, String startAfter, long maxB AzureUri parsed = AzureUri.parse(uri); String container = parsed.container(); String keyPattern = parsed.key(); - // base = uri with the key portion stripped, preserving the original scheme/host syntax. - String base = uri.substring(0, uri.length() - keyPattern.length()); - Pattern matcher = Pattern.compile(globToRegex(keyPattern)); String listKeyPrefix = longestNonGlobPrefix(keyPattern); - String listPrefixUri = base + listKeyPrefix; + String listPrefixUri = parsed.withKey(listKeyPrefix).toString(); List files = new ArrayList<>(); long totalSize = 0L; @@ -428,7 +414,7 @@ public GlobListing globListWithLimit(Location path, String startAfter, long maxB break outer; } files.add(new FileEntry( - Location.of(base + key), + Location.of(parsed.withKey(key).toString()), obj.getSize(), false, obj.getModificationTime(), @@ -587,24 +573,24 @@ static String globToRegex(String glob) { /** Lazy-paginating FileIterator over Azure list results. */ private class AzureFileIterator implements FileIterator { private final String prefix; + private final AzureUri prefixUri; private String continuationToken; private List buffer = new ArrayList<>(); private int bufferIdx = 0; private boolean done = false; - AzureFileIterator(String prefix) { + AzureFileIterator(String prefix) throws IOException { this.prefix = prefix; + this.prefixUri = AzureUri.parse(prefix); } @Override public boolean hasNext() throws IOException { - if (bufferIdx < buffer.size()) { - return true; - } - if (done) { - return false; + // A page can contain only directory markers (or no blobs) and still have a + // continuation token. Filtering that page must not hide later files. + while (bufferIdx >= buffer.size() && !done) { + fetchNextPage(); } - fetchNextPage(); return bufferIdx < buffer.size(); } @@ -625,7 +611,7 @@ private void fetchNextPage() throws IOException { if (obj.getKey().endsWith(DIR_MARKER_SUFFIX)) { continue; } - Location loc = Location.of(rebuildUri(prefix, obj.getKey())); + Location loc = Location.of(prefixUri.withKey(obj.getKey()).toString()); buffer.add(new FileEntry(loc, obj.getSize(), false, obj.getModificationTime(), List.of())); } if (page.isTruncated()) { diff --git a/fe/fe-filesystem/fe-filesystem-azure/src/main/java/org/apache/doris/filesystem/azure/AzureFileSystemProperties.java b/fe/fe-filesystem/fe-filesystem-azure/src/main/java/org/apache/doris/filesystem/azure/AzureFileSystemProperties.java index e4221d19cc1a0c..269672a4f00240 100644 --- a/fe/fe-filesystem/fe-filesystem-azure/src/main/java/org/apache/doris/filesystem/azure/AzureFileSystemProperties.java +++ b/fe/fe-filesystem/fe-filesystem-azure/src/main/java/org/apache/doris/filesystem/azure/AzureFileSystemProperties.java @@ -18,26 +18,32 @@ package org.apache.doris.filesystem.azure; import org.apache.doris.filesystem.FileSystemType; +import org.apache.doris.filesystem.Location; import org.apache.doris.filesystem.properties.BackendStorageKind; import org.apache.doris.filesystem.properties.BackendStorageProperties; import org.apache.doris.filesystem.properties.FileSystemProperties; import org.apache.doris.filesystem.properties.FsCacheKeys; import org.apache.doris.filesystem.properties.HadoopStorageProperties; import org.apache.doris.filesystem.properties.StorageKind; +import org.apache.doris.filesystem.spi.AzureBlobEndpointSignals; import org.apache.doris.foundation.property.ConnectorPropertiesUtils; import org.apache.doris.foundation.property.ConnectorProperty; import org.apache.doris.foundation.property.ParamRules; +import org.apache.doris.foundation.property.StoragePropertiesException; import org.apache.commons.lang3.BooleanUtils; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; +import java.io.IOException; import java.lang.reflect.Field; import java.net.URI; import java.net.URISyntaxException; +import java.time.Clock; import java.util.Collections; import java.util.HashMap; import java.util.LinkedHashSet; +import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Optional; @@ -46,9 +52,9 @@ /** * Provider-owned Azure Blob Storage properties. * - *

The public aliases, endpoint formatting, shared-key validation, and backend - * map match fe-core AzureProperties. Legacy uppercase aliases remain accepted so - * existing FE filesystem callers can migrate through {@link AzureFileSystemProvider#bind(Map)}. + *

The public aliases, endpoint formatting, and shared-key validation retain fe-core Azure + * compatibility. Backend maps use provider-owned {@code AZURE_*} keys. Existing uppercase input + * aliases remain accepted, but new backend fields are output-only, not aliases for user properties. */ public final class AzureFileSystemProperties implements FileSystemProperties, BackendStorageProperties, HadoopStorageProperties { @@ -62,13 +68,28 @@ public final class AzureFileSystemProperties public static final String OAUTH_ACCOUNT_HOST = "azure.oauth2_account_host"; public static final String TENANT_ID = "azure.oauth2_client_tenant_id"; public static final String AUTH_TYPE = "azure.auth_type"; + public static final String SAS_TOKEN = "azure.sas_token"; + public static final String SAS_EXPIRY_MS = "azure.sas_expiry_ms"; public static final String CONTAINER = "container"; public static final String USE_PATH_STYLE = "use_path_style"; public static final String FORCE_PARSING_BY_STANDARD_URI = "force_parsing_by_standard_uri"; public static final String SHARED_KEY_AUTH = "SharedKey"; + public static final String SAS_AUTH = "SAS"; public static final String OAUTH2_AUTH = "OAuth2"; - public static final String AZURE_ENDPOINT_TEMPLATE = "https://%s.blob.core.windows.net"; + // Provider-owned backend keys. Azure keeps the existing FILE_S3 wire slot for compatibility, + // while these names prevent Azure credentials from inheriting AWS/S3 parameter semantics. + public static final String BACKEND_AUTH_TYPE = "AZURE_AUTH_TYPE"; + public static final String BACKEND_ENDPOINT = "AZURE_ENDPOINT"; + public static final String BACKEND_ACCOUNT_NAME = "AZURE_ACCOUNT_NAME"; + public static final String BACKEND_ACCOUNT_KEY = "AZURE_ACCOUNT_KEY"; + public static final String BACKEND_CONTAINER = "AZURE_CONTAINER"; + public static final String BACKEND_SAS_TOKEN = "AZURE_SAS_TOKEN"; + public static final String BACKEND_SAS_EXPIRY_MS = "AZURE_SAS_EXPIRY_MS"; + public static final String BACKEND_CLIENT_ID = "AZURE_CLIENT_ID"; + public static final String BACKEND_CLIENT_SECRET = "AZURE_CLIENT_SECRET"; + public static final String BACKEND_TENANT_ID = "AZURE_TENANT_ID"; + public static final String BACKEND_OAUTH_SERVER_URI = "AZURE_OAUTH_SERVER_URI"; private static final String[] AZURE_BLOB_HOST_SUFFIXES = { "blob.core.windows.net", @@ -77,24 +98,27 @@ public final class AzureFileSystemProperties "blob.core.cloudapi.de" }; + private static final List PROVIDER_ENDPOINT_ALIASES = List.of(ENDPOINT, "AZURE_ENDPOINT"); + private static final Set PROVIDER_ACCOUNT_ALIASES = Set.of( + ACCOUNT_NAME, "azure.access_key", "AZURE_ACCOUNT_NAME"); + private static final Set LEGACY_ENDPOINT_ALIASES = Set.of( + "s3.endpoint", "AWS_ENDPOINT", "endpoint", "ENDPOINT", "AZURE_ENDPOINT"); + // In each @ConnectorProperty below, the first name is the canonical key, kept as a - // constant because other code references it. The remaining literal names are legacy - // aliases accepted for compatibility only and referenced nowhere else, so they are not - // promoted to constants. + // constant because other code references it. The remaining literal names are input aliases + // kept for compatibility, not new backend-protocol fields; they are not promoted to constants. @ConnectorProperty(names = {ENDPOINT, "s3.endpoint", "AWS_ENDPOINT", "endpoint", "ENDPOINT", "AZURE_ENDPOINT"}, required = false, description = "The endpoint of Azure Blob Storage.") private String endpoint = ""; - @ConnectorProperty(names = {ACCOUNT_NAME, "azure.access_key", "s3.access_key", - "AWS_ACCESS_KEY", "ACCESS_KEY", "access_key", "AZURE_ACCOUNT_NAME"}, + @ConnectorProperty(names = {ACCOUNT_NAME, "azure.access_key", "AZURE_ACCOUNT_NAME"}, required = false, description = "The Azure storage account name.") private String accountName = ""; - @ConnectorProperty(names = {ACCOUNT_KEY, "azure.secret_key", "s3.secret_key", - "AWS_SECRET_KEY", "secret_key", "SECRET_KEY", "AZURE_ACCOUNT_KEY"}, + @ConnectorProperty(names = {ACCOUNT_KEY, "azure.secret_key", "AZURE_ACCOUNT_KEY"}, required = false, sensitive = true, description = "The Azure storage account key.") @@ -113,7 +137,8 @@ public final class AzureFileSystemProperties @ConnectorProperty(names = {OAUTH_SERVER_URI}, required = false, - description = "The Azure OAuth2 token endpoint.") + description = "The HTTPS Entra token endpoint: /{tenant}/oauth2/token or /{tenant}/oauth2/v2.0/token." + + " The native SDK uses its authority and tenant, not an arbitrary token path.") private String oauthServerUri = ""; @ConnectorProperty(names = {OAUTH_ACCOUNT_HOST}, @@ -129,7 +154,18 @@ public final class AzureFileSystemProperties @ConnectorProperty(names = {AUTH_TYPE}, required = false, description = "The auth type of Azure Blob Storage.") - private String azureAuthType = SHARED_KEY_AUTH; + private String azureAuthType = ""; + + @ConnectorProperty(names = {SAS_TOKEN, "azure.sas-token"}, + required = false, + sensitive = true, + description = "A provider-issued Azure SAS token.") + private String sasToken = ""; + + @ConnectorProperty(names = {SAS_EXPIRY_MS, "azure.sas-token-expires-at-ms"}, + required = false, + description = "The expiry time of the Azure SAS token in Unix milliseconds.") + private String sasExpiryMs = ""; @ConnectorProperty(names = {CONTAINER, "azure.bucket", "azure.container", "s3.bucket", "AZURE_CONTAINER", "AZURE_BUCKET", "AWS_BUCKET"}, @@ -149,38 +185,406 @@ public final class AzureFileSystemProperties private final Map rawProperties; private final Map matchedProperties; + private final AzureAuthType authType; + private final AzureAccountHost accountHost; + private final AzureSasToken sasCredential; + private final boolean explicitEndpoint; + private final Clock clock; + private final Object hadoopBackendPropertiesLock = new Object(); + private volatile Map cachedHadoopBackendProperties; - private AzureFileSystemProperties(Map rawProperties) { + private AzureFileSystemProperties(Map rawProperties, Clock clock) { + this.clock = clock; // Defensive copy before wrapping: unmodifiableMap alone is only a read-only view, // so without the copy later mutations of the caller's map would leak through. this.rawProperties = Collections.unmodifiableMap(new HashMap<>(rawProperties)); - this.matchedProperties = Collections.unmodifiableMap(collectMatchedProperties(rawProperties)); - ConnectorPropertiesUtils.bindConnectorProperties(this, rawProperties); - normalize(); + Map bindingProperties = azureBindingProperties(rawProperties); + Map matched = collectMatchedProperties(bindingProperties); + ConnectorPropertiesUtils.bindConnectorProperties(this, bindingProperties); + this.authType = resolveAuthType(); + // S3/generic endpoint aliases belong to the legacy SharedKey compatibility path. Native + // SAS/OAuth2 bindings must derive the endpoint from the Azure account unless the caller + // supplied the provider-owned azure.endpoint; otherwise a sibling S3 provider can silently + // redirect Azure requests to its endpoint. + if ((authType != AzureAuthType.SHARED_KEY || hasModernAzureCredentialField(matched)) + && !hasMatchedProperty(matched, PROVIDER_ENDPOINT_ALIASES)) { + endpoint = ""; + } + this.explicitEndpoint = StringUtils.isNotBlank(endpoint); + bindLegacySharedKey(rawProperties, matched); + this.matchedProperties = Collections.unmodifiableMap(matched); + azureAuthType = authType.propertyValue(); + this.accountHost = resolveAccountHostModel(); + endpoint = formatAzureEndpoint(endpoint, accountHost); + this.sasCredential = authType == AzureAuthType.SAS && StringUtils.isNotBlank(sasToken) + ? AzureSasToken.of(sasToken, parseSasExpiry(sasExpiryMs)) : null; + if (sasCredential != null) { + sasExpiryMs = sasCredential.expiresAt() + .map(expiry -> Long.toString(expiry.toEpochMilli())).orElse(""); + } } public static AzureFileSystemProperties of(Map properties) { - AzureFileSystemProperties props = new AzureFileSystemProperties(properties); + return of(properties, Clock.systemUTC()); + } + + static AzureFileSystemProperties of(Map properties, Clock clock) { + AzureFileSystemProperties props = new AzureFileSystemProperties(properties, clock); + props.validate(); + return props; + } + + static AzureFileSystemProperties withVendedSas(AzureVendedSas sas, + Map credentials, Map catalogProperties) { + AzureFileSystemProperties props = new AzureFileSystemProperties(sas, credentials, catalogProperties); props.validate(); return props; } + static AzureFileSystemProperties withFileIOSharedKey(AzureFileIOSharedKey sharedKey, + Map catalogProperties) { + AzureFileSystemProperties props = new AzureFileSystemProperties(sharedKey, catalogProperties); + props.validate(); + return props; + } + + private AzureFileSystemProperties(AzureFileIOSharedKey sharedKey, Map catalogProperties) { + this.clock = Clock.systemUTC(); + // The FileIO's complete SharedKey group replaces every static authentication dialect. + // Reuse only the typed connection defaults, never an intermediate backend credential map. + Map connection = connectionProperties(catalogProperties); + ConnectorPropertiesUtils.bindConnectorProperties(this, connection); + this.explicitEndpoint = StringUtils.isNotBlank(endpoint); + this.authType = AzureAuthType.SHARED_KEY; + this.azureAuthType = authType.propertyValue(); + validateVendedAccountAgainstLegacyAliases(catalogProperties, sharedKey.accountName()); + if (StringUtils.isNotBlank(accountName) && !accountName.equalsIgnoreCase(sharedKey.accountName())) { + throw new StoragePropertiesException("Azure FileIO SharedKey account does not match the catalog account"); + } + this.accountName = sharedKey.accountName(); + this.accountKey = sharedKey.accountKey(); + this.accountHost = resolveAccountHostModel(); + if (StringUtils.isNotBlank(endpoint) + && AzureBlobEndpointSignals.isAzureBlobEndpoint(endpoint, catalogProperties) + && !accountName.equalsIgnoreCase(accountHost.accountName())) { + throw new StoragePropertiesException("Azure FileIO SharedKey account does not match the catalog endpoint"); + } + this.endpoint = formatAzureEndpoint(endpoint, accountHost); + this.sasCredential = null; + connection.putAll(sharedKey.properties()); + this.rawProperties = Collections.unmodifiableMap(new HashMap<>(connection)); + this.matchedProperties = this.rawProperties; + } + + private AzureFileSystemProperties(AzureVendedSas sas, Map credentials, + Map catalogProperties) { + this.clock = Clock.systemUTC(); + // Only connection defaults survive credential replacement. Do not rebind old key/OAuth2 + // aliases or round-trip the SAS through the BE's AZURE_* protocol. + Map connection = connectionProperties(catalogProperties, false); + // The SAS suffix owns the account/endpoint. Other stores' location fields and the old + // static identity in the FileIO map must not override its scope. Only connection options + // are overlaid, by typed field rather than by alias spelling. + for (Field field : ConnectorPropertiesUtils.getConnectorProperties(AzureFileSystemProperties.class)) { + if (Set.of("usePathStyle", "forceParsingByStandardUrl").contains(field.getName())) { + String matched = ConnectorPropertiesUtils.getMatchedPropertyName(field, credentials); + if (matched != null) { + for (String alias : field.getAnnotation(ConnectorProperty.class).names()) { + connection.remove(alias); + } + connection.put(matched, credentials.get(matched)); + } + } + } + ConnectorPropertiesUtils.bindConnectorProperties(this, connection); + this.explicitEndpoint = StringUtils.isNotBlank(endpoint); + this.authType = AzureAuthType.SAS; + this.azureAuthType = authType.propertyValue(); + this.accountHost = sas.accountHost(); + validateVendedAccountAgainstLegacyAliases(catalogProperties, accountHost.accountName()); + if (StringUtils.isNotBlank(accountName) + && !accountName.equalsIgnoreCase(accountHost.accountName())) { + throw new StoragePropertiesException("Azure vended SAS account does not match the catalog account"); + } + if (StringUtils.isNotBlank(endpoint)) { + AzureAccountHost configuredHost = AzureAccountHost.parse(endpoint); + if (AzureBlobEndpointSignals.isAzureBlobEndpoint(endpoint, catalogProperties) + && !configuredHost.blobHost().equalsIgnoreCase(accountHost.blobHost())) { + throw new StoragePropertiesException("Azure vended SAS account does not match the catalog endpoint"); + } + } + this.accountName = accountHost.accountName(); + this.endpoint = formatAzureEndpoint(endpoint, accountHost); + this.sasCredential = sas.token(); + this.sasToken = sasCredential.value(); + this.sasExpiryMs = sasCredential.expiresAt() + .map(expiry -> Long.toString(expiry.toEpochMilli())).orElse(""); + for (Map.Entry entry : credentials.entrySet()) { + String key = entry.getKey(); + if (key != null && Set.of("adls.container", "adls.container-name", "azure.container", "azure.bucket") + .contains(key.toLowerCase(Locale.ROOT))) { + String scopedContainer = entry.getValue(); + if (StringUtils.isBlank(scopedContainer) + || (StringUtils.isNotBlank(container) && !container.equals(scopedContainer.trim()))) { + throw new StoragePropertiesException("Azure vended SAS container does not match the binding"); + } + this.container = scopedContainer.trim(); + connection.put(key, scopedContainer); + } + } + connection.putAll(sas.properties()); + this.rawProperties = Collections.unmodifiableMap(new HashMap<>(connection)); + this.matchedProperties = this.rawProperties; + } + + private static Map connectionProperties(Map properties) { + return connectionProperties(properties, true); + } + + private static Map connectionProperties(Map properties, + boolean includeLegacyEndpointAliases) { + Map result = new HashMap<>(); + Set fields = Set.of("endpoint", "accountName", "container", "usePathStyle", + "forceParsingByStandardUrl"); + for (Field field : ConnectorPropertiesUtils.getConnectorProperties(AzureFileSystemProperties.class)) { + if (fields.contains(field.getName())) { + String matched = field.getName().equals("endpoint") + ? matchedEndpointProperty(field, properties, includeLegacyEndpointAliases) + : ConnectorPropertiesUtils.getMatchedPropertyName(field, properties); + if (matched != null) { + result.put(matched, properties.get(matched)); + } + } + } + return result; + } + + private static String matchedEndpointProperty(Field field, Map properties, + boolean includeLegacyEndpointAliases) { + String providerEndpoint = firstNonBlankKey(properties, PROVIDER_ENDPOINT_ALIASES); + if (providerEndpoint != null) { + return providerEndpoint; + } + return includeLegacyEndpointAliases + ? ConnectorPropertiesUtils.getMatchedPropertyName(field, properties) : null; + } + @Override public void validate() { - new ParamRules() - .check(() -> !isSharedKeyAuth() && !isOauth2Auth(), - "Unsupported Azure auth_type: " + azureAuthType) - .check(() -> isSharedKeyAuth() - && (StringUtils.isBlank(accountName) || StringUtils.isBlank(accountKey)), - "When auth_type is SharedKey, account_name and account_key are required.") - .check(() -> isOauth2Auth() - && (StringUtils.isBlank(oauthAccountHost) + ParamRules rules = new ParamRules(); + switch (authType) { + case SHARED_KEY: + rules.check(() -> StringUtils.isBlank(accountName) || StringUtils.isBlank(accountKey), + "When auth_type is SharedKey, account_name and account_key are required.") + .check(() -> StringUtils.isNotBlank(sasToken), + "When auth_type is SharedKey, sas_token must not be set."); + break; + case SAS: + rules.check(() -> StringUtils.isBlank(sasToken), + "When auth_type is SAS, sas_token is required.") + .check(() -> StringUtils.isNotBlank(accountKey), + "When auth_type is SAS, account_key must not be set.") + .check(() -> hasOauth2CredentialMaterial(), + "When auth_type is SAS, OAuth2 credential material must not be set."); + break; + case OAUTH2: + rules.check(() -> StringUtils.isBlank(oauthAccountHost) || StringUtils.isBlank(clientId) || StringUtils.isBlank(clientSecret) - || StringUtils.isBlank(oauthServerUri)), + || StringUtils.isBlank(oauthServerUri), "When auth_type is OAuth2, oauth2_account_host, oauth2_client_id, " - + "oauth2_client_secret, and oauth2_server_uri are required.") - .validate("Invalid Azure filesystem properties"); + + "oauth2_client_secret, and oauth2_server_uri are required."); + rules.check(() -> StringUtils.isNotBlank(accountKey), + "When auth_type is OAuth2, account_key must not be set.") + .check(() -> StringUtils.isNotBlank(sasToken), + "When auth_type is OAuth2, sas_token must not be set."); + break; + default: + throw new IllegalStateException("Unhandled Azure auth type: " + authType); + } + rules.validate("Invalid Azure filesystem properties"); + if (authType == AzureAuthType.OAUTH2) { + validateOAuthServerUri(); + if (StringUtils.isNotBlank(accountName) && !accountName.equalsIgnoreCase(accountHost.accountName())) { + throw new StoragePropertiesException("Azure OAuth2 account name does not match the account host"); + } + // Standard Azure endpoints identify the account as well as the transport. A custom + // proxy does not: its exact HTTP origin is checked when binding a location instead. + if (explicitEndpoint && AzureBlobEndpointSignals.isAzureBlobEndpoint(endpoint, rawProperties) + && !accountHost.blobHost().equalsIgnoreCase(AzureAccountHost.parse(endpoint).blobHost())) { + throw new StoragePropertiesException("Azure OAuth2 account host does not match the storage endpoint"); + } + } + } + + private void validateOAuthServerUri() { + final URI uri; + try { + uri = URI.create(oauthServerUri); + } catch (IllegalArgumentException e) { + throw new StoragePropertiesException("Invalid Azure OAuth2 server URI"); + } + String path = uri.getPath(); + if (!"https".equalsIgnoreCase(uri.getScheme()) || uri.getHost() == null + || uri.getRawUserInfo() != null || uri.getRawQuery() != null + || uri.getRawFragment() != null + || path == null || !path.matches("/[^/]+/oauth2(/v2\\.0)?/token")) { + throw new StoragePropertiesException( + "Azure OAuth2 server URI must be an HTTPS Entra token endpoint"); + } + } + + private void bindLegacySharedKey(Map properties, Map matched) { + if (authType != AzureAuthType.SHARED_KEY) { + return; + } + // A provider-owned account/key is an explicit modern binding. Even when the routing layer + // also leaves provider=azure in the map, never fill its missing key from a sibling S3 + // provider. Legacy fallback is only for maps that have no Azure account/key fields. + if (hasModernAzureCredentialField(matched)) { + return; + } + // Old Azure resources used S3 properties or generic uppercase wire fields. Keep that + // entry point explicitly qualified; an Azure SAS or an incomplete modern SharedKey + // binding must never borrow a different provider's account/secret from the catalog map. + boolean legacyAzure = "azure".equalsIgnoreCase(properties.get("provider")) + || "AZURE".equalsIgnoreCase(properties.get("_STORAGE_TYPE_")) + || Boolean.parseBoolean(properties.get("fs.azure.support")) + || matched.containsKey("AZURE_ACCOUNT_NAME") + || (StringUtils.isBlank(accountName) + && Set.of("AWS_ACCESS_KEY", "ACCESS_KEY", "access_key").stream() + .anyMatch(key -> StringUtils.isNotBlank(properties.get(key)))) + || Set.of("s3.endpoint", "AWS_ENDPOINT", "endpoint", "ENDPOINT", "AZURE_ENDPOINT").stream() + .map(properties::get).filter(StringUtils::isNotBlank) + .anyMatch(value -> AzureBlobEndpointSignals.isAzureBlobEndpoint(value, properties)); + if (!legacyAzure) { + return; + } + if (hasS3ProviderMarker(properties)) { + return; + } + if (StringUtils.isBlank(accountName)) { + accountName = legacyValue(properties, matched, + "s3.access_key", "AWS_ACCESS_KEY", "ACCESS_KEY", "access_key"); + } + if (StringUtils.isBlank(accountKey)) { + if (matched.containsKey("AZURE_ACCOUNT_NAME") && !matched.containsKey(ACCOUNT_NAME)) { + // AZURE_ACCOUNT_NAME is a historical input alias. Preserve its old pairing with + // the provider-qualified s3.secret_key, but do not absorb an AWS wire secret + // merely because an Azure account alias is present. + accountKey = legacyValue(properties, matched, "s3.secret_key"); + } else { + accountKey = legacyValue(properties, matched, + "s3.secret_key", "AWS_SECRET_KEY", "secret_key", "SECRET_KEY"); + } + } + } + + private static boolean hasModernAzureCredentialField(Map matched) { + // Only canonical azure.* fields are modern bindings. Uppercase AZURE_* names are + // historical aliases and must continue to participate in the legacy SharedKey fallback + // when paired with an old s3.secret_key. + return Set.of(ENDPOINT, ACCOUNT_NAME, ACCOUNT_KEY, AUTH_TYPE).stream() + .anyMatch(matched::containsKey); + } + + private static boolean hasS3ProviderMarker(Map properties) { + return "s3".equalsIgnoreCase(properties.get("provider")) + || "S3".equalsIgnoreCase(properties.get("_STORAGE_TYPE_")) + || Boolean.parseBoolean(properties.get("fs.s3.support")); + } + + private static boolean hasMatchedProperty(Map properties, Iterable names) { + for (String name : names) { + if (properties.containsKey(name)) { + return true; + } + } + return false; + } + + private static void validateVendedAccountAgainstLegacyAliases(Map properties, + String vendedAccount) { + boolean azureBinding = "azure".equalsIgnoreCase(properties.get("provider")) + || "AZURE".equalsIgnoreCase(properties.get("_STORAGE_TYPE_")) + || Boolean.parseBoolean(properties.get("fs.azure.support")) + || hasProviderOwnedAzureIdentity(properties) + || LEGACY_ENDPOINT_ALIASES + .stream().map(properties::get).filter(StringUtils::isNotBlank) + .anyMatch(value -> AzureBlobEndpointSignals.isAzureBlobEndpoint(value, properties)); + if (!azureBinding) { + return; + } + // Provider-owned Azure fields are resolved by the typed binding and validated by the + // constructor. Generic aliases are only legacy Azure inputs when no provider-owned + // identity is present; otherwise they may belong to a sibling S3 binding. + if (hasProviderOwnedAzureIdentity(properties)) { + return; + } + for (String key : new String[] {"s3.access_key", "AWS_ACCESS_KEY", "ACCESS_KEY", "access_key"}) { + String legacyAccount = properties.get(key); + if (StringUtils.isNotBlank(legacyAccount) + && !legacyAccount.trim().equalsIgnoreCase(vendedAccount)) { + throw new StoragePropertiesException( + "Azure vended credential account does not match the legacy account"); + } + } + for (String key : LEGACY_ENDPOINT_ALIASES) { + String legacyEndpoint = properties.get(key); + if (StringUtils.isBlank(legacyEndpoint) + || !AzureBlobEndpointSignals.isAzureBlobEndpoint(legacyEndpoint, properties)) { + continue; + } + AzureAccountHost legacyHost = AzureAccountHost.parse(legacyEndpoint); + if (!legacyHost.accountName().equalsIgnoreCase(vendedAccount)) { + throw new StoragePropertiesException( + "Azure vended credential account does not match the legacy endpoint"); + } + } + } + + private static boolean hasProviderOwnedAzureIdentity(Map properties) { + return firstNonBlankKey(properties, PROVIDER_ACCOUNT_ALIASES) != null + || firstNonBlankKey(properties, PROVIDER_ENDPOINT_ALIASES) != null + || StringUtils.isNotBlank(properties.get(ACCOUNT_KEY)) + || StringUtils.isNotBlank(properties.get(AUTH_TYPE)); + } + + private static String firstNonBlankKey(Map properties, Iterable names) { + for (String name : names) { + String value = properties.get(name); + if (StringUtils.isNotBlank(value)) { + return name; + } + } + return null; + } + + private static Map azureBindingProperties(Map properties) { + String providerEndpoint = firstNonBlankKey(properties, PROVIDER_ENDPOINT_ALIASES); + if (providerEndpoint == null) { + return properties; + } + Map result = new HashMap<>(properties); + LEGACY_ENDPOINT_ALIASES.forEach(result::remove); + result.put(providerEndpoint, properties.get(providerEndpoint)); + return result; + } + + private static String legacyValue(Map properties, Map matched, String... names) { + for (String name : names) { + String value = properties.get(name); + if (StringUtils.isNotBlank(value)) { + matched.put(name, value); + return value; + } + } + return ""; + } + + @Override + public void validateForAccess() { + validateSasExpiry(clock); } @Override @@ -219,39 +623,136 @@ public Optional toBackendProperties() { return Optional.of(this); } + @Override + public Optional resolveBackendProperties(String normalizedUri) { + AzureUri uri = parseAndValidateUri(normalizedUri); + if (isOneLake(uri)) { + return Optional.of(new BackendStorageProperties() { + @Override + public BackendStorageKind backendKind() { + return BackendStorageKind.HDFS; + } + + @Override + public Map toMap() { + return hadoopBackendProperties(); + } + }); + } + return toBackendProperties(); + } + @Override public Optional toHadoopProperties() { return Optional.of(this); } + @Override + public Optional toIcebergHadoopProperties() { + // Default REST metadata uses ADLSFileIO for SAS, after choosing the current load's + // credentials. Eagerly materializing its old ABFS config would reject an expired + // static SAS before a fresh vended credential can replace it. + return authType == AzureAuthType.SAS ? Optional.empty() : toHadoopProperties(); + } + + @Override + public Map toIcebergFileIOConnectionProperties() { + if (accountHost == null) { + throw new StoragePropertiesException("Azure Iceberg FileIO requires an account name or endpoint"); + } + // The SDK applies endpoint() after credentials; a SAS in its query would replace the + // selected credential. Never let the endpoint become a second auth source. + URI fileIOEndpoint = URI.create(endpoint); + if (fileIOEndpoint.getRawUserInfo() != null || fileIOEndpoint.getRawQuery() != null + || fileIOEndpoint.getRawFragment() != null) { + throw new StoragePropertiesException( + "Azure Iceberg FileIO endpoint must not contain userinfo, query or fragment"); + } + + Map properties = new HashMap<>(); + // Iceberg's connection-string property is an endpoint URL, not an SDK connection string. + for (String host : new String[] {accountHost.dfsHost(), accountHost.blobHost()}) { + properties.put("adls.connection-string." + host, endpoint); + } + return Collections.unmodifiableMap(properties); + } + + @Override + public Map toIcebergFileIOProperties() { + validateForAccess(); + if (isOauth2Auth()) { + // Iceberg's ADLSFileIO cannot construct this per-catalog client-secret identity. + // Its Hadoop FileIO consumes our separate Hadoop view; native BE auth is unchanged. + return Map.of("io-impl", "org.apache.iceberg.hadoop.HadoopFileIO"); + } + Map properties = new HashMap<>(toIcebergFileIOConnectionProperties()); + switch (authType) { + case SHARED_KEY: + properties.put("adls.auth.shared-key.account.name", accountName); + properties.put("adls.auth.shared-key.account.key", accountKey); + break; + case SAS: + // Iceberg 1.10.1 AzureProperties looks up SAS by the full ADLSLocation host. + for (String host : new String[] {accountHost.dfsHost(), accountHost.blobHost()}) { + properties.put("adls.sas-token." + host, sasCredential.value()); + if (StringUtils.isNotBlank(sasExpiryMs)) { + properties.put("adls.sas-token-expires-at-ms." + host, sasExpiryMs); + } + } + break; + default: + throw new IllegalStateException("Unhandled Azure auth type: " + authType); + } + return Collections.unmodifiableMap(properties); + } + @Override public BackendStorageKind backendKind() { - return BackendStorageKind.S3_COMPATIBLE; + // Native provider identity is independent of the existing FILE_S3 Thrift reader slot. + return BackendStorageKind.NATIVE; } @Override public Map toMap() { - // OAuth2 has no AK/SK equivalent that the BE S3-compatible adapter can consume; - // OAuth2 access currently only works through the Hadoop ABFS connector (the Iceberg - // REST catalog / OneLake path), so hand over the fs.azure.* OAuth config instead of - // S3-style params. Mirrors legacy AzureProperties.getBackendConfigProperties(); - // native-SDK OAuth2 support may replace this in the future. - if (isOauth2Auth()) { - return oauth2BackendProperties(); + validateForAccess(); + // Keep Azure's native credential vocabulary at the FE→BE boundary. The BE still receives + // FILE_S3, but its provider marker dispatches this map to the Azure SDK. In particular, + // an Azure SAS is not an AWS session token. + Map azureProps = new HashMap<>(); + azureProps.put("provider", "azure"); + azureProps.put(BACKEND_AUTH_TYPE, authType.name()); + switch (authType) { + case SHARED_KEY: + azureProps.put(BACKEND_ACCOUNT_KEY, accountKey); + break; + case SAS: + azureProps.put(BACKEND_SAS_TOKEN, sasCredential.value()); + if (StringUtils.isNotBlank(sasExpiryMs)) { + azureProps.put(BACKEND_SAS_EXPIRY_MS, sasExpiryMs); + } + break; + case OAUTH2: + azureProps.put(BACKEND_CLIENT_ID, clientId); + azureProps.put(BACKEND_CLIENT_SECRET, clientSecret); + resolveTenantId().ifPresent(value -> azureProps.put(BACKEND_TENANT_ID, value)); + azureProps.put(BACKEND_OAUTH_SERVER_URI, oauthServerUri); + break; + default: + throw new IllegalStateException("Unhandled Azure auth type: " + authType); + } + if (StringUtils.isNotBlank(endpoint)) { + azureProps.put(BACKEND_ENDPOINT, endpoint); } - Map s3Props = new HashMap<>(); - s3Props.put("AWS_ENDPOINT", endpoint); - s3Props.put("AWS_REGION", "dummy_region"); - s3Props.put("AWS_ACCESS_KEY", accountName); - s3Props.put("AWS_SECRET_KEY", accountKey); - s3Props.put("AWS_NEED_OVERRIDE_ENDPOINT", "true"); - s3Props.put("provider", "azure"); - s3Props.put("use_path_style", usePathStyle); - return Collections.unmodifiableMap(s3Props); + String backendAccountName = resolveBackendAccountName(); + if (StringUtils.isNotBlank(backendAccountName)) { + azureProps.put(BACKEND_ACCOUNT_NAME, backendAccountName); + } + return Collections.unmodifiableMap(azureProps); } @Override public Map toHadoopConfigurationMap() { + validateForAccess(); Map cfg = new HashMap<>(); // No blanket ABFS/WASB cache disabling: the Doris-patched FileSystem keys its cache by the // per-scheme credential fingerprint below, so different credentials never share an @@ -262,49 +763,56 @@ public Map toHadoopConfigurationMap() { cfg.put(key, value); } }); - if (isOauth2Auth()) { - cfg.put("fs.azure.account.auth.type." + oauthAccountHost, "OAuth"); - cfg.put("fs.azure.account.oauth.provider.type." + oauthAccountHost, - "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider"); - cfg.put("fs.azure.account.oauth2.client.id." + oauthAccountHost, clientId); - cfg.put("fs.azure.account.oauth2.client.secret." + oauthAccountHost, clientSecret); - cfg.put("fs.azure.account.oauth2.client.endpoint." + oauthAccountHost, oauthServerUri); - } else { - for (String suffix : normalizedAzureBlobHostSuffixes()) { - cfg.put("fs.azure.account.key." + accountName + "." + suffix, accountKey); - } - cfg.put("fs.azure.account.key", accountKey); + switch (authType) { + case SHARED_KEY: + for (String suffix : normalizedAzureBlobHostSuffixes()) { + cfg.put("fs.azure.account.key." + accountName + "." + suffix, accountKey); + } + cfg.put("fs.azure.account.key", accountKey); + break; + case SAS: + String accountHost = resolveAccountHost(); + if (StringUtils.isNotBlank(accountHost)) { + cfg.put("fs.azure.account.auth.type." + accountHost, SAS_AUTH); + cfg.put("fs.azure.sas.fixed.token." + accountHost, sasCredential.value()); + } + break; + case OAUTH2: + cfg.put("fs.azure.account.auth.type." + oauthAccountHost, "OAuth"); + cfg.put("fs.azure.account.oauth.provider.type." + oauthAccountHost, + "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider"); + cfg.put("fs.azure.account.oauth2.client.id." + oauthAccountHost, clientId); + cfg.put("fs.azure.account.oauth2.client.secret." + oauthAccountHost, clientSecret); + cfg.put("fs.azure.account.oauth2.client.endpoint." + oauthAccountHost, oauthServerUri); + break; + default: + throw new IllegalStateException("Unhandled Azure auth type: " + authType); } return Collections.unmodifiableMap(cfg); } /** - * BE property map for OAuth2, key-for-key equal to what legacy fe-core - * {@code AzureProperties.getBackendConfigProperties()} produced: it dumped its whole - * {@link Configuration}, so on top of {@link #toHadoopConfigurationMap()} the map also carries - * hadoop's {@code core-default.xml} and any {@code core-site.xml} reachable on the FE - * classpath ({@code start_fe.sh} puts {@code ${DORIS_HOME}/conf} and {@code ${HADOOP_CONF_DIR}} - * there, so this is a real operator-facing channel, not just hadoop's built-in defaults). - * - *

The live consumer is Microsoft Fabric OneLake: {@code LocationPath.getTFileTypeForBE()} - * routes {@code abfs[s]://...dfs.fabric.microsoft.com} to {@code FILE_HDFS}, and BE's - * {@code hdfs_builder.cpp} feeds every entry of this map into its JNI hadoop builder — which is - * how the {@code fs.azure.account.oauth2.*} keys reach the ABFS connector. - * - *

Ordering is load-bearing and mirrors the legacy sequence: hadoop defaults first, then the - * provider's own {@code fs.azure.*} view (which carries the per-scheme cache fingerprint), then - * user {@code fs.*} passthrough (so an explicit user value wins), then cache-disable - * normalization last. - * - *

The plugin bundles hadoop, but {@code FileSystemPluginManager.FS_PARENT_FIRST_PREFIXES} - * makes {@code org.apache.hadoop.} parent-first, so whenever the FE host ships hadoop this - * resolves to the host copy and the bundled one is only a {@code findClass} fallback. That also - * means the XML defaults below come from wherever the context classloader finds them — hadoop's - * {@code Configuration} looks up {@code core-default.xml}/{@code core-site.xml} through the - * TCCL, not through this class's loader. A host that stops shipping hadoop would therefore need - * the TCCL pinned to the plugin loader here, not just the bundled jars. + * Legacy Hadoop configuration view retained for genuine Fabric OneLake locations. OneLake + * remains explicitly FILE_HDFS-routed; ordinary Azure ABFS paths use the native fields in + * {@link #toMap()}. */ - private Map oauth2BackendProperties() { + private Map hadoopBackendProperties() { + // Caching configuration must not extend the credential's validity on later accesses. + validateForAccess(); + Map cached = cachedHadoopBackendProperties; + if (cached == null) { + synchronized (hadoopBackendPropertiesLock) { + cached = cachedHadoopBackendProperties; + if (cached == null) { + cached = buildHadoopBackendProperties(); + cachedHadoopBackendProperties = cached; + } + } + } + return cached; + } + + private Map buildHadoopBackendProperties() { Configuration conf = new Configuration(); toHadoopConfigurationMap().forEach(conf::set); rawProperties.forEach((key, value) -> { @@ -315,9 +823,6 @@ private Map oauth2BackendProperties() { for (String scheme : legacyCacheSchemes()) { String key = "fs." + scheme + ".impl.disable.cache"; String userValue = rawProperties.get(key); - // No blanket disable any more (the fingerprint in toHadoopConfigurationMap() isolates - // credentials instead); an explicit user value is still honored, normalized to - // true/false ("yes"/"1" would reach BE verbatim through the fs.* passthrough above). if (StringUtils.isNotBlank(userValue)) { conf.setBoolean(key, BooleanUtils.toBoolean(userValue)); } @@ -327,6 +832,167 @@ private Map oauth2BackendProperties() { return Collections.unmodifiableMap(dump); } + /** + * Keeps Azure's account authority and object path intact for the BE native reader. The old + * fe-core adapter rewrote these locations to {@code s3://container/path}, which discarded the + * account host and forced Azure data through an HDFS/S3 compatibility interpretation. + */ + @Override + public String validateAndNormalizeUri(String path) { + AzureUri parsed = parseAndValidateUri(path); + int delimiter = path.indexOf("://"); + return path.substring(0, delimiter).equals(parsed.scheme()) + ? path : parsed.scheme() + path.substring(delimiter); + } + + @Override + public boolean matchesLocationPrefix(String rawLocation, String rawPrefix) { + AzureUri location = parseAndValidateUri(rawLocation); + AzureUri prefix = parseAndValidateUri(rawPrefix); + // Legacy SharedKey s3:// locations take their account from this binding. Native Azure + // authorities compare by Blob host so DFS/Blob spelling does not change storage identity. + AzureAccountHost locationHost = scopeAccountHost(location); + AzureAccountHost prefixHost = scopeAccountHost(prefix); + return locationHost.blobHost().equalsIgnoreCase(prefixHost.blobHost()) + && location.container().equals(prefix.container()) + && (prefix.key().isEmpty() + || Location.of(location.key()).startsWith(Location.of(prefix.key()))); + } + + private AzureAccountHost scopeAccountHost(AzureUri uri) { + // parseAndValidateUri has already checked the exact HTTP origin. A custom endpoint's + // hostname is a transport address, not an account; its identity belongs to this binding. + return accountInAuthority(uri) ? uri.accountHost().orElse(accountHost) : accountHost; + } + + private AzureUri parseAndValidateUri(String path) { + if (StringUtils.isBlank(path)) { + throw new StoragePropertiesException("Path cannot be null or empty"); + } + int delimiter = path.indexOf("://"); + if (delimiter <= 0) { + throw new StoragePropertiesException("Azure URI must contain a scheme"); + } + String scheme = path.substring(0, delimiter).toLowerCase(Locale.ROOT); + if (!(scheme.equals("wasb") || scheme.equals("wasbs") + || scheme.equals("abfs") || scheme.equals("abfss") + || scheme.equals("http") || scheme.equals("https") + || scheme.equals("s3"))) { + throw new StoragePropertiesException("Unsupported Azure URI scheme"); + } + if (scheme.equals("s3") && !isSharedKeyAuth()) { + throw new StoragePropertiesException("Azure SAS/OAuth2 data access requires an Azure URI"); + } + // Parse account/container/path now so malformed locations fail before a scan reaches BE; + // return the original path (apart from a case-insensitive scheme) to preserve object keys. + try { + AzureUri parsed = AzureUri.parse(path); + validateLocationBinding(parsed); + if (scheme.equals("http") || scheme.equals("https")) { + URI uriEndpoint = httpAuthority(parsed.accountHost().orElseThrow().blobEndpoint()); + URI configuredEndpoint = httpAuthority(endpoint); + if (!claimsUri(path) || uriEndpoint == null || configuredEndpoint == null + || !sameOrigin(uriEndpoint, configuredEndpoint)) { + throw new StoragePropertiesException("Azure URI endpoint does not match the binding"); + } + } + return parsed; + } catch (IOException e) { + throw new StoragePropertiesException("Invalid Azure URI", e); + } + } + + private void validateLocationBinding(AzureUri uri) { + // Custom HTTP endpoint hosts do not encode a storage account name. Their exact origin + // is checked separately; treating the first DNS label as an account rejects valid proxies. + boolean accountInAuthority = accountInAuthority(uri); + if (accountInAuthority && StringUtils.isNotBlank(accountName) && StringUtils.isNotBlank(uri.accountName()) + && !StringUtils.equalsIgnoreCase(accountName, uri.accountName())) { + throw new StoragePropertiesException( + "Azure URI account does not match configured account_name"); + } + if (accountInAuthority && accountHost != null && uri.accountHost().isPresent()) { + AzureAccountHost uriHost = uri.accountHost().get(); + if (accountHost.isAzureCloudHost() && uriHost.isAzureCloudHost() + && !accountHost.blobHost().equalsIgnoreCase(uriHost.blobHost())) { + throw new StoragePropertiesException("Azure URI account host does not match the binding"); + } + } + if (StringUtils.isNotBlank(container) && !StringUtils.equals(container, uri.container())) { + throw new StoragePropertiesException( + "Azure URI container does not match configured container"); + } + } + + private boolean accountInAuthority(AzureUri uri) { + return !(uri.scheme().equals("http") || uri.scheme().equals("https")) + || AzureBlobEndpointSignals.isAzureBlobEndpoint( + uri.accountHost().orElseThrow().blobEndpoint(), rawProperties); + } + + @Override + public boolean claimsUri(String uri) { + // Only parse the authority here. An invalid key (for example %GG) on an Azure host + // must still select this binding and fail strict validation instead of routing as HTTP. + URI authority = httpAuthority(uri); + if (authority == null) { + return false; + } + if (AzureBlobEndpointSignals.isAzureBlobEndpoint(authority.toString(), rawProperties)) { + return true; + } + URI configuredEndpoint = explicitEndpoint ? httpAuthority(endpoint) : null; + return configuredEndpoint != null && sameOrigin(authority, configuredEndpoint); + } + + private static boolean isOneLake(AzureUri uri) { + return (uri.scheme().equals("abfs") || uri.scheme().equals("abfss")) + && uri.accountHost().filter(host -> host.isDfsHost() + && "fabric.microsoft.com".equalsIgnoreCase(host.cloudSuffix())).isPresent(); + } + + private static URI httpAuthority(String value) { + if (StringUtils.isBlank(value)) { + return null; + } + int delimiter = value.indexOf("://"); + if (delimiter <= 0) { + return null; + } + String scheme = value.substring(0, delimiter).toLowerCase(Locale.ROOT); + if (!scheme.equals("http") && !scheme.equals("https")) { + return null; + } + int authorityEnd = value.length(); + for (char separator : new char[] {'/', '?', '#'}) { + int index = value.indexOf(separator, delimiter + 3); + if (index >= 0) { + authorityEnd = Math.min(authorityEnd, index); + } + } + try { + URI authority = new URI(scheme + value.substring(delimiter, authorityEnd)); + if (authority.getHost() == null || authority.getRawUserInfo() != null + || authority.getPort() > 65535) { + return null; + } + return authority; + } catch (URISyntaxException e) { + // URI syntax errors contain the original input, potentially including credentials. + return null; + } + } + + private static boolean sameOrigin(URI first, URI second) { + return first.getScheme().equalsIgnoreCase(second.getScheme()) + && first.getHost().equalsIgnoreCase(second.getHost()) + && effectivePort(first) == effectivePort(second); + } + + private static int effectivePort(URI uri) { + return uri.getPort() >= 0 ? uri.getPort() : uri.getScheme().equalsIgnoreCase("https") ? 443 : 80; + } + public String getEndpoint() { return endpoint; } @@ -339,6 +1005,14 @@ public String getAccountKey() { return accountKey; } + public String getSasToken() { + return sasCredential == null ? sasToken : sasCredential.value(); + } + + public String getSasExpiryMs() { + return sasExpiryMs; + } + public String getClientId() { return clientId; } @@ -363,6 +1037,10 @@ public String getAzureAuthType() { return azureAuthType; } + public AzureAuthType authType() { + return authType; + } + public String getContainer() { return container; } @@ -376,11 +1054,15 @@ public String getForceParsingByStandardUrl() { } public boolean isSharedKeyAuth() { - return SHARED_KEY_AUTH.equalsIgnoreCase(azureAuthType); + return authType == AzureAuthType.SHARED_KEY; + } + + public boolean isSasAuth() { + return authType == AzureAuthType.SAS; } public boolean isOauth2Auth() { - return OAUTH2_AUTH.equalsIgnoreCase(azureAuthType); + return authType == AzureAuthType.OAUTH2; } public Optional resolveTenantId() { @@ -410,35 +1092,74 @@ public Optional resolveTenantId() { } } - private void normalize() { - endpoint = formatAzureEndpoint(endpoint, accountName, oauthAccountHost); + private AzureAuthType resolveAuthType() { if (StringUtils.isNotBlank(azureAuthType)) { - if (SHARED_KEY_AUTH.equalsIgnoreCase(azureAuthType)) { - azureAuthType = SHARED_KEY_AUTH; - } else if (OAUTH2_AUTH.equalsIgnoreCase(azureAuthType)) { - azureAuthType = OAUTH2_AUTH; - } + return AzureAuthType.parse(azureAuthType); + } + // Infer only when the property binder found no nonblank auth type, respecting its alias + // precedence and case-sensitive key matching. OAuth2 continues to require an explicit type. + boolean hasSas = StringUtils.isNotBlank(sasToken); + boolean hasSharedKey = StringUtils.isNotBlank(accountKey); + if (hasSas && hasSharedKey) { + throw new StoragePropertiesException( + "Azure auth_type is required when SharedKey and SAS credential materials are both set; " + + "cannot infer an authentication mode"); } + return hasSas ? AzureAuthType.SAS : AzureAuthType.SHARED_KEY; } - private static String formatAzureEndpoint(String endpoint, String accountName, String accountHost) { - if (StringUtils.isBlank(endpoint)) { - if (StringUtils.isNotBlank(accountName)) { - return String.format(AZURE_ENDPOINT_TEMPLATE, accountName); - } - return addHttpsScheme(accountHost); + private boolean hasOauth2CredentialMaterial() { + return StringUtils.isNotBlank(clientId) || StringUtils.isNotBlank(clientSecret) + || StringUtils.isNotBlank(oauthServerUri) || StringUtils.isNotBlank(oauthAccountHost) + || StringUtils.isNotBlank(tenantId); + } + + private AzureAccountHost resolveAccountHostModel() { + if (authType == AzureAuthType.OAUTH2 && StringUtils.isNotBlank(oauthAccountHost)) { + return AzureAccountHost.parse(oauthAccountHost); } - return addHttpsScheme(endpoint); + if (StringUtils.isNotBlank(endpoint)) { + return AzureAccountHost.parse(endpoint); + } + if (StringUtils.isNotBlank(accountName)) { + return AzureAccountHost.fromAccountName(accountName); + } + return null; } - private static String addHttpsScheme(String endpoint) { - if (StringUtils.isBlank(endpoint)) { - return ""; + private String resolveBackendAccountName() { + if (StringUtils.isNotBlank(accountName)) { + return accountName; } - if (endpoint.contains("://")) { - return endpoint; + return accountHost == null ? "" : accountHost.accountName(); + } + + private String resolveAccountHost() { + return accountHost == null ? "" : accountHost.dfsHost(); + } + + void validateSasExpiry(Clock clock) { + if (isSasAuth()) { + sasCredential.validateNotExpired(clock); + } + } + + private static Long parseSasExpiry(String expiry) { + if (StringUtils.isBlank(expiry)) { + return null; + } + try { + return Long.parseLong(expiry.trim()); + } catch (NumberFormatException e) { + throw new StoragePropertiesException("Invalid Azure SAS expiry value"); + } + } + + private static String formatAzureEndpoint(String endpoint, AzureAccountHost accountHost) { + if (StringUtils.isBlank(endpoint)) { + return accountHost == null ? "" : accountHost.blobEndpoint(); } - return "https://" + endpoint; + return AzureAccountHost.parse(endpoint).blobEndpoint(); } private static Set normalizedAzureBlobHostSuffixes() { diff --git a/fe/fe-filesystem/fe-filesystem-azure/src/main/java/org/apache/doris/filesystem/azure/AzureFileSystemProvider.java b/fe/fe-filesystem/fe-filesystem-azure/src/main/java/org/apache/doris/filesystem/azure/AzureFileSystemProvider.java index d76f0917675470..411d2266a7731e 100644 --- a/fe/fe-filesystem/fe-filesystem-azure/src/main/java/org/apache/doris/filesystem/azure/AzureFileSystemProvider.java +++ b/fe/fe-filesystem/fe-filesystem-azure/src/main/java/org/apache/doris/filesystem/azure/AzureFileSystemProvider.java @@ -23,9 +23,8 @@ import org.apache.doris.foundation.property.ConnectorPropertiesUtils; import java.io.IOException; -import java.util.Arrays; -import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; /** @@ -33,9 +32,8 @@ * *

Registered via META-INF/services/org.apache.doris.filesystem.spi.FileSystemProvider. * - *

Identified by the presence of {@code AZURE_ACCOUNT_NAME}, {@code azure.account_name}, - * or an endpoint that contains a known Azure Blob Storage host suffix from one of the - * sovereign clouds. + *

Identified by the presence of Azure provider-owned account/SAS properties, or an endpoint + * whose host matches a known Azure Blob/DFS suffix from one of the sovereign clouds. */ public class AzureFileSystemProvider implements FileSystemProvider { @@ -44,21 +42,12 @@ public class AzureFileSystemProvider implements FileSystemProvider AZURE_BLOB_HOST_SUFFIXES = Arrays.asList( - "blob.core.windows.net", - "blob.core.chinacloudapi.cn", - "blob.core.usgovcloudapi.net", - "blob.core.cloudapi.de"); - @Override public boolean supports(Map properties) { if (isExplicitAzure(properties)) { @@ -67,16 +56,11 @@ public boolean supports(Map properties) { if (firstPresent(properties, ACCOUNT_NAME_KEYS) != null) { return true; } - String endpoint = firstPresent(properties, ENDPOINT_KEYS); - if (endpoint == null) { - return false; - } - for (String suffix : AZURE_BLOB_HOST_SUFFIXES) { - if (endpoint.contains(suffix)) { - return true; - } + if (firstPresent(properties, SAS_TOKEN_KEYS) != null) { + return true; } - return false; + String endpoint = firstPresent(properties, ENDPOINT_KEYS); + return endpoint != null && AzureBlobEndpointSignals.isAzureBlobEndpoint(endpoint, properties); } @Override @@ -84,6 +68,22 @@ public AzureFileSystemProperties bind(Map properties) { return AzureFileSystemProperties.of(properties); } + @Override + public Optional bindVended( + Map credentials, Map catalogProperties) { + Optional sas = AzureVendedSas.parse(credentials); + // A vended credential may be bound after the catalog has already selected this provider. + // Also retain catalog defaults recognized only by the raw compatibility hook, such as a + // historical AZURE_ENDPOINT standard host, without widening the routing guess contract. + Map connectionDefaults = supportsExplicit(catalogProperties) + || supportsGuess(catalogProperties) || supports(catalogProperties) ? catalogProperties : Map.of(); + if (sas.isPresent()) { + return Optional.of(AzureFileSystemProperties.withVendedSas(sas.get(), credentials, connectionDefaults)); + } + return AzureFileIOSharedKey.parse(credentials) + .map(sharedKey -> AzureFileSystemProperties.withFileIOSharedKey(sharedKey, connectionDefaults)); + } + @Override public FileSystem create(AzureFileSystemProperties properties) throws IOException { return new AzureFileSystem(new AzureObjStorage(properties)); @@ -109,7 +109,9 @@ public boolean supportsGuess(Map properties) { // (endpoint alias list, host extraction, dot-anchored endsWith, probe-injected live // suffix list) is shared with the S3-compatible fallback providers via // AzureBlobEndpointSignals so their mutual exclusion can never drift from this claim. - if ("azure".equalsIgnoreCase(properties.get(PROVIDER_KEY))) { + if ("azure".equalsIgnoreCase(properties.get(PROVIDER_KEY)) + || firstPresent(properties, ACCOUNT_NAME_KEYS) != null + || firstPresent(properties, SAS_TOKEN_KEYS) != null) { return true; } return AzureBlobEndpointSignals.guessIsAzureBlobEndpoint(properties); @@ -127,7 +129,12 @@ public String name() { @Override public Set sensitivePropertyKeys() { - return ConnectorPropertiesUtils.getSensitiveKeys(AzureFileSystemProperties.class); + Set keys = ConnectorPropertiesUtils.getSensitiveKeys(AzureFileSystemProperties.class); + // Wire secrets must stay masked independently of which spellings the input binder accepts. + keys.addAll(Set.of(AzureFileSystemProperties.BACKEND_ACCOUNT_KEY, + AzureFileSystemProperties.BACKEND_CLIENT_SECRET, AzureFileSystemProperties.BACKEND_SAS_TOKEN, + "s3.secret_key", "AWS_SECRET_KEY", "secret_key", "SECRET_KEY")); + return keys; } private boolean isExplicitAzure(Map properties) { diff --git a/fe/fe-filesystem/fe-filesystem-azure/src/main/java/org/apache/doris/filesystem/azure/AzureObjStorage.java b/fe/fe-filesystem/fe-filesystem-azure/src/main/java/org/apache/doris/filesystem/azure/AzureObjStorage.java index 837e739abd184b..1e8a2a37d11cc3 100644 --- a/fe/fe-filesystem/fe-filesystem-azure/src/main/java/org/apache/doris/filesystem/azure/AzureObjStorage.java +++ b/fe/fe-filesystem/fe-filesystem-azure/src/main/java/org/apache/doris/filesystem/azure/AzureObjStorage.java @@ -49,6 +49,7 @@ import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.charset.StandardCharsets; +import java.time.Clock; import java.time.OffsetDateTime; import java.util.ArrayList; import java.util.Base64; @@ -75,6 +76,7 @@ public class AzureObjStorage implements ObjStorage { private static final int SESSION_EXPIRE_SECONDS = 3600; private final AzureFileSystemProperties properties; + private final Clock clock; private volatile BlobServiceClient client; public AzureObjStorage(Map properties) { @@ -82,11 +84,17 @@ public AzureObjStorage(Map properties) { } public AzureObjStorage(AzureFileSystemProperties properties) { + this(properties, Clock.systemUTC()); + } + + AzureObjStorage(AzureFileSystemProperties properties, Clock clock) { this.properties = properties; + this.clock = clock; } @Override public BlobServiceClient getClient() throws IOException { + properties.validateSasExpiry(clock); if (client == null) { synchronized (this) { if (client == null) { @@ -98,24 +106,30 @@ public BlobServiceClient getClient() throws IOException { } protected BlobServiceClient buildClient() throws IOException { + properties.validateSasExpiry(clock); String endpoint = requireProperty( properties.getEndpoint(), AzureFileSystemProperties.ENDPOINT, "Azure endpoint"); BlobServiceClientBuilder builder = new BlobServiceClientBuilder().endpoint(endpoint); - if (properties.isSharedKeyAuth()) { - String accountName = requireProperty( - properties.getAccountName(), AzureFileSystemProperties.ACCOUNT_NAME, "Azure account name"); - String accountKey = requireProperty( - properties.getAccountKey(), AzureFileSystemProperties.ACCOUNT_KEY, "Azure account key"); - builder.credential(new StorageSharedKeyCredential(accountName, accountKey)); - } else { - String tenantId = properties.resolveTenantId() - .orElseThrow(() -> new IOException("Azure tenant id is required for OAuth2 native SDK access")); - builder.credential(new ClientSecretCredentialBuilder() - .tenantId(tenantId) - .clientId(properties.getClientId()) - .clientSecret(properties.getClientSecret()) - .build()); + switch (properties.authType()) { + case SHARED_KEY: + builder.credential(new StorageSharedKeyCredential( + properties.getAccountName(), properties.getAccountKey())); + break; + case SAS: + builder.sasToken(properties.getSasToken()); + break; + case OAUTH2: + String tenantId = properties.resolveTenantId() + .orElseThrow(() -> new IOException("Azure tenant id is required for OAuth2 native SDK access")); + builder.credential(new ClientSecretCredentialBuilder() + .tenantId(tenantId) + .clientId(properties.getClientId()) + .clientSecret(properties.getClientSecret()) + .build()); + break; + default: + throw new IllegalStateException("Unhandled Azure auth type: " + properties.authType()); } return builder.buildClient(); } diff --git a/fe/fe-filesystem/fe-filesystem-azure/src/main/java/org/apache/doris/filesystem/azure/AzureSasToken.java b/fe/fe-filesystem/fe-filesystem-azure/src/main/java/org/apache/doris/filesystem/azure/AzureSasToken.java new file mode 100644 index 00000000000000..71986362b51758 --- /dev/null +++ b/fe/fe-filesystem/fe-filesystem-azure/src/main/java/org/apache/doris/filesystem/azure/AzureSasToken.java @@ -0,0 +1,144 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.filesystem.azure; + +import org.apache.doris.foundation.property.StoragePropertiesException; + +import java.net.URLDecoder; +import java.nio.charset.StandardCharsets; +import java.time.Clock; +import java.time.DateTimeException; +import java.time.Instant; +import java.time.OffsetDateTime; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; +import java.util.Optional; + +/** Normalized Azure SAS token with optional, validated expiry information. */ +public final class AzureSasToken { + + private final String value; + private final Optional expiresAt; + + private AzureSasToken(String value, Optional expiresAt) { + this.value = value; + this.expiresAt = expiresAt; + } + + /** + * Removes only transport prefixes from a SAS token. The signed query contents are not + * decoded or re-encoded. + */ + public static AzureSasToken of(String raw, Long explicitExpiryMs) { + if (raw == null || raw.isBlank()) { + throw new StoragePropertiesException("Azure SAS token must not be empty"); + } + if (raw.indexOf('\r') >= 0 || raw.indexOf('\n') >= 0) { + throw new StoragePropertiesException("Azure SAS token must not contain a line break"); + } + String normalized = raw.trim(); + while (normalized.startsWith("?") || normalized.startsWith("&")) { + normalized = normalized.substring(1); + } + if (normalized.isEmpty()) { + throw new StoragePropertiesException("Azure SAS token must not be empty"); + } + + Optional explicitExpiry = Optional.ofNullable(explicitExpiryMs) + .map(AzureSasToken::instantFromEpochMillis); + Optional tokenExpiry = findTokenExpiry(normalized); + Optional effectiveExpiry = minExpiry(explicitExpiry, tokenExpiry); + return new AzureSasToken(normalized, effectiveExpiry); + } + + public String value() { + return value; + } + + public Optional expiresAt() { + return expiresAt; + } + + public boolean isExpired(Clock clock) { + return expiresAt.map(expiry -> !expiry.isAfter(Instant.now(clock))).orElse(false); + } + + /** + * Rejects a known expired token. Without expiry information Doris cannot prevalidate its + * lifetime; the storage service still decides whether to accept it. + */ + public void validateNotExpired(Clock clock) { + if (isExpired(clock)) { + throw new StoragePropertiesException("Azure SAS credential is expired"); + } + } + + private static Instant instantFromEpochMillis(Long expiryMs) { + if (expiryMs <= 0) { + throw new StoragePropertiesException("Azure SAS expiry must be a positive Unix timestamp"); + } + return Instant.ofEpochMilli(expiryMs); + } + + private static Optional findTokenExpiry(String token) { + Optional expiry = Optional.empty(); + for (String field : token.split("&", -1)) { + int separator = field.indexOf('='); + if (separator <= 0 || !field.substring(0, separator).equals("se")) { + continue; + } + String encodedExpiry = field.substring(separator + 1); + if (encodedExpiry.isEmpty()) { + throw new StoragePropertiesException("Azure SAS credential has an empty expiry"); + } + if (expiry.isPresent()) { + throw new StoragePropertiesException("Azure SAS credential has multiple expiry fields"); + } + expiry = Optional.of(parseTokenExpiry(encodedExpiry)); + } + return expiry; + } + + private static Instant parseTokenExpiry(String encodedExpiry) { + try { + String decodedExpiry = URLDecoder.decode(encodedExpiry, StandardCharsets.UTF_8); + Instant expiry; + try { + expiry = OffsetDateTime.parse(decodedExpiry, DateTimeFormatter.ISO_OFFSET_DATE_TIME).toInstant(); + } catch (DateTimeParseException e) { + expiry = Instant.parse(decodedExpiry); + } + // The native protocol represents expiry as signed Unix milliseconds. + expiry.toEpochMilli(); + return expiry; + } catch (IllegalArgumentException | DateTimeException | ArithmeticException e) { + // Decode/parse exceptions include the raw input. Do not propagate it into logs. + throw new StoragePropertiesException("Azure SAS credential has an invalid expiry"); + } + } + + private static Optional minExpiry(Optional first, Optional second) { + if (first.isEmpty()) { + return second; + } + if (second.isEmpty()) { + return first; + } + return Optional.of(first.get().isBefore(second.get()) ? first.get() : second.get()); + } +} diff --git a/fe/fe-filesystem/fe-filesystem-azure/src/main/java/org/apache/doris/filesystem/azure/AzureUri.java b/fe/fe-filesystem/fe-filesystem-azure/src/main/java/org/apache/doris/filesystem/azure/AzureUri.java index 2668d1bb458f1c..cf74172dc27c58 100644 --- a/fe/fe-filesystem/fe-filesystem-azure/src/main/java/org/apache/doris/filesystem/azure/AzureUri.java +++ b/fe/fe-filesystem/fe-filesystem-azure/src/main/java/org/apache/doris/filesystem/azure/AzureUri.java @@ -17,14 +17,18 @@ package org.apache.doris.filesystem.azure; +import org.apache.doris.foundation.property.StoragePropertiesException; + import java.io.IOException; import java.net.URLDecoder; import java.net.URLEncoder; import java.nio.charset.StandardCharsets; +import java.util.Locale; +import java.util.Optional; import java.util.regex.Pattern; /** - * Parsed Azure Blob Storage URI. + * Parsed Azure storage location or HTTP(S) URL. * *

Supported formats: *

    @@ -45,13 +49,15 @@ public final class AzureUri { Pattern.compile("^[a-z0-9](?:[a-z0-9-]{1,61}[a-z0-9])?$"); private final String scheme; - private final String accountName; + private final String authority; + private final AzureAccountHost accountHost; private final String container; private final String key; - private AzureUri(String scheme, String accountName, String container, String key) { + private AzureUri(String scheme, String authority, AzureAccountHost accountHost, String container, String key) { this.scheme = scheme; - this.accountName = accountName; + this.authority = authority; + this.accountHost = accountHost; this.container = container; this.key = key; } @@ -60,8 +66,12 @@ private AzureUri(String scheme, String accountName, String container, String key * Parses an Azure path into its components. * *

    Strips any URL query string (everything after the first {@code ?}) and - * fragment (everything after the first {@code #}) before extracting the key, - * then percent-decodes the key as UTF-8. The container name is validated + * fragment (everything after the first {@code #}) before extracting the key. + * Iceberg ADLSFileIO passes ABFS/WASB object names literally to the Java SDK, + * including percent sequences; the SDK encodes them for transport. HTTP(S) + * keys are URL paths and are percent-decoded once as UTF-8. The existing S3 + * compatibility decoding is unchanged. Literal {@code +} characters and path + * separators are preserved. The container name is validated * against Azure's naming rules and is NOT decoded (Azure containers are * required to be ASCII per the storage service contract). * @@ -74,10 +84,10 @@ public static AzureUri parse(String path) throws IOException { throw new IOException("Azure path must not be null or empty"); } int schemeEnd = path.indexOf("://"); - if (schemeEnd < 0) { - throw new IOException("Cannot parse Azure URI without scheme: " + path); + if (schemeEnd <= 0) { + throw new IOException("Cannot parse Azure URI without scheme"); } - String scheme = path.substring(0, schemeEnd).toLowerCase(); + String scheme = path.substring(0, schemeEnd).toLowerCase(Locale.ROOT); String rest = path.substring(schemeEnd + 3); // Strip query and fragment so they do not pollute the key segment. int queryIdx = rest.indexOf('?'); @@ -90,43 +100,44 @@ public static AzureUri parse(String path) throws IOException { } AzureUri parsed; - if (scheme.equals("wasb") || scheme.equals("wasbs") - || scheme.equals("abfs") || scheme.equals("abfss")) { - parsed = parseWasbAbfs(scheme, rest, path); + if (isAdlsScheme(scheme)) { + parsed = parseWasbAbfs(scheme, rest); } else if (scheme.equals("https") || scheme.equals("http")) { parsed = parseHttps(scheme, rest); } else if (scheme.equals("s3") || scheme.equals("s3a") || scheme.equals("s3n")) { parsed = parseS3Compat(scheme, rest); } else { - throw new IOException("Unsupported Azure URI scheme '" + scheme + "' in path: " + path); + throw new IOException("Unsupported Azure URI scheme"); } if (!CONTAINER_NAME_PATTERN.matcher(parsed.container).matches()) { - throw new IOException("Invalid Azure container name: " + parsed.container); + throw new IOException("Invalid Azure container name"); } return parsed; } - private static AzureUri parseWasbAbfs(String scheme, String rest, String original) throws IOException { + private static AzureUri parseWasbAbfs(String scheme, String rest) throws IOException { // wasb://container@account.blob.core.windows.net/path - int atIdx = rest.indexOf('@'); - if (atIdx < 0) { - throw new IOException("Invalid Azure URI format (missing '@'): " + original); + int slashIdx = rest.indexOf('/'); + String authority = slashIdx < 0 ? rest : rest.substring(0, slashIdx); + int atIdx = authority.indexOf('@'); + if (atIdx <= 0 || atIdx == authority.length() - 1 || authority.indexOf('@', atIdx + 1) >= 0) { + throw new IOException("Invalid Azure URI authority: expected container@account-host"); } - String container = rest.substring(0, atIdx); - String hostAndPath = rest.substring(atIdx + 1); - int slashIdx = hostAndPath.indexOf('/'); - String host = slashIdx < 0 ? hostAndPath : hostAndPath.substring(0, slashIdx); - String key = slashIdx < 0 ? "" : hostAndPath.substring(slashIdx + 1); - String accountName = host.contains(".") ? host.substring(0, host.indexOf('.')) : host; - return new AzureUri(scheme, accountName, container, decodeKey(key)); + String container = authority.substring(0, atIdx); + AzureAccountHost accountHost = parseAccountHost(authority.substring(atIdx + 1)); + String key = slashIdx < 0 ? "" : rest.substring(slashIdx + 1); + return new AzureUri(scheme, authority, accountHost, container, key); } - private static AzureUri parseHttps(String scheme, String rest) { + private static AzureUri parseHttps(String scheme, String rest) throws IOException { // https://account.blob.core.windows.net/container/path int slashIdx = rest.indexOf('/'); String host = slashIdx < 0 ? rest : rest.substring(0, slashIdx); + if (host.isEmpty() || host.indexOf('@') >= 0) { + throw new IOException("Invalid Azure URI account host"); + } String pathAfterHost = slashIdx < 0 ? "" : rest.substring(slashIdx + 1); - String accountName = host.contains(".") ? host.substring(0, host.indexOf('.')) : host; + AzureAccountHost accountHost = parseAccountHost(scheme + "://" + host); int containerEnd = pathAfterHost.indexOf('/'); String container; String key; @@ -137,10 +148,10 @@ private static AzureUri parseHttps(String scheme, String rest) { container = pathAfterHost.substring(0, containerEnd); key = pathAfterHost.substring(containerEnd + 1); } - return new AzureUri(scheme, accountName, container, decodeKey(key)); + return new AzureUri(scheme, host, accountHost, container, decodeKey(key)); } - private static AzureUri parseS3Compat(String scheme, String rest) { + private static AzureUri parseS3Compat(String scheme, String rest) throws IOException { // s3://container/key — S3-compatibility mode; accountName from properties int slashIdx = rest.indexOf('/'); String container; @@ -152,14 +163,33 @@ private static AzureUri parseS3Compat(String scheme, String rest) { container = rest.substring(0, slashIdx); key = rest.substring(slashIdx + 1); } - return new AzureUri(scheme, "", container, decodeKey(key)); + return new AzureUri(scheme, container, null, container, decodeKey(key)); } - private static String decodeKey(String raw) { + private static AzureAccountHost parseAccountHost(String host) throws IOException { + try { + return AzureAccountHost.parse(host); + } catch (StoragePropertiesException e) { + throw new IOException("Invalid Azure URI account host"); + } + } + + private static String decodeKey(String raw) throws IOException { if (raw.isEmpty()) { return raw; } - return URLDecoder.decode(raw, StandardCharsets.UTF_8); + try { + // URLDecoder uses HTML form rules, but '+' is a literal character in an object path. + return URLDecoder.decode(raw.replace("+", "%2B"), StandardCharsets.UTF_8); + } catch (IllegalArgumentException e) { + // The decoder error may contain the input. Do not retain URI credential material. + throw new IOException("Invalid percent encoding in Azure object path"); + } + } + + private static boolean isAdlsScheme(String scheme) { + return scheme.equals("abfs") || scheme.equals("abfss") + || scheme.equals("wasb") || scheme.equals("wasbs"); } public String scheme() { @@ -167,7 +197,12 @@ public String scheme() { } public String accountName() { - return accountName; + return accountHost == null ? "" : accountHost.accountName(); + } + + /** The URI's account and cloud suffix; absent for legacy S3-compatible locations. */ + public Optional accountHost() { + return Optional.ofNullable(accountHost); } public String container() { @@ -178,16 +213,24 @@ public String key() { return key; } + /** Replaces only the raw SDK object name, retaining the original container and authority. */ + AzureUri withKey(String objectKey) { + return new AzureUri(scheme, authority, accountHost, container, objectKey); + } + /** - * Renders this URI in a canonical {@code scheme://container@accountName/key} form. + * Renders this URI with its original scheme and complete authority, including the cloud suffix. * - *

    The key is percent-encoded with UTF-8 so that any reserved or non-ASCII - * characters round-trip safely through SDK calls. Path separators ({@code /}) - * are preserved literally; spaces are emitted as {@code %20} (not {@code +}). + *

    ABFS/WASB names remain literal, matching ADLSLocation and avoiding a second + * encoding of percent characters on the next parse. HTTP(S) and the existing + * S3-compatible rendering percent-encode the key as UTF-8, preserving path + * separators ({@code /}) and emitting spaces as {@code %20} (not {@code +}). */ @Override public String toString() { - return scheme + "://" + container + "@" + accountName + "/" + encodeKey(key); + String containerPath = scheme.equals("http") || scheme.equals("https") ? "/" + container : ""; + String renderedKey = isAdlsScheme(scheme) ? key : encodeKey(key); + return scheme + "://" + authority + containerPath + "/" + renderedKey; } private static String encodeKey(String raw) { diff --git a/fe/fe-filesystem/fe-filesystem-azure/src/main/java/org/apache/doris/filesystem/azure/AzureVendedSas.java b/fe/fe-filesystem/fe-filesystem-azure/src/main/java/org/apache/doris/filesystem/azure/AzureVendedSas.java new file mode 100644 index 00000000000000..4668d1c965744d --- /dev/null +++ b/fe/fe-filesystem/fe-filesystem-azure/src/main/java/org/apache/doris/filesystem/azure/AzureVendedSas.java @@ -0,0 +1,132 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.filesystem.azure; + +import org.apache.doris.foundation.property.StoragePropertiesException; + +import java.net.URI; +import java.net.URISyntaxException; +import java.util.Collections; +import java.util.HashMap; +import java.util.Locale; +import java.util.Map; +import java.util.Optional; + +/** Iceberg's account-scoped SAS input, parsed before applying any catalog defaults. */ +final class AzureVendedSas { + private static final String TOKEN_PREFIX = "adls.sas-token."; + private static final String EXPIRY_PREFIX = "adls.sas-token-expires-at-ms."; + + private final AzureAccountHost accountHost; + private final AzureSasToken token; + private final Map properties; + + private AzureVendedSas(String accountHost, String token, String expiry, Map properties) { + this.accountHost = AzureAccountHost.parse(accountHost); + Long expiryMs = null; + if (expiry != null) { + try { + expiryMs = Long.parseLong(expiry.trim()); + } catch (NumberFormatException e) { + throw new StoragePropertiesException("Invalid Azure ADLS SAS expiry value"); + } + } + this.token = AzureSasToken.of(token, expiryMs); + this.properties = Collections.unmodifiableMap(new HashMap<>(properties)); + } + + static Optional parse(Map credentials) { + String accountHost = null; + String token = null; + String expiry = null; + Map matched = new HashMap<>(); + for (Map.Entry entry : credentials.entrySet()) { + String key = entry.getKey(); + if (key == null) { + continue; + } + String lowerKey = key.toLowerCase(Locale.ROOT); + boolean isToken = lowerKey.startsWith(TOKEN_PREFIX); + boolean isExpiry = lowerKey.startsWith(EXPIRY_PREFIX); + if (!isToken && !isExpiry) { + continue; + } + String host = parseHost(key.substring(isToken ? TOKEN_PREFIX.length() : EXPIRY_PREFIX.length())); + if (accountHost != null && !accountHost.equals(host)) { + throw new StoragePropertiesException( + "Azure ADLS SAS token and expiry must refer to one account host"); + } + accountHost = host; + String value = entry.getValue(); + if (value == null || value.isBlank()) { + throw new StoragePropertiesException("Azure ADLS SAS token and expiry must not be empty"); + } + if (isToken) { + if (token != null && !token.equals(value)) { + throw new StoragePropertiesException("Conflicting Azure ADLS SAS tokens for one account host"); + } + token = value; + } else { + if (expiry != null && !expiry.equals(value)) { + throw new StoragePropertiesException("Conflicting Azure ADLS SAS expiry values"); + } + expiry = value; + } + matched.put(key, value); + } + if (accountHost == null) { + if (credentials.keySet().stream().anyMatch("adls.token"::equalsIgnoreCase)) { + throw new StoragePropertiesException( + "Azure vended access tokens are not supported; an account-scoped SAS token is required"); + } + return Optional.empty(); + } + if (token == null) { + throw new StoragePropertiesException("Azure ADLS SAS expiry was supplied without a SAS token"); + } + return Optional.of(new AzureVendedSas(accountHost, token, expiry, matched)); + } + + private static String parseHost(String host) { + try { + // The property suffix is an account host, not a URL or an object path. Reject query, + // userinfo and port syntax here without including the credential-bearing input. + URI uri = new URI("https://" + host); + if (uri.getHost() == null || !host.equalsIgnoreCase(uri.getHost())) { + throw new StoragePropertiesException("Invalid Azure ADLS SAS account host"); + } + // DFS and Blob address the same account; only canonicalize the grouping key. + // Retain the original property names for Iceberg's exact-host credential lookup. + return AzureAccountHost.parse(uri.getHost().toLowerCase(Locale.ROOT)).dfsHost(); + } catch (URISyntaxException e) { + throw new StoragePropertiesException("Invalid Azure ADLS SAS account host"); + } + } + + AzureAccountHost accountHost() { + return accountHost; + } + + AzureSasToken token() { + return token; + } + + Map properties() { + return properties; + } +} diff --git a/fe/fe-filesystem/fe-filesystem-azure/src/test/java/org/apache/doris/filesystem/azure/AzureAccountHostTest.java b/fe/fe-filesystem/fe-filesystem-azure/src/test/java/org/apache/doris/filesystem/azure/AzureAccountHostTest.java new file mode 100644 index 00000000000000..f1244a425d4fdd --- /dev/null +++ b/fe/fe-filesystem/fe-filesystem-azure/src/test/java/org/apache/doris/filesystem/azure/AzureAccountHostTest.java @@ -0,0 +1,108 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.filesystem.azure; + +import org.apache.doris.foundation.property.StoragePropertiesException; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.ValueSource; + +class AzureAccountHostTest { + + @ParameterizedTest + @CsvSource({ + "account.dfs.core.windows.net, account, core.windows.net, true", + "account.blob.core.windows.net, account, core.windows.net, false", + "https://account.dfs.core.chinacloudapi.cn, account, core.chinacloudapi.cn, true", + "account.dfs.core.usgovcloudapi.net, account, core.usgovcloudapi.net, true", + "account.blob.core.usgovcloudapi.net, account, core.usgovcloudapi.net, false", + "account.dfs.core.cloudapi.de, account, core.cloudapi.de, true", + "account.blob.core.cloudapi.de, account, core.cloudapi.de, false" + }) + void parse_extractsAccountAndCloudSuffix(String input, String account, String suffix, boolean dfs) { + AzureAccountHost host = AzureAccountHost.parse(input); + + Assertions.assertEquals(account, host.accountName()); + Assertions.assertEquals(suffix, host.cloudSuffix()); + Assertions.assertEquals(dfs, host.isDfsHost()); + Assertions.assertEquals("account.dfs." + suffix, host.dfsHost()); + Assertions.assertEquals("account.blob." + suffix, host.blobHost()); + Assertions.assertEquals("https://account.blob." + suffix, host.blobEndpoint()); + } + + @Test + void fromAccountName_defaultsToPublicCloud() { + AzureAccountHost host = AzureAccountHost.fromAccountName("account"); + + Assertions.assertEquals("account.dfs.core.windows.net", host.dfsHost()); + Assertions.assertEquals("https://account.blob.core.windows.net", host.blobEndpoint()); + } + + @Test + void parse_preservesUnknownEndpointSuffix() { + AzureAccountHost host = AzureAccountHost.parse("account.blob.example.test"); + + Assertions.assertEquals("account", host.accountName()); + Assertions.assertEquals("example.test", host.cloudSuffix()); + Assertions.assertEquals("https://account.blob.example.test", host.blobEndpoint()); + } + + @Test + void blobEndpoint_preservesTransportAndEncodedPathWhenConvertingDfsHost() { + AzureAccountHost host = AzureAccountHost.parse( + "http://account.dfs.core.chinacloudapi.cn:10000/proxy%2Fpath//a+b"); + + Assertions.assertEquals("account.blob.core.chinacloudapi.cn", host.blobHost()); + Assertions.assertEquals("http://account.blob.core.chinacloudapi.cn:10000/proxy%2Fpath//a+b", + host.blobEndpoint()); + } + + @ParameterizedTest + @ValueSource(strings = {"https://storage.example.test:8443/proxy%2Fpath", + "http://127.0.0.1:10000/devstoreaccount1", "http://localhost:10000"}) + void parse_preservesCustomEndpointWithoutInventingServiceLabels(String endpoint) { + AzureAccountHost host = AzureAccountHost.parse(endpoint); + + Assertions.assertEquals(endpoint, host.blobEndpoint()); + Assertions.assertEquals(host.blobHost(), host.dfsHost()); + Assertions.assertEquals("", host.cloudSuffix()); + } + + @Test + void parse_rejectsEmptyHost() { + Assertions.assertThrows(StoragePropertiesException.class, () -> AzureAccountHost.parse(" ")); + } + + @ParameterizedTest + @ValueSource(strings = {"https://proxy.dfs.internal", "http://proxy.dfs.internal:10000/base%2Fpath", + "https://account.dfs.core.windows.net.proxy.test:8443", "https://onelake.dfs.fabric.microsoft.com"}) + void blobEndpoint_preservesNonAzureDfsEndpoints(String endpoint) { + Assertions.assertEquals(endpoint, AzureAccountHost.parse(endpoint).blobEndpoint()); + } + + @ParameterizedTest + @ValueSource(strings = {"core.windows.net", "core.chinacloudapi.cn", "core.usgovcloudapi.net", "core.cloudapi.de"}) + void blobEndpoint_convertsOfficialDfsHostsWithPorts(String suffix) { + String endpoint = "https://account.dfs." + suffix + ":8443/base%2Fpath"; + Assertions.assertEquals("https://account.blob." + suffix + ":8443/base%2Fpath", + AzureAccountHost.parse(endpoint).blobEndpoint()); + } +} diff --git a/fe/fe-filesystem/fe-filesystem-azure/src/test/java/org/apache/doris/filesystem/azure/AzureAdlsLocationContractTest.java b/fe/fe-filesystem/fe-filesystem-azure/src/test/java/org/apache/doris/filesystem/azure/AzureAdlsLocationContractTest.java new file mode 100644 index 00000000000000..18c76659f30134 --- /dev/null +++ b/fe/fe-filesystem/fe-filesystem-azure/src/test/java/org/apache/doris/filesystem/azure/AzureAdlsLocationContractTest.java @@ -0,0 +1,72 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.filesystem.azure; + +import com.azure.core.http.HttpClient; +import com.azure.storage.blob.BlobClient; +import com.azure.storage.blob.BlobContainerClient; +import com.azure.storage.blob.BlobContainerClientBuilder; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.mockito.Mockito; + +import java.io.IOException; +import java.lang.reflect.Constructor; +import java.lang.reflect.Method; +import java.net.URI; +import java.util.List; + +class AzureAdlsLocationContractTest { + + @ParameterizedTest + @CsvSource({"abfs,dfs", "abfss,dfs", "wasb,blob", "wasbs,blob"}) + void icebergAdlsLocationAndSdkPreservePercentSequences(String scheme, String service) + throws ReflectiveOperationException, IOException { + // ADLSLocation is package-private. Keep this dependency and reflective + // access in tests, without adding Iceberg types to the provider API. + Class locationClass = Class.forName("org.apache.iceberg.azure.adlsv2.ADLSLocation"); + Constructor constructor = locationClass.getDeclaredConstructor(String.class); + constructor.setAccessible(true); + Method path = locationClass.getDeclaredMethod("path"); + path.setAccessible(true); + + HttpClient httpClient = Mockito.mock(HttpClient.class); + BlobContainerClient container = new BlobContainerClientBuilder() + .endpoint("https://account.blob.core.windows.net/container") + .httpClient(httpClient) + .buildClient(); + for (String key : List.of("data/p=a%2Fb/file.parquet", "data/p=a%20b/file.parquet", + "data/p=a%252Fb/file.parquet", "data/p=a+b/http://example//file", + "data/p=100%/file.parquet", "data/p=%2/%GG/file.parquet")) { + String location = scheme + "://container@account." + service + ".core.windows.net/" + key; + String icebergKey = (String) path.invoke(constructor.newInstance(location)); + AzureUri nativeLocation = AzureUri.parse(location); + + Assertions.assertEquals(key, icebergKey, location); + Assertions.assertEquals(icebergKey, nativeLocation.key(), location); + Assertions.assertEquals(location, nativeLocation.toString()); + // This is the Blob SDK object-name API also used by DataLakeFileClient. + // Decode its URL once to check the name that reaches Azure, without I/O. + BlobClient sdkClient = container.getBlobClient(icebergKey); + Assertions.assertEquals(key, sdkClient.getBlobName()); + Assertions.assertEquals("/container/" + key, URI.create(sdkClient.getBlobUrl()).getPath()); + } + Mockito.verifyNoInteractions(httpClient); + } +} diff --git a/fe/fe-filesystem/fe-filesystem-azure/src/test/java/org/apache/doris/filesystem/azure/AzureAuthTypeTest.java b/fe/fe-filesystem/fe-filesystem-azure/src/test/java/org/apache/doris/filesystem/azure/AzureAuthTypeTest.java new file mode 100644 index 00000000000000..0cfdc557bd3187 --- /dev/null +++ b/fe/fe-filesystem/fe-filesystem-azure/src/test/java/org/apache/doris/filesystem/azure/AzureAuthTypeTest.java @@ -0,0 +1,72 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.filesystem.azure; + +import org.apache.doris.foundation.property.StoragePropertiesException; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.NullAndEmptySource; +import org.junit.jupiter.params.provider.ValueSource; + +class AzureAuthTypeTest { + + @ParameterizedTest + @CsvSource({ + "SharedKey, SHARED_KEY", + "sharedkey, SHARED_KEY", + "SHARED_KEY, SHARED_KEY", + "shared_key, SHARED_KEY", + "' SharedKey ', SHARED_KEY", + "SAS, SAS", + "sas, SAS", + "OAuth2, OAUTH2", + "OAUTH2, OAUTH2", + "oauth2, OAUTH2" + }) + void parse_acceptsLegacyAndNativeValues(String input, AzureAuthType expected) { + Assertions.assertEquals(expected, AzureAuthType.parse(input)); + } + + @ParameterizedTest + @NullAndEmptySource + @ValueSource(strings = {" ", "unknown", "OAuth", "ManagedIdentity", "WorkloadIdentity"}) + void parse_rejectsUnsupportedValues(String input) { + StoragePropertiesException exception = Assertions.assertThrows(StoragePropertiesException.class, + () -> AzureAuthType.parse(input)); + + Assertions.assertTrue(exception.getMessage().contains("Unsupported Azure auth_type")); + } + + @Test + void parse_doesNotEchoInvalidInput() { + StoragePropertiesException exception = Assertions.assertThrows(StoragePropertiesException.class, + () -> AzureAuthType.parse("credential-in-wrong-property")); + + Assertions.assertFalse(exception.getMessage().contains("credential-in-wrong-property")); + } + + @Test + void propertyValue_preservesLegacyValues() { + Assertions.assertEquals(AzureFileSystemProperties.SHARED_KEY_AUTH, AzureAuthType.SHARED_KEY.propertyValue()); + Assertions.assertEquals(AzureFileSystemProperties.SAS_AUTH, AzureAuthType.SAS.propertyValue()); + Assertions.assertEquals(AzureFileSystemProperties.OAUTH2_AUTH, AzureAuthType.OAUTH2.propertyValue()); + } +} diff --git a/fe/fe-filesystem/fe-filesystem-azure/src/test/java/org/apache/doris/filesystem/azure/AzureBackendViewTest.java b/fe/fe-filesystem/fe-filesystem-azure/src/test/java/org/apache/doris/filesystem/azure/AzureBackendViewTest.java new file mode 100644 index 00000000000000..0ef6437ffd2e49 --- /dev/null +++ b/fe/fe-filesystem/fe-filesystem-azure/src/test/java/org/apache/doris/filesystem/azure/AzureBackendViewTest.java @@ -0,0 +1,315 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.filesystem.azure; + +import org.apache.doris.filesystem.properties.BackendStorageKind; +import org.apache.doris.filesystem.properties.BackendStorageProperties; +import org.apache.doris.filesystem.properties.FileSystemProperties; +import org.apache.doris.filesystem.properties.FsCacheKeys; +import org.apache.doris.foundation.property.StoragePropertiesException; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.NullAndEmptySource; +import org.junit.jupiter.params.provider.ValueSource; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +class AzureBackendViewTest { + + private static final String ACCOUNT_HOST = "account.dfs.core.windows.net"; + private static final String ONELAKE_HOST = "onelake.dfs.fabric.microsoft.com"; + private static final String TOKEN = "sig=test-signature&se=2100-01-01T00%3A00%3A00Z"; + private static final String TOKEN_ENDPOINT = "https://login.microsoftonline.com/tenant/oauth2/token"; + + @ParameterizedTest + @ValueSource(strings = {"SHARED_KEY", "SAS", "OAUTH2"}) + void nativeViewContainsOnlyTheSelectedAuthenticationVocabulary(String authType) { + AzureFileSystemProperties properties = new AzureFileSystemProvider().bind(input(authType, ACCOUNT_HOST)); + Map expected = new HashMap<>(Map.of( + "provider", "azure", "AZURE_AUTH_TYPE", authType, + "AZURE_ACCOUNT_NAME", "account", "AZURE_ENDPOINT", "https://account.blob.core.windows.net")); + switch (authType) { + case "SHARED_KEY": + expected.put("AZURE_ACCOUNT_KEY", "shared-key"); + break; + case "SAS": + expected.put("AZURE_SAS_TOKEN", TOKEN); + expected.put("AZURE_SAS_EXPIRY_MS", "4102444800000"); + break; + case "OAUTH2": + expected.put("AZURE_CLIENT_ID", "client"); + expected.put("AZURE_CLIENT_SECRET", "client-secret"); + expected.put("AZURE_TENANT_ID", "tenant"); + expected.put("AZURE_OAUTH_SERVER_URI", TOKEN_ENDPOINT); + break; + default: + throw new AssertionError("Unexpected test authentication type"); + } + + Assertions.assertEquals(BackendStorageKind.NATIVE, properties.backendKind()); + Assertions.assertEquals(expected, properties.toMap()); + for (String uri : List.of( + "abfss://container@" + ACCOUNT_HOST + "/dir/http://example/file", + "wasbs://container@account.blob.core.windows.net/dir/file", + "https://account.blob.core.windows.net/container/dir/file")) { + FileSystemProperties binding = properties; + BackendStorageProperties backend = binding.resolveBackendProperties( + binding.validateAndNormalizeUri(uri)).orElseThrow(); + Assertions.assertSame(properties, backend); + Assertions.assertEquals(BackendStorageKind.NATIVE, backend.backendKind()); + Assertions.assertEquals(expected, backend.toMap()); + Assertions.assertThrows(UnsupportedOperationException.class, () -> backend.toMap().put("extra", "value")); + } + // Input aliases remain available to legacy FE clients, but never leak into the native map. + Assertions.assertEquals("container", properties.getContainer()); + Assertions.assertEquals("true", properties.getUsePathStyle()); + Assertions.assertTrue(properties.toHadoopConfigurationMap().containsKey("fs.azure.readaheadqueue.depth")); + } + + @ParameterizedTest + @ValueSource(strings = {"SAS", "OAUTH2"}) + void nativeSasAndOauthRequireAnAzureUri(String authType) { + AzureFileSystemProperties properties = new AzureFileSystemProvider().bind(input(authType, ACCOUNT_HOST)); + for (String scheme : List.of("s3", "S3")) { + String uri = scheme + "://container/dir/file"; + StoragePropertiesException normalizeError = Assertions.assertThrows(StoragePropertiesException.class, + () -> properties.validateAndNormalizeUri(uri)); + StoragePropertiesException viewError = Assertions.assertThrows(StoragePropertiesException.class, + () -> properties.resolveBackendProperties(uri)); + Assertions.assertEquals("Azure SAS/OAuth2 data access requires an Azure URI", normalizeError.getMessage()); + Assertions.assertEquals(normalizeError.getMessage(), viewError.getMessage()); + Assertions.assertNull(normalizeError.getCause()); + Assertions.assertNull(viewError.getCause()); + } + } + + @Test + void legacySharedKeyRetainsS3UriAndCredentialAliases() { + AzureFileSystemProperties properties = new AzureFileSystemProvider().bind(Map.of( + "fs.azure.support", "true", "AWS_ACCESS_KEY", "account", "AWS_SECRET_KEY", "shared-key", + "AWS_ENDPOINT", "https://account.blob.core.windows.net")); + String normalized = properties.validateAndNormalizeUri("S3://container/dir/http://example/file"); + BackendStorageProperties view = properties.resolveBackendProperties(normalized).orElseThrow(); + + Assertions.assertEquals("s3://container/dir/http://example/file", normalized); + Assertions.assertSame(properties, view); + Assertions.assertEquals(BackendStorageKind.NATIVE, view.backendKind()); + Assertions.assertEquals("SHARED_KEY", view.toMap().get("AZURE_AUTH_TYPE")); + Assertions.assertEquals("shared-key", view.toMap().get("AZURE_ACCOUNT_KEY")); + } + + @ParameterizedTest + @ValueSource(strings = {"SHARED_KEY", "SAS", "OAUTH2"}) + void oneLakeSelectsHadoopWithoutMutatingTheNativeView(String authType) { + Map raw = input(authType, ONELAKE_HOST); + raw.remove("AZURE_CONTAINER"); + AzureFileSystemProperties properties = new AzureFileSystemProvider().bind(raw); + Map nativeBefore = properties.toMap(); + Map cachedHadoop = properties.resolveBackendProperties( + "abfss://workspace@" + ONELAKE_HOST + "/lakehouse/Tables/first-file").orElseThrow().toMap(); + for (String scheme : List.of("abfs", "abfss", "ABFSS")) { + String uri = scheme + "://workspace@" + ONELAKE_HOST + "/lakehouse/Tables/dir//file%2Fname"; + String normalized = properties.validateAndNormalizeUri(uri); + Assertions.assertEquals(uri.replace("ABFSS://", "abfss://"), normalized); + BackendStorageProperties backend = properties.resolveBackendProperties(normalized).orElseThrow(); + Map hadoop = backend.toMap(); + Assertions.assertSame(cachedHadoop, hadoop); + Assertions.assertSame(hadoop, backend.toMap()); + Assertions.assertEquals(BackendStorageKind.HDFS, backend.backendKind()); + Assertions.assertEquals("8", hadoop.get("fs.azure.readaheadqueue.depth")); + Assertions.assertEquals("false", hadoop.get("fs.abfss.impl.disable.cache")); + Assertions.assertEquals("file:///", hadoop.get("fs.defaultFS")); + Assertions.assertEquals(properties.fsCacheFingerprint(), + hadoop.get(FsCacheKeys.fsCacheKeyProperty("abfss"))); + Assertions.assertFalse(hadoop.containsKey("provider")); + Assertions.assertFalse(hadoop.keySet().stream().anyMatch(key -> key.startsWith("AZURE_"))); + switch (authType) { + case "SHARED_KEY": + Assertions.assertEquals("shared-key", hadoop.get("fs.azure.account.key")); + break; + case "SAS": + Assertions.assertEquals("SAS", hadoop.get("fs.azure.account.auth.type." + ONELAKE_HOST)); + Assertions.assertEquals(TOKEN, hadoop.get("fs.azure.sas.fixed.token." + ONELAKE_HOST)); + break; + case "OAUTH2": + Assertions.assertEquals("OAuth", hadoop.get("fs.azure.account.auth.type." + ONELAKE_HOST)); + Assertions.assertEquals("client", hadoop.get("fs.azure.account.oauth2.client.id." + ONELAKE_HOST)); + Assertions.assertEquals("client-secret", + hadoop.get("fs.azure.account.oauth2.client.secret." + ONELAKE_HOST)); + Assertions.assertEquals(TOKEN_ENDPOINT, + hadoop.get("fs.azure.account.oauth2.client.endpoint." + ONELAKE_HOST)); + break; + default: + throw new AssertionError("Unexpected test authentication type"); + } + Assertions.assertThrows(UnsupportedOperationException.class, () -> hadoop.put("extra", "value")); + } + Assertions.assertEquals(BackendStorageKind.NATIVE, + properties.toBackendProperties().orElseThrow().backendKind()); + Assertions.assertEquals(nativeBefore, properties.toMap()); + } + + @ParameterizedTest + @ValueSource(strings = {"core.windows.net", "core.chinacloudapi.cn", "core.usgovcloudapi.net", "core.cloudapi.de"}) + void claimsKnownAzureHttpHostsWithoutRegisteringGenericHttpSchemes(String suffix) { + FileSystemProperties binding = new AzureFileSystemProvider().bind(input("SAS", ACCOUNT_HOST)); + + Assertions.assertEquals(Set.of("abfs", "abfss", "wasb", "wasbs"), binding.getSupportedSchemes()); + for (String service : List.of("blob", "dfs")) { + for (String scheme : List.of("http", "HTTPS")) { + Assertions.assertTrue(binding.claimsUri(scheme + "://account." + service + "." + suffix + + "/container/file?sig=test-signature")); + } + } + } + + @ParameterizedTest + @NullAndEmptySource + @ValueSource(strings = { + "https://example.test/container/file", + "https://account.blob.core.windows.net.example.test/container/file", + "https://accountblob.core.windows.net/container/file", + "https://example.test/account.blob.core.windows.net/container/file", + "https://account.blob.core.windows.net@example.test/container/file", + "https://user@account.blob.core.windows.net/container/file", + "https://account.blob.core.windows.net:99999/container/file", + "https://onelake.dfs.fabric.microsoft.com/container/file", + "https://bucket.s3.amazonaws.com/container/file", + "ftp://account.blob.core.windows.net/container/file", + "abfss://container@account.dfs.core.windows.net/file" + }) + void doesNotClaimOtherProvidersOrLookalikeHosts(String uri) { + FileSystemProperties binding = new AzureFileSystemProvider().bind(input("SAS", ACCOUNT_HOST)); + + Assertions.assertFalse(binding.claimsUri(uri)); + } + + @ParameterizedTest + @ValueSource(strings = { + "https://account.blob.core.windows.net/container/%GG?sig=test-signature", + "https://account.blob.core.windows.net/InvalidContainer/file?sig=test-signature" + }) + void claimedHostWithAnInvalidPathFailsStrictlyWithoutLeakingTheQuery(String uri) { + FileSystemProperties binding = new AzureFileSystemProvider().bind(input("SAS", ACCOUNT_HOST)); + + Assertions.assertTrue(binding.claimsUri(uri)); + StoragePropertiesException exception = Assertions.assertThrows(StoragePropertiesException.class, + () -> binding.validateAndNormalizeUri(uri)); + for (Throwable cause = exception; cause != null; cause = cause.getCause()) { + Assertions.assertFalse(cause.getMessage().contains("test-signature")); + Assertions.assertFalse(cause.getMessage().contains(uri)); + } + } + + @Test + void claimDoesNotBypassAccountContainerOrEndpointValidation() { + FileSystemProperties binding = new AzureFileSystemProvider().bind(input("SAS", ACCOUNT_HOST)); + for (String uri : List.of( + "https://other.blob.core.windows.net/container/file", + "https://account.blob.core.chinacloudapi.cn/container/file", + "https://account.blob.core.windows.net/other/file", + "https://account.blob.core.windows.net:8443/container/file", + "http://account.blob.core.windows.net/container/file")) { + Assertions.assertTrue(binding.claimsUri(uri)); + Assertions.assertThrows(StoragePropertiesException.class, () -> binding.validateAndNormalizeUri(uri)); + Assertions.assertThrows(StoragePropertiesException.class, () -> binding.resolveBackendProperties(uri)); + } + } + + @ParameterizedTest + @CsvSource({ + "https://storage.example.test:8443, https://STORAGE.EXAMPLE.TEST:8443/container/file, true", + "https://storage.example.test:443, https://storage.example.test/container/file, true", + "http://127.0.0.1:10000, http://127.0.0.1:10000/container/file, true", + "https://storage.example.test:8443, https://storage.example.test/container/file, false", + "https://storage.example.test:8443, http://storage.example.test:8443/container/file, false", + "https://storage.example.test:8443, https://storage.example.test.evil:8443/container/file, false", + "https://storage.example.test:8443, https://other.example.test:8443/container/file, false" + }) + void customHttpClaimRequiresTheExplicitEndpointOrigin(String endpoint, String uri, boolean expectedClaim) { + Map raw = input("SHARED_KEY", ACCOUNT_HOST); + raw.put("azure.endpoint", endpoint); + AzureFileSystemProperties binding = new AzureFileSystemProvider().bind(raw); + + Assertions.assertEquals(expectedClaim, binding.claimsUri(uri)); + if (expectedClaim) { + Assertions.assertEquals(uri, binding.validateAndNormalizeUri(uri)); + Assertions.assertEquals(BackendStorageKind.NATIVE, + binding.resolveBackendProperties(uri).orElseThrow().backendKind()); + // The custom hostname's first label is not a replacement credential account. + Assertions.assertEquals("account", binding.toMap().get("AZURE_ACCOUNT_NAME")); + } else { + Assertions.assertThrows(StoragePropertiesException.class, () -> binding.validateAndNormalizeUri(uri)); + } + } + + @Test + void customOauthAccountHostWithoutAnExplicitEndpointDoesNotClaimHttp() { + Map raw = input("OAUTH2", "account.example.test"); + raw.remove("azure.endpoint"); + FileSystemProperties binding = new AzureFileSystemProvider().bind(raw); + + Assertions.assertFalse(binding.claimsUri("https://account.example.test/container/file")); + } + + @Test + void customEndpointAlsoWorksWithTypedVendedSasWithoutGuessingItsAccount() { + AzureFileSystemProperties binding = new AzureFileSystemProvider().bindVended( + Map.of("adls.sas-token." + ACCOUNT_HOST, TOKEN), + Map.of("fs.azure.support", "true", "azure.endpoint", "https://proxy.blob.example.test:8443")) + .orElseThrow(); + String uri = "https://proxy.blob.example.test:8443/container/file"; + + Assertions.assertTrue(binding.claimsUri(uri)); + Assertions.assertEquals(uri, binding.validateAndNormalizeUri(uri)); + Assertions.assertEquals("account", binding.resolveBackendProperties(uri).orElseThrow() + .toMap().get("AZURE_ACCOUNT_NAME")); + } + + private static Map input(String authType, String accountHost) { + Map raw = new HashMap<>(Map.of( + "fs.azure.support", "true", "azure.auth_type", authType, + "azure.endpoint", "https://" + accountHost, + "azure.account_name", accountHost.substring(0, accountHost.indexOf('.')), + "AZURE_CONTAINER", "container", "use_path_style", "true", + "fs.azure.readaheadqueue.depth", "8", "fs.abfss.impl.disable.cache", "no")); + switch (authType) { + case "SHARED_KEY": + raw.put("azure.account_key", "shared-key"); + break; + case "SAS": + raw.put("azure.sas_token", TOKEN); + break; + case "OAUTH2": + raw.put("azure.oauth2_account_host", accountHost); + raw.put("azure.oauth2_client_id", "client"); + raw.put("azure.oauth2_client_secret", "client-secret"); + raw.put("azure.oauth2_server_uri", TOKEN_ENDPOINT); + break; + default: + throw new AssertionError("Unexpected test authentication type"); + } + return raw; + } +} diff --git a/fe/fe-filesystem/fe-filesystem-azure/src/test/java/org/apache/doris/filesystem/azure/AzureCredentialScopeTest.java b/fe/fe-filesystem/fe-filesystem-azure/src/test/java/org/apache/doris/filesystem/azure/AzureCredentialScopeTest.java new file mode 100644 index 00000000000000..7a2cf47b8141ea --- /dev/null +++ b/fe/fe-filesystem/fe-filesystem-azure/src/test/java/org/apache/doris/filesystem/azure/AzureCredentialScopeTest.java @@ -0,0 +1,235 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.filesystem.azure; + +import org.apache.doris.foundation.property.StoragePropertiesException; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.ValueSource; + +import java.time.Clock; +import java.time.Instant; +import java.time.ZoneOffset; +import java.util.Map; + +class AzureCredentialScopeTest { + private static final String ABFS_ROOT = "abfss://container@account.dfs.core.windows.net/"; + private static final String HTTPS_ROOT = "https://account.blob.core.windows.net/container/"; + + @ParameterizedTest + @CsvSource({ + "data/file.parquet, data/file.parquet, true", + "data/partition/file.parquet, data, true", + "database/file.parquet, data, false", + "data, data/, false", + "data/, data/, true", + "data/file.parquet, data/, true", + "Data/file.parquet, data/, false" + }) + void matchesOnlyTheSameObjectOrDescendants(String key, String prefix, boolean matches) { + AzureFileSystemProperties properties = sasProperties(); + + Assertions.assertEquals(matches, properties.matchesLocationPrefix(ABFS_ROOT + key, ABFS_ROOT + prefix)); + Assertions.assertEquals(matches, properties.matchesLocationPrefix(HTTPS_ROOT + key, HTTPS_ROOT + prefix)); + } + + @ParameterizedTest + @ValueSource(strings = { + "abfss://container@account.dfs.core.windows.net", + "abfss://container@account.dfs.core.windows.net/", + "https://account.blob.core.windows.net/container", + "https://account.blob.core.windows.net/container/" + }) + void containerRootIncludesEveryObject(String prefix) { + AzureFileSystemProperties properties = sasProperties(); + + Assertions.assertTrue(properties.matchesLocationPrefix(ABFS_ROOT + "data/file.parquet", prefix)); + Assertions.assertTrue(properties.matchesLocationPrefix(ABFS_ROOT, prefix)); + } + + @Test + void literalLeadingSlashIsNotTheContainerRoot() { + AzureFileSystemProperties properties = sasProperties(); + + Assertions.assertFalse(properties.matchesLocationPrefix(ABFS_ROOT + "data/file", ABFS_ROOT + "/")); + Assertions.assertTrue(properties.matchesLocationPrefix(ABFS_ROOT + "/data/file", ABFS_ROOT + "/")); + } + + @Test + void differentContainersNeverShareAScope() { + AzureFileSystemProperties properties = sasProperties(); + String otherContainer = "abfss://other-container@account.dfs.core.windows.net/data/"; + + Assertions.assertFalse(properties.matchesLocationPrefix(ABFS_ROOT + "data/file", otherContainer)); + Assertions.assertFalse(properties.matchesLocationPrefix(otherContainer + "file", ABFS_ROOT + "data/")); + } + + @ParameterizedTest + @ValueSource(strings = { + "abfss://container@other.dfs.core.windows.net/data/", + "abfss://container@account.dfs.core.chinacloudapi.cn/data/" + }) + void bothLocationsMustBelongToTheBoundAccountAndCloud(String otherIdentity) { + AzureFileSystemProperties properties = sasProperties(); + + Assertions.assertThrows(StoragePropertiesException.class, + () -> properties.matchesLocationPrefix(ABFS_ROOT + "data/file", otherIdentity)); + Assertions.assertThrows(StoragePropertiesException.class, + () -> properties.matchesLocationPrefix(otherIdentity + "file", ABFS_ROOT + "data/")); + } + + @ParameterizedTest + @ValueSource(strings = { + "abfs://container@account.dfs.core.windows.net/data/file", + "abfss://container@account.dfs.core.windows.net/data/file", + "wasb://container@account.blob.core.windows.net/data/file", + "wasbs://container@account.blob.core.windows.net/data/file", + "ABFSS://container@ACCOUNT.DFS.CORE.WINDOWS.NET/data/file", + "https://account.blob.core.windows.net/container/data/file" + }) + void dfsAndBlobSpellingsUseTheSameStorageIdentity(String location) { + Assertions.assertTrue(sasProperties().matchesLocationPrefix(location, ABFS_ROOT + "data/")); + } + + @ParameterizedTest + @CsvSource({ + "data/a%2Fb, data/a%252Fb, true", + "data/a%2Fb, data/a%2Fb, false", + "data/a%2Fb, data/a%25252Fb, false", + "data/a/b, data/a%2Fb, true", + "data/a+b, data/a+b, true", + "data/a+b, data/a%2Bb, true", + "data/a+b, data/a%20b, false", + "data/a%2Bb, data/a%252Bb, true" + }) + void httpsDecodesOnceAndAbfsKeepsPercentSequencesLiteral( + String literalKey, String encodedKey, boolean matches) { + AzureFileSystemProperties properties = sasProperties(); + + Assertions.assertEquals(matches, properties.matchesLocationPrefix( + HTTPS_ROOT + encodedKey + "/file", ABFS_ROOT + literalKey + "/")); + Assertions.assertEquals(matches, properties.matchesLocationPrefix( + ABFS_ROOT + literalKey + "/file", HTTPS_ROOT + encodedKey + "/")); + } + + @ParameterizedTest + @CsvSource({ + "data//file, data//, true", + "data/file, data//, false", + "data/./file, data/./, true", + "data/file, data/./, false", + "data/../file, data/, true", + "data/../file, data/../, true", + "file, data/../, false", + "data/http://example/file, data/http://example/, true", + "data/http:/example/file, data/http://example/, false" + }) + void separatorsAndDotSegmentsRemainObjectNameCharacters(String key, String prefix, boolean matches) { + AzureFileSystemProperties properties = sasProperties(); + + Assertions.assertEquals(matches, properties.matchesLocationPrefix(ABFS_ROOT + key, ABFS_ROOT + prefix)); + Assertions.assertEquals(matches, properties.matchesLocationPrefix(HTTPS_ROOT + key, HTTPS_ROOT + prefix)); + } + + @Test + void scopeComparisonDoesNotAccessAnExpiredCredential() { + AzureFileSystemProperties properties = AzureFileSystemProperties.of(Map.of( + "azure.auth_type", "SAS", "azure.account_name", "account", + "azure.sas_token", "sig=scope-test", "azure.sas_expiry_ms", "1"), + Clock.fixed(Instant.parse("2026-01-01T00:00:00Z"), ZoneOffset.UTC)); + + Assertions.assertThrows(StoragePropertiesException.class, properties::validateForAccess); + Assertions.assertTrue(properties.matchesLocationPrefix(ABFS_ROOT + "data/file", ABFS_ROOT + "data/")); + Assertions.assertFalse(properties.matchesLocationPrefix(ABFS_ROOT + "other/file", ABFS_ROOT + "data/")); + Assertions.assertThrows(StoragePropertiesException.class, properties::validateForAccess); + } + + @Test + void legacySharedKeyLocationsUseTheBindingAccountAndKeepTheirExistingDecoding() { + AzureFileSystemProperties properties = AzureFileSystemProperties.of(Map.of( + "azure.account_name", "account", "azure.account_key", "scope-test-key")); + + Assertions.assertTrue(properties.matchesLocationPrefix("s3://container/data/a%252Fb/file", + ABFS_ROOT + "data/a%2Fb/")); + Assertions.assertFalse(properties.matchesLocationPrefix("s3://container/data/a/b/file", + ABFS_ROOT + "data/a%2Fb/")); + Assertions.assertTrue(properties.matchesLocationPrefix(ABFS_ROOT + "data/file", "s3://container/data/")); + } + + @Test + void customHttpsEndpointUsesItsBoundAccountForAnAbfsCredentialPrefix() { + AzureFileSystemProperties properties = customEndpointProperties(); + String location = "https://proxy.blob.example.test:8443/container/data/file"; + Assertions.assertEquals(location, properties.validateAndNormalizeUri(location)); + + Assertions.assertTrue(properties.matchesLocationPrefix(location, ABFS_ROOT + "data/")); + Assertions.assertTrue(properties.matchesLocationPrefix(ABFS_ROOT + "data/file", + "https://proxy.blob.example.test:8443/container/data/")); + } + + @ParameterizedTest + @ValueSource(strings = { + "https://other.blob.example.test:8443", + "https://proxy.blob.example.test.other.invalid:8443", + "https://proxy.blob.example.test:443", + "http://proxy.blob.example.test:8443" + }) + void customEndpointScopeStillRequiresTheExactHttpOrigin(String otherOrigin) { + AzureFileSystemProperties properties = customEndpointProperties(); + + Assertions.assertThrows(StoragePropertiesException.class, () -> properties.matchesLocationPrefix( + otherOrigin + "/container/data/file", ABFS_ROOT + "data/")); + Assertions.assertThrows(StoragePropertiesException.class, () -> properties.matchesLocationPrefix( + ABFS_ROOT + "data/file", otherOrigin + "/container/data/")); + } + + @Test + void customEndpointScopeKeepsContainerAndObjectBoundaries() { + AzureFileSystemProperties properties = customEndpointProperties(); + + Assertions.assertFalse(properties.matchesLocationPrefix( + "https://proxy.blob.example.test:8443/container/database/file", ABFS_ROOT + "data/")); + Assertions.assertFalse(properties.matchesLocationPrefix( + "https://proxy.blob.example.test:8443/other/data/file", ABFS_ROOT + "data/")); + } + + @Test + void anUnrecognizedAbfsAuthorityIsNotAnAliasForTheBoundAccount() { + AzureFileSystemProperties properties = customEndpointProperties(); + String otherRoot = "abfss://container@account.example.test/"; + + Assertions.assertFalse(properties.matchesLocationPrefix(otherRoot + "data/file", ABFS_ROOT + "data/")); + Assertions.assertFalse(properties.matchesLocationPrefix(ABFS_ROOT + "data/file", otherRoot + "data/")); + } + + private static AzureFileSystemProperties customEndpointProperties() { + return new AzureFileSystemProvider().bindVended( + Map.of("adls.sas-token.account.dfs.core.windows.net", "sig=scope-test"), + Map.of("fs.azure.support", "true", "azure.endpoint", "https://proxy.blob.example.test:8443")) + .orElseThrow(); + } + + private static AzureFileSystemProperties sasProperties() { + return AzureFileSystemProperties.of(Map.of( + "azure.auth_type", "SAS", "azure.account_name", "account", + "azure.sas_token", "sig=scope-test")); + } +} diff --git a/fe/fe-filesystem/fe-filesystem-azure/src/test/java/org/apache/doris/filesystem/azure/AzureFileSystemPropertiesTest.java b/fe/fe-filesystem/fe-filesystem-azure/src/test/java/org/apache/doris/filesystem/azure/AzureFileSystemPropertiesTest.java index 7a15c98503960b..95a6c58aea9313 100644 --- a/fe/fe-filesystem/fe-filesystem-azure/src/test/java/org/apache/doris/filesystem/azure/AzureFileSystemPropertiesTest.java +++ b/fe/fe-filesystem/fe-filesystem-azure/src/test/java/org/apache/doris/filesystem/azure/AzureFileSystemPropertiesTest.java @@ -23,17 +23,94 @@ import org.apache.doris.filesystem.properties.BackendStorageProperties; import org.apache.doris.filesystem.properties.FsCacheKeys; import org.apache.doris.filesystem.properties.StorageKind; +import org.apache.doris.filesystem.properties.StorageProperties; +import org.apache.doris.foundation.property.StoragePropertiesException; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.NullSource; +import org.junit.jupiter.params.provider.ValueSource; import java.io.IOException; +import java.time.Clock; +import java.time.Instant; +import java.time.ZoneOffset; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; class AzureFileSystemPropertiesTest { + @ParameterizedTest + @NullSource + @ValueSource(strings = {"SharedKey", "SHARED_KEY"}) + void bind_preservesSharedKeyWithUnusedLegacyOAuth2Fields(String authType) { + Map input = new HashMap<>(Map.of( + "azure.account_name", "account", + "azure.account_key", "shared-key", + "azure.oauth2_account_host", "not a valid Azure host", + "azure.oauth2_client_id", "legacy-client", + "azure.oauth2_client_secret", "legacy-secret", + "azure.oauth2_client_tenant_id", "legacy-tenant", + "azure.oauth2_server_uri", "not a valid token endpoint")); + if (authType != null) { + input.put("azure.auth_type", authType); + } + AzureFileSystemProperties properties = AzureFileSystemProperties.of(input); + + Assertions.assertEquals(AzureAuthType.SHARED_KEY, properties.authType()); + Assertions.assertEquals("not a valid Azure host", properties.getOauthAccountHost()); + Assertions.assertEquals("legacy-client", properties.getClientId()); + Assertions.assertEquals("legacy-secret", properties.getClientSecret()); + Assertions.assertEquals("https://account.blob.core.windows.net", properties.getEndpoint()); + Assertions.assertEquals(Map.of( + "provider", "azure", + "AZURE_AUTH_TYPE", "SHARED_KEY", + "AZURE_ACCOUNT_NAME", "account", + "AZURE_ENDPOINT", "https://account.blob.core.windows.net", + "AZURE_ACCOUNT_KEY", "shared-key"), properties.toMap()); + Map hadoop = properties.toHadoopConfigurationMap(); + Assertions.assertEquals("shared-key", hadoop.get("fs.azure.account.key.account.blob.core.windows.net")); + Assertions.assertFalse(hadoop.keySet().stream().anyMatch(key -> key.contains("oauth"))); + Assertions.assertFalse(hadoop.values().stream().anyMatch(value -> value.contains("legacy-"))); + Assertions.assertFalse(properties.toString().contains("legacy-secret")); + } + + @Test + void validateAndNormalizeUri_preservesCloudAndOriginalObjectPath() { + AzureFileSystemProperties properties = AzureFileSystemProperties.of(Map.of( + "azure.auth_type", "SAS", "azure.sas_token", "sig=temporary", + "azure.endpoint", "https://account.dfs.core.chinacloudapi.cn", "container", "container")); + String path = "abfss://container@account.dfs.core.chinacloudapi.cn//dir/http://example/a+b%252Ffile"; + + Assertions.assertEquals(path, properties.validateAndNormalizeUri(path)); + Assertions.assertEquals("https://account.blob.core.chinacloudapi.cn/container/dir/file", + properties.validateAndNormalizeUri("https://account.blob.core.chinacloudapi.cn/container/dir/file")); + StoragePropertiesException mismatch = Assertions.assertThrows(StoragePropertiesException.class, + () -> properties.validateAndNormalizeUri( + "abfss://container@account.dfs.core.windows.net/dir/file")); + Assertions.assertEquals("Azure URI account host does not match the binding", mismatch.getMessage()); + // Only legacy SharedKey access supports S3-style locations without an account authority. + Assertions.assertThrows(StoragePropertiesException.class, + () -> properties.validateAndNormalizeUri("s3://container/dir/file")); + } + + @Test + void validateAndNormalizeUri_doesNotIncludeSasInErrors() { + AzureFileSystemProperties properties = AzureFileSystemProperties.of(Map.of( + "azure.account_name", "account", "azure.account_key", "key")); + StoragePropertiesException error = Assertions.assertThrows(StoragePropertiesException.class, + () -> properties.validateAndNormalizeUri( + "https://account.blob.core.windows.net/container/%GG?sig=secret-signature")); + + Assertions.assertEquals("Invalid Azure URI", error.getMessage()); + Assertions.assertEquals("Invalid percent encoding in Azure object path", error.getCause().getMessage()); + Assertions.assertNull(error.getCause().getCause()); + } + @Test void bind_usesFeCoreAzureAliasOrder() { AzureFileSystemProperties properties = AzureFileSystemProperties.of(Map.of( @@ -48,6 +125,100 @@ void bind_usesFeCoreAzureAliasOrder() { Assertions.assertEquals("azure-key", properties.getAccountKey()); } + @Test + void sharedKey_doesNotUseS3SecretWhenProviderMarkerIsPresent() { + Assertions.assertThrows(IllegalArgumentException.class, () -> AzureFileSystemProperties.of(Map.of( + "provider", "azure", "azure.auth_type", "SharedKey", "azure.account_name", "account", + "s3.endpoint", "https://s3.example.test", "s3.access_key", "s3-account", + "s3.secret_key", "s3-secret"))); + } + + @Test + void sas_prefersAzureEndpointAliasOverSiblingS3Endpoint() { + AzureFileSystemProperties properties = AzureFileSystemProperties.of(Map.of( + "provider", "azure", "azure.auth_type", "SAS", "azure.account_name", "account", + "azure.sas_token", "sig=temporary", "AZURE_ENDPOINT", "https://proxy.example.test", + "s3.endpoint", "https://s3.example.test")); + + Assertions.assertEquals("https://proxy.example.test", properties.getEndpoint()); + } + + @ParameterizedTest + @ValueSource(strings = { + "https://other.blob.core.windows.net", + "https://account.blob.core.chinacloudapi.cn" + }) + void oauth2_rejectsAccountHostAndEndpointConflictsAtBinding(String endpoint) { + StoragePropertiesException error = Assertions.assertThrows(StoragePropertiesException.class, + () -> AzureFileSystemProperties.of(oauth2Properties(endpoint))); + + Assertions.assertEquals("Azure OAuth2 account host does not match the storage endpoint", error.getMessage()); + Assertions.assertNull(error.getCause()); + } + + @ParameterizedTest + @ValueSource(strings = {"https://login.example.test/tenant/custom/token", + "ftp://login.example.test/tenant/oauth2/token", + "http://login.example.test/tenant/oauth2/token", + "https://private-material@login.example.test/tenant/oauth2/token", + "https://login.example.test/tenant/oauth2/token?sig=private-material", + "https://login.example.test/tenant/oauth2/token#private-material"}) + void oauth2_rejectsUnsupportedTokenEndpointsWithoutEchoingCredentials(String tokenEndpoint) { + Map input = oauth2Properties("https://account.blob.core.windows.net"); + input.put("azure.oauth2_server_uri", tokenEndpoint); + StoragePropertiesException error = Assertions.assertThrows(StoragePropertiesException.class, + () -> AzureFileSystemProperties.of(input)); + Assertions.assertNull(error.getCause()); + Assertions.assertFalse(error.getMessage().contains("private-material")); + } + + @ParameterizedTest + @ValueSource(strings = {"https://login.microsoftonline.com/tenant/oauth2/token", + "https://login.microsoftonline.us/tenant/oauth2/v2.0/token"}) + void oauth2_acceptsSupportedEntraTokenEndpoints(String tokenEndpoint) { + Map input = oauth2Properties("https://account.blob.core.windows.net"); + input.put("azure.oauth2_server_uri", tokenEndpoint); + Assertions.assertEquals(tokenEndpoint, + AzureFileSystemProperties.of(input).toMap().get("AZURE_OAUTH_SERVER_URI")); + } + + private static Map oauth2Properties(String endpoint) { + return new HashMap<>(Map.of( + "azure.auth_type", "OAuth2", "azure.endpoint", endpoint, + "azure.oauth2_account_host", "account.dfs.core.windows.net", + "azure.oauth2_client_id", "client-id", "azure.oauth2_client_secret", "client-secret", + "azure.oauth2_server_uri", "https://login.microsoftonline.com/tenant/oauth2/token")); + } + + @ParameterizedTest + @CsvSource({ + "AZURE_AUTH_TYPE, SAS", + "AZURE_SAS_TOKEN, sig=wire-only", + "AZURE_SAS_EXPIRY_MS, 1", + "AZURE_OAUTH_SERVER_URI, https://login.microsoftonline.com/wire-tenant/oauth2/token", + "AZURE_OAUTH_ACCOUNT_HOST, wire-account.dfs.core.windows.net" + }) + void bind_doesNotAcceptNewBackendFieldsAsInputAliases(String key, String value) { + Map input = new HashMap<>(Map.of( + "azure.account_name", "account", "azure.account_key", "catalog-key")); + input.put(key, value); + + AzureFileSystemProperties properties = AzureFileSystemProperties.of(input); + + Assertions.assertEquals(input, properties.rawProperties()); + Assertions.assertEquals(Map.of("azure.account_name", "account", "azure.account_key", "catalog-key"), + properties.matchedProperties()); + Assertions.assertEquals(AzureAuthType.SHARED_KEY, properties.authType()); + Assertions.assertEquals("", properties.getSasToken()); + Assertions.assertEquals("", properties.getSasExpiryMs()); + Assertions.assertEquals("", properties.getOauthServerUri()); + Assertions.assertEquals("", properties.getOauthAccountHost()); + Assertions.assertEquals(Map.of( + "provider", "azure", "AZURE_AUTH_TYPE", "SHARED_KEY", + "AZURE_ACCOUNT_NAME", "account", "AZURE_ACCOUNT_KEY", "catalog-key", + "AZURE_ENDPOINT", "https://account.blob.core.windows.net"), properties.toMap()); + } + @Test void toString_masksCredentialsAndNeverLeaksPlaintext() { AzureFileSystemProperties properties = AzureFileSystemProperties.of(Map.of( @@ -73,6 +244,7 @@ void provider_sensitivePropertyKeysCoverSecretsButNotAccountName() { Assertions.assertTrue(keys.contains("azure.secret_key"), keys.toString()); Assertions.assertTrue(keys.contains("AZURE_ACCOUNT_KEY"), keys.toString()); Assertions.assertTrue(keys.contains("AZURE_CLIENT_SECRET"), keys.toString()); + Assertions.assertTrue(keys.contains("AZURE_SAS_TOKEN"), keys.toString()); Assertions.assertFalse(keys.contains("AZURE_ACCOUNT_NAME"), keys.toString()); Assertions.assertFalse(keys.contains("azure.access_key"), keys.toString()); } @@ -84,6 +256,50 @@ void bind_formatsEndpointFromAccountNameWhenEndpointMissing() { "azure.account_key", "key")); Assertions.assertEquals("https://myaccount.blob.core.windows.net", properties.getEndpoint()); + Assertions.assertEquals(AzureAuthType.SHARED_KEY, properties.authType()); + Assertions.assertEquals("SharedKey", properties.getAzureAuthType()); + } + + @ParameterizedTest + @ValueSource(strings = {"core.windows.net", "core.chinacloudapi.cn", "core.usgovcloudapi.net", "core.cloudapi.de"}) + void bind_convertsDfsEndpointToBlobEndpointForNativeClient(String suffix) { + AzureFileSystemProperties properties = AzureFileSystemProperties.of(Map.of( + "azure.endpoint", "account.dfs." + suffix, + "azure.account_name", "account", + "azure.account_key", "key")); + + Assertions.assertEquals("https://account.blob." + suffix, properties.getEndpoint()); + Assertions.assertEquals("key", properties.toHadoopConfigurationMap() + .get("fs.azure.account.key.account.blob." + suffix)); + } + + @ParameterizedTest + @CsvSource({ + "http://account.dfs.core.windows.net:10000/proxy%2Fpath, http://account.blob.core.windows.net:10000/proxy%2Fpath", + "http://127.0.0.1:10000/account, http://127.0.0.1:10000/account", + "https://storage.example.test:8443, https://storage.example.test:8443" + }) + void bind_preservesExplicitEndpointTransport(String input, String expected) { + AzureFileSystemProperties properties = AzureFileSystemProperties.of(Map.of( + "azure.endpoint", input, + "azure.account_name", "account", + "azure.account_key", "key")); + + Assertions.assertEquals(expected, properties.getEndpoint()); + Assertions.assertEquals(expected, properties.toMap().get("AZURE_ENDPOINT")); + } + + @Test + void bind_usesEndpointAccountAndCloudSuffixForSas() { + AzureFileSystemProperties properties = AzureFileSystemProperties.of(Map.of( + "azure.endpoint", "account.dfs.core.usgovcloudapi.net", + "azure.sas_token", "sig=temporary")); + + Assertions.assertEquals("account", properties.toMap().get("AZURE_ACCOUNT_NAME")); + Assertions.assertEquals("https://account.blob.core.usgovcloudapi.net", + properties.toMap().get("AZURE_ENDPOINT")); + Assertions.assertEquals("sig=temporary", properties.toHadoopConfigurationMap() + .get("fs.azure.sas.fixed.token.account.dfs.core.usgovcloudapi.net")); } @Test @@ -99,6 +315,117 @@ void bind_acceptsLegacyUppercaseKeysForExistingAzureCallers() { Assertions.assertEquals("https://legacy-account.blob.core.windows.net", properties.getEndpoint()); } + @Test + void bind_preservesHistoricalUppercaseAzureEndpointAlias() { + AzureFileSystemProperties properties = AzureFileSystemProperties.of(Map.of( + "AZURE_ENDPOINT", "https://proxy.example.test:8443", + "AZURE_ACCOUNT_NAME", "account", + "AZURE_ACCOUNT_KEY", "key")); + + Assertions.assertEquals("https://proxy.example.test:8443", properties.getEndpoint()); + Assertions.assertEquals("https://proxy.example.test:8443", + properties.toMap().get("AZURE_ENDPOINT")); + } + + @Test + void bind_preservesCustomDfsLabelEndpointForAbfsLocations() { + AzureFileSystemProperties properties = AzureFileSystemProperties.of(Map.of( + "provider", "azure", "azure.endpoint", "https://proxy.dfs.internal", + "azure.account_name", "account", "azure.account_key", "key")); + + Assertions.assertDoesNotThrow(() -> properties.validateAndNormalizeUri( + "abfss://container@account.dfs.core.windows.net/dir/file.parquet")); + } + + @Test + void bind_preservesLegacyAccountAliasFallbackWithoutProviderMarker() { + AzureFileSystemProperties properties = AzureFileSystemProperties.of(Map.of( + "AZURE_ACCOUNT_NAME", "account", "s3.secret_key", "legacy-key")); + + Assertions.assertEquals("account", properties.getAccountName()); + Assertions.assertEquals("legacy-key", properties.getAccountKey()); + } + + @Test + void bind_doesNotUseLegacyS3SecretWhenS3ProviderIsExplicit() { + Assertions.assertThrows(IllegalArgumentException.class, () -> AzureFileSystemProperties.of(Map.of( + "AZURE_ACCOUNT_NAME", "account", "s3.secret_key", "s3-secret", + "fs.s3.support", "true"))); + } + + @ParameterizedTest + @ValueSource(strings = {"SharedKey", "sharedkey", "SHARED_KEY", "shared_key", " SHARED_KEY "}) + void bind_normalizesSharedKeyAuthType(String authType) { + AzureFileSystemProperties properties = AzureFileSystemProperties.of(Map.of( + "azure.auth_type", authType, + "azure.account_name", "account", + "azure.account_key", "key")); + + Assertions.assertEquals(AzureAuthType.SHARED_KEY, properties.authType()); + Assertions.assertEquals("SharedKey", properties.getAzureAuthType()); + Assertions.assertEquals("SHARED_KEY", properties.toMap().get("AZURE_AUTH_TYPE")); + Assertions.assertEquals("key", properties.toHadoopConfigurationMap() + .get("fs.azure.account.key.account.blob.core.windows.net")); + } + + @Test + void bind_infersSasWhenAuthTypeIsAbsent() { + AzureFileSystemProperties properties = AzureFileSystemProperties.of(Map.of( + "azure.account_name", "account", + "azure.sas_token", "?sig=temporary")); + + Assertions.assertEquals(AzureAuthType.SAS, properties.authType()); + Assertions.assertEquals("SAS", properties.getAzureAuthType()); + Assertions.assertEquals("sig=temporary", properties.toHadoopConfigurationMap() + .get("fs.azure.sas.fixed.token.account.dfs.core.windows.net")); + } + + @Test + void bind_rejectsUnknownExplicitAuthTypeWithoutSasFallback() { + StoragePropertiesException exception = Assertions.assertThrows(StoragePropertiesException.class, + () -> AzureFileSystemProperties.of(Map.of( + "azure.auth_type", "unknown", + "azure.sas_token", "sig=temporary"))); + + Assertions.assertTrue(exception.getMessage().contains("Unsupported Azure auth_type")); + } + + @Test + void bind_rejectsExplicitSharedKeyWithSasToken() { + IllegalArgumentException exception = Assertions.assertThrows(IllegalArgumentException.class, + () -> AzureFileSystemProperties.of(Map.of( + "azure.auth_type", "SHARED_KEY", + "azure.account_name", "account", + "azure.account_key", "key", + "azure.sas_token", "sig=temporary"))); + + Assertions.assertTrue(exception.getMessage().contains("sas_token must not be set")); + } + + @Test + void bind_rejectsExplicitSasWithoutTokenInsteadOfUsingSharedKey() { + IllegalArgumentException exception = Assertions.assertThrows(IllegalArgumentException.class, + () -> AzureFileSystemProperties.of(Map.of( + "azure.auth_type", "SAS", + "azure.account_name", "account", + "azure.account_key", "key"))); + + Assertions.assertTrue(exception.getMessage().contains("When auth_type is SAS, sas_token is required")); + } + + @Test + void bind_rejectsIncompleteOAuth2InsteadOfUsingSharedKeyOrSas() { + IllegalArgumentException exception = Assertions.assertThrows(IllegalArgumentException.class, + () -> AzureFileSystemProperties.of(Map.of( + "azure.auth_type", "OAUTH2", + "azure.account_name", "account", + "azure.account_key", "key", + "azure.sas_token", "sig=temporary", + "azure.oauth2_client_id", "client-id"))); + + Assertions.assertTrue(exception.getMessage().contains("When auth_type is OAuth2")); + } + @Test void toBackendProperties_matchesFeCoreAzureSharedKeyMap() { AzureFileSystemProperties properties = AzureFileSystemProperties.of(Map.of( @@ -110,50 +437,159 @@ void toBackendProperties_matchesFeCoreAzureSharedKeyMap() { BackendStorageProperties backend = properties.toBackendProperties().orElseThrow(); Map backendMap = backend.toMap(); - Assertions.assertEquals(BackendStorageKind.S3_COMPATIBLE, backend.backendKind()); - Assertions.assertEquals("https://account.blob.core.windows.net", backendMap.get("AWS_ENDPOINT")); - Assertions.assertEquals("dummy_region", backendMap.get("AWS_REGION")); - Assertions.assertEquals("account", backendMap.get("AWS_ACCESS_KEY")); - Assertions.assertEquals("key", backendMap.get("AWS_SECRET_KEY")); - Assertions.assertEquals("true", backendMap.get("AWS_NEED_OVERRIDE_ENDPOINT")); - Assertions.assertEquals("azure", backendMap.get("provider")); - Assertions.assertEquals("true", backendMap.get("use_path_style")); - Assertions.assertFalse(backendMap.keySet().stream().anyMatch(keyName -> keyName.startsWith("AZURE_"))); - // The hadoop Configuration dump is the OAuth2 arm ONLY: SharedKey must stay the exact - // 7-key S3-style map, so a regression that applies the dump unconditionally shows up here. - Assertions.assertEquals(7, backendMap.size(), backendMap.toString()); + Assertions.assertEquals(BackendStorageKind.NATIVE, backend.backendKind()); + Assertions.assertEquals(Map.of( + "provider", "azure", + "AZURE_AUTH_TYPE", "SHARED_KEY", + "AZURE_ENDPOINT", "https://account.blob.core.windows.net", + "AZURE_ACCOUNT_NAME", "account", + "AZURE_ACCOUNT_KEY", "key"), backendMap); + } + + @Test + void toBackendProperties_emitsNativeSasCredentialsAndExpiry() { + AzureFileSystemProperties properties = AzureFileSystemProperties.of(Map.of( + "azure.auth_type", "SAS", + "AZURE_ENDPOINT", "account.blob.core.windows.net", + "AZURE_ACCOUNT_NAME", "account", + "AZURE_CONTAINER", "container", + "azure.sas_token", "?sv=2024-01-01&sig=temporary", + "azure.sas_expiry_ms", "4102444800000")); + + Map backendMap = properties.toBackendProperties().orElseThrow().toMap(); + + Assertions.assertEquals(Map.of( + "provider", "azure", + "AZURE_AUTH_TYPE", "SAS", + "AZURE_ENDPOINT", "https://account.blob.core.windows.net", + "AZURE_ACCOUNT_NAME", "account", + "AZURE_SAS_TOKEN", "sv=2024-01-01&sig=temporary", + "AZURE_SAS_EXPIRY_MS", "4102444800000"), backendMap); + } + + @Test + void bind_normalizesSasTokenAndReadsTokenExpiry() { + AzureFileSystemProperties properties = AzureFileSystemProperties.of(Map.of( + "azure.auth_type", "SAS", + "azure.account_name", "account", + "azure.sas_token", "?sv=2024-01-01&se=2100-01-01T00:00:00Z&sig=temporary")); + + Assertions.assertEquals("sv=2024-01-01&se=2100-01-01T00:00:00Z&sig=temporary", + properties.getSasToken()); + Assertions.assertEquals(properties.getSasToken(), properties.toMap().get("AZURE_SAS_TOKEN")); + Assertions.assertEquals("4102444800000", properties.getSasExpiryMs()); + Assertions.assertEquals("4102444800000", properties.toMap().get("AZURE_SAS_EXPIRY_MS")); + } + + @ParameterizedTest + @CsvSource({"4102444800000, 4102444800000", "4133980800000, 4102531200000"}) + void toMap_emitsEarlierExplicitOrTokenExpiry(String explicitExpiry, String expectedExpiry) { + AzureFileSystemProperties properties = AzureFileSystemProperties.of(Map.of( + "azure.sas_token", "se=2100-01-02T00%3A00%3A00Z&sig=a+b%2Fc", + "azure.sas_expiry_ms", explicitExpiry)); + + Assertions.assertEquals(expectedExpiry, properties.getSasExpiryMs()); + Assertions.assertEquals(expectedExpiry, properties.toMap().get("AZURE_SAS_EXPIRY_MS")); + Assertions.assertEquals("se=2100-01-02T00%3A00%3A00Z&sig=a+b%2Fc", + properties.toMap().get("AZURE_SAS_TOKEN")); + } + + @Test + void toMap_doesNotInventUnknownSasExpiry() { + AzureFileSystemProperties properties = AzureFileSystemProperties.of(Map.of( + "azure.sas_token", "si=stored-access-policy&sig=temporary")); + + Assertions.assertEquals("", properties.getSasExpiryMs()); + Assertions.assertDoesNotThrow(properties::validateForAccess); + Assertions.assertFalse(properties.toMap().containsKey("AZURE_SAS_EXPIRY_MS")); + } + + @Test + void bind_defersExpiredSasUntilCredentialOutput() { + AzureFileSystemProperties properties = AzureFileSystemProperties.of(Map.of( + "azure.auth_type", "SAS", + "AZURE_ENDPOINT", "account.blob.core.windows.net", + "azure.sas_token", "sv=2024-01-01&sig=expired", + "azure.sas_expiry_ms", "1")); + + StorageProperties bound = properties; + Assertions.assertDoesNotThrow(bound::validate); + StoragePropertiesException accessException = Assertions.assertThrows(StoragePropertiesException.class, + bound::validateForAccess); + Assertions.assertEquals("Azure SAS credential is expired", accessException.getMessage()); + Assertions.assertNull(accessException.getCause()); + + StoragePropertiesException exception = Assertions.assertThrows(StoragePropertiesException.class, + () -> properties.toMap()); + + Assertions.assertTrue(exception.getMessage().contains("expired"), exception.getMessage()); + Assertions.assertThrows(StoragePropertiesException.class, properties::toHadoopConfigurationMap); + } + + @Test + void validateForAccess_acceptsUnexpiredSas() { + StorageProperties properties = AzureFileSystemProperties.of(Map.of( + "azure.sas_token", "sig=temporary", + "azure.sas_expiry_ms", "4102444800000")); + + Assertions.assertDoesNotThrow(properties::validateForAccess); + } + + @Test + void validateSasExpiry_acceptsInjectedClockBeforeExpiry() { + AzureFileSystemProperties properties = AzureFileSystemProperties.of(Map.of( + "azure.auth_type", "SAS", + "azure.sas_token", "sv=2024-01-01&se=2026-01-02T00:00:00Z&sig=x")); + + properties.validateSasExpiry(Clock.fixed( + Instant.parse("2026-01-01T00:00:00Z"), ZoneOffset.UTC)); + Assertions.assertThrows(StoragePropertiesException.class, + () -> properties.validateSasExpiry(Clock.fixed( + Instant.parse("2026-01-02T00:00:00Z"), ZoneOffset.UTC))); + } + + @Test + void bind_rejectsConflictingExplicitCredentialMaterials() { + IllegalArgumentException exception = Assertions.assertThrows(IllegalArgumentException.class, + () -> AzureFileSystemProperties.of(Map.of( + "azure.auth_type", "OAuth2", + "azure.oauth2_account_host", "account.dfs.core.windows.net", + "azure.oauth2_client_id", "client-id", + "azure.oauth2_client_secret", "client-secret", + "azure.oauth2_server_uri", "https://login.microsoftonline.com/tenant/oauth2/token", + "azure.sas_token", "sig=temporary"))); + + Assertions.assertTrue(exception.getMessage().contains("sas_token must not be set"), exception.getMessage()); } /** - * Pins the OAuth2 backend map that moved here from fe-core {@code StorageAdapter} (which built a - * hadoop {@code Configuration} and dumped it). BE's live consumer is Microsoft Fabric OneLake: - * {@code abfs[s]://...dfs.fabric.microsoft.com} routes to {@code FILE_HDFS} and every entry is - * fed into BE's JNI hadoop builder, so dropping the hadoop-resolved keys silently changes what - * the ABFS connector is configured with. + * The full Hadoop configuration belongs only to the URI-selected OneLake backend view. */ @Test - void toBackendProperties_oauth2DumpsHadoopResolvedConfig() { + void resolveBackendProperties_oneLakeDumpsHadoopResolvedConfig() { AzureFileSystemProperties properties = AzureFileSystemProperties.of(Map.of( - "azure.endpoint", "account.blob.core.windows.net", "azure.auth_type", "OAuth2", - "azure.oauth2_account_host", "myaccount.dfs.core.windows.net", + "azure.oauth2_account_host", "onelake.dfs.fabric.microsoft.com", "azure.oauth2_client_id", "client-id", "azure.oauth2_client_secret", "client-secret", "azure.oauth2_server_uri", "https://login.microsoftonline.com/tenant/oauth2/token")); - Map backendMap = properties.toBackendProperties().orElseThrow().toMap(); + BackendStorageProperties backend = properties.resolveBackendProperties( + "abfss://workspace@onelake.dfs.fabric.microsoft.com/lakehouse/Tables/file").orElseThrow(); + Map backendMap = backend.toMap(); + Assertions.assertEquals(BackendStorageKind.HDFS, backend.backendKind()); // 1. The OAuth config the ABFS connector actually authenticates with. Assertions.assertEquals("OAuth", - backendMap.get("fs.azure.account.auth.type.myaccount.dfs.core.windows.net")); + backendMap.get("fs.azure.account.auth.type.onelake.dfs.fabric.microsoft.com")); Assertions.assertEquals("org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider", - backendMap.get("fs.azure.account.oauth.provider.type.myaccount.dfs.core.windows.net")); + backendMap.get("fs.azure.account.oauth.provider.type.onelake.dfs.fabric.microsoft.com")); Assertions.assertEquals("client-id", - backendMap.get("fs.azure.account.oauth2.client.id.myaccount.dfs.core.windows.net")); + backendMap.get("fs.azure.account.oauth2.client.id.onelake.dfs.fabric.microsoft.com")); Assertions.assertEquals("client-secret", - backendMap.get("fs.azure.account.oauth2.client.secret.myaccount.dfs.core.windows.net")); + backendMap.get("fs.azure.account.oauth2.client.secret.onelake.dfs.fabric.microsoft.com")); Assertions.assertEquals("https://login.microsoftonline.com/tenant/oauth2/token", - backendMap.get("fs.azure.account.oauth2.client.endpoint.myaccount.dfs.core.windows.net")); + backendMap.get("fs.azure.account.oauth2.client.endpoint.onelake.dfs.fabric.microsoft.com")); // 2. hadoop core-default.xml is merged in. This is the whole reason the module compiles // against hadoop-common: a plain key-value map would carry the OAuth keys above but none @@ -163,17 +599,56 @@ void toBackendProperties_oauth2DumpsHadoopResolvedConfig() { Assertions.assertTrue(backendMap.size() > 100, "expected a resolved hadoop config, got " + backendMap.size() + " keys"); - // 3. Never S3-style: OAuth2 has no AK/SK the BE S3 adapter could consume. - Assertions.assertFalse(backendMap.containsKey("AWS_ACCESS_KEY"), backendMap.toString()); - Assertions.assertFalse(backendMap.containsKey("provider"), backendMap.toString()); + // 3. The selected Hadoop view never carries native Azure authentication fields. + Assertions.assertFalse(backendMap.containsKey("provider")); + Assertions.assertFalse(backendMap.keySet().stream().anyMatch(key -> key.startsWith("AZURE_"))); } @Test - void toBackendProperties_oauth2PassesUserFsKeysThroughAndNormalizesCacheFlags() { + void toHadoopConfigurationMap_preservesOneLakeOAuth2Host() { + AzureFileSystemProperties properties = AzureFileSystemProperties.of(Map.of( + "azure.auth_type", "OAuth2", + "azure.oauth2_account_host", "onelake.dfs.fabric.microsoft.com", + "azure.oauth2_client_id", "client-id", + "azure.oauth2_client_secret", "client-secret", + "azure.oauth2_server_uri", "https://login.microsoftonline.com/tenant/oauth2/token")); + + Map hadoop = properties.toHadoopConfigurationMap(); + Assertions.assertEquals("OAuth", hadoop.get("fs.azure.account.auth.type.onelake.dfs.fabric.microsoft.com")); + Assertions.assertEquals("client-id", + hadoop.get("fs.azure.account.oauth2.client.id.onelake.dfs.fabric.microsoft.com")); + Assertions.assertEquals("client-secret", + hadoop.get("fs.azure.account.oauth2.client.secret.onelake.dfs.fabric.microsoft.com")); + Assertions.assertEquals("OAuth", + properties.resolveBackendProperties("abfss://workspace@onelake.dfs.fabric.microsoft.com/file") + .orElseThrow().toMap().get("fs.azure.account.auth.type.onelake.dfs.fabric.microsoft.com")); + Assertions.assertFalse(properties.toMap().keySet().stream().anyMatch(key -> key.startsWith("fs."))); + Assertions.assertFalse(hadoop.keySet().stream().anyMatch(key -> key.contains(".blob.fabric.microsoft.com"))); + } + + @ParameterizedTest + @ValueSource(strings = {"OAuth2", "OAUTH2", "oauth2"}) + void bind_preservesLegacyOAuth2ClientAliases(String authType) { + AzureFileSystemProperties properties = AzureFileSystemProperties.of(Map.of( + "azure.auth_type", authType, + "azure.oauth2_account_host", "account.dfs.core.windows.net", + "AZURE_CLIENT_ID", "client-id", + "AZURE_CLIENT_SECRET", "client-secret", + "azure.oauth2_server_uri", "https://login.microsoftonline.com/tenant/oauth2/token")); + + Assertions.assertTrue(properties.isOauth2Auth()); + Assertions.assertEquals(AzureAuthType.OAUTH2, properties.authType()); + Assertions.assertEquals("OAuth2", properties.getAzureAuthType()); + Assertions.assertEquals("account.dfs.core.windows.net", properties.getOauthAccountHost()); + Assertions.assertEquals("https://login.microsoftonline.com/tenant/oauth2/token", + properties.getOauthServerUri()); + } + + @Test + void resolveBackendProperties_oneLakePassesUserFsKeysThroughAndNormalizesCacheFlags() { AzureFileSystemProperties properties = AzureFileSystemProperties.of(Map.of( - "azure.endpoint", "account.blob.core.windows.net", "azure.auth_type", "OAuth2", - "azure.oauth2_account_host", "myaccount.dfs.core.windows.net", + "azure.oauth2_account_host", "onelake.dfs.fabric.microsoft.com", "azure.oauth2_client_id", "client-id", "azure.oauth2_client_secret", "client-secret", "azure.oauth2_server_uri", "https://login.microsoftonline.com/tenant/oauth2/token", @@ -182,7 +657,8 @@ void toBackendProperties_oauth2PassesUserFsKeysThroughAndNormalizesCacheFlags() // explicit cache flag in a spelling only BooleanUtils understands "fs.abfss.impl.disable.cache", "no")); - Map backendMap = properties.toBackendProperties().orElseThrow().toMap(); + Map backendMap = properties.resolveBackendProperties( + "abfss://workspace@onelake.dfs.fabric.microsoft.com/file").orElseThrow().toMap(); Assertions.assertEquals("8", backendMap.get("fs.azure.readaheadqueue.depth")); // Explicit user value wins, but normalized to true/false — "no" must not reach BE verbatim. diff --git a/fe/fe-filesystem/fe-filesystem-azure/src/test/java/org/apache/doris/filesystem/azure/AzureFileSystemProviderTest.java b/fe/fe-filesystem/fe-filesystem-azure/src/test/java/org/apache/doris/filesystem/azure/AzureFileSystemProviderTest.java index 0358e2b3e4adb4..c4ec0e9e79533a 100644 --- a/fe/fe-filesystem/fe-filesystem-azure/src/test/java/org/apache/doris/filesystem/azure/AzureFileSystemProviderTest.java +++ b/fe/fe-filesystem/fe-filesystem-azure/src/test/java/org/apache/doris/filesystem/azure/AzureFileSystemProviderTest.java @@ -49,6 +49,23 @@ void supports_returnsFalseForPlainAwsAccessKeyWithoutAzureSignal() { Assertions.assertFalse(provider.supports(props)); } + @Test + void supports_recognizesProviderOwnedSasToken() { + Map props = new HashMap<>(); + props.put("azure.sas_token", "sv=2024-01-01&sig=temporary"); + Assertions.assertTrue(provider.supports(props)); + Assertions.assertTrue(provider.supportsGuess(props)); + } + + @Test + void supports_doesNotRouteByTheBackendSasField() { + Map props = Map.of("AZURE_SAS_TOKEN", "sig=wire-only"); + + Assertions.assertFalse(provider.supports(props)); + Assertions.assertFalse(provider.supportsGuess(props)); + Assertions.assertTrue(provider.sensitivePropertyKeys().contains("AZURE_SAS_TOKEN")); + } + @Test void supports_returnsFalseForUnknownEndpointSuffix() { Map props = new HashMap<>(); @@ -56,6 +73,41 @@ void supports_returnsFalseForUnknownEndpointSuffix() { Assertions.assertFalse(provider.supports(props)); } + @Test + void supportsExplicit_requiresRawAzureSupportFlag() { + Assertions.assertTrue(provider.supportsExplicit(Map.of("fs.azure.support", "true"))); + Assertions.assertFalse(provider.supportsExplicit(Map.of("fs.azure.support", "false"))); + Assertions.assertFalse(provider.supportsExplicit(Map.of("provider", "azure"))); + Assertions.assertFalse(provider.supportsExplicit(Map.of("_STORAGE_TYPE_", "AZURE"))); + Assertions.assertFalse(provider.supportsExplicit(Map.of("AZURE_ACCOUNT_NAME", "account"))); + Assertions.assertFalse(provider.supportsExplicit(Map.of("fs.s3.support", "true"))); + } + + @Test + void supportsGuess_doesNotTreatConvertedStorageMarkerAsExplicitInput() { + Assertions.assertFalse(provider.supportsGuess(Map.of("_STORAGE_TYPE_", "AZURE"))); + Assertions.assertTrue(provider.supportsGuess(Map.of("provider", "azure"))); + } + + @ParameterizedTest + @ValueSource(strings = {"https://ACCOUNT.DFS.CORE.WINDOWS.NET", + "https://account.dfs.core.chinacloudapi.cn", "https://ACCOUNT.BLOB.CORE.USGOVCLOUDAPI.NET"}) + void supportsGuess_recognizesDfsAndCaseInsensitiveAzureHosts(String endpoint) { + Assertions.assertTrue(provider.supportsGuess(Map.of("azure.endpoint", endpoint))); + Assertions.assertTrue(provider.supports(Map.of("azure.endpoint", endpoint))); + Assertions.assertTrue(provider.supports(Map.of("AZURE_ENDPOINT", endpoint))); + } + + @ParameterizedTest + @ValueSource(strings = {"https://foo.blob.core.windows.net.proxy.test", + "https://foo.dfs.core.windows.net.proxy.test", "https://proxy.test/blob.core.windows.net", + "https://proxy.test?endpoint=account.blob.core.windows.net"}) + void supports_matchesOnlyTheActualHostSuffix(String endpoint) { + Assertions.assertFalse(provider.supports(Map.of("azure.endpoint", endpoint))); + Assertions.assertFalse(provider.supports(Map.of("AZURE_ENDPOINT", endpoint))); + Assertions.assertFalse(provider.supportsGuess(Map.of("azure.endpoint", endpoint))); + } + // F21 — provider must recognise all four Azure sovereign-cloud blob host suffixes. @ParameterizedTest @ValueSource(strings = { diff --git a/fe/fe-filesystem/fe-filesystem-azure/src/test/java/org/apache/doris/filesystem/azure/AzureFileSystemTest.java b/fe/fe-filesystem/fe-filesystem-azure/src/test/java/org/apache/doris/filesystem/azure/AzureFileSystemTest.java index 596f1c88ff0b1f..b597bdf36cdb86 100644 --- a/fe/fe-filesystem/fe-filesystem-azure/src/test/java/org/apache/doris/filesystem/azure/AzureFileSystemTest.java +++ b/fe/fe-filesystem/fe-filesystem-azure/src/test/java/org/apache/doris/filesystem/azure/AzureFileSystemTest.java @@ -19,6 +19,7 @@ import org.apache.doris.filesystem.DorisOutputFile; import org.apache.doris.filesystem.FileEntry; +import org.apache.doris.filesystem.FileIterator; import org.apache.doris.filesystem.GlobListing; import org.apache.doris.filesystem.Location; import org.apache.doris.filesystem.spi.RemoteObject; @@ -28,6 +29,9 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import org.mockito.ArgumentCaptor; import org.mockito.ArgumentMatchers; import org.mockito.Mockito; @@ -102,6 +106,111 @@ void delete_recursive_doesNotTouchSiblingBlobWithSameName() throws IOException { Mockito.verify(mockStorage, Mockito.never()).deleteObject("wasbs://c@a.host/foo"); } + @ParameterizedTest + @ValueSource(strings = { + "https://account.blob.core.windows.net/container/", + "http://account.blob.core.windows.net/container/", + "abfss://container@account.dfs.core.windows.net/", + "wasbs://container@account.blob.core.windows.net/", + "s3://container/" + }) + void listAndRecursiveDelete_preserveContainerAndRawKeysAcrossPages(String root) throws IOException { + String prefix = root + "dir/"; + List keys = List.of("dir/a%2Fb +中文.parquet", "dir/100%.parquet"); + Mockito.when(mockStorage.listObjects(prefix, null)).thenReturn(new RemoteObjects( + List.of(new RemoteObject(keys.get(0), "", null, 1L, 0L)), true, "next")); + Mockito.when(mockStorage.listObjects(prefix, "next")).thenReturn(new RemoteObjects( + List.of(new RemoteObject(keys.get(1), "", null, 2L, 0L)), false, null)); + + List files = fs.listFiles(Location.of(prefix)); + Assertions.assertEquals(2, files.size()); + for (int i = 0; i < keys.size(); i++) { + AzureUri uri = AzureUri.parse(files.get(i).location().uri()); + Assertions.assertEquals("container", uri.container()); + Assertions.assertEquals(keys.get(i), uri.key()); + Assertions.assertTrue(files.get(i).location().uri().startsWith(root)); + } + + fs.delete(Location.of(prefix), true); + + ArgumentCaptor deleted = ArgumentCaptor.forClass(String.class); + Mockito.verify(mockStorage, Mockito.times(2)).deleteObject(deleted.capture()); + Assertions.assertEquals(files.stream().map(file -> file.location().uri()).toList(), deleted.getAllValues()); + } + + @Test + void httpsGlobListing_preservesEncodedPrefixAndReturnedKeys() throws IOException { + String root = "https://account.blob.core.windows.net/container/"; + String prefix = root + "dir%252Fdata/"; + String key = "dir%2Fdata/a%2Fb +中文.parquet"; + Mockito.when(mockStorage.listObjects(prefix, null)).thenReturn(new RemoteObjects( + List.of(new RemoteObject(key, "", null, 1L, 0L)), false, null)); + + for (List files : List.of( + fs.listFiles(Location.of(prefix + "*.parquet")), + fs.globListWithLimit(Location.of(prefix + "*.parquet"), null, 0, 0).getFiles())) { + Assertions.assertEquals(1, files.size()); + AzureUri uri = AzureUri.parse(files.get(0).location().uri()); + Assertions.assertEquals("container", uri.container()); + Assertions.assertEquals(key, uri.key()); + } + } + + @Test + void httpsSingleLevelGlobDecodesEncodedBasenameBeforeMatching() throws IOException { + String prefix = "https://account.blob.core.windows.net/container/dir%20/"; + String key = "dir /file name.parquet"; + Mockito.when(mockStorage.listObjects(prefix, null)).thenReturn(new RemoteObjects( + List.of(new RemoteObject(key, "", null, 1L, 0L)), false, null)); + + List files = fs.listFiles(Location.of(prefix + "file%20*.parquet")); + + Assertions.assertEquals(1, files.size()); + Assertions.assertEquals(key, AzureUri.parse(files.get(0).location().uri()).key()); + Mockito.verify(mockStorage).listObjects(prefix, null); + } + + @Test + void httpsDirectoryOperations_appendSlashBeforeQueryAndFragment() throws IOException { + String root = "https://account.blob.core.windows.net/container/"; + String prefix = root + "dir%252Fdata/"; + String key = "dir%2Fdata/a%2Fb.parquet"; + Mockito.when(mockStorage.listObjects(prefix, null)).thenReturn(new RemoteObjects( + List.of(new RemoteObject(key, "", null, 1L, 0L)), false, null)); + Location location = Location.of(root + "dir%252Fdata?ignored=query#fragment"); + + try (FileIterator it = fs.list(location)) { + Assertions.assertTrue(it.hasNext()); + Assertions.assertEquals(key, AzureUri.parse(it.next().location().uri()).key()); + Assertions.assertFalse(it.hasNext()); + } + fs.delete(location, true); + + Mockito.verify(mockStorage, Mockito.times(2)).listObjects(prefix, null); + Mockito.verify(mockStorage).deleteObject(prefix + "a%252Fb.parquet"); + Mockito.verifyNoMoreInteractions(mockStorage); + } + + @ParameterizedTest + @ValueSource(strings = { + "https://account.blob.core.windows.net/container", + "https://account.blob.core.windows.net/container/", + "abfss://container@account.dfs.core.windows.net", + "abfss://container@account.dfs.core.windows.net/" + }) + void containerRoot_usesAnEmptyObjectPrefix(String location) throws IOException { + String prefix = location.endsWith("/") ? location : location + "/"; + Mockito.when(mockStorage.listObjects(prefix, null)).thenReturn(new RemoteObjects( + List.of(new RemoteObject("file.parquet", "", null, 1L, 0L)), false, null)); + + Assertions.assertEquals(prefix + "file.parquet", fs.listFiles(Location.of(location)).get(0).location().uri()); + fs.delete(Location.of(location), true); + + Mockito.verify(mockStorage, Mockito.times(2)).listObjects(prefix, null); + Mockito.verify(mockStorage).deleteObject(prefix + "file.parquet"); + Mockito.verifyNoMoreInteractions(mockStorage); + } + @Test void delete_nonRecursive_throwsIfDirectoryNotEmpty() throws IOException { RemoteObjects page = new RemoteObjects( @@ -528,6 +637,50 @@ void list_skipsDirectoryMarkerEntries() throws IOException { Assertions.assertEquals("wasbs://c@a.host/dir/b.csv", entries.get(1).location().uri()); } + @Test + void list_continuesPastMarkerOnlyAndEmptyPagesUntilRealEof() throws IOException { + String prefix = "abfss://container@account.dfs.core.windows.net/dir/"; + Mockito.when(mockStorage.listObjects(prefix, null)).thenReturn(new RemoteObjects( + List.of(new RemoteObject("dir/", "", null, 0L, 0L)), true, "empty-page")); + Mockito.when(mockStorage.listObjects(prefix, "empty-page")) + .thenReturn(new RemoteObjects(List.of(), true, "file-page")); + Mockito.when(mockStorage.listObjects(prefix, "file-page")).thenReturn(new RemoteObjects( + List.of(new RemoteObject("dir/file.parquet", "", null, 1L, 0L)), true, "last-page")); + Mockito.when(mockStorage.listObjects(prefix, "last-page")).thenReturn(new RemoteObjects( + List.of(new RemoteObject("dir/marker/", "", null, 0L, 0L)), false, null)); + + try (FileIterator it = fs.list(Location.of(prefix))) { + Assertions.assertTrue(it.hasNext()); + Assertions.assertTrue(it.hasNext()); + Assertions.assertEquals(prefix + "file.parquet", it.next().location().uri()); + Assertions.assertFalse(it.hasNext()); + Assertions.assertFalse(it.hasNext()); + Assertions.assertThrows(java.util.NoSuchElementException.class, it::next); + } + Mockito.verify(mockStorage).listObjects(prefix, null); + Mockito.verify(mockStorage).listObjects(prefix, "empty-page"); + Mockito.verify(mockStorage).listObjects(prefix, "file-page"); + Mockito.verify(mockStorage).listObjects(prefix, "last-page"); + Mockito.verifyNoMoreInteractions(mockStorage); + } + + @Test + void directoryChecks_doNotTreatAMarkerOnlyFirstPageAsEmpty() throws IOException { + String directory = "https://account.blob.core.windows.net/container/dir"; + Mockito.when(mockStorage.headObject(directory)).thenThrow(new FileNotFoundException()); + Mockito.when(mockStorage.listObjects(directory + "/", null)).thenReturn(new RemoteObjects( + List.of(new RemoteObject("dir/", "", null, 0L, 0L)), true, "next")); + Mockito.when(mockStorage.listObjects(directory + "/", "next")).thenReturn(new RemoteObjects( + List.of(new RemoteObject("dir/file.parquet", "", null, 1L, 0L)), false, null)); + + Assertions.assertTrue(fs.exists(Location.of(directory))); + IOException error = Assertions.assertThrows(IOException.class, () -> fs.delete(Location.of(directory), false)); + + Assertions.assertTrue(error.getMessage().contains("Directory not empty")); + Mockito.verify(mockStorage, Mockito.never()).deleteObject(ArgumentMatchers.anyString()); + Mockito.verify(mockStorage, Mockito.times(2)).listObjects(directory + "/", "next"); + } + // --------------------------------------------------------------------- // F16 — AzureFileIterator.next() throws NoSuchElementException when exhausted // --------------------------------------------------------------------- diff --git a/fe/fe-filesystem/fe-filesystem-azure/src/test/java/org/apache/doris/filesystem/azure/AzureIcebergFileIOPropertiesTest.java b/fe/fe-filesystem/fe-filesystem-azure/src/test/java/org/apache/doris/filesystem/azure/AzureIcebergFileIOPropertiesTest.java new file mode 100644 index 00000000000000..a6205460679493 --- /dev/null +++ b/fe/fe-filesystem/fe-filesystem-azure/src/test/java/org/apache/doris/filesystem/azure/AzureIcebergFileIOPropertiesTest.java @@ -0,0 +1,268 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.filesystem.azure; + +import org.apache.doris.foundation.property.StoragePropertiesException; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.params.provider.ValueSource; +import org.mockito.Mockito; + +import java.time.Clock; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Stream; + +class AzureIcebergFileIOPropertiesTest { + @Test + void defaultIcebergFileIODoesNotAccessAnUnusedExpiredHadoopSas() { + AzureFileSystemProperties storage = AzureFileSystemProperties.of(Map.of( + "azure.account_name", "account", "azure.sas_token", "sig=expired-static-test-signature", + "azure.sas_expiry_ms", "1")); + + Assertions.assertTrue(storage.toIcebergHadoopProperties().isEmpty()); + Assertions.assertThrows(StoragePropertiesException.class, storage::toHadoopConfigurationMap, + "real Hadoop consumers must retain their original expiry validation"); + Assertions.assertThrows(StoragePropertiesException.class, storage::toIcebergFileIOProperties, + "the selected FileIO credential still must be checked before access"); + } + + @Test + void expiredSasExposesOnlyCustomFileIOConnectionsWithoutAccessingAuthentication() { + AzureFileSystemProperties storage = AzureFileSystemProperties.of(Map.of( + "azure.account_name", "account", "azure.sas_token", "sig=expired-static-test-signature", + "azure.sas_expiry_ms", "1", + "azure.endpoint", "http://account.dfs.core.windows.net:10000/proxy%2Fpath")); + + Map connections = storage.toIcebergFileIOConnectionProperties(); + + Assertions.assertEquals(Map.of( + "adls.connection-string.account.dfs.core.windows.net", + "http://account.blob.core.windows.net:10000/proxy%2Fpath", + "adls.connection-string.account.blob.core.windows.net", + "http://account.blob.core.windows.net:10000/proxy%2Fpath"), connections); + Assertions.assertThrows(UnsupportedOperationException.class, () -> connections.put("io-impl", "other")); + StoragePropertiesException failure = Assertions.assertThrows(StoragePropertiesException.class, + storage::toIcebergFileIOProperties); + Assertions.assertEquals("Azure SAS credential is expired", failure.getMessage()); + } + + @Test + void legacySharedKeyEmitsOnlyOfficialFileIOCredentialsAndEndpoints() { + AzureFileSystemProperties storage = AzureFileSystemProperties.of(Map.of( + "AZURE_ACCOUNT_NAME", "account", "AZURE_ACCOUNT_KEY", "test-key", + "azure.endpoint", "http://account.dfs.core.windows.net:10000/proxy%2Fpath")); + + Map output = storage.toIcebergFileIOProperties(); + + Assertions.assertEquals(Map.of( + "adls.auth.shared-key.account.name", "account", + "adls.auth.shared-key.account.key", "test-key", + "adls.connection-string.account.dfs.core.windows.net", + "http://account.blob.core.windows.net:10000/proxy%2Fpath", + "adls.connection-string.account.blob.core.windows.net", + "http://account.blob.core.windows.net:10000/proxy%2Fpath"), output); + Assertions.assertThrows(UnsupportedOperationException.class, () -> output.put("io-impl", "other")); + Assertions.assertEquals(Map.of( + "adls.connection-string.account.dfs.core.windows.net", + "http://account.blob.core.windows.net:10000/proxy%2Fpath", + "adls.connection-string.account.blob.core.windows.net", + "http://account.blob.core.windows.net:10000/proxy%2Fpath"), + storage.toIcebergFileIOConnectionProperties()); + Assertions.assertEquals(storage.toHadoopConfigurationMap(), + storage.toIcebergHadoopProperties().orElseThrow().toHadoopConfigurationMap()); + } + + @ParameterizedTest + @CsvSource({"4102444800000,4102444800000", "4133980800000,4102531200000"}) + void sasEmitsTheEarlierExpiryWithoutReencodingTheToken(String explicitExpiry, String expectedExpiry) { + String token = "se=2100-01-02T00%3A00%3A00Z&sig=a+b%2Fc"; + AzureFileSystemProperties storage = AzureFileSystemProperties.of(Map.of( + "azure.account_name", "account", "azure.sas_token", "?" + token, + "azure.sas_expiry_ms", explicitExpiry)); + + Map output = storage.toIcebergFileIOProperties(); + + Assertions.assertEquals(Map.of( + "adls.sas-token.account.dfs.core.windows.net", token, + "adls.sas-token.account.blob.core.windows.net", token, + "adls.sas-token-expires-at-ms.account.dfs.core.windows.net", expectedExpiry, + "adls.sas-token-expires-at-ms.account.blob.core.windows.net", expectedExpiry, + "adls.connection-string.account.dfs.core.windows.net", "https://account.blob.core.windows.net", + "adls.connection-string.account.blob.core.windows.net", "https://account.blob.core.windows.net"), + output); + Assertions.assertThrows(UnsupportedOperationException.class, () -> output.put("adls.token", "other")); + } + + @Test + void sasWithUnknownExpiryDoesNotInventAnExpiryProperty() { + String token = "si=stored-access-policy&sig=test-signature"; + AzureFileSystemProperties storage = AzureFileSystemProperties.of(Map.of( + "azure.account_name", "account", "azure.sas_token", token)); + + Assertions.assertEquals(Map.of( + "adls.sas-token.account.dfs.core.windows.net", token, + "adls.sas-token.account.blob.core.windows.net", token, + "adls.connection-string.account.dfs.core.windows.net", "https://account.blob.core.windows.net", + "adls.connection-string.account.blob.core.windows.net", "https://account.blob.core.windows.net"), + storage.toIcebergFileIOProperties()); + } + + @Test + void cachedBindingRejectsFileIOCredentialOutputAtExpiry() { + Clock clock = Mockito.mock(Clock.class); + Instant expiry = Instant.parse("2026-01-02T00:00:00Z"); + Mockito.when(clock.instant()).thenReturn(expiry.minusMillis(1)); + AzureFileSystemProperties storage = AzureFileSystemProperties.of(Map.of( + "azure.account_name", "account", + "azure.sas_token", "se=2026-01-02T00:00:00Z&sig=private-material"), clock); + Assertions.assertDoesNotThrow(storage::toIcebergFileIOProperties); + + Mockito.when(clock.instant()).thenReturn(expiry); + + StoragePropertiesException error = Assertions.assertThrows(StoragePropertiesException.class, + storage::toIcebergFileIOProperties); + Assertions.assertEquals("Azure SAS credential is expired", error.getMessage()); + Assertions.assertNull(error.getCause()); + Assertions.assertDoesNotThrow(storage::validate); + } + + @ParameterizedTest + @ValueSource(strings = {"core.windows.net", "core.chinacloudapi.cn", "core.usgovcloudapi.net"}) + void fileIOSasAliasesRebindToTheSameNativeCredentials(String cloudSuffix) { + Map input = Map.of( + "azure.account_name", "account", + "azure.endpoint", "https://account.blob." + cloudSuffix, + "azure.sas_token", "sig=test-signature&se=2100-01-01T00:00:00Z", + "azure.sas_expiry_ms", "4070908800000"); + AzureFileSystemProvider provider = new AzureFileSystemProvider(); + AzureFileSystemProperties storage = provider.bind(input); + Map fileIOProperties = storage.toIcebergFileIOProperties(); + + AzureFileSystemProperties rebound = provider.bindVended(fileIOProperties, input).orElseThrow(); + + Assertions.assertEquals(storage.toMap(), rebound.toMap()); + Assertions.assertEquals(fileIOProperties, rebound.toIcebergFileIOProperties()); + Assertions.assertEquals("4070908800000", rebound.toMap().get("AZURE_SAS_EXPIRY_MS")); + } + + @ParameterizedTest + @MethodSource("conflictingFileIOAliases") + void rebindRejectsConflictingFileIOAliasesWithoutStaticFallback(Map credentials) { + AzureFileSystemProvider provider = new AzureFileSystemProvider(); + Map catalog = Map.of("azure.account_name", "account", "azure.account_key", "test-key"); + List> entries = new ArrayList<>(credentials.entrySet()); + for (boolean reverse : List.of(false, true)) { + if (reverse) { + Collections.reverse(entries); + } + Map ordered = new LinkedHashMap<>(); + entries.forEach(entry -> ordered.put(entry.getKey(), entry.getValue())); + StoragePropertiesException error = Assertions.assertThrows(StoragePropertiesException.class, + () -> provider.bindVended(ordered, catalog)); + Assertions.assertFalse(error.getMessage().contains("private-material")); + Assertions.assertNull(error.getCause()); + } + } + + @ParameterizedTest + @ValueSource(strings = {"account.dfs.core.windows.net", "onelake.dfs.fabric.microsoft.com"}) + void clientSecretOAuthRequiresHadoopFileIOWithoutChangingOtherCredentialViews(String host) { + AzureFileSystemProperties storage = AzureFileSystemProperties.of(Map.of( + "azure.auth_type", "OAuth2", + "azure.oauth2_account_host", host, + "azure.oauth2_client_id", "test-client", + "azure.oauth2_client_secret", "test-client-secret", + "azure.oauth2_server_uri", "https://login.microsoftonline.com/test-tenant/oauth2/token")); + Map nativeProperties = storage.toMap(); + Map hadoopProperties = storage.toHadoopConfigurationMap(); + + Map output = storage.toIcebergFileIOProperties(); + Assertions.assertEquals(Map.of("io-impl", "org.apache.iceberg.hadoop.HadoopFileIO"), output); + Assertions.assertThrows(UnsupportedOperationException.class, () -> output.put("adls.token", "other")); + + Assertions.assertEquals(nativeProperties, storage.toMap()); + Assertions.assertEquals("OAUTH2", nativeProperties.get("AZURE_AUTH_TYPE")); + Assertions.assertEquals(hadoopProperties, storage.toHadoopConfigurationMap()); + Assertions.assertEquals(hadoopProperties, + storage.toIcebergHadoopProperties().orElseThrow().toHadoopConfigurationMap()); + Assertions.assertEquals("test-client-secret", + hadoopProperties.get("fs.azure.account.oauth2.client.secret." + host)); + } + + private static Stream> conflictingFileIOAliases() { + String dfsToken = "adls.sas-token.account.dfs.core.windows.net"; + String blobToken = "adls.sas-token.account.blob.core.windows.net"; + String token = "sig=private-material"; + return Stream.of( + Map.of(dfsToken, token, blobToken, "sig=other-private-material"), + Map.of(dfsToken, token, blobToken, token, + "adls.sas-token-expires-at-ms.account.dfs.core.windows.net", "4102444800000", + "adls.sas-token-expires-at-ms.account.blob.core.windows.net", "4133980800000"), + Map.of(dfsToken, token, "adls.sas-token.other.blob.core.windows.net", token), + Map.of(dfsToken, token, "adls.sas-token.account.blob.core.chinacloudapi.cn", token)); + } + + @Test + void fileIOOutputRejectsIncompleteSasConnectionWithoutChangingGeneralBinding() { + AzureFileSystemProperties storage = AzureFileSystemProperties.of( + Map.of("azure.sas_token", "sig=private-material")); + Assertions.assertDoesNotThrow(storage::validate); + + StoragePropertiesException error = Assertions.assertThrows(StoragePropertiesException.class, + storage::toIcebergFileIOProperties); + + Assertions.assertNull(error.getCause()); + Assertions.assertFalse(error.getMessage().contains("private-material")); + Assertions.assertFalse(error.getMessage().contains("https://")); + StoragePropertiesException connectionError = Assertions.assertThrows(StoragePropertiesException.class, + storage::toIcebergFileIOConnectionProperties); + Assertions.assertEquals(error.getMessage(), connectionError.getMessage()); + Assertions.assertNull(connectionError.getCause()); + } + + @ParameterizedTest + @MethodSource("invalidEndpointConnections") + void bindingRejectsInvalidEndpointsBeforeFileIOOutput(Map input) { + StoragePropertiesException error = Assertions.assertThrows(StoragePropertiesException.class, + () -> AzureFileSystemProperties.of(input)); + Assertions.assertNull(error.getCause()); + Assertions.assertFalse(error.getMessage().contains("private-material")); + Assertions.assertFalse(error.getMessage().contains("https://")); + } + + private static Stream> invalidEndpointConnections() { + return Stream.of( + "https://account.blob.core.windows.net?sig=private-material", + "https://private-material@account.blob.core.windows.net", + "https://account.blob.core.windows.net#private-material", + "ftp://account.blob.core.windows.net").flatMap(endpoint -> Stream.of( + Map.of("azure.account_name", "account", "azure.account_key", "test-key", + "azure.endpoint", endpoint), + Map.of("azure.account_name", "account", "azure.sas_token", "sig=bound-sas", + "azure.endpoint", endpoint))); + } +} diff --git a/fe/fe-filesystem/fe-filesystem-azure/src/test/java/org/apache/doris/filesystem/azure/AzureObjStorageExtensionTest.java b/fe/fe-filesystem/fe-filesystem-azure/src/test/java/org/apache/doris/filesystem/azure/AzureObjStorageExtensionTest.java index 2945b141cef296..0dafdbf40f42a5 100644 --- a/fe/fe-filesystem/fe-filesystem-azure/src/test/java/org/apache/doris/filesystem/azure/AzureObjStorageExtensionTest.java +++ b/fe/fe-filesystem/fe-filesystem-azure/src/test/java/org/apache/doris/filesystem/azure/AzureObjStorageExtensionTest.java @@ -20,6 +20,7 @@ import org.apache.doris.filesystem.UploadPartResult; import org.apache.doris.filesystem.spi.RemoteObjects; import org.apache.doris.filesystem.spi.RequestBody; +import org.apache.doris.foundation.property.StoragePropertiesException; import com.azure.storage.blob.BlobClient; import com.azure.storage.blob.BlobContainerClient; @@ -35,7 +36,10 @@ import java.io.ByteArrayInputStream; import java.io.IOException; import java.nio.charset.StandardCharsets; +import java.time.Clock; +import java.time.Instant; import java.time.OffsetDateTime; +import java.time.ZoneOffset; import java.util.Arrays; import java.util.Base64; import java.util.Collections; @@ -122,18 +126,93 @@ void getPresignedUrl_passesParsedUriToGenerateSasUrl() throws Exception { } @Test - void getPresignedUrl_fallsBackToAwsSecretKey() throws Exception { + void getPresignedUrl_rejectsAwsSecretKeyAsAzureAccountKey() { Map props = new HashMap<>(); props.put("AZURE_ACCOUNT_NAME", "myaccount"); - // No AZURE_ACCOUNT_KEY, but AWS_SECRET_KEY is present (S3-compat config) + // No AZURE_ACCOUNT_KEY, but AWS_SECRET_KEY is present from a sibling S3 binding. props.put("AWS_SECRET_KEY", "dGVzdA=="); - TestableAzureObjStorage storage = new TestableAzureObjStorage(props, null); - storage.stubbedSasUrl = "https://sas-with-fallback-key"; + Assertions.assertThrows(IllegalArgumentException.class, + () -> new TestableAzureObjStorage(props, null)); + } + + @Test + void buildClient_acceptsProviderSasInput() throws Exception { + Map props = new HashMap<>(); + props.put("azure.auth_type", "SAS"); + props.put("AZURE_ENDPOINT", "account.blob.core.windows.net"); + props.put("AZURE_ACCOUNT_NAME", "account"); + props.put("azure.sas_token", "?sv=2024-01-01&sig=temporary"); + props.put("azure.sas_expiry_ms", "4102444800000"); + + AzureObjStorage storage = new AzureObjStorage(props); + + Assertions.assertNotNull(storage.buildClient()); + } + + @Test + void buildClient_acceptsSharedKeyCredential() throws Exception { + AzureObjStorage storage = new AzureObjStorage(Map.of( + "AZURE_ACCOUNT_NAME", "account", + "AZURE_ACCOUNT_KEY", "dGVzdA==")); + + Assertions.assertNotNull(storage.buildClient()); + } - String result = storage.getPresignedUrl( - "wasb://mycontainer@myaccount.blob.core.windows.net/blob"); - Assertions.assertEquals("https://sas-with-fallback-key", result); + @Test + void buildClient_acceptsCanonicalSharedKeyAuthType() throws Exception { + AzureObjStorage storage = new AzureObjStorage(Map.of( + "AZURE_AUTH_TYPE", "SHARED_KEY", + "AZURE_ACCOUNT_NAME", "account", + "AZURE_ACCOUNT_KEY", "dGVzdA==")); + + BlobServiceClient client = storage.buildClient(); + + Assertions.assertEquals("https://account.blob.core.windows.net", client.getAccountUrl()); + } + + @Test + void getClient_rechecksExpiryBeforeReturningCachedSasClient() throws Exception { + Clock clock = Mockito.mock(Clock.class); + Mockito.when(clock.instant()).thenReturn(Instant.parse("2026-01-01T00:00:00Z")); + AzureObjStorage storage = new AzureObjStorage(AzureFileSystemProperties.of(Map.of( + "azure.account_name", "account", + "azure.sas_token", "se=2026-01-02T00:00:00Z&sig=temporary")), clock); + + BlobServiceClient client = storage.getClient(); + Assertions.assertNotNull(client); + Assertions.assertSame(client, storage.getClient()); + + Mockito.when(clock.instant()).thenReturn(Instant.parse("2026-01-02T00:00:00Z")); + Assertions.assertThrows(StoragePropertiesException.class, storage::getClient); + } + + @Test + void buildClient_rejectsSasExpiredSinceBinding() { + AzureFileSystemProperties properties = AzureFileSystemProperties.of(Map.of( + "azure.account_name", "account", + "azure.sas_token", "sig=temporary", + "azure.sas_expiry_ms", "1")); + AzureObjStorage storage = new AzureObjStorage(properties, + Clock.fixed(Instant.ofEpochMilli(1), ZoneOffset.UTC)); + + Assertions.assertThrows(StoragePropertiesException.class, storage::buildClient); + Assertions.assertThrows(StoragePropertiesException.class, storage::getClient); + } + + @Test + void buildClient_acceptsExplicitOAuth2Credential() throws Exception { + AzureObjStorage storage = new AzureObjStorage(Map.of( + "azure.auth_type", "OAUTH2", + "azure.endpoint", "account.blob.core.windows.net", + "azure.oauth2_account_host", "account.dfs.core.windows.net", + "azure.oauth2_client_id", "client-id", + "azure.oauth2_client_secret", "client-secret", + "azure.oauth2_server_uri", "https://login.microsoftonline.com/tenant/oauth2/token")); + + BlobServiceClient client = storage.buildClient(); + + Assertions.assertEquals("https://account.blob.core.windows.net", client.getAccountUrl()); } // ------------------------------------------------------------------ diff --git a/fe/fe-filesystem/fe-filesystem-azure/src/test/java/org/apache/doris/filesystem/azure/AzureSasTokenTest.java b/fe/fe-filesystem/fe-filesystem-azure/src/test/java/org/apache/doris/filesystem/azure/AzureSasTokenTest.java new file mode 100644 index 00000000000000..dc58a36a41566a --- /dev/null +++ b/fe/fe-filesystem/fe-filesystem-azure/src/test/java/org/apache/doris/filesystem/azure/AzureSasTokenTest.java @@ -0,0 +1,117 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.filesystem.azure; + +import org.apache.doris.foundation.property.StoragePropertiesException; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.time.Clock; +import java.time.Instant; +import java.time.ZoneOffset; + +class AzureSasTokenTest { + + private static final Clock NOW = Clock.fixed(Instant.parse("2026-01-01T00:00:00Z"), ZoneOffset.UTC); + + @Test + void of_stripsTransportPrefixesWithoutChangingSignedContents() { + AzureSasToken token = AzureSasToken.of(" ?&sv=2024-01-01&sig=a+b%2Fc ", null); + + Assertions.assertEquals("sv=2024-01-01&sig=a+b%2Fc", token.value()); + Assertions.assertTrue(token.expiresAt().isEmpty()); + } + + @Test + void of_readsEncodedTokenExpiry() { + AzureSasToken token = AzureSasToken.of("sv=2024-01-01&se=2026-01-02T03%3A04%3A05Z&sig=x", null); + + Assertions.assertEquals(Instant.parse("2026-01-02T03:04:05Z"), token.expiresAt().orElseThrow()); + Assertions.assertFalse(token.isExpired(NOW)); + } + + @Test + void of_readsEncodedOffsetWithoutChangingSignature() { + String raw = "se=2026-01-02T08%3A00%3A00%2B08%3A00&sig=a+b%2Fc"; + AzureSasToken token = AzureSasToken.of(raw, null); + + Assertions.assertEquals(Instant.parse("2026-01-02T00:00:00Z"), token.expiresAt().orElseThrow()); + Assertions.assertEquals(raw, token.value()); + } + + @Test + void of_usesEarlierOfExplicitAndTokenExpiry() { + AzureSasToken token = AzureSasToken.of( + "sv=2024-01-01&se=2026-01-03T00:00:00Z&sig=x", + Instant.parse("2026-01-02T00:00:00Z").toEpochMilli()); + + Assertions.assertEquals(Instant.parse("2026-01-02T00:00:00Z"), token.expiresAt().orElseThrow()); + + AzureSasToken earlierToken = AzureSasToken.of( + "se=2026-01-02T00:00:00Z&sig=x", Instant.parse("2026-01-03T00:00:00Z").toEpochMilli()); + Assertions.assertEquals(Instant.parse("2026-01-02T00:00:00Z"), earlierToken.expiresAt().orElseThrow()); + } + + @Test + void isExpired_handlesBoundary() { + AzureSasToken token = AzureSasToken.of("sv=2024-01-01&se=2026-01-01T00:00:00Z&sig=x", null); + + Assertions.assertTrue(token.isExpired(NOW)); + Assertions.assertThrows(StoragePropertiesException.class, () -> token.validateNotExpired(NOW)); + } + + @Test + void of_rejectsMalformedExpiryAndLineBreak() { + Assertions.assertThrows(StoragePropertiesException.class, + () -> AzureSasToken.of("sv=2024-01-01&se=not-a-date&sig=x", null)); + Assertions.assertThrows(StoragePropertiesException.class, + () -> AzureSasToken.of("sv=2024-01-01\n&sig=x", null)); + } + + @ParameterizedTest + @ValueSource(strings = {"\nsig=x", "sig=x\r\n", "?\r\nsig=x"}) + void of_rejectsLineBreaksBeforeTrimming(String raw) { + Assertions.assertThrows(StoragePropertiesException.class, () -> AzureSasToken.of(raw, null)); + } + + @ParameterizedTest + @ValueSource(strings = {"se=not-a-date-secret", "se=%GG-secret", "se=%", "se=", + "se=%2B999999999-12-31T23%3A59%3A59Z"}) + void of_rejectsInvalidExpiryWithoutEchoingInput(String raw) { + StoragePropertiesException exception = Assertions.assertThrows(StoragePropertiesException.class, + () -> AzureSasToken.of(raw + "&sig=secret", null)); + + Assertions.assertFalse(exception.getMessage().contains("secret")); + Assertions.assertNull(exception.getCause()); + } + + @Test + void of_rejectsAmbiguousDuplicateExpiry() { + Assertions.assertThrows(StoragePropertiesException.class, + () -> AzureSasToken.of("se=2026-01-01T00:00:00Z&se=2027-01-01T00:00:00Z&sig=x", null)); + } + + @Test + void of_rejectsNonPositiveExplicitExpiry() { + Assertions.assertThrows(StoragePropertiesException.class, + () -> AzureSasToken.of("sv=2024-01-01&sig=x", 0L)); + } +} diff --git a/fe/fe-filesystem/fe-filesystem-azure/src/test/java/org/apache/doris/filesystem/azure/AzureUriTest.java b/fe/fe-filesystem/fe-filesystem-azure/src/test/java/org/apache/doris/filesystem/azure/AzureUriTest.java index e394b2e3ee9e21..94d505f8c012e7 100644 --- a/fe/fe-filesystem/fe-filesystem-azure/src/test/java/org/apache/doris/filesystem/azure/AzureUriTest.java +++ b/fe/fe-filesystem/fe-filesystem-azure/src/test/java/org/apache/doris/filesystem/azure/AzureUriTest.java @@ -19,6 +19,9 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.ValueSource; import java.io.IOException; @@ -90,7 +93,7 @@ void parseWasbNoPath() throws IOException { @Test void toStringReconstructsUri() throws IOException { AzureUri uri = AzureUri.parse("wasbs://mycontainer@myaccount.blob.core.windows.net/path/key"); - Assertions.assertEquals("wasbs://mycontainer@myaccount/path/key", uri.toString()); + Assertions.assertEquals("wasbs://mycontainer@myaccount.blob.core.windows.net/path/key", uri.toString()); } @Test @@ -123,10 +126,10 @@ void wasbMissingAtSignThrows() { // --------------------------------------------------------------------- @Test - void parse_decodesPercentEncodedKey() throws IOException { + void parse_preservesLiteralAdlsPercentSequences() throws IOException { AzureUri uri = AzureUri.parse( "wasbs://mycontainer@myaccount.blob.core.windows.net/dir/with%20space/a%2Bb.csv"); - Assertions.assertEquals("dir/with space/a+b.csv", uri.key()); + Assertions.assertEquals("dir/with%20space/a%2Bb.csv", uri.key()); Assertions.assertEquals("mycontainer", uri.container()); } @@ -168,13 +171,164 @@ void parse_emptyKeyWithTrailingSlash() throws IOException { } @Test - void toString_percentEncodesKey_keepsSlashes() throws IOException { - // Input contains percent-encoded space (%20) and percent-encoded '+' (%2B); - // round-tripping through parse/toString must preserve them and keep '/' literal. + void toString_preservesLiteralAdlsName() throws IOException { + // ADLSLocation treats these percent sequences as object-name characters. + // Rendering must not change them before the next SDK call. AzureUri uri = AzureUri.parse( "wasbs://mycontainer@myaccount.blob.core.windows.net/dir/with%20space/a%2Bb.csv"); Assertions.assertEquals( - "wasbs://mycontainer@myaccount/dir/with%20space/a%2Bb.csv", + "wasbs://mycontainer@myaccount.blob.core.windows.net/dir/with%20space/a%2Bb.csv", uri.toString()); } + + @ParameterizedTest + @CsvSource({ + "abfs, dfs, core.windows.net", + "abfss, dfs, core.chinacloudapi.cn", + "abfss, dfs, core.usgovcloudapi.net", + "abfss, dfs, core.cloudapi.de", + "wasb, blob, core.windows.net", + "wasbs, blob, core.chinacloudapi.cn", + "wasbs, blob, core.usgovcloudapi.net", + "wasbs, blob, core.cloudapi.de" + }) + void parse_preservesAccountHostAndCloudSuffix(String scheme, String service, String suffix) throws IOException { + String location = scheme + "://container@account." + service + "." + suffix + "/dir/file"; + AzureUri uri = AzureUri.parse(location); + AzureAccountHost host = uri.accountHost().orElseThrow(); + + Assertions.assertEquals("account", uri.accountName()); + Assertions.assertEquals("account", host.accountName()); + Assertions.assertEquals(suffix, host.cloudSuffix()); + Assertions.assertEquals("account.dfs." + suffix, host.dfsHost()); + Assertions.assertEquals("https://account.blob." + suffix, host.blobEndpoint()); + Assertions.assertEquals(location, uri.toString()); + } + + @ParameterizedTest + @ValueSource(strings = {"core.windows.net", "core.chinacloudapi.cn", "core.usgovcloudapi.net", "core.cloudapi.de"}) + void parse_preservesHttpsAuthorityAndContainerPath(String suffix) throws IOException { + String endpoint = "https://account.blob." + suffix + ":8443"; + String location = endpoint + "/container/dir/file"; + AzureUri uri = AzureUri.parse(location); + + Assertions.assertEquals("account", uri.accountName()); + Assertions.assertEquals(suffix, uri.accountHost().orElseThrow().cloudSuffix()); + Assertions.assertEquals(endpoint, uri.accountHost().orElseThrow().blobEndpoint()); + Assertions.assertEquals("container", uri.container()); + Assertions.assertEquals("dir/file", uri.key()); + Assertions.assertEquals(location, uri.toString()); + } + + @Test + void parse_preservesExplicitHttpTransportAndPort() throws IOException { + AzureUri uri = AzureUri.parse("http://account.dfs.core.chinacloudapi.cn:10000/container/dir/file"); + + Assertions.assertEquals("http://account.blob.core.chinacloudapi.cn:10000", + uri.accountHost().orElseThrow().blobEndpoint()); + Assertions.assertEquals("http://account.dfs.core.chinacloudapi.cn:10000/container/dir/file", uri.toString()); + } + + @Test + void parse_preservesCustomAuthorityWithoutInventingCloudSuffix() throws IOException { + String location = "wasbs://container@storage.example.test:8443/dir/file"; + AzureUri uri = AzureUri.parse(location); + + Assertions.assertEquals("storage", uri.accountName()); + Assertions.assertEquals("", uri.accountHost().orElseThrow().cloudSuffix()); + Assertions.assertEquals("storage.example.test", uri.accountHost().orElseThrow().blobHost()); + Assertions.assertEquals(location, uri.toString()); + } + + @Test + void parse_preservesOneLakeAuthority() throws IOException { + String location = "abfss://workspace@onelake.dfs.fabric.microsoft.com/lakehouse/Tables/data/file"; + AzureUri uri = AzureUri.parse(location); + + Assertions.assertEquals("onelake", uri.accountName()); + Assertions.assertEquals("onelake.dfs.fabric.microsoft.com", uri.accountHost().orElseThrow().dfsHost()); + Assertions.assertEquals("workspace", uri.container()); + Assertions.assertEquals("lakehouse/Tables/data/file", uri.key()); + // Parsing a location must not rewrite OneLake to a Blob endpoint or choose its reader. + Assertions.assertEquals(location, uri.toString()); + } + + @ParameterizedTest + @ValueSource(strings = {"s3", "s3a", "s3n"}) + void parse_preservesLegacyS3CompatibleLocations(String scheme) throws IOException { + String location = scheme + "://container/dir/file"; + AzureUri uri = AzureUri.parse(location); + + Assertions.assertEquals("", uri.accountName()); + Assertions.assertTrue(uri.accountHost().isEmpty()); + Assertions.assertEquals("container", uri.container()); + Assertions.assertEquals("dir/file", uri.key()); + Assertions.assertEquals(location, uri.toString()); + } + + @ParameterizedTest + @ValueSource(strings = {"https://account.blob.core.windows.net/container/", "s3://container/"}) + void parse_decodesPathOnceWithoutChangingPlusOrSeparators(String prefix) throws IOException { + AzureUri uri = AzureUri.parse(prefix + "/dir//http://example/a+b%2Bc%252F%20file%3F%23"); + + Assertions.assertEquals("/dir//http://example/a+b+c%2F file?#", uri.key()); + AzureUri roundTrip = AzureUri.parse(uri.toString()); + Assertions.assertEquals(uri.key(), roundTrip.key()); + Assertions.assertEquals(uri.accountName(), roundTrip.accountName()); + Assertions.assertEquals(uri.container(), roundTrip.container()); + } + + @ParameterizedTest + @ValueSource(strings = {"abfs", "abfss", "wasb", "wasbs"}) + void parse_preservesAdlsNamesWithoutPercentDecoding(String scheme) throws IOException { + String key = "/dir//http://example/p=a%2Fb/a+b%2Bc%252F%20file%3F%23"; + String location = scheme + "://container@account.dfs.core.windows.net/" + key; + AzureUri uri = AzureUri.parse(location); + + Assertions.assertEquals(key, uri.key()); + Assertions.assertEquals(location, uri.toString()); + Assertions.assertEquals(key, AzureUri.parse(uri.toString()).key()); + } + + @ParameterizedTest + @ValueSource(strings = {"%", "%2", "%GG"}) + void parse_acceptsLiteralPercentCharactersInAdlsNames(String name) throws IOException { + String location = "abfss://container@account.dfs.core.windows.net/" + name; + AzureUri uri = AzureUri.parse(location); + + Assertions.assertEquals(name, uri.key()); + Assertions.assertEquals(location, uri.toString()); + } + + @Test + void parse_normalizesSchemeOnly() throws IOException { + AzureUri uri = AzureUri.parse("ABFSS://container@account.dfs.core.windows.net/Dir/File"); + + Assertions.assertEquals("abfss", uri.scheme()); + Assertions.assertEquals("Dir/File", uri.key()); + Assertions.assertEquals("abfss://container@account.dfs.core.windows.net/Dir/File", uri.toString()); + } + + @ParameterizedTest + @ValueSource(strings = {"abfss://container@/file", "abfss://@account.dfs.core.windows.net/file", + "abfss://container@account@other.dfs.core.windows.net/file", + "abfss://container/file@account.dfs.core.windows.net/file", + "https:///container/file", "https://user@account.blob.core.windows.net/container/file"}) + void parse_rejectsMalformedAuthorityWithoutLeakingQuery(String location) { + IOException error = Assertions.assertThrows(IOException.class, + () -> AzureUri.parse(location + "?sig=secret-signature")); + + Assertions.assertFalse(error.getMessage().contains("secret-signature")); + Assertions.assertNull(error.getCause()); + } + + @ParameterizedTest + @ValueSource(strings = {"%", "%2", "%GG"}) + void parse_reportsMalformedHttpEscapeWithoutLeakingQuery(String malformedEscape) { + IOException error = Assertions.assertThrows(IOException.class, () -> AzureUri.parse( + "https://account.blob.core.windows.net/container/" + malformedEscape + "?sig=secret-signature")); + + Assertions.assertEquals("Invalid percent encoding in Azure object path", error.getMessage()); + Assertions.assertNull(error.getCause()); + } } diff --git a/fe/fe-filesystem/fe-filesystem-azure/src/test/java/org/apache/doris/filesystem/azure/AzureVendedCredentialsTest.java b/fe/fe-filesystem/fe-filesystem-azure/src/test/java/org/apache/doris/filesystem/azure/AzureVendedCredentialsTest.java new file mode 100644 index 00000000000000..902d7242513b0e --- /dev/null +++ b/fe/fe-filesystem/fe-filesystem-azure/src/test/java/org/apache/doris/filesystem/azure/AzureVendedCredentialsTest.java @@ -0,0 +1,362 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.filesystem.azure; + +import org.apache.doris.foundation.property.StoragePropertiesException; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.NullAndEmptySource; +import org.junit.jupiter.params.provider.ValueSource; + +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.Map; + +class AzureVendedCredentialsTest { + + private static final String HOST = "account.dfs.core.windows.net"; + private static final String TOKEN_KEY = "adls.sas-token." + HOST; + private static final String EXPIRY_KEY = "adls.sas-token-expires-at-ms." + HOST; + private static final String TOKEN = "sv=2024-01-01&sig=secret-signature%2B%2F%3D"; + private static final String EXPIRING_TOKEN = TOKEN + "&se=2100-01-01T00%3A00%3A00Z"; + + private final AzureFileSystemProvider provider = new AzureFileSystemProvider(); + + @Test + void bindVended_bindsAccountScopedSasDirectlyToTypedProperties() { + AzureFileSystemProperties properties = provider.bindVended(Map.of(TOKEN_KEY, EXPIRING_TOKEN), + Map.of()).orElseThrow(); + + Assertions.assertEquals(AzureAuthType.SAS, properties.authType()); + Assertions.assertEquals("account", properties.getAccountName()); + Assertions.assertEquals("https://account.blob.core.windows.net", properties.getEndpoint()); + Assertions.assertEquals(Map.of(TOKEN_KEY, EXPIRING_TOKEN), properties.rawProperties()); + Assertions.assertEquals(properties.rawProperties(), properties.matchedProperties()); + Assertions.assertEquals(Map.of( + "provider", "azure", + "AZURE_AUTH_TYPE", "SAS", + "AZURE_ACCOUNT_NAME", "account", + "AZURE_ENDPOINT", "https://account.blob.core.windows.net", + "AZURE_SAS_TOKEN", EXPIRING_TOKEN, + "AZURE_SAS_EXPIRY_MS", "4102444800000"), properties.toMap()); + Assertions.assertFalse(properties.toString().contains("secret-signature")); + } + + @ParameterizedTest + @ValueSource(strings = {"SharedKey", "OAuth2", "SAS", "obsolete-auth-type"}) + void bindVended_replacesTheWholeOldCredentialGroup(String oldAuthType) { + Map catalog = new HashMap<>(); + catalog.put("provider", "azure"); + catalog.put("azure.account_name", "account"); + catalog.put("azure.endpoint", "https://account.dfs.core.windows.net"); + catalog.put("azure.auth_type", oldAuthType); + catalog.put("azure.account_key", "old-shared-key"); + catalog.put("AZURE_ACCOUNT_KEY", "old-uppercase-key"); + catalog.put("azure.oauth2_account_host", "old-account.dfs.core.chinacloudapi.cn"); + catalog.put("azure.oauth2_client_id", "old-client-id"); + catalog.put("azure.oauth2_client_secret", "old-client-secret"); + catalog.put("azure.oauth2_server_uri", "https://login.microsoftonline.com/old-tenant/oauth2/token"); + catalog.put("azure.oauth2_client_tenant_id", "old-tenant"); + catalog.put("azure.sas_token", "sig=old-signature"); + catalog.put("azure.sas_expiry_ms", "1"); + catalog.put("fs.azure.sas.fixed.token." + HOST, "sig=old-hadoop-signature"); + Map credentials = new HashMap<>(catalog); + credentials.put(TOKEN_KEY, TOKEN); + + AzureFileSystemProperties properties = provider.bindVended(credentials, catalog).orElseThrow(); + + Assertions.assertEquals(AzureAuthType.SAS, properties.authType()); + Assertions.assertEquals("", properties.getAccountKey()); + Assertions.assertEquals("", properties.getClientId()); + Assertions.assertEquals("", properties.getClientSecret()); + Assertions.assertEquals("", properties.getOauthAccountHost()); + Assertions.assertEquals("", properties.getOauthServerUri()); + Assertions.assertEquals("", properties.getTenantId()); + Assertions.assertEquals("", properties.getSasExpiryMs()); + Assertions.assertEquals(Map.of( + "provider", "azure", + "AZURE_AUTH_TYPE", "SAS", + "AZURE_ACCOUNT_NAME", "account", + "AZURE_ENDPOINT", "https://account.blob.core.windows.net", + "AZURE_SAS_TOKEN", TOKEN), properties.toMap()); + Assertions.assertFalse(properties.rawProperties().values().stream().anyMatch(value -> value.contains("old-"))); + Assertions.assertEquals(TOKEN, + properties.toHadoopConfigurationMap().get("fs.azure.sas.fixed.token." + HOST)); + } + + @ParameterizedTest + @ValueSource(strings = {"?", "&", "?&&"}) + void bindVended_normalizesTransportPrefixWithoutReencodingSignature(String prefix) { + AzureFileSystemProperties properties = provider.bindVended(Map.of(TOKEN_KEY, prefix + EXPIRING_TOKEN), + Map.of()).orElseThrow(); + + Assertions.assertEquals(EXPIRING_TOKEN, properties.getSasToken()); + Assertions.assertEquals(EXPIRING_TOKEN, properties.toMap().get("AZURE_SAS_TOKEN")); + } + + @ParameterizedTest + @CsvSource({"4102444700000, 4102444700000", "4200000000000, 4102444800000"}) + void bindVended_usesTheEarlierKnownExpiry(String explicitExpiry, String expectedExpiry) { + AzureFileSystemProperties properties = provider.bindVended( + Map.of(TOKEN_KEY, EXPIRING_TOKEN, EXPIRY_KEY, explicitExpiry), Map.of()).orElseThrow(); + + Assertions.assertEquals(expectedExpiry, properties.getSasExpiryMs()); + Assertions.assertEquals(expectedExpiry, properties.toMap().get("AZURE_SAS_EXPIRY_MS")); + } + + @Test + void bindVended_defersKnownExpiryUntilAccessAndNeverFallsBack() { + AzureFileSystemProperties properties = provider.bindVended(Map.of(TOKEN_KEY, TOKEN, EXPIRY_KEY, "1"), + Map.of("azure.account_name", "account", "azure.account_key", "old-shared-key")).orElseThrow(); + + Assertions.assertEquals(AzureAuthType.SAS, properties.authType()); + Assertions.assertEquals("", properties.getAccountKey()); + Assertions.assertDoesNotThrow(properties::validate); + StoragePropertiesException error = Assertions.assertThrows(StoragePropertiesException.class, + properties::validateForAccess); + Assertions.assertEquals("Azure SAS credential is expired", error.getMessage()); + Assertions.assertThrows(StoragePropertiesException.class, properties::toMap); + Assertions.assertThrows(StoragePropertiesException.class, properties::toHadoopConfigurationMap); + } + + @Test + void bindVended_doesNotMutateInputsOrRetainMutableViews() { + Map credentials = new HashMap<>(Map.of(TOKEN_KEY, TOKEN)); + Map catalog = new HashMap<>(Map.of( + "azure.account_name", "account", "azure.account_key", "old-shared-key", "container", "container")); + Map originalCredentials = new HashMap<>(credentials); + Map originalCatalog = new HashMap<>(catalog); + AzureFileSystemProperties properties = provider.bindVended(credentials, catalog).orElseThrow(); + + Assertions.assertEquals(originalCredentials, credentials); + Assertions.assertEquals(originalCatalog, catalog); + credentials.put(TOKEN_KEY, "sig=replacement"); + catalog.put("container", "replacement-container"); + Assertions.assertEquals(TOKEN, properties.getSasToken()); + Assertions.assertEquals(TOKEN, properties.rawProperties().get(TOKEN_KEY)); + Assertions.assertEquals("container", properties.getContainer()); + Assertions.assertThrows(UnsupportedOperationException.class, + () -> properties.rawProperties().put(TOKEN_KEY, "sig=replacement")); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void bindVended_rejectsMultipleAccountsIndependentOfMapOrder(boolean reverseOrder) { + Map credentials = new LinkedHashMap<>(); + String otherKey = "adls.sas-token.other.dfs.core.windows.net"; + credentials.put(reverseOrder ? otherKey : TOKEN_KEY, TOKEN); + credentials.put(reverseOrder ? TOKEN_KEY : otherKey, TOKEN); + + assertRejected(credentials); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void bindVended_rejectsExpiryForAnotherHostIndependentOfMapOrder(boolean expiryFirst) { + Map credentials = new LinkedHashMap<>(); + String otherExpiry = "adls.sas-token-expires-at-ms.account.dfs.core.chinacloudapi.cn"; + credentials.put(expiryFirst ? otherExpiry : TOKEN_KEY, expiryFirst ? "4102444800000" : TOKEN); + credentials.put(expiryFirst ? TOKEN_KEY : otherExpiry, expiryFirst ? TOKEN : "4102444800000"); + + assertRejected(credentials); + } + + @Test + void bindVended_rejectsConflictingTokensWithDifferentKeyCase() { + assertRejected(Map.of(TOKEN_KEY, TOKEN, + "ADLS.SAS-TOKEN.ACCOUNT.DFS.CORE.WINDOWS.NET", "sig=other-secret-signature")); + } + + @ParameterizedTest + @NullAndEmptySource + @ValueSource(strings = {" ", "\t", "?&&"}) + void bindVended_rejectsMissingTokenWithoutStaticFallback(String token) { + Map credentials = new HashMap<>(); + credentials.put(TOKEN_KEY, token); + + assertRejected(credentials); + } + + @ParameterizedTest + @NullAndEmptySource + @ValueSource(strings = {" ", "0", "-1", "9223372036854775808", "secret-signature"}) + void bindVended_rejectsMalformedExplicitExpiryWithoutLeakingValue(String expiry) { + Map credentials = new HashMap<>(); + credentials.put(TOKEN_KEY, TOKEN); + credentials.put(EXPIRY_KEY, expiry); + + assertRejected(credentials); + } + + @Test + void bindVended_rejectsExpiryWithoutToken() { + assertRejected(Map.of(EXPIRY_KEY, "4102444800000")); + } + + @ParameterizedTest + @ValueSource(strings = {"se=", "se=secret-signature", "se=%GG", "sig=secret-signature\r\n"}) + void bindVended_rejectsMalformedSasWithoutLeakingInput(String token) { + assertRejected(Map.of(TOKEN_KEY, token)); + } + + @ParameterizedTest + @ValueSource(strings = {"adls.token", "ADLS.TOKEN"}) + void bindVended_rejectsUnsupportedVendedAccessToken(String key) { + StoragePropertiesException error = assertRejected(Map.of(key, "secret-signature")); + + Assertions.assertTrue(error.getMessage().contains("access tokens are not supported")); + } + + @ParameterizedTest + @ValueSource(strings = {"adls.container", "ADLS.CONTAINER", "adls.container-name", + "azure.container", "AZURE.BUCKET"}) + void bindVended_preservesAndValidatesContainerScope(String containerKey) { + AzureFileSystemProperties properties = provider.bindVended( + Map.of(TOKEN_KEY, TOKEN, containerKey, " container "), Map.of()).orElseThrow(); + + Assertions.assertEquals("container", properties.getContainer()); + Assertions.assertFalse(properties.toMap().containsKey("AZURE_CONTAINER")); + Assertions.assertEquals("abfss://container@" + HOST + "/file", + properties.validateAndNormalizeUri("abfss://container@" + HOST + "/file")); + Assertions.assertThrows(StoragePropertiesException.class, + () -> properties.validateAndNormalizeUri("abfss://other@" + HOST + "/file")); + } + + @Test + void bindVended_rejectsConflictingContainerScopesAcrossAliases() { + assertRejected(Map.of(TOKEN_KEY, TOKEN, "ADLS.CONTAINER", "first", "azure.bucket", "second")); + Assertions.assertThrows(StoragePropertiesException.class, () -> provider.bindVended( + Map.of(TOKEN_KEY, TOKEN, "ADLS.CONTAINER", "other"), + Map.of("provider", "azure", "azure.bucket", "container"))); + } + + @ParameterizedTest + @CsvSource({"azure.account_name, other", "azure.endpoint, https://other.blob.core.windows.net", + "azure.endpoint, https://account.blob.core.chinacloudapi.cn"}) + void bindVended_rejectsCatalogAccountOrCloudMismatch(String key, String value) { + StoragePropertiesException error = Assertions.assertThrows(StoragePropertiesException.class, + () -> provider.bindVended(Map.of(TOKEN_KEY, TOKEN), Map.of("provider", "azure", key, value))); + + Assertions.assertTrue(error.getMessage().contains("does not match")); + Assertions.assertNull(error.getCause()); + } + + @Test + void bindVended_rejectsLegacyAccountConflictWhenReplacingCredentials() { + StoragePropertiesException error = Assertions.assertThrows(StoragePropertiesException.class, + () -> provider.bindVended(Map.of(TOKEN_KEY, TOKEN), Map.of( + "provider", "azure", "s3.access_key", "old-account"))); + + Assertions.assertEquals("Azure vended credential account does not match the legacy account", + error.getMessage()); + } + + @Test + void bindVended_prefersProviderEndpointOverSiblingS3Endpoint() { + AzureFileSystemProperties properties = provider.bindVended(Map.of(TOKEN_KEY, TOKEN), Map.of( + "provider", "azure", + "azure.endpoint", "https://proxy.example.test:8443", + "s3.endpoint", "https://old-account.blob.core.windows.net")).orElseThrow(); + + Assertions.assertEquals("https://proxy.example.test:8443", properties.getEndpoint()); + } + + @Test + void bindVended_rejectsUppercaseLegacyEndpointAccountConflictWhenReplacingCredentials() { + StoragePropertiesException error = Assertions.assertThrows(StoragePropertiesException.class, + () -> provider.bindVended(Map.of(TOKEN_KEY, TOKEN), Map.of( + "AZURE_ENDPOINT", "https://old-account.blob.core.windows.net"))); + + Assertions.assertTrue(error.getMessage().contains("does not match")); + } + + @Test + void bindVended_ignoresSiblingS3AccountWhenAzureIdentityIsExplicit() { + Assertions.assertDoesNotThrow(() -> provider.bindVended(Map.of(TOKEN_KEY, TOKEN), Map.of( + "provider", "azure", "azure.account_name", "account", + "s3.access_key", "s3-account"))); + } + + @Test + void bindVended_preservesHistoricalUppercaseEndpointWithAccountAlias() { + AzureFileSystemProperties properties = provider.bindVended(Map.of(TOKEN_KEY, TOKEN), Map.of( + "AZURE_ENDPOINT", "https://proxy.example.test:8443", + "AZURE_ACCOUNT_NAME", "account")).orElseThrow(); + + Assertions.assertEquals("https://proxy.example.test:8443", properties.getEndpoint()); + } + + @ParameterizedTest + @ValueSource(strings = {"http://localhost:10000/devstoreaccount1", + "https://storage.example.test:8443/proxy%2Fpath"}) + void bindVended_inheritsExplicitAzureCustomEndpointAndConnectionOptions(String endpoint) { + AzureFileSystemProperties properties = provider.bindVended(Map.of(TOKEN_KEY, TOKEN), Map.of( + "fs.azure.support", "true", "azure.endpoint", endpoint, + "container", "container", "use_path_style", "true", + "force_parsing_by_standard_uri", "true")).orElseThrow(); + + Assertions.assertEquals(endpoint, properties.getEndpoint()); + Assertions.assertEquals("container", properties.getContainer()); + Assertions.assertEquals("true", properties.getUsePathStyle()); + Assertions.assertEquals("true", properties.getForceParsingByStandardUrl()); + } + + @Test + void bindVended_inheritsDfsEndpointWithUppercaseHost() { + AzureFileSystemProperties properties = provider.bindVended(Map.of(TOKEN_KEY, TOKEN), Map.of( + "azure.endpoint", "http://ACCOUNT.DFS.CORE.WINDOWS.NET:8443/proxy%2Fpath", + "container", "container")).orElseThrow(); + + Assertions.assertEquals("http://ACCOUNT.blob.CORE.WINDOWS.NET:8443/proxy%2Fpath", properties.getEndpoint()); + Assertions.assertEquals("container", properties.getContainer()); + } + + @ParameterizedTest + @CsvSource({"use_path_style, s3.path-style-access", "s3.path-style-access, use_path_style"}) + void bindVended_overridesConnectionOptionByFieldInsteadOfAlias(String catalogKey, String vendedKey) { + AzureFileSystemProperties properties = provider.bindVended(Map.of( + TOKEN_KEY, TOKEN, vendedKey, "true", "force_parsing_by_standard_uri", "true"), Map.of( + "provider", "azure", catalogKey, "false", "force_parsing_by_standard_uri", "false")) + .orElseThrow(); + + Assertions.assertEquals("true", properties.getUsePathStyle()); + Assertions.assertEquals("true", properties.getForceParsingByStandardUrl()); + Assertions.assertFalse(properties.toMap().containsKey("use_path_style")); + Assertions.assertFalse(properties.rawProperties().containsKey(catalogKey)); + } + + @Test + void bindVended_doesNotRecognizeS3Credentials() { + Assertions.assertTrue(provider.bindVended(Map.of( + "s3.access-key-id", "aws-key", "s3.secret-access-key", "aws-secret", "s3.session-token", "aws-token"), + Map.of("provider", "azure")).isEmpty()); + } + + private StoragePropertiesException assertRejected(Map credentials) { + StoragePropertiesException error = Assertions.assertThrows(StoragePropertiesException.class, + () -> provider.bindVended(credentials, + Map.of("azure.account_name", "account", "azure.account_key", "old-shared-key"))); + Assertions.assertFalse(error.getMessage().contains("secret-signature")); + Assertions.assertFalse(error.getMessage().contains("old-shared-key")); + Assertions.assertNull(error.getCause()); + return error; + } +} diff --git a/fe/fe-filesystem/fe-filesystem-hdfs/src/main/java/org/apache/doris/filesystem/hdfs/HdfsFileSystemProvider.java b/fe/fe-filesystem/fe-filesystem-hdfs/src/main/java/org/apache/doris/filesystem/hdfs/HdfsFileSystemProvider.java index bf8d3ff28b6333..e5064ec8456a26 100644 --- a/fe/fe-filesystem/fe-filesystem-hdfs/src/main/java/org/apache/doris/filesystem/hdfs/HdfsFileSystemProvider.java +++ b/fe/fe-filesystem/fe-filesystem-hdfs/src/main/java/org/apache/doris/filesystem/hdfs/HdfsFileSystemProvider.java @@ -73,10 +73,19 @@ public boolean supports(Map properties) { @Override public HdfsProperties bind(Map properties) { + return bind(properties, false); + } + + @Override + public HdfsProperties bindDefault(Map properties) { + return bind(properties, true); + } + + private HdfsProperties bind(Map properties, boolean syntheticDefault) { // Resolve raw user properties through the migrated HdfsProperties so that typed auth // params (hdfs.authentication.*) are translated to Hadoop keys, xml resources are // loaded, and defaults are injected — instead of passing raw keys straight through. - HdfsProperties hdfsProperties = new HdfsProperties(properties); + HdfsProperties hdfsProperties = new HdfsProperties(properties, syntheticDefault); hdfsProperties.initNormalizeAndCheckProps(); return hdfsProperties; } @@ -89,7 +98,7 @@ public FileSystem create(HdfsProperties properties) throws IOException { private static final Set GUESS_HINT_KEYS = Set.of("hdfs.authentication.type", "hadoop.security.authentication", "hadoop.username", "fs.defaultFS", "hdfs.authentication.kerberos.principal", "hadoop.kerberos.principal", - "dfs.nameservices", "hdfs.config.resources"); + "dfs.nameservices", "hdfs.config.resources", "hadoop.config.resources"); @Override public boolean supportsExplicit(Map properties) { diff --git a/fe/fe-filesystem/fe-filesystem-hdfs/src/main/java/org/apache/doris/filesystem/hdfs/properties/HdfsProperties.java b/fe/fe-filesystem/fe-filesystem-hdfs/src/main/java/org/apache/doris/filesystem/hdfs/properties/HdfsProperties.java index 9c7c611011eed1..41c7a4dad16745 100644 --- a/fe/fe-filesystem/fe-filesystem-hdfs/src/main/java/org/apache/doris/filesystem/hdfs/properties/HdfsProperties.java +++ b/fe/fe-filesystem/fe-filesystem-hdfs/src/main/java/org/apache/doris/filesystem/hdfs/properties/HdfsProperties.java @@ -54,8 +54,21 @@ public class HdfsProperties extends HdfsCompatibleProperties { private String dfsNameServices = ""; + private final boolean syntheticDefault; + public HdfsProperties(Map origProps) { + this(origProps, false); + } + + /** The provider sets the origin only when the registry explicitly requests its default binding. */ + public HdfsProperties(Map origProps, boolean syntheticDefault) { super(origProps); + this.syntheticDefault = syntheticDefault; + } + + @Override + public boolean isSyntheticDefault() { + return syntheticDefault; } @Override diff --git a/fe/fe-filesystem/fe-filesystem-hdfs/src/test/java/org/apache/doris/filesystem/hdfs/HdfsFileSystemProviderTest.java b/fe/fe-filesystem/fe-filesystem-hdfs/src/test/java/org/apache/doris/filesystem/hdfs/HdfsFileSystemProviderTest.java index 42f0f2ff15c105..2f82d4fa19bd64 100644 --- a/fe/fe-filesystem/fe-filesystem-hdfs/src/test/java/org/apache/doris/filesystem/hdfs/HdfsFileSystemProviderTest.java +++ b/fe/fe-filesystem/fe-filesystem-hdfs/src/test/java/org/apache/doris/filesystem/hdfs/HdfsFileSystemProviderTest.java @@ -17,9 +17,15 @@ package org.apache.doris.filesystem.hdfs; +import org.apache.doris.filesystem.hdfs.properties.HdfsProperties; +import org.apache.doris.filesystem.properties.StorageProperties; + import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.HashMap; import java.util.Map; @@ -75,4 +81,55 @@ void schemeMatchIsCaseInsensitive() { Assertions.assertTrue(provider.supports(props("fs.defaultFS", "HDFS://ns"))); Assertions.assertTrue(provider.supports(props("fs.defaultFS", "ViewFS://cluster"))); } + + @Test + void defaultBindingKeepsTypedPropertiesAndConfiguration() { + Map raw = Map.of("azure.account_name", "account", "azure.account_key", "key"); + HdfsProperties normal = provider.bind(raw); + HdfsProperties fallback = provider.bindDefault(raw); + StorageProperties storage = fallback; + + Assertions.assertFalse(normal.isSyntheticDefault()); + Assertions.assertTrue(storage.isSyntheticDefault()); + Assertions.assertEquals(HdfsProperties.class, fallback.getClass()); + Assertions.assertEquals(raw, fallback.rawProperties()); + Assertions.assertEquals(normal.matchedProperties(), fallback.matchedProperties()); + Assertions.assertEquals(normal.toBackendProperties().get().toMap(), + fallback.toBackendProperties().get().toMap()); + Assertions.assertEquals(normal.toHadoopProperties().get().toHadoopConfigurationMap(), + fallback.toHadoopProperties().get().toHadoopConfigurationMap()); + Assertions.assertEquals(normal.fsCacheFingerprint(), fallback.fsCacheFingerprint()); + } + + @Test + void normalBindingsNeverInferSyntheticOriginFromMissingHints() { + Assertions.assertFalse(provider.bind(Map.of()).isSyntheticDefault()); + Assertions.assertFalse(provider.bind(Map.of("fs.hdfs.support", "true")).isSyntheticDefault()); + HdfsProperties hdfsUri = provider.bind(Map.of("uri", "hdfs://namenode/warehouse")); + Assertions.assertFalse(hdfsUri.isSyntheticDefault()); + Assertions.assertEquals("hdfs://namenode", hdfsUri.getBackendConfigProperties().get("fs.defaultFS")); + } + + @Test + void recognizesAndLoadsActualHadoopResources(@TempDir Path tmp) throws Exception { + Path xml = tmp.resolve("storage-site.xml"); + Files.writeString(xml, "" + + "fs.s3a.connection.ssl.enabledfalse" + + ""); + Map raw = Map.of("hadoop.config.resources", xml.toString()); + + Assertions.assertTrue(provider.supportsGuess(raw)); + HdfsProperties properties = provider.bind(raw); + + Assertions.assertFalse(properties.isSyntheticDefault()); + Assertions.assertEquals("false", properties.getBackendConfigProperties().get("fs.s3a.connection.ssl.enabled")); + Assertions.assertEquals("false", + properties.toHadoopConfigurationMap().get("fs.s3a.connection.ssl.enabled")); + } + + @Test + void retainsLegacyResourceGuessHintWithoutLoadingFilesDuringGuess() { + Assertions.assertTrue(provider.supportsGuess(Map.of("hdfs.config.resources", "legacy-site.xml"))); + Assertions.assertTrue(provider.supportsGuess(Map.of("hadoop.config.resources", "missing-site.xml"))); + } } diff --git a/fe/fe-filesystem/fe-filesystem-spi/src/main/java/org/apache/doris/filesystem/spi/FileSystemProvider.java b/fe/fe-filesystem/fe-filesystem-spi/src/main/java/org/apache/doris/filesystem/spi/FileSystemProvider.java index 7e4859dbb147a5..ad0d20b1afb92b 100644 --- a/fe/fe-filesystem/fe-filesystem-spi/src/main/java/org/apache/doris/filesystem/spi/FileSystemProvider.java +++ b/fe/fe-filesystem/fe-filesystem-spi/src/main/java/org/apache/doris/filesystem/spi/FileSystemProvider.java @@ -27,6 +27,7 @@ import java.util.Collections; import java.util.EnumSet; import java.util.Map; +import java.util.Optional; import java.util.Set; /** @@ -92,6 +93,32 @@ default P bind(Map properties) { name() + " does not support typed FileSystemProperties binding yet."); } + /** + * Binds a registry-supplied default when no user configuration matched this storage family. + * Providers that distinguish fallback bindings override this hook and return their normal + * typed model with {@link org.apache.doris.filesystem.properties.StorageProperties#isSyntheticDefault()} + * set to true. The origin must not be encoded in raw properties or runtime configuration maps. + * The default preserves normal binding for providers without fallback-origin support. + */ + default P bindDefault(Map properties) { + return bind(properties); + } + + /** + * Binds externally issued credentials directly into this provider's typed model. + * The catalog properties supply connection defaults, not fallback authentication material. + * Authentication is replaced as a group; implementations must retain and validate any scope + * carried by the credential. This hook performs no I/O and must not mutate either input. + * + *

    Empty means the credential format is not recognized by this provider. Recognized but + * invalid credentials must throw a credential-safe exception, never return empty or fall back + * to another identity. Time-dependent checks belong to the returned model's + * {@link org.apache.doris.filesystem.properties.StorageProperties#validateForAccess()}.

    + */ + default Optional

    bindVended(Map credentials, Map catalogProperties) { + return Optional.empty(); + } + /** * Creates a FileSystem instance from validated typed properties. * diff --git a/fe/fe-filesystem/fe-filesystem-spi/src/test/java/org/apache/doris/filesystem/spi/FileSystemPluginSurfaceTest.java b/fe/fe-filesystem/fe-filesystem-spi/src/test/java/org/apache/doris/filesystem/spi/FileSystemPluginSurfaceTest.java index 16c91831c14485..40ea46a843ae21 100644 --- a/fe/fe-filesystem/fe-filesystem-spi/src/test/java/org/apache/doris/filesystem/spi/FileSystemPluginSurfaceTest.java +++ b/fe/fe-filesystem/fe-filesystem-spi/src/test/java/org/apache/doris/filesystem/spi/FileSystemPluginSurfaceTest.java @@ -17,6 +17,9 @@ package org.apache.doris.filesystem.spi; +import org.apache.doris.filesystem.properties.FileSystemProperties; +import org.apache.doris.filesystem.properties.StorageProperties; + import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -28,6 +31,7 @@ import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.List; +import java.util.Properties; import java.util.TreeSet; /** @@ -65,9 +69,23 @@ public class FileSystemPluginSurfaceTest { private static final String BASELINE_RESOURCE = "/filesystem-plugin-surface.txt"; + @Test + public void filesystemApiMajorTracksTheRecordedSurfaceChange() throws IOException { + Properties version = new Properties(); + try (InputStream in = FileSystemProvider.class.getResourceAsStream( + "/META-INF/doris/filesystem-plugin-api-version.properties")) { + Assertions.assertNotNull(in, "missing filesystem plugin API version resource"); + version.load(in); + } + // Provider-owned Iceberg FileIO properties extend the shared storage API. + Assertions.assertEquals("7.0", version.getProperty("api.version")); + } + /** The types a filesystem plugin implements or calls. Everything reachable on them is the contract. */ private static final List> FROZEN_TYPES = Arrays.asList( FileSystemProvider.class, + FileSystemProperties.class, + StorageProperties.class, ObjFileSystem.class, ObjStorage.class, org.apache.doris.extension.spi.Plugin.class, diff --git a/fe/fe-filesystem/fe-filesystem-spi/src/test/java/org/apache/doris/filesystem/spi/FileSystemProviderTypedPropertiesTest.java b/fe/fe-filesystem/fe-filesystem-spi/src/test/java/org/apache/doris/filesystem/spi/FileSystemProviderTypedPropertiesTest.java index 4d614f22bd3396..37058a69e3b4c3 100644 --- a/fe/fe-filesystem/fe-filesystem-spi/src/test/java/org/apache/doris/filesystem/spi/FileSystemProviderTypedPropertiesTest.java +++ b/fe/fe-filesystem/fe-filesystem-spi/src/test/java/org/apache/doris/filesystem/spi/FileSystemProviderTypedPropertiesTest.java @@ -46,6 +46,16 @@ void provider_canBindAndCreateWithTypedProperties() throws IOException { Assertions.assertSame(properties, ((TestFileSystem) fileSystem).properties); } + @Test + void defaultBindingRetainsTypedProviderCompatibility() throws IOException { + TypedProvider provider = new TypedProvider(); + TestProperties properties = provider.bindDefault(Map.of("test.key", "default-value")); + + Assertions.assertEquals("default-value", properties.value); + Assertions.assertFalse(properties.isSyntheticDefault()); + Assertions.assertSame(properties, ((TestFileSystem) provider.createUntyped(properties)).properties); + } + private static class TypedProvider implements FileSystemProvider { @Override public boolean supports(Map properties) { diff --git a/fe/fe-filesystem/fe-filesystem-spi/src/test/resources/filesystem-plugin-surface.txt b/fe/fe-filesystem/fe-filesystem-spi/src/test/resources/filesystem-plugin-surface.txt index 819b07ca74430e..0ab31a9295640b 100644 --- a/fe/fe-filesystem/fe-filesystem-spi/src/test/resources/filesystem-plugin-surface.txt +++ b/fe/fe-filesystem/fe-filesystem-spi/src/test/resources/filesystem-plugin-surface.txt @@ -5,7 +5,45 @@ org.apache.doris.extension.spi.PluginFactory#create():org.apache.doris.extension org.apache.doris.extension.spi.PluginFactory#create(org.apache.doris.extension.spi.PluginContext):org.apache.doris.extension.spi.Plugin org.apache.doris.extension.spi.PluginFactory#description():java.lang.String org.apache.doris.extension.spi.PluginFactory#name():java.lang.String +org.apache.doris.filesystem.properties.FileSystemProperties#claimsUri(java.lang.String):boolean +org.apache.doris.filesystem.properties.FileSystemProperties#fsCacheFingerprint():java.lang.String +org.apache.doris.filesystem.properties.FileSystemProperties#getSupportedSchemes():java.util.Set +org.apache.doris.filesystem.properties.FileSystemProperties#isSyntheticDefault():boolean +org.apache.doris.filesystem.properties.FileSystemProperties#kind():org.apache.doris.filesystem.properties.StorageKind +org.apache.doris.filesystem.properties.FileSystemProperties#legacyCacheSchemes():java.util.Set +org.apache.doris.filesystem.properties.FileSystemProperties#matchedProperties():java.util.Map +org.apache.doris.filesystem.properties.FileSystemProperties#matchesLocationPrefix(java.lang.String,java.lang.String):boolean +org.apache.doris.filesystem.properties.FileSystemProperties#providerName():java.lang.String +org.apache.doris.filesystem.properties.FileSystemProperties#rawProperties():java.util.Map +org.apache.doris.filesystem.properties.FileSystemProperties#resolveBackendProperties(java.lang.String):java.util.Optional +org.apache.doris.filesystem.properties.FileSystemProperties#storageFamilyName():java.lang.String +org.apache.doris.filesystem.properties.FileSystemProperties#toBackendProperties():java.util.Optional +org.apache.doris.filesystem.properties.FileSystemProperties#toHadoopProperties():java.util.Optional +org.apache.doris.filesystem.properties.FileSystemProperties#toIcebergFileIOConnectionProperties():java.util.Map +org.apache.doris.filesystem.properties.FileSystemProperties#toIcebergFileIOProperties():java.util.Map +org.apache.doris.filesystem.properties.FileSystemProperties#toIcebergHadoopProperties():java.util.Optional +org.apache.doris.filesystem.properties.FileSystemProperties#type():org.apache.doris.filesystem.FileSystemType +org.apache.doris.filesystem.properties.FileSystemProperties#validate():void +org.apache.doris.filesystem.properties.FileSystemProperties#validateAndGetUri(java.util.Map):java.lang.String +org.apache.doris.filesystem.properties.FileSystemProperties#validateAndNormalizeUri(java.lang.String):java.lang.String +org.apache.doris.filesystem.properties.FileSystemProperties#validateForAccess():void +org.apache.doris.filesystem.properties.StorageProperties#isSyntheticDefault():boolean +org.apache.doris.filesystem.properties.StorageProperties#kind():org.apache.doris.filesystem.properties.StorageKind +org.apache.doris.filesystem.properties.StorageProperties#matchedProperties():java.util.Map +org.apache.doris.filesystem.properties.StorageProperties#matchesLocationPrefix(java.lang.String,java.lang.String):boolean +org.apache.doris.filesystem.properties.StorageProperties#providerName():java.lang.String +org.apache.doris.filesystem.properties.StorageProperties#rawProperties():java.util.Map +org.apache.doris.filesystem.properties.StorageProperties#toBackendProperties():java.util.Optional +org.apache.doris.filesystem.properties.StorageProperties#toHadoopProperties():java.util.Optional +org.apache.doris.filesystem.properties.StorageProperties#toIcebergFileIOConnectionProperties():java.util.Map +org.apache.doris.filesystem.properties.StorageProperties#toIcebergFileIOProperties():java.util.Map +org.apache.doris.filesystem.properties.StorageProperties#toIcebergHadoopProperties():java.util.Optional +org.apache.doris.filesystem.properties.StorageProperties#type():org.apache.doris.filesystem.FileSystemType +org.apache.doris.filesystem.properties.StorageProperties#validate():void +org.apache.doris.filesystem.properties.StorageProperties#validateForAccess():void org.apache.doris.filesystem.spi.FileSystemProvider#bind(java.util.Map):org.apache.doris.filesystem.properties.FileSystemProperties +org.apache.doris.filesystem.spi.FileSystemProvider#bindDefault(java.util.Map):org.apache.doris.filesystem.properties.FileSystemProperties +org.apache.doris.filesystem.spi.FileSystemProvider#bindVended(java.util.Map,java.util.Map):java.util.Optional org.apache.doris.filesystem.spi.FileSystemProvider#capabilities(org.apache.doris.filesystem.properties.FileSystemProperties):java.util.Set org.apache.doris.filesystem.spi.FileSystemProvider#create():org.apache.doris.extension.spi.Plugin org.apache.doris.filesystem.spi.FileSystemProvider#create(java.util.Map):org.apache.doris.filesystem.FileSystem diff --git a/fe/fe-filesystem/pom.xml b/fe/fe-filesystem/pom.xml index a8368afa785571..b8dbfd4b1b28d8 100644 --- a/fe/fe-filesystem/pom.xml +++ b/fe/fe-filesystem/pom.xml @@ -53,7 +53,7 @@ under the License. bumping all five families. See plan-doc/designs/2026-07-29-plugin-api-version-check-design.md. --> - 1.0 + 7.0 From a4eed504e5f2e0b483f6e5e9f9054f8db05bae96 Mon Sep 17 00:00:00 2001 From: xylaaaaa <2392805527@qq.com> Date: Sun, 13 Sep 2026 20:01:21 +0800 Subject: [PATCH 04/10] [feature](fe) Add request-local storage binding resolution ### What problem does this PR solve? Related PR: #67470 Problem Summary: Keep the Connector Storage SPI and FE engine storage-adapter/context integration together with their tests. This is a history-only regrouping of the native Azure PR. The final seven-commit series preserves the exact code tree of azure-native-before-squash-20260913550a886487e. ### Release note None ### Check List (For Author) - Test: Manual test / Unit Test - Final tree and PR diff are verified identical to the pre-squash snapshot - Existing final-snapshot verification: 711 selected FE tests passed - Tests were not rerun for this history-only rewrite - Behavior changed: No (relative to the pre-squash PR snapshot) - Does this need documentation: No --- .../connector/spi/ConnectorStorageAccess.java | 77 +++++++ .../spi/ConnectorStorageAccessResolver.java | 72 +++++++ .../spi/ConnectorStorageContext.java | 54 ++++- .../spi/scan/ConnectorScanRange.java | 9 + .../spi/ConnectorPluginSurfaceTest.java | 13 +- .../spi/ConnectorStorageAccessTest.java | 107 ++++++++++ .../spi/ConnectorStorageContextTest.java | 71 +++++++ .../resources/connector-plugin-surface.txt | 78 +++++++ fe/fe-connector/pom.xml | 2 +- .../doris/cloud/storage/ObjectInfo.java | 2 +- .../cloud/storage/ObjectInfoAdapter.java | 16 +- .../doris/common/util/LocationPath.java | 43 +++- .../connector/DefaultConnectorContext.java | 155 +++++++++++--- .../credentials/CredentialUtils.java | 10 +- .../datasource/split/PluginDrivenSplit.java | 2 + .../datasource/storage/StorageAdapter.java | 184 ++++++++++++---- .../datasource/storage/StorageUriUtils.java | 111 +--------- .../doris/fs/FileSystemPluginManager.java | 73 ++++++- .../cloud/storage/ObjectInfoAdapterTest.java | 101 +++++++++ .../util/DatasourcePrintableMapTest.java | 21 ++ .../doris/common/util/LocationPathTest.java | 59 ++++- ...faultConnectorContextNormalizeUriTest.java | 20 ++ .../DefaultConnectorContextVendTest.java | 165 +++++++++++++- .../AzureCatalogPropertyPersistTest.java | 153 +++++++++++++ .../credentials/CredentialUtilsTest.java | 13 ++ .../FileQueryScanNodeStorageAccessTest.java | 192 +++++++++++++++++ .../StorageAdapterAccessValidationTest.java | 171 +++++++++++++++ .../StorageAdapterBindingOriginTest.java | 164 ++++++++++++++ .../doris/fs/FileSystemPluginManagerTest.java | 201 +++++++++++++++++- .../PluginApiVersionWiringTest.java | 34 ++- 30 files changed, 2145 insertions(+), 228 deletions(-) create mode 100644 fe/fe-connector/fe-connector-spi/src/main/java/org/apache/doris/connector/spi/ConnectorStorageAccess.java create mode 100644 fe/fe-connector/fe-connector-spi/src/main/java/org/apache/doris/connector/spi/ConnectorStorageAccessResolver.java create mode 100644 fe/fe-connector/fe-connector-spi/src/test/java/org/apache/doris/connector/spi/ConnectorStorageAccessTest.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/AzureCatalogPropertyPersistTest.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/scan/FileQueryScanNodeStorageAccessTest.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/storage/StorageAdapterAccessValidationTest.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/storage/StorageAdapterBindingOriginTest.java diff --git a/fe/fe-connector/fe-connector-spi/src/main/java/org/apache/doris/connector/spi/ConnectorStorageAccess.java b/fe/fe-connector/fe-connector-spi/src/main/java/org/apache/doris/connector/spi/ConnectorStorageAccess.java new file mode 100644 index 00000000000000..a5e1f234fa925f --- /dev/null +++ b/fe/fe-connector/fe-connector-spi/src/main/java/org/apache/doris/connector/spi/ConnectorStorageAccess.java @@ -0,0 +1,77 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.spi; + +import org.apache.doris.filesystem.properties.BackendStorageKind; + +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Objects; + +/** + * One location resolved against one request-local storage binding and credential generation. + * + *

    Consumers must use its URI, reader and backend properties together, without merging another + * catalog-wide credential map. The reader is a Thrift enum name so this SPI remains RPC-neutral. + * This value contains execution credentials: keep it request-local and never publish it in metadata, + * profiles or logs. Its diagnostic representation deliberately omits both the URI and properties. + */ +public final class ConnectorStorageAccess { + private final String providerName; + private final String normalizedUri; + private final BackendStorageKind backendKind; + private final String backendFileType; + private final Map backendProperties; + + public ConnectorStorageAccess(String providerName, String normalizedUri, BackendStorageKind backendKind, + String backendFileType, Map backendProperties) { + this.providerName = Objects.requireNonNull(providerName, "providerName is required"); + this.normalizedUri = Objects.requireNonNull(normalizedUri, "normalizedUri is required"); + this.backendKind = Objects.requireNonNull(backendKind, "backendKind is required"); + this.backendFileType = Objects.requireNonNull(backendFileType, "backendFileType is required"); + this.backendProperties = Collections.unmodifiableMap(new LinkedHashMap<>( + Objects.requireNonNull(backendProperties, "backendProperties is required"))); + } + + public String getProviderName() { + return providerName; + } + + public String getNormalizedUri() { + return normalizedUri; + } + + public BackendStorageKind getBackendKind() { + return backendKind; + } + + public String getBackendFileType() { + return backendFileType; + } + + public Map getBackendProperties() { + return backendProperties; + } + + @Override + public String toString() { + return "ConnectorStorageAccess{provider=" + providerName + ", backendKind=" + backendKind + + ", backendFileType=" + backendFileType + ", location/credentials=}"; + } +} diff --git a/fe/fe-connector/fe-connector-spi/src/main/java/org/apache/doris/connector/spi/ConnectorStorageAccessResolver.java b/fe/fe-connector/fe-connector-spi/src/main/java/org/apache/doris/connector/spi/ConnectorStorageAccessResolver.java new file mode 100644 index 00000000000000..666c6cfa0ddb43 --- /dev/null +++ b/fe/fe-connector/fe-connector-spi/src/main/java/org/apache/doris/connector/spi/ConnectorStorageAccessResolver.java @@ -0,0 +1,72 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.spi; + +import java.util.Collections; +import java.util.Objects; +import java.util.Set; +import java.util.TreeSet; +import java.util.function.BiPredicate; +import java.util.function.Function; + +/** + * Request-local storage resolver with an immutable, case-insensitive snapshot of its bound providers. + * + *

    Provider lookup does not resolve a URI or access credentials. A connector can select the relevant + * storage contract without requiring its metadata root to be a data-file location. Resolution remains + * engine-owned, and this object must not be cached across requests or credential generations. + */ +public final class ConnectorStorageAccessResolver implements Function { + private final Set providerNames; + private final Function resolver; + private final BiPredicate locationPrefixMatcher; + + public ConnectorStorageAccessResolver(Set providerNames, + Function resolver) { + this(providerNames, resolver, (rawLocation, rawPrefix) -> { + throw new UnsupportedOperationException("Storage prefix matching is not supported by this resolver"); + }); + } + + public ConnectorStorageAccessResolver(Set providerNames, + Function resolver, + BiPredicate locationPrefixMatcher) { + Set snapshot = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); + snapshot.addAll(Objects.requireNonNull(providerNames, "providerNames is required")); + this.providerNames = Collections.unmodifiableSet(snapshot); + this.resolver = Objects.requireNonNull(resolver, "resolver is required"); + this.locationPrefixMatcher = Objects.requireNonNull(locationPrefixMatcher, + "locationPrefixMatcher is required"); + } + + /** Returns whether this request has the named provider, without resolving any location. */ + public boolean hasProvider(String providerName) { + return providerNames.contains(Objects.requireNonNull(providerName, "providerName is required")); + } + + /** Resolves the supplied location through the captured engine resolver. */ + @Override + public ConnectorStorageAccess apply(String rawUri) { + return resolver.apply(rawUri); + } + + /** Matches a location prefix through the same provider snapshot, without accessing credentials. */ + public boolean matchesLocationPrefix(String rawLocation, String rawPrefix) { + return locationPrefixMatcher.test(rawLocation, rawPrefix); + } +} diff --git a/fe/fe-connector/fe-connector-spi/src/main/java/org/apache/doris/connector/spi/ConnectorStorageContext.java b/fe/fe-connector/fe-connector-spi/src/main/java/org/apache/doris/connector/spi/ConnectorStorageContext.java index 691631eee87d4c..c53c317b7a274d 100644 --- a/fe/fe-connector/fe-connector-spi/src/main/java/org/apache/doris/connector/spi/ConnectorStorageContext.java +++ b/fe/fe-connector/fe-connector-spi/src/main/java/org/apache/doris/connector/spi/ConnectorStorageContext.java @@ -49,9 +49,9 @@ public interface ConnectorStorageContext { /** - * The context for a catalog whose storage the engine does not manage. Every method keeps its interface - * default, so a connector reaching a service that is not there gets the same benign answer it would get - * from a context that simply did not override it. + * The context for a catalog whose storage the engine does not manage. Methods retain their interface + * defaults. In particular, resolved data access is unavailable: a URI scheme alone cannot establish + * a storage identity or its credentials. */ ConnectorStorageContext NOOP = new ConnectorStorageContext() { }; @@ -60,7 +60,8 @@ public interface ConnectorStorageContext { * Normalizes raw per-table vended cloud-storage credentials (the token map a REST catalog * returns, e.g. {@code fs.oss.accessKeyId} / {@code s3.access-key}) into the BE-facing storage * property map ({@code AWS_ACCESS_KEY} / {@code AWS_SECRET_KEY} / {@code AWS_TOKEN} / - * {@code AWS_ENDPOINT} / {@code AWS_REGION}). The connector extracts the raw token from the live + * {@code AWS_ENDPOINT} / {@code AWS_REGION}; Azure uses its provider-owned + * {@code AZURE_*} equivalents). The connector extracts the raw token from the live * table (paimon SDK only); the engine performs the same {@code StorageProperties} normalization * it uses for static catalog credentials (the connector cannot import fe-core). * @@ -139,6 +140,26 @@ default UnaryOperator newStorageUriNormalizer(Map rawVen return rawUri -> normalizeStorageUri(rawUri, rawVendedCredentials); } + /** + * Captures one request's storage bindings and resolves each location to its matching provider, + * normalized URI, reader and backend credential view together. Vended authentication replaces the + * matching static authentication group before credentials are accessed; unrelated bindings are not + * flattened into the result. The returned resolver must not be cached across requests or credential + * generations. A write creates it after acquiring the write-authorized table, not from a read token. + * Its provider snapshot permits binding-presence checks without resolving a metadata root as a data URI. + * + *

    The engine implements binding and routing. Contexts without that service fail explicitly rather + * than inventing an authenticated data path from a URI scheme. Existing normalization-only hooks + * remain available to connectors that do not consume this resolved-access contract. + * + * @param rawVendedCredentials raw request credentials; null/empty selects static bindings + * @return a resolver with a provider snapshot, whose access values contain only the selected binding's properties + */ + default ConnectorStorageAccessResolver newStorageAccessResolver( + Map rawVendedCredentials) { + throw new UnsupportedOperationException("Storage access resolution is unavailable for this catalog"); + } + /** * Resolves the BE-facing file type (a {@code TFileType} enum name, e.g. {@code "FILE_S3"}) for a raw * storage URI a connector emits (e.g. an iceberg write output path). A write-side analogue of @@ -203,7 +224,8 @@ default List getBrokerAddresses() { /** * Returns the catalog's static storage credentials/config normalized to BE-canonical scan * properties: object-store creds as {@code AWS_ACCESS_KEY} / {@code AWS_SECRET_KEY} / - * {@code AWS_TOKEN} / {@code AWS_ENDPOINT} / {@code AWS_REGION}, and HDFS config as the resolved + * {@code AWS_TOKEN} / {@code AWS_ENDPOINT} / {@code AWS_REGION} (Azure uses provider-owned + * {@code AZURE_*} keys), and HDFS config as the resolved * {@code hadoop.*} / {@code dfs.*} keys (user overrides plus the legacy-derived defaults). The * engine runs the same {@code CredentialUtils.getBackendPropertiesFromStorageMap} that legacy / * iceberg / hive use over the catalog's parsed {@code StorageProperties} map — the single source of @@ -271,6 +293,28 @@ default List getStorageProperties() { return Collections.emptyList(); } + /** + * Captures one request's effective typed storage bindings. Provider-owned vended authentication + * replaces the matching static authentication group before it is accessed; unrelated storage + * bindings remain available. This uses the same binding contract as + * {@link #newStorageAccessResolver(Map)}, without selecting a data URI or emitting backend credentials. + * A connector can consume the selected provider's {@link StorageProperties#toIcebergFileIOProperties()} + * directly, without translating a backend credential map back into a FileIO dialect. + * + *

    The returned list is immutable and request-local, not a cache across credential generations. + * Access-time credential checks still belong to the properties' consumer-facing views. The default + * supports static bindings only and explicitly rejects vended credentials rather than ignoring them. + * + * @param rawVendedCredentials raw request credentials; null/empty selects static bindings + * @return the effective typed storage bindings for this request + */ + default List resolveStorageProperties(Map rawVendedCredentials) { + if (rawVendedCredentials == null || rawVendedCredentials.isEmpty()) { + return List.copyOf(getStorageProperties()); + } + throw new UnsupportedOperationException("Vended storage binding is unavailable for this catalog"); + } + /** * Returns the engine's {@link FileSystem} for this catalog — a scheme-routing handle backed by the * catalog's parsed {@link #getStorageProperties() storage properties} and the registered fe-filesystem diff --git a/fe/fe-connector/fe-connector-spi/src/main/java/org/apache/doris/connector/spi/scan/ConnectorScanRange.java b/fe/fe-connector/fe-connector-spi/src/main/java/org/apache/doris/connector/spi/scan/ConnectorScanRange.java index 572a78ba1e0e78..331265a528b902 100644 --- a/fe/fe-connector/fe-connector-spi/src/main/java/org/apache/doris/connector/spi/scan/ConnectorScanRange.java +++ b/fe/fe-connector/fe-connector-spi/src/main/java/org/apache/doris/connector/spi/scan/ConnectorScanRange.java @@ -46,6 +46,15 @@ default Optional getPath() { return Optional.empty(); } + /** + * Reader selected by the storage binding that supplied this range's URI and scan-node + * credentials, expressed as a BE file-type enum name. Empty preserves legacy scheme routing + * for connectors that do not resolve storage access through the engine. + */ + default Optional getBackendFileType() { + return Optional.empty(); + } + /** Returns the byte offset to start reading from. */ default long getStart() { return 0; diff --git a/fe/fe-connector/fe-connector-spi/src/test/java/org/apache/doris/connector/spi/ConnectorPluginSurfaceTest.java b/fe/fe-connector/fe-connector-spi/src/test/java/org/apache/doris/connector/spi/ConnectorPluginSurfaceTest.java index fa459f99076f11..d4941ba1c1f736 100644 --- a/fe/fe-connector/fe-connector-spi/src/test/java/org/apache/doris/connector/spi/ConnectorPluginSurfaceTest.java +++ b/fe/fe-connector/fe-connector-spi/src/test/java/org/apache/doris/connector/spi/ConnectorPluginSurfaceTest.java @@ -20,10 +20,13 @@ import org.apache.doris.connector.spi.handle.ConnectorColumnHandle; import org.apache.doris.connector.spi.handle.ConnectorWriteHandle; import org.apache.doris.connector.spi.scan.ConnectorScanPlanProvider; +import org.apache.doris.connector.spi.scan.ConnectorScanRange; import org.apache.doris.connector.spi.scan.ScanNodePropertyKeys; import org.apache.doris.connector.spi.write.ConnectorChangelogMode; import org.apache.doris.connector.spi.write.ConnectorRowLevelDmlRequest; import org.apache.doris.connector.spi.write.ConnectorWritePlanProvider; +import org.apache.doris.filesystem.properties.FileSystemProperties; +import org.apache.doris.filesystem.properties.StorageProperties; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -87,8 +90,8 @@ public void connectorApiMajorTracksTheRecordedSurfaceChange() throws IOException Assertions.assertNotNull(in, "missing connector plugin API version resource"); version.load(in); } - // Major 13 carries textual write-input semantics on ConnectorColumn. - Assertions.assertEquals("13.0", version.getProperty("api.version")); + // Major 14 adds native storage bindings to the existing textual write-input contract. + Assertions.assertEquals("14.0", version.getProperty("api.version")); } /** Root entry points plus provider/handle types returned to connector plugins. */ @@ -96,6 +99,9 @@ public void connectorApiMajorTracksTheRecordedSurfaceChange() throws IOException ConnectorProvider.class, ConnectorContext.class, ConnectorSession.class, + ConnectorStorageContext.class, + ConnectorStorageAccess.class, + ConnectorStorageAccessResolver.class, Connector.class, ConnectorColumnHandle.class, ConnectorColumn.class, @@ -103,11 +109,14 @@ public void connectorApiMajorTracksTheRecordedSurfaceChange() throws IOException org.apache.doris.connector.spi.mvcc.ConnectorMvccSnapshot.class, org.apache.doris.connector.spi.mvcc.ConnectorMvccSnapshot.Builder.class, ConnectorScanPlanProvider.class, + ConnectorScanRange.class, ConnectorWriteHandle.class, ConnectorChangelogMode.class, ConnectorRowLevelDmlRequest.class, org.apache.doris.connector.spi.write.ConnectorWriteDistribution.class, ConnectorWritePlanProvider.class, + FileSystemProperties.class, + StorageProperties.class, org.apache.doris.extension.spi.Plugin.class, org.apache.doris.extension.spi.PluginFactory.class, org.apache.doris.extension.spi.PluginContext.class); diff --git a/fe/fe-connector/fe-connector-spi/src/test/java/org/apache/doris/connector/spi/ConnectorStorageAccessTest.java b/fe/fe-connector/fe-connector-spi/src/test/java/org/apache/doris/connector/spi/ConnectorStorageAccessTest.java new file mode 100644 index 00000000000000..6b632df95a9931 --- /dev/null +++ b/fe/fe-connector/fe-connector-spi/src/test/java/org/apache/doris/connector/spi/ConnectorStorageAccessTest.java @@ -0,0 +1,107 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.spi; + +import org.apache.doris.filesystem.properties.BackendStorageKind; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +public class ConnectorStorageAccessTest { + + @Test + public void resolvedIdentityAndCredentialsBelongToOneSnapshot() { + String uri = "abfss://container@account.dfs.core.windows.net/data/file.parquet"; + Map original = new LinkedHashMap<>(Map.of( + "provider", "azure", "AZURE_AUTH_TYPE", "SAS", "AZURE_SAS_TOKEN", "test-sas-token")); + ConnectorStorageAccess access = new ConnectorStorageAccess( + "azure", uri, BackendStorageKind.NATIVE, "FILE_S3", original); + + Assertions.assertEquals("azure", access.getProviderName()); + Assertions.assertEquals(uri, access.getNormalizedUri()); + Assertions.assertEquals(BackendStorageKind.NATIVE, access.getBackendKind()); + Assertions.assertEquals("FILE_S3", access.getBackendFileType()); + Assertions.assertNotSame(original, access.getBackendProperties()); + + original.put("AZURE_SAS_TOKEN", "next-generation-token"); + original.clear(); + Assertions.assertEquals(Map.of("provider", "azure", "AZURE_AUTH_TYPE", "SAS", + "AZURE_SAS_TOKEN", "test-sas-token"), access.getBackendProperties()); + } + + @Test + public void backendPropertiesCannotBeChangedThroughTheReturnedMapOrEntries() { + ConnectorStorageAccess access = new ConnectorStorageAccess("azure", + "abfss://container@account.dfs.core.windows.net/data/file.parquet", + BackendStorageKind.NATIVE, "FILE_S3", Map.of("AZURE_SAS_TOKEN", "test-sas-token")); + Map properties = access.getBackendProperties(); + + Assertions.assertThrows(UnsupportedOperationException.class, + () -> properties.put("AZURE_SAS_TOKEN", "replacement-token")); + Assertions.assertThrows(UnsupportedOperationException.class, + () -> properties.remove("AZURE_SAS_TOKEN")); + Assertions.assertThrows(UnsupportedOperationException.class, properties::clear); + Assertions.assertThrows(UnsupportedOperationException.class, + () -> properties.entrySet().iterator().next().setValue("replacement-token")); + Assertions.assertEquals(Map.of("AZURE_SAS_TOKEN", "test-sas-token"), access.getBackendProperties()); + } + + @Test + public void diagnosticStringOmitsTheEntireLocationAndPropertyMap() { + ConnectorStorageAccess access = new ConnectorStorageAccess("azure", + "https://account.blob.core.windows.net/private-location" + + "?sig=uri-secret-sentinel#fragment-secret-sentinel", + BackendStorageKind.NATIVE, "FILE_S3", Map.of( + "AZURE_SAS_TOKEN", "test-sas-secret", + "AZURE_CLIENT_SECRET", "test-client-secret", + "unrecognized.secret.key", "test-unrecognized-secret")); + + String diagnostic = access.toString(); + Assertions.assertTrue(diagnostic.contains("azure")); + Assertions.assertTrue(diagnostic.contains("NATIVE")); + Assertions.assertTrue(diagnostic.contains("FILE_S3")); + Assertions.assertTrue(diagnostic.contains("")); + for (String privateValue : List.of("account.blob.core.windows.net", "private-location", + "uri-secret-sentinel", "fragment-secret-sentinel", "AZURE_SAS_TOKEN", + "AZURE_CLIENT_SECRET", "unrecognized.secret.key", "test-sas-secret", + "test-client-secret", "test-unrecognized-secret")) { + Assertions.assertFalse(diagnostic.contains(privateValue)); + } + } + + @Test + public void noopAndDefaultContextsRejectAccessResolutionBeforeReturningAResolver() { + ConnectorStorageContext defaultContext = new ConnectorStorageContext() { + }; + List> credentialSets = Arrays.asList(null, Map.of(), + Map.of("adls.sas-token.account", "test-vended-token")); + for (ConnectorStorageContext context : List.of(ConnectorStorageContext.NOOP, defaultContext)) { + for (Map credentials : credentialSets) { + UnsupportedOperationException error = Assertions.assertThrows(UnsupportedOperationException.class, + () -> context.newStorageAccessResolver(credentials)); + Assertions.assertEquals("Storage access resolution is unavailable for this catalog", + error.getMessage()); + } + } + } +} diff --git a/fe/fe-connector/fe-connector-spi/src/test/java/org/apache/doris/connector/spi/ConnectorStorageContextTest.java b/fe/fe-connector/fe-connector-spi/src/test/java/org/apache/doris/connector/spi/ConnectorStorageContextTest.java index dd16dd86044dd1..20f5c74d65ca7e 100644 --- a/fe/fe-connector/fe-connector-spi/src/test/java/org/apache/doris/connector/spi/ConnectorStorageContextTest.java +++ b/fe/fe-connector/fe-connector-spi/src/test/java/org/apache/doris/connector/spi/ConnectorStorageContextTest.java @@ -17,11 +17,14 @@ package org.apache.doris.connector.spi; +import org.apache.doris.filesystem.FileSystemType; +import org.apache.doris.filesystem.properties.StorageKind; import org.apache.doris.filesystem.properties.StorageProperties; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.function.UnaryOperator; @@ -42,6 +45,38 @@ public void getStorageProperties_defaultsToEmptyList() { Assertions.assertNotNull(storage, "getStorageProperties() must never return null"); Assertions.assertTrue(storage.isEmpty(), "default getStorageProperties() must be empty so connectors without storage are unaffected"); + Assertions.assertTrue(ConnectorStorageContext.NOOP.resolveStorageProperties(null).isEmpty()); + Assertions.assertTrue(ConnectorStorageContext.NOOP.resolveStorageProperties(Map.of()).isEmpty()); + } + + @Test + public void resolveStorageProperties_defaultCapturesAnImmutableStaticList() { + StorageProperties storage = new StaticStorageProperties(); + List staticBindings = new ArrayList<>(List.of(storage)); + ConnectorStorageContext context = staticContext(staticBindings); + + List fromNull = context.resolveStorageProperties(null); + List fromEmpty = context.resolveStorageProperties(Map.of()); + staticBindings.clear(); + + Assertions.assertEquals(List.of(storage), fromNull); + Assertions.assertEquals(List.of(storage), fromEmpty); + Assertions.assertThrows(UnsupportedOperationException.class, fromNull::clear); + Assertions.assertThrows(UnsupportedOperationException.class, () -> fromEmpty.add(storage)); + Assertions.assertTrue(context.resolveStorageProperties(Map.of()).isEmpty(), + "a new request must capture the current static bindings, not reuse the old list"); + } + + @Test + public void resolveStorageProperties_defaultNeverIgnoresVendedCredentials() { + for (ConnectorStorageContext context : List.of(ConnectorStorageContext.NOOP, + staticContext(List.of(new StaticStorageProperties())))) { + UnsupportedOperationException failure = Assertions.assertThrows(UnsupportedOperationException.class, + () -> context.resolveStorageProperties(Map.of("adls.sas-token.account", "private-material"))); + + Assertions.assertEquals("Vended storage binding is unavailable for this catalog", failure.getMessage()); + Assertions.assertNull(failure.getCause()); + } } @Test @@ -85,4 +120,40 @@ public void remainingDefaultsAreBenign() { Assertions.assertDoesNotThrow(() -> ctx.cleanupEmptyManagedLocation("s3://bucket/db/t", List.of()), "cleanup is cosmetic and must never fail a drop"); } + + private static ConnectorStorageContext staticContext(List bindings) { + return new ConnectorStorageContext() { + @Override + public List getStorageProperties() { + return bindings; + } + }; + } + + private static final class StaticStorageProperties implements StorageProperties { + @Override + public String providerName() { + return "LOCAL"; + } + + @Override + public StorageKind kind() { + return StorageKind.LOCAL; + } + + @Override + public FileSystemType type() { + return FileSystemType.FILE; + } + + @Override + public Map rawProperties() { + return Map.of(); + } + + @Override + public Map matchedProperties() { + return Map.of(); + } + } } diff --git a/fe/fe-connector/fe-connector-spi/src/test/resources/connector-plugin-surface.txt b/fe/fe-connector/fe-connector-spi/src/test/resources/connector-plugin-surface.txt index f0ae0af27d158c..1e1687134157e9 100644 --- a/fe/fe-connector/fe-connector-spi/src/test/resources/connector-plugin-surface.txt +++ b/fe/fe-connector/fe-connector-spi/src/test/resources/connector-plugin-surface.txt @@ -102,6 +102,30 @@ org.apache.doris.connector.spi.ConnectorSession#getTimeZone():java.lang.String org.apache.doris.connector.spi.ConnectorSession#getUser():java.lang.String org.apache.doris.connector.spi.ConnectorSession#isExternalScanTaskReuseEnabled():boolean org.apache.doris.connector.spi.ConnectorSession#setCurrentTransaction(org.apache.doris.connector.spi.handle.ConnectorTransaction):void +org.apache.doris.connector.spi.ConnectorStorageAccess#getBackendFileType():java.lang.String +org.apache.doris.connector.spi.ConnectorStorageAccess#getBackendKind():org.apache.doris.filesystem.properties.BackendStorageKind +org.apache.doris.connector.spi.ConnectorStorageAccess#getBackendProperties():java.util.Map +org.apache.doris.connector.spi.ConnectorStorageAccess#getNormalizedUri():java.lang.String +org.apache.doris.connector.spi.ConnectorStorageAccess#getProviderName():java.lang.String +org.apache.doris.connector.spi.ConnectorStorageAccess#toString():java.lang.String +org.apache.doris.connector.spi.ConnectorStorageAccessResolver#andThen(java.util.function.Function):java.util.function.Function +org.apache.doris.connector.spi.ConnectorStorageAccessResolver#apply(java.lang.String):org.apache.doris.connector.spi.ConnectorStorageAccess +org.apache.doris.connector.spi.ConnectorStorageAccessResolver#compose(java.util.function.Function):java.util.function.Function +org.apache.doris.connector.spi.ConnectorStorageAccessResolver#hasProvider(java.lang.String):boolean +org.apache.doris.connector.spi.ConnectorStorageAccessResolver#matchesLocationPrefix(java.lang.String,java.lang.String):boolean +org.apache.doris.connector.spi.ConnectorStorageContext#cleanupEmptyManagedLocation(java.lang.String,java.util.List):void +org.apache.doris.connector.spi.ConnectorStorageContext#getBackendFileType(java.lang.String,java.util.Map):java.lang.String +org.apache.doris.connector.spi.ConnectorStorageContext#getBackendStorageProperties():java.util.Map +org.apache.doris.connector.spi.ConnectorStorageContext#getBrokerAddresses():java.util.List +org.apache.doris.connector.spi.ConnectorStorageContext#getFileSystem(org.apache.doris.connector.spi.ConnectorSession):org.apache.doris.filesystem.FileSystem +org.apache.doris.connector.spi.ConnectorStorageContext#getStorageProperties():java.util.List +org.apache.doris.connector.spi.ConnectorStorageContext#newStorageAccessResolver(java.util.Map):org.apache.doris.connector.spi.ConnectorStorageAccessResolver +org.apache.doris.connector.spi.ConnectorStorageContext#newStorageUriNormalizer(java.util.Map):java.util.function.UnaryOperator +org.apache.doris.connector.spi.ConnectorStorageContext#normalizeStorageUri(java.lang.String):java.lang.String +org.apache.doris.connector.spi.ConnectorStorageContext#normalizeStorageUri(java.lang.String,java.util.Map):java.lang.String +org.apache.doris.connector.spi.ConnectorStorageContext#resolveStorageProperties(java.util.Map):java.util.List +org.apache.doris.connector.spi.ConnectorStorageContext#testBackendStorageConnectivity(int,java.util.Map):void +org.apache.doris.connector.spi.ConnectorStorageContext#vendStorageCredentials(java.util.Map):java.util.Map org.apache.doris.connector.spi.ConnectorTableSchema#equals(java.lang.Object):boolean org.apache.doris.connector.spi.ConnectorTableSchema#getColumns():java.util.List org.apache.doris.connector.spi.ConnectorTableSchema#getProperties():java.util.Map @@ -168,6 +192,24 @@ org.apache.doris.connector.spi.scan.ConnectorScanPlanProvider#supportsSystemTabl org.apache.doris.connector.spi.scan.ConnectorScanPlanProvider#supportsSystemTableTimeTravel():boolean org.apache.doris.connector.spi.scan.ConnectorScanPlanProvider#supportsTableSample():boolean org.apache.doris.connector.spi.scan.ConnectorScanPlanProvider#usesHiveParquetInt96TimeZone():boolean +org.apache.doris.connector.spi.scan.ConnectorScanRange#getBackendFileType():java.util.Optional +org.apache.doris.connector.spi.scan.ConnectorScanRange#getFileFormat():java.lang.String +org.apache.doris.connector.spi.scan.ConnectorScanRange#getFileSize():long +org.apache.doris.connector.spi.scan.ConnectorScanRange#getHosts():java.util.List +org.apache.doris.connector.spi.scan.ConnectorScanRange#getLength():long +org.apache.doris.connector.spi.scan.ConnectorScanRange#getModificationTime():long +org.apache.doris.connector.spi.scan.ConnectorScanRange#getPartitionValues():java.util.Map +org.apache.doris.connector.spi.scan.ConnectorScanRange#getPath():java.util.Optional +org.apache.doris.connector.spi.scan.ConnectorScanRange#getProperties():java.util.Map +org.apache.doris.connector.spi.scan.ConnectorScanRange#getPushDownRowCount():long +org.apache.doris.connector.spi.scan.ConnectorScanRange#getSelfSplitWeight():long +org.apache.doris.connector.spi.scan.ConnectorScanRange#getStart():long +org.apache.doris.connector.spi.scan.ConnectorScanRange#getTableFormatType():java.lang.String +org.apache.doris.connector.spi.scan.ConnectorScanRange#getTargetSplitSize():long +org.apache.doris.connector.spi.scan.ConnectorScanRange#isNativeReadRange():boolean +org.apache.doris.connector.spi.scan.ConnectorScanRange#isPartitionBearing():boolean +org.apache.doris.connector.spi.scan.ConnectorScanRange#isSingleUse():boolean +org.apache.doris.connector.spi.scan.ConnectorScanRange#populateRangeParams(org.apache.doris.thrift.TTableFormatFileDesc,org.apache.doris.thrift.TFileRangeDesc):void org.apache.doris.connector.spi.scan.ScanNodePropertyKeys#field:FILE_FORMAT_TYPE:java.lang.String=file_format_type org.apache.doris.connector.spi.scan.ScanNodePropertyKeys#field:LOCATION_PREFIX:java.lang.String=location. org.apache.doris.connector.spi.scan.ScanNodePropertyKeys#field:PATH_PARTITION_KEYS:java.lang.String=path_partition_keys @@ -247,3 +289,39 @@ org.apache.doris.extension.spi.PluginFactory#create():org.apache.doris.extension org.apache.doris.extension.spi.PluginFactory#create(org.apache.doris.extension.spi.PluginContext):org.apache.doris.extension.spi.Plugin org.apache.doris.extension.spi.PluginFactory#description():java.lang.String org.apache.doris.extension.spi.PluginFactory#name():java.lang.String +org.apache.doris.filesystem.properties.FileSystemProperties#claimsUri(java.lang.String):boolean +org.apache.doris.filesystem.properties.FileSystemProperties#fsCacheFingerprint():java.lang.String +org.apache.doris.filesystem.properties.FileSystemProperties#getSupportedSchemes():java.util.Set +org.apache.doris.filesystem.properties.FileSystemProperties#isSyntheticDefault():boolean +org.apache.doris.filesystem.properties.FileSystemProperties#kind():org.apache.doris.filesystem.properties.StorageKind +org.apache.doris.filesystem.properties.FileSystemProperties#legacyCacheSchemes():java.util.Set +org.apache.doris.filesystem.properties.FileSystemProperties#matchedProperties():java.util.Map +org.apache.doris.filesystem.properties.FileSystemProperties#matchesLocationPrefix(java.lang.String,java.lang.String):boolean +org.apache.doris.filesystem.properties.FileSystemProperties#providerName():java.lang.String +org.apache.doris.filesystem.properties.FileSystemProperties#rawProperties():java.util.Map +org.apache.doris.filesystem.properties.FileSystemProperties#resolveBackendProperties(java.lang.String):java.util.Optional +org.apache.doris.filesystem.properties.FileSystemProperties#storageFamilyName():java.lang.String +org.apache.doris.filesystem.properties.FileSystemProperties#toBackendProperties():java.util.Optional +org.apache.doris.filesystem.properties.FileSystemProperties#toHadoopProperties():java.util.Optional +org.apache.doris.filesystem.properties.FileSystemProperties#toIcebergFileIOConnectionProperties():java.util.Map +org.apache.doris.filesystem.properties.FileSystemProperties#toIcebergFileIOProperties():java.util.Map +org.apache.doris.filesystem.properties.FileSystemProperties#toIcebergHadoopProperties():java.util.Optional +org.apache.doris.filesystem.properties.FileSystemProperties#type():org.apache.doris.filesystem.FileSystemType +org.apache.doris.filesystem.properties.FileSystemProperties#validate():void +org.apache.doris.filesystem.properties.FileSystemProperties#validateAndGetUri(java.util.Map):java.lang.String +org.apache.doris.filesystem.properties.FileSystemProperties#validateAndNormalizeUri(java.lang.String):java.lang.String +org.apache.doris.filesystem.properties.FileSystemProperties#validateForAccess():void +org.apache.doris.filesystem.properties.StorageProperties#isSyntheticDefault():boolean +org.apache.doris.filesystem.properties.StorageProperties#kind():org.apache.doris.filesystem.properties.StorageKind +org.apache.doris.filesystem.properties.StorageProperties#matchedProperties():java.util.Map +org.apache.doris.filesystem.properties.StorageProperties#matchesLocationPrefix(java.lang.String,java.lang.String):boolean +org.apache.doris.filesystem.properties.StorageProperties#providerName():java.lang.String +org.apache.doris.filesystem.properties.StorageProperties#rawProperties():java.util.Map +org.apache.doris.filesystem.properties.StorageProperties#toBackendProperties():java.util.Optional +org.apache.doris.filesystem.properties.StorageProperties#toHadoopProperties():java.util.Optional +org.apache.doris.filesystem.properties.StorageProperties#toIcebergFileIOConnectionProperties():java.util.Map +org.apache.doris.filesystem.properties.StorageProperties#toIcebergFileIOProperties():java.util.Map +org.apache.doris.filesystem.properties.StorageProperties#toIcebergHadoopProperties():java.util.Optional +org.apache.doris.filesystem.properties.StorageProperties#type():org.apache.doris.filesystem.FileSystemType +org.apache.doris.filesystem.properties.StorageProperties#validate():void +org.apache.doris.filesystem.properties.StorageProperties#validateForAccess():void diff --git a/fe/fe-connector/pom.xml b/fe/fe-connector/pom.xml index 9bf53c1050bfa3..913f9ba889077e 100644 --- a/fe/fe-connector/pom.xml +++ b/fe/fe-connector/pom.xml @@ -55,7 +55,7 @@ under the License. of the latter two means bumping this property as well (and fe-extension-spi means bumping all five families). --> - 13.0 + 14.0 diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/storage/ObjectInfo.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/storage/ObjectInfo.java index dc3959f01d1624..8a144aa4c46bdd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/storage/ObjectInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/storage/ObjectInfo.java @@ -139,7 +139,7 @@ public String toString() { + ", roleName='" + roleName + '\'' + ", arn='" + arn + '\'' + ", externalId='" + externalId + '\'' - + ", token='" + token + '\'' + + ", token='******'" + '}'; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/storage/ObjectInfoAdapter.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/storage/ObjectInfoAdapter.java index e38aacfa6a6ba5..895ab9d93d3ae9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/storage/ObjectInfoAdapter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/storage/ObjectInfoAdapter.java @@ -179,13 +179,21 @@ private static Map buildS3CompatibleProps(ObjectInfo obj) { private static Map buildAzureProps(ObjectInfo obj) { Map props = new HashMap<>(); - // Azure maps ak→accountName, sk→accountKey + // Azure maps ak to the account identifier, regardless of the authentication mode. putIfNotBlank(props, "azure.account_name", obj.getAk()); - putIfNotBlank(props, "azure.account_key", obj.getSk()); putIfNotBlank(props, "azure.endpoint", obj.getEndpoint()); putIfNotBlank(props, "azure.container", obj.getBucket()); - // SAS token (populated after getStsToken) - putIfNotBlank(props, "azure.sas_token", obj.getToken()); + // A supplied SAS replaces the SharedKey credential; it is not an AWS-style session + // token that accompanies a key pair. Do not discard malformed nonempty tokens here: + // the Azure provider must reject them instead of falling back to the account key. + // This converts supplied credentials only; Azure getStsToken is not implemented. + if (obj.getToken() != null && !obj.getToken().isEmpty()) { + props.put("azure.auth_type", "SAS"); + props.put("azure.sas_token", obj.getToken()); + } else { + props.put("azure.auth_type", "SharedKey"); + putIfNotBlank(props, "azure.account_key", obj.getSk()); + } putIfNotBlank(props, STS_ROLE_NAME_KEY, obj.getRoleName()); putIfNotBlank(props, STS_ROLE_ARN_KEY, obj.getArn()); putIfNotBlank(props, STS_EXTERNAL_ID_KEY, obj.getExternalId()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/LocationPath.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/LocationPath.java index 32e35dcf1e44c4..1344c3d37e902a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/LocationPath.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/LocationPath.java @@ -24,6 +24,7 @@ import org.apache.doris.datasource.storage.StorageUriUtils; import org.apache.doris.filesystem.FileSystemType; import org.apache.doris.filesystem.Location; +import org.apache.doris.filesystem.properties.BackendStorageKind; import org.apache.doris.foundation.property.StoragePropertiesException; import org.apache.doris.thrift.TFileType; @@ -134,7 +135,7 @@ public static LocationPath ofAdapters(String location, normalize = false; } if (normalize) { - storageAdapter = findStorageAdapter(type, schema, storageAdaptersMap); + storageAdapter = findStorageAdapter(type, schema, location, storageAdaptersMap); if (storageAdapter == null) { throw new UserException("No storage properties found for schema: " + schema); @@ -175,9 +176,8 @@ public static LocationPath ofAdapters(String location, try { return LocationPath.ofAdapters(location, storageAdaptersMap, true); } catch (UserException | StoragePropertiesException e) { - // the facade throws unchecked StoragePropertiesException where legacy threw checked - // UserException — keep the legacy location context in the wrapped message - throw new StoragePropertiesException("Failed to create LocationPath for location: " + location, e); + // A storage URI may carry a SAS query. Do not include it in diagnostic context. + throw new StoragePropertiesException("Failed to resolve storage location", e); } } @@ -298,8 +298,22 @@ private static String extractScheme(String location) { * @param storageAdaptersMap a map of available storage types to their bindings * @return a matching {@link StorageAdapter} if found; otherwise, {@code null} */ - private static StorageAdapter findStorageAdapter(StorageTypeId type, String schema, + private static StorageAdapter findStorageAdapter(StorageTypeId type, String schema, String location, Map storageAdaptersMap) { + // Only an already-bound provider may claim a generic URI (for example an Azure Blob + // HTTPS URL). A claim still goes through that provider's strict location validation. + StorageAdapter claimed = null; + for (StorageAdapter candidate : storageAdaptersMap.values()) { + if (candidate.getSpiProperties().claimsUri(location)) { + if (claimed != null) { + throw new StoragePropertiesException("Multiple storage bindings claim the location"); + } + claimed = candidate; + } + } + if (claimed != null) { + return claimed; + } // Step 1: Try direct match by type StorageAdapter adapter = storageAdaptersMap.get(type); if (adapter != null) { @@ -323,6 +337,12 @@ private static StorageAdapter findStorageAdapter(StorageTypeId type, String sche return storageAdaptersMap.get(StorageTypeId.OZONE); } + // Azure is provider-specific. Falling back to a generic S3 binding would discard the + // account@host authority and can silently select an unrelated credential set. + if (type == StorageTypeId.AZURE) { + return null; + } + // Step 4: Compatibility fallback based on schema // In previous configurations, the schema name may not strictly match the actual storage type. // For example, a COS storage might use the "s3" schema, or an S3 storage might use the "cos" schema. @@ -375,6 +395,10 @@ public static TFileType getTFileTypeForBE(String location) { } public TFileType getTFileTypeForBE() { + if (storageAdapter != null) { + return storageAdapter.getBackendFileType( + storageAdapter.resolveBackendProperties(normalizedLocation).backendKind()); + } if (("abfs".equals(schema) || "abfss".equals(schema)) && StorageUriUtils.isOneLakeLocation(normalizedLocation)) { return TFileType.FILE_HDFS; @@ -405,6 +429,15 @@ public Location toStorageLocation() { public FileSystemType getFileSystemType() { + if (storageAdapter != null) { + BackendStorageKind kind = storageAdapter.resolveBackendProperties(normalizedLocation).backendKind(); + if (kind == BackendStorageKind.HDFS) { + return FileSystemType.HDFS; + } + if (kind == BackendStorageKind.NATIVE) { + return storageAdapter.getSpiProperties().type(); + } + } if (("abfs".equals(schema) || "abfss".equals(schema)) && StorageUriUtils.isOneLakeLocation(normalizedLocation)) { return FileSystemType.HDFS; diff --git a/fe/fe-core/src/main/java/org/apache/doris/connector/DefaultConnectorContext.java b/fe/fe-core/src/main/java/org/apache/doris/connector/DefaultConnectorContext.java index f519f6de28b368..c8fff109a009c2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/connector/DefaultConnectorContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/connector/DefaultConnectorContext.java @@ -30,6 +30,8 @@ import org.apache.doris.connector.spi.ConnectorContext; import org.apache.doris.connector.spi.ConnectorHttpSecurityHook; import org.apache.doris.connector.spi.ConnectorSession; +import org.apache.doris.connector.spi.ConnectorStorageAccess; +import org.apache.doris.connector.spi.ConnectorStorageAccessResolver; import org.apache.doris.connector.spi.ConnectorStorageContext; import org.apache.doris.datasource.CatalogIf; import org.apache.doris.datasource.ExternalCatalog; @@ -40,11 +42,15 @@ import org.apache.doris.filesystem.FileIterator; import org.apache.doris.filesystem.FileSystem; import org.apache.doris.filesystem.Location; +import org.apache.doris.filesystem.properties.BackendStorageKind; +import org.apache.doris.filesystem.properties.BackendStorageProperties; +import org.apache.doris.filesystem.properties.StorageProperties; import org.apache.doris.fs.FileSystemFactory; import org.apache.doris.fs.SpiSwitchingFileSystem; import org.apache.doris.kerberos.ExecutionAuthenticator; import org.apache.doris.system.Backend; import org.apache.doris.thrift.BackendService; +import org.apache.doris.thrift.TFileType; import org.apache.doris.thrift.TNetworkAddress; import org.apache.doris.thrift.TStatusCode; import org.apache.doris.thrift.TStorageBackendType; @@ -61,9 +67,11 @@ import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.concurrent.Callable; import java.util.function.Function; import java.util.function.Supplier; @@ -96,8 +104,8 @@ public class DefaultConnectorContext implements ConnectorContext, ConnectorStora // (FIX-URI-NORMALIZE). Invoked at scan time only (catalog fully initialized). Empty for ctors // that do not wire it — those callers (non-plugin catalogs) never invoke normalizeStorageUri. private final Supplier> storagePropertiesSupplier; - // Supplies the catalog's effective raw storage map (persisted props + derived defaults, empty when the - // connector supplies vended credentials) for direct fe-filesystem binding in getStorageProperties() + // Supplies the catalog's effective raw storage map (persisted props + derived defaults) + // for direct fe-filesystem binding in getStorageProperties() and request-local resolution // (design S2): no fe-core StorageProperties parse on the connector storage path. Empty for ctors that do // not wire it (non-plugin / 2-3-4-arg) — those yield an empty storage list, correct parity. private final Supplier> rawStoragePropsSupplier; @@ -224,55 +232,66 @@ public T executeAuthenticated(Callable task) throws Exception { @Override public Map vendStorageCredentials(Map rawVendedCredentials) { - // Map the per-table vended token to the BE-facing AWS_* properties. Fail-soft (empty) on any - // error, matching the legacy provider, so a malformed token degrades gracefully rather than - // killing the scan. The outer try also covers getBackendPropertiesFromStorageMap so the - // fail-soft boundary is byte-identical to the pre-refactor method; buildVendedStorageMap shares - // the typed-map build with normalizeStorageUri (single source of truth — no drift). - try { - Map map = buildVendedStorageMap(rawVendedCredentials); - return map == null ? Collections.emptyMap() - : CredentialUtils.getBackendPropertiesFromStorageMap(map); - } catch (Exception e) { - LOG.warn("Failed to normalize vended credentials", e); - return Collections.emptyMap(); - } + return withVendedStorageMap(rawVendedCredentials, + CredentialUtils::getBackendPropertiesFromStorageMap, Collections.emptyMap()); } /** - * Builds the vended {@link StorageAdapter} typed map from a raw per-table token: filter to - * cloud-storage props, run {@link StorageAdapter#ofAll} (normalizes arbitrary token key - * shapes + derives region/endpoint), then index by {@link StorageTypeId}. Mirrors the - * legacy vended-credentials normalization tail exactly, so the BE-credential overlay - * ({@link #vendStorageCredentials}) and the URI normalization ({@link #normalizeStorageUri(String, - * Map)}) derive the SAME credentials from the SAME token — no drift. Returns {@code null} when the - * token is null/empty, yields no cloud-storage props, or normalization throws — replicating the - * legacy "return null → fall back to the base/static map" contract. + * Shares provider binding between backend credentials, URI normalization and reader selection. + * Returns null only when no usable vended properties exist and callers should use the static map. */ private Map buildVendedStorageMap( Map rawVendedCredentials) { + return withVendedStorageMap(rawVendedCredentials, Function.identity(), null); + } + + /** + * Provider-owned vended credentials and their use stay outside the legacy fail-soft boundary. In + * particular, a SAS can bind successfully for replay/URI normalization and expire before its backend + * map is emitted: that access-time failure must not become an empty credential overlay. Providers + * without a vended dialect retain their existing filtered-property binding and failure behavior. + */ + private T withVendedStorageMap(Map rawVendedCredentials, + Function, T> useBindings, T noBindings) { if (rawVendedCredentials == null || rawVendedCredentials.isEmpty()) { - return null; + return noBindings; + } + return withVendedStorageMap(rawVendedCredentials, rawStoragePropsSupplier.get(), useBindings, noBindings); + } + + private T withVendedStorageMap(Map rawVendedCredentials, + Map rawCatalogProperties, + Function, T> useBindings, T noBindings) { + if (rawVendedCredentials == null || rawVendedCredentials.isEmpty()) { + return noBindings; + } + Optional> providerBindings = StorageAdapter.ofVended( + rawVendedCredentials, rawCatalogProperties); + if (providerBindings.isPresent()) { + return useBindings.apply(indexStorageBindings(providerBindings.get())); } try { Map filtered = CredentialUtils.filterCloudStorageProperties(rawVendedCredentials); if (filtered.isEmpty()) { - return null; + return noBindings; } - List vended = StorageAdapter.ofAll(filtered); - return vended.stream() - .collect(Collectors.toMap(StorageAdapter::getType, Function.identity())); + return useBindings.apply(indexStorageBindings(StorageAdapter.ofAll(filtered))); } catch (Exception e) { LOG.warn("Failed to normalize vended credentials", e); - return null; + return noBindings; } } + private static Map indexStorageBindings(List bindings) { + return bindings.stream().collect(Collectors.toMap(StorageAdapter::getType, Function.identity())); + } + @Override public Map getBackendStorageProperties() { // Mirror legacy PaimonScanNode.getLocationProperties(): translate the catalog's parsed - // storage-adapter map into BE-canonical scan keys (AWS_* for object stores, hadoop/dfs for - // HDFS) via the SAME CredentialUtils.getBackendPropertiesFromStorageMap legacy/iceberg/hive use + // storage-adapter map into BE-canonical scan keys (AWS_* for generic object stores, AZURE_* + // for Azure, hadoop/dfs for HDFS) via the SAME + // CredentialUtils.getBackendPropertiesFromStorageMap legacy/iceberg/hive use // — single source of truth, no drift. The map is already validated at catalog creation, so this // does not throw; an empty map (non-plugin ctor / local-FS warehouse) yields an empty result // (no overlay) — correct parity, unlike normalizeStorageUri which must fail-loud on a bad path. @@ -342,9 +361,9 @@ public List getStorage // storage as typed fe-filesystem StorageProperties (from which it derives its Hadoop/HiveConf config // and BE creds without importing fe-core), sourcing the raw map straight from the catalog's raw // storage supplier -- no fe-core StorageProperties.createAll round-trip via getOrigProps(). The raw - // supplier already merges the catalog's derived storage defaults (warehouse -> fs.defaultFS) and - // honors the vended gate (empty for a REST/vended catalog). An empty map (non-plugin ctor / - // REST-vended / credential-less warehouse) yields an empty list -- no static storage, correct parity. + // supplier already merges the catalog's derived storage defaults (warehouse -> fs.defaultFS). + // Per-request vended replacement belongs to the request-local resolution hooks, not this view. + // An empty map (non-plugin ctor / credential-less warehouse) yields an empty storage list. Map rawCatalogProps = rawStoragePropsSupplier.get(); if (rawCatalogProps == null || rawCatalogProps.isEmpty()) { return Collections.emptyList(); @@ -352,6 +371,12 @@ public List getStorage return FileSystemFactory.bindAllStorageProperties(rawCatalogProps); } + @Override + public List resolveStorageProperties(Map rawVendedCredentials) { + return snapshotStorageBindings(rawVendedCredentials).values().stream() + .map(StorageAdapter::getSpiProperties).collect(Collectors.toUnmodifiableList()); + } + @Override public FileSystem getFileSystem(ConnectorSession session) { // Engine-owned, per-catalog scheme-routing FileSystem, lazily built and cached so repeated scans reuse @@ -438,7 +463,7 @@ public String normalizeStorageUri(String rawUri, Map rawVendedCr @Override public UnaryOperator newStorageUriNormalizer(Map rawVendedCredentials) { // PERF: the vended token is scan-invariant, so derive the effective storage map (the expensive - // buildVendedStorageMap = StorageProperties.createAll + hadoop config build) ONCE per scan and reuse + // provider binding / legacy raw-property conversion) ONCE per scan and reuse // it for every per-file normalize, instead of rebuilding it per data/delete file. Each application is // byte-identical to normalizeStorageUri(rawUri, token): the SAME empty-uri short-circuit, the SAME // vended-replaces-static precedence, the SAME fail-loud LocationPath. The derivation is done LAZILY on @@ -466,6 +491,66 @@ public String apply(String rawUri) { }; } + private Map snapshotStorageBindings(Map rawVendedCredentials) { + Map rawCatalog = new HashMap<>(rawStoragePropsSupplier.get()); + Map rawVended = rawVendedCredentials == null + ? Collections.emptyMap() : new HashMap<>(rawVendedCredentials); + // Bind the new credential generation before consulting any static backend view. In + // particular, an expired static SAS must not prevent a valid fresh SAS from replacing it. + Map vended = withVendedStorageMap( + rawVended, rawCatalog, Function.identity(), Collections.emptyMap()); + Map bindings; + if (rawCatalog.isEmpty()) { + // Lightweight contexts may expose only pre-bound adapters. Production plugin catalogs + // expose raw properties, so their request uses one catalog generation below. + bindings = new LinkedHashMap<>(storagePropertiesSupplier.get()); + } else { + bindings = new LinkedHashMap<>(indexStorageBindings(StorageAdapter.ofAllExcept(rawCatalog, + vended.values().stream().filter(StorageAdapter::isExplicitlyConfigured) + .map(binding -> binding.getSpiProperties().providerName()).collect(Collectors.toSet())))); + } + vended.forEach((type, binding) -> { + // Legacy dialect binding can prepend a synthetic HDFS default. It must not erase a + // real catalog HDFS identity when a separate object-store credential is refreshed. + if (binding.isExplicitlyConfigured() || !bindings.containsKey(type)) { + bindings.put(type, binding); + } + }); + return Collections.unmodifiableMap(bindings); + } + + @Override + public ConnectorStorageAccessResolver newStorageAccessResolver( + Map rawVendedCredentials) { + Map snapshot = snapshotStorageBindings(rawVendedCredentials); + // Construction completes before publication. Every application only reads immutable bindings, + // so properties planning and a streaming scan pump may share this request-local resolver. + return new ConnectorStorageAccessResolver(snapshot.values().stream() + .map(binding -> binding.getSpiProperties().providerName()).collect(Collectors.toSet()), rawUri -> { + LocationPath location = LocationPath.ofAdapters(rawUri, snapshot); + String normalized = location.getNormalizedLocation(); + StorageAdapter adapter = location.getStorageAdapter(); + if (adapter == null) { + // LocationPath intentionally skips storage binding for local files. + if (location.getTFileTypeForBE() != TFileType.FILE_LOCAL) { + throw new IllegalStateException("Storage access requires a provider binding"); + } + return new ConnectorStorageAccess("LOCAL", normalized, BackendStorageKind.LOCAL, + TFileType.FILE_LOCAL.name(), Collections.emptyMap()); + } + BackendStorageProperties view = adapter.resolveBackendProperties(normalized); + return new ConnectorStorageAccess(adapter.getSpiProperties().providerName(), normalized, + view.backendKind(), adapter.getBackendFileType(view.backendKind()).name(), + adapter.getBackendConfigProperties(view)); + }, (rawLocation, rawPrefix) -> { + StorageAdapter adapter = LocationPath.ofAdapters(rawLocation, snapshot).getStorageAdapter(); + if (adapter == null) { + throw new IllegalStateException("Storage prefix matching requires a provider binding"); + } + return adapter.getSpiProperties().matchesLocationPrefix(rawLocation, rawPrefix); + }); + } + @Override public String getBackendFileType(String rawUri, Map rawVendedCredentials) { // Same LocationPath build as normalizeStorageUri (vended-aware), then read the BE file type from diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/credentials/CredentialUtils.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/credentials/CredentialUtils.java index 2c5e734d21962b..d800cf879fceec 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/credentials/CredentialUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/credentials/CredentialUtils.java @@ -23,6 +23,7 @@ import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; +import java.util.Locale; import java.util.Map; import java.util.Set; @@ -61,12 +62,19 @@ public static Map filterCloudStorageProperties(Map filtered = new HashMap<>(); rawVendedCredentials.entrySet().stream() .filter(entry -> entry.getKey() != null && entry.getValue() != null) - .filter(entry -> CLOUD_STORAGE_PREFIXES.stream().anyMatch(prefix -> entry.getKey().startsWith(prefix))) + .filter(entry -> isSupportedCloudStorageKey(entry.getKey())) .forEach(entry -> filtered.put(entry.getKey(), entry.getValue())); return filtered; } + private static boolean isSupportedCloudStorageKey(String key) { + String lowerKey = key.toLowerCase(Locale.ROOT); + return CLOUD_STORAGE_PREFIXES.stream() + .anyMatch(prefix -> lowerKey.startsWith(prefix.toLowerCase(Locale.ROOT))) + || "provider".equals(lowerKey); + } + /** * Extract backend properties from StorageAdapter map * Reference: CatalogProperty.getBackendStorageProperties() diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/split/PluginDrivenSplit.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/split/PluginDrivenSplit.java index 2504b09adf0b9f..d6a8906307e8f6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/split/PluginDrivenSplit.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/split/PluginDrivenSplit.java @@ -19,6 +19,7 @@ import org.apache.doris.common.util.LocationPath; import org.apache.doris.connector.spi.scan.ConnectorScanRange; +import org.apache.doris.thrift.TFileType; import java.util.ArrayList; import java.util.List; @@ -45,6 +46,7 @@ public PluginDrivenSplit(ConnectorScanRange scanRange) { scanRange.getHosts().toArray(new String[0]), buildPartitionValues(scanRange)); this.connectorScanRange = scanRange; + scanRange.getBackendFileType().ifPresent(fileType -> this.locationType = TFileType.valueOf(fileType)); // FIX-A1: thread the connector's proportional split weight into the FileSplit scheduling fields so // FederationBackendPolicy distributes by size (legacy parity) instead of uniform standard() weight. // Set ONLY when the connector provides BOTH a weight (>= 0; 0 is a real weight) and a positive diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/storage/StorageAdapter.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/storage/StorageAdapter.java index 0c42c095d33cff..a246b134b3d716 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/storage/StorageAdapter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/storage/StorageAdapter.java @@ -19,6 +19,8 @@ import org.apache.doris.common.Config; import org.apache.doris.datasource.property.common.AwsCredentialsProviderMode; +import org.apache.doris.filesystem.properties.BackendStorageKind; +import org.apache.doris.filesystem.properties.BackendStorageProperties; import org.apache.doris.filesystem.properties.FileSystemProperties; import org.apache.doris.filesystem.properties.FsCacheKeys; import org.apache.doris.filesystem.properties.HadoopStorageProperties; @@ -27,6 +29,7 @@ import org.apache.doris.foundation.property.StoragePropertiesException; import org.apache.doris.foundation.security.ExecutionAuthenticator; import org.apache.doris.fs.FileSystemPluginManager; +import org.apache.doris.thrift.TFileType; import com.google.common.collect.ImmutableMap; import org.apache.commons.lang3.StringUtils; @@ -39,6 +42,7 @@ import java.util.Locale; import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.Set; /** @@ -49,9 +53,9 @@ * {@link FileSystemPluginManager#bindPrimary}/{@code bindAll}. Its public surface mirrors the * legacy typed storage-properties contract exactly — backend map, storage name, schemas, type * — so consumers can migrate mechanically. Building a hadoop {@code Configuration} is NOT part - * of that surface: the one legacy consumer (the Azure OAuth2 backend map) is now served by + * of that surface: Azure backend parameters are emitted by * {@code AzureFileSystemProperties.toMap()} inside fe-filesystem-azure, which keeps fe-core - * source hadoop-free. + * source hadoop-free and preserves the account authority for native reads. * Every known SPI-vs-fe-core drift from the master plan's §2.4 parity ledger is reconciled here * (or in the SPI implementation, where noted); each reconciliation carries a * "align fe-core" comment referencing the ledger item.

    @@ -162,6 +166,34 @@ public static List ofAll(Map origProps) { return result; } + /** Builds the remaining static bindings from the same raw snapshot used for vending. */ + public static List ofAllExcept(Map origProps, Set replacedProviders) { + List bindings = + manager().bindAllExcept(withHadoopConfigDir(origProps), replacedProviders); + List result = new ArrayList<>(bindings.size()); + for (FileSystemProperties binding : bindings) { + result.add(new StorageAdapter(binding, origProps)); + } + return result; + } + + /** + * Wraps provider-owned vended bindings directly, without serializing a backend map and rebinding it + * as catalog properties. Empty means the dialect was not recognized; invalid recognized credentials + * propagate instead of falling back to another identity. + */ + public static Optional> ofVended(Map credentials, + Map catalogProperties) { + return manager().bindVended(withHadoopConfigDir(credentials), withHadoopConfigDir(catalogProperties)) + .map(bindings -> { + List result = new ArrayList<>(bindings.size()); + for (FileSystemProperties binding : bindings) { + result.add(new StorageAdapter(binding, binding.rawProperties())); + } + return result; + }); + } + /** * Binds one specific provider by name, bypassing routing entirely — the facade twin of the * legacy per-dialect factories ({@code S3Properties.of}, {@code OSSProperties.of}, @@ -308,20 +340,11 @@ public ExecutionAuthenticator getExecutionAuthenticator() { /** * Legacy {@code HdfsProperties.isExplicitlyConfigured()}: {@code false} only for the * default-HDFS fallback binding that {@code bindAll}/legacy {@code createAll} auto-prepends - * when nothing HDFS-like matched. A binding whose own provider matches the raw props - * (explicit {@code fs.x.support} flag or guess heuristics) is explicit; non-HDFS types were - * always explicit in fe-core. + * when nothing HDFS-like matched. The provider records that origin when binding; callers + * must not rerun routing heuristics against raw properties to reconstruct it. */ public boolean isExplicitlyConfigured() { - if (type != StorageTypeId.HDFS) { - return true; - } - for (FileSystemProvider provider : manager().getProviders()) { - if (providerKey.equalsIgnoreCase(provider.name())) { - return provider.supportsExplicit(origProps) || provider.supportsGuess(origProps); - } - } - return true; + return !spi.isSyntheticDefault(); } /** @@ -334,18 +357,15 @@ public AwsCredentialsProviderMode getAwsCredentialsProviderMode() { } public String validateAndNormalizeUri(String uri) { - // Align fe-core AbstractS3CompatibleProperties/AzureProperties: the SPI S3/Azure typed - // props do not normalize URIs (compat schemes like cos:// must become s3:// before the - // path reaches BE or a concrete filesystem), so the facade owns the legacy logic. + // S3-compatible providers still use the fe-core compatibility normalization (cos/oss/obs + // aliases become s3://). Azure owns its URI handling in the provider so account@host + // authorities remain available to the native Azure client. if (spi instanceof S3CompatibleFileSystemProperties) { return StorageUriUtils.validateAndNormalizeS3Uri(uri, ((S3CompatibleFileSystemProperties) spi).getUsePathStyle(), forceParsingByStandardUriValue, "OSS".equals(providerKey)); } - if ("AZURE".equals(providerKey)) { - return StorageUriUtils.validateAndNormalizeAzureUri(uri); - } if (type == StorageTypeId.HDFS && StorageUriUtils.isJfsLocation(uri)) { // Align fe-core: the legacy HDFS typed class accepted {hdfs, viewfs, jfs} while the // HDFS plugin's scheme identity excludes jfs (it has its own plugin), so the facade @@ -356,15 +376,12 @@ public String validateAndNormalizeUri(String uri) { } public String validateAndGetUri(Map loadProps) { - // Align fe-core: the S3/Azure legacy classes return the RAW uri value (no normalization) - // and throw when the map is empty or has no uri key; the SPI default would silently - // return null instead. + // Align fe-core: the S3 legacy class returns the RAW uri value (no normalization) and + // throws when the map is empty or has no uri key; the SPI default would silently return + // null instead. Azure's provider validates and preserves its URI directly. if (spi instanceof S3CompatibleFileSystemProperties) { return StorageUriUtils.validateAndGetS3Uri(loadProps); } - if ("AZURE".equals(providerKey)) { - return StorageUriUtils.validateAndGetAzureUri(loadProps); - } if (type == StorageTypeId.HDFS && loadProps != null) { // jfs leg of the HDFS binding (see validateAndNormalizeUri above): a single // case-insensitive `uri` entry with a jfs scheme is normalized by the facade. @@ -422,23 +439,100 @@ public Map getBrokerParams() { } /** - * Backend (BE) configuration map, key-for-key equal to the legacy typed classes plus the + * Backend (BE) configuration map. Hadoop-compatible views retain the * per-scheme {@code doris.fs.cache.key.} entries identifying the credential set it was - * built from (see {@link FsCacheKeys}). + * built from (see {@link FsCacheKeys}); native views contain only their provider's wire properties. * *

    The returned map is a defensive copy: the Broker/Local/Http branch of * {@link #computeBackendConfigProperties()} hands back the caller's own raw property map, which * must not gain the injected keys nor any caller's later edits. + * Credential validity is checked on every access, independently of the map cache. */ public Map getBackendConfigProperties() { + spi.validateForAccess(); if (backendConfigProperties == null) { Map props = new HashMap<>(computeBackendConfigProperties()); - FsCacheKeys.putFsCacheKeys(props, spi); + if (spi.toBackendProperties().map(BackendStorageProperties::backendKind).orElse(null) + != BackendStorageKind.NATIVE) { + FsCacheKeys.putFsCacheKeys(props, spi); + } backendConfigProperties = props; } return new HashMap<>(backendConfigProperties); } + /** Selects a backend view from this binding, after URI validation and normalization. */ + public BackendStorageProperties resolveBackendProperties(String normalizedUri) { + return spi.resolveBackendProperties(normalizedUri).orElseGet(() -> { + BackendStorageKind kind; + switch (type) { + case BROKER: + kind = BackendStorageKind.BROKER; + break; + case LOCAL: + kind = BackendStorageKind.LOCAL; + break; + case HTTP: + kind = BackendStorageKind.NATIVE; + break; + default: + throw new IllegalStateException("Provider " + providerKey + " exposes no backend properties"); + } + return new BackendStorageProperties() { + @Override + public BackendStorageKind backendKind() { + return kind; + } + + @Override + public Map toMap() { + return new HashMap<>(origProps); + } + }; + }); + } + + /** Wire routing is engine-owned; the provider chooses the native/Hadoop view, not a URI guess. */ + public TFileType getBackendFileType(BackendStorageKind kind) { + switch (kind) { + case S3_COMPATIBLE: + return TFileType.FILE_S3; + case HDFS: + return TFileType.FILE_HDFS; + case BROKER: + return TFileType.FILE_BROKER; + case LOCAL: + return TFileType.FILE_LOCAL; + case NATIVE: + // Azure retains the existing FILE_S3 transport and selects its client with provider=azure. + if (type == StorageTypeId.AZURE) { + return TFileType.FILE_S3; + } + if (type == StorageTypeId.HTTP) { + return TFileType.FILE_HTTP; + } + throw new IllegalStateException("No native BE reader registered for provider " + providerKey); + default: + throw new IllegalStateException("Unhandled backend storage kind: " + kind); + } + } + + /** Emits only the selected view; native parameters never acquire Hadoop filesystem cache keys. */ + public Map getBackendConfigProperties(BackendStorageProperties view) { + if (view == spi.toBackendProperties().orElse(null) + || type == StorageTypeId.BROKER || type == StorageTypeId.LOCAL || type == StorageTypeId.HTTP) { + // Reuse the immutable default view's map and its access-time expiry check. Native + // scans call this once per file, so binding fingerprints must not be recomputed there. + return getBackendConfigProperties(); + } + spi.validateForAccess(); + Map props = new HashMap<>(alignBackendConfigProperties(view.toMap())); + if (view.backendKind() != BackendStorageKind.NATIVE) { + FsCacheKeys.putFsCacheKeys(props, spi); + } + return props; + } + /** * Stable fingerprint of this storage identity, computed by the bound SPI properties from their * class name and matched (credential-carrying) user properties — see {@link FsCacheKeys}. It is @@ -455,14 +549,6 @@ private Map computeBackendConfigProperties() { case HTTP: // Align fe-core: Broker/Local/Http return the raw user properties verbatim. return origProps; - case AZURE: - // Provider-owned, both auth types (the OAuth2 map is a hadoop Configuration dump - // built inside fe-filesystem-azure). Routed out here so it never reaches the - // S3-family alignment below, exactly as before. - return spi.toBackendProperties() - .orElseThrow(() -> new IllegalStateException( - "Provider " + providerKey + " exposes no backend properties")) - .toMap(); default: break; } @@ -470,6 +556,10 @@ private Map computeBackendConfigProperties() { .orElseThrow(() -> new IllegalStateException( "Provider " + providerKey + " exposes no backend properties")) .toMap(); + return alignBackendConfigProperties(base); + } + + private Map alignBackendConfigProperties(Map base) { if (spi instanceof S3CompatibleFileSystemProperties) { return alignS3FamilyBackendMap((S3CompatibleFileSystemProperties) spi, base); } @@ -547,17 +637,19 @@ private String feCoreS3CredentialsProviderType() { } /** - * Align fe-core AzureProperties.initNormalizeAndCheckProps: the temporary fe-core-only - * restriction that OAuth2 is supported only for the Iceberg REST catalog. This check reads - * catalog-level keys the SPI never sees, so it stays in the facade. + * Align the Azure catalog restriction that OAuth2 is supported only for the Iceberg REST + * catalog. This check reads catalog-level keys the SPI never sees, so it stays in the facade. */ private void checkAzureOauth2OnlyForIcebergRest() { - // Align fe-core AzureProperties exactly: the REST-only gate compared CASE-SENSITIVELY - // ("OAuth2".equals(...)) while the rest of the class used equalsIgnoreCase — so a - // lowercase "oauth2" slipped past this gate in legacy (and old images may carry it). - // Replicate the asymmetry; tightening it would break replay of such images. - if (!"AZURE".equals(providerKey) - || !"OAuth2".equals(origProps.getOrDefault("azure.auth_type", "SharedKey"))) { + if (!"AZURE".equals(providerKey)) { + return; + } + String authType = origProps.entrySet().stream() + .filter(entry -> "azure.auth_type".equalsIgnoreCase(entry.getKey())) + .map(Map.Entry::getValue) + .findFirst() + .orElse("SharedKey"); + if (!"OAuth2".equalsIgnoreCase(authType)) { return; } boolean hasIcebergType = origProps.entrySet().stream() diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/storage/StorageUriUtils.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/storage/StorageUriUtils.java index 2f3043bee8ddcf..6691c3d2d00e3e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/storage/StorageUriUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/storage/StorageUriUtils.java @@ -36,8 +36,8 @@ /** * fe-core-only URI validation/normalization glue for {@link StorageAdapter}. * - *

    Ports the URI subset of the legacy {@code S3PropertyUtils} and {@code AzurePropertyUtils} - * pure functions so the facade does not depend on the to-be-deleted + *

    Ports the URI subset of the legacy {@code S3PropertyUtils} and the remaining + * location classifier from {@code AzurePropertyUtils} so the facade does not depend on the to-be-deleted * legacy typed storage package (deleted in Phase D). Behavior is kept identical, with one deliberate * exception: the legacy full-parse path could throw the checked {@code UserException} (from * {@link S3URI#create}); the facade surface is unchecked, so that exception is rethrown as @@ -211,31 +211,10 @@ public static String validateAndGetS3Uri(Map props) { return uriOptional.get(); } - /** - * Port of legacy {@code AzurePropertyUtils.validateAndNormalizeUri}: accepts Azure Blob - * URI schemes only, keeps OneLake locations as-is, and converts everything else to the - * unified {@code s3://container/path} style. - */ - public static String validateAndNormalizeAzureUri(String path) { - if (StringUtils.isBlank(path)) { - throw new StoragePropertiesException("Path cannot be null or empty"); - } - // Only accept Azure Blob Storage-related URI schemes - if (!(path.startsWith("wasb://") || path.startsWith("wasbs://") - || path.startsWith("abfs://") || path.startsWith("abfss://") - || path.startsWith("https://") || path.startsWith("http://") - || path.startsWith("s3://"))) { - throw new StoragePropertiesException("Unsupported Azure URI scheme: " + path); - } - if (isOneLakeLocation(path)) { - return path; - } - return convertAzureToS3Style(path); - } - /** * Port of legacy {@code AzurePropertyUtils.isOneLakeLocation}: true when the location is a - * Microsoft Fabric OneLake abfs/abfss URI (kept as-is instead of s3-style normalization). + * Microsoft Fabric OneLake abfs/abfss URI. OneLake remains Hadoop-routed until native OAuth2 + * support is complete; ordinary Azure ABFS locations are routed through FILE_S3. */ public static boolean isOneLakeLocation(String location) { return ONELAKE_PATTERN.matcher(location).matches(); @@ -275,86 +254,4 @@ public static String validateAndNormalizeJfsUri(String location) { } } - private static String convertAzureToS3Style(String uri) { - if (StringUtils.isBlank(uri)) { - throw new StoragePropertiesException("URI is blank"); - } - if (uri.startsWith("s3://")) { - return uri; - } - // Handle Azure HDFS-style URIs (wasb://, wasbs://, abfs://, abfss://) - if (uri.startsWith("wasb://") || uri.startsWith("wasbs://") - || uri.startsWith("abfs://") || uri.startsWith("abfss://")) { - - // Example: wasbs://container@account.blob.core.windows.net/path/file.txt - String schemeRemoved = uri.replaceFirst("^[a-z]+s?://", ""); - int atIndex = schemeRemoved.indexOf('@'); - if (atIndex < 0) { - throw new StoragePropertiesException("Invalid Azure URI, missing '@': " + uri); - } - - // Extract container name (before '@') - String container = schemeRemoved.substring(0, atIndex); - - // Extract remaining part after '@' - String remainder = schemeRemoved.substring(atIndex + 1); - int slashIndex = remainder.indexOf('/'); - - // Extract the path part if it exists - String path = (slashIndex != -1) ? remainder.substring(slashIndex + 1) : ""; - - // Normalize to s3-style URI: s3:/// - return StringUtils.isBlank(path) - ? String.format("s3://%s", container) - : String.format("s3://%s/%s", container, path); - } - - // Handle HTTPS/HTTP Azure Blob Storage URLs - if (uri.startsWith("https://") || uri.startsWith("http://")) { - try { - URI parsed = new URI(uri); - String host = parsed.getHost(); - String path = parsed.getPath(); - - if (StringUtils.isBlank(host)) { - throw new StoragePropertiesException("Invalid Azure HTTPS URI, missing host: " + uri); - } - - // Path usually looks like: // - String[] parts = path.split("/", 3); - if (parts.length < 2) { - throw new StoragePropertiesException("Invalid Azure Blob URL, missing container: " + uri); - } - - String container = parts[1]; - String remainder = (parts.length == 3) ? parts[2] : ""; - - // Convert HTTPS URL to s3-style format - return StringUtils.isBlank(remainder) - ? String.format("s3://%s", container) - : String.format("s3://%s/%s", container, remainder); - - } catch (URISyntaxException e) { - throw new StoragePropertiesException("Invalid HTTPS URI: " + uri, e); - } - } - - throw new StoragePropertiesException("Unsupported Azure URI scheme: " + uri); - } - - /** - * Port of legacy {@code AzurePropertyUtils.validateAndGetUri}: extracts the raw URI string - * from the given props map (case-insensitive {@code uri} key). - */ - public static String validateAndGetAzureUri(Map props) { - if (props == null || props.isEmpty()) { - throw new StoragePropertiesException("Properties map cannot be null or empty"); - } - - return props.entrySet().stream() - .filter(e -> URI_KEY.equalsIgnoreCase(e.getKey())) - .map(Map.Entry::getValue) - .findFirst() - .orElseThrow(() -> new StoragePropertiesException("Properties must contain 'uri' key")); - } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/fs/FileSystemPluginManager.java b/fe/fe-core/src/main/java/org/apache/doris/fs/FileSystemPluginManager.java index 4c3d59548307cf..6e81feedf6b5c9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/fs/FileSystemPluginManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/fs/FileSystemPluginManager.java @@ -41,11 +41,15 @@ import java.util.Arrays; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; +import java.util.Locale; import java.util.Map; +import java.util.Optional; import java.util.ServiceLoader; import java.util.Set; import java.util.concurrent.CopyOnWriteArrayList; +import java.util.stream.Collectors; /** * Manages lifecycle of FileSystemProvider plugins. @@ -270,12 +274,70 @@ public FileSystemProperties bindPrimary(Map properties) { * default HDFS binding is inserted at index 0 (fe-core's default-HDFS fallback). */ public List bindAll(Map properties) { + return bindAll(properties, Collections.emptySet(), true); + } + + /** + * Binds a raw catalog snapshot except for providers already replaced by request credentials. + * Skipping occurs before static binding/validation, not after constructing the obsolete identity. + */ + public List bindAllExcept(Map properties, Set replacedProviders) { + Set skipped = replacedProviders.stream().map(name -> name.toUpperCase(Locale.ROOT)) + .collect(Collectors.toSet()); + return bindAll(properties, skipped, true); + } + + /** + * Binds credentials using provider-owned vended dialects before raw-property routing. Empty means + * that no provider recognized the dialect; recognized but invalid credentials must propagate the + * provider's exception. Other matching providers retain their normal explicit/guess routing, but a + * vended binding never creates a synthetic default HDFS entry. + */ + public Optional> bindVended(Map credentials, + Map catalogProperties) { + List result = new ArrayList<>(); + Set matchedProviders = new HashSet<>(); + for (StorageRegistry.Provider meta : StorageRegistry.Provider.values()) { + FileSystemProvider provider = providerByName(meta.name()); + if (provider != null) { + bindVendedProvider(provider, credentials, catalogProperties, result, matchedProviders); + } + } + for (FileSystemProvider provider : providers) { + if (!StorageRegistry.Provider.byName(provider.name()).isPresent()) { + bindVendedProvider(provider, credentials, catalogProperties, result, matchedProviders); + } + } + if (result.isEmpty()) { + return Optional.empty(); + } + result.addAll(bindAll(credentials, matchedProviders, false)); + return Optional.of(result); + } + + private static void bindVendedProvider(FileSystemProvider provider, Map credentials, + Map catalogProperties, List result, Set matchedProviders) { + String providerName = provider.name().toUpperCase(Locale.ROOT); + if (matchedProviders.contains(providerName)) { + return; + } + provider.bindVended(credentials, catalogProperties).ifPresent(binding -> { + result.add(binding); + matchedProviders.add(providerName); + }); + } + + private List bindAll(Map properties, Set skippedProviders, + boolean includeDefaultHdfs) { boolean useGuess = !hasAnyExplicitFsSupport(properties); Map probeView = withProbeContext(properties); List result = new ArrayList<>(); - boolean ossHdfsMatched = false; - boolean hdfsFamilyMatched = false; + boolean ossHdfsMatched = skippedProviders.contains("OSS_HDFS"); + boolean hdfsFamilyMatched = skippedProviders.contains("HDFS") || skippedProviders.contains("JFS"); for (StorageRegistry.Provider meta : StorageRegistry.Provider.values()) { + if (skippedProviders.contains(meta.name())) { + continue; + } FileSystemProvider provider = providerByName(meta.name()); if (provider == null) { continue; @@ -307,7 +369,8 @@ public List bindAll(Map properties) { } // Unlisted (out-of-tree) providers append after the known set, registration order. for (FileSystemProvider provider : providers) { - if (StorageRegistry.Provider.byName(provider.name()).isPresent()) { + if (StorageRegistry.Provider.byName(provider.name()).isPresent() + || skippedProviders.contains(provider.name().toUpperCase(Locale.ROOT))) { continue; } FileSystemProperties bound = tryBindUnlisted(provider, properties, useGuess, probeView); @@ -315,10 +378,10 @@ public List bindAll(Map properties) { result.add(bound); } } - if (useGuess && !hdfsFamilyMatched) { + if (includeDefaultHdfs && useGuess && !hdfsFamilyMatched) { FileSystemProvider hdfs = providerByName("HDFS"); if (hdfs != null) { - result.add(0, hdfs.bind(properties)); + result.add(0, hdfs.bindDefault(properties)); } } return result; diff --git a/fe/fe-core/src/test/java/org/apache/doris/cloud/storage/ObjectInfoAdapterTest.java b/fe/fe-core/src/test/java/org/apache/doris/cloud/storage/ObjectInfoAdapterTest.java index ed26ad4a0aab41..4b3e474601db50 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/cloud/storage/ObjectInfoAdapterTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/cloud/storage/ObjectInfoAdapterTest.java @@ -21,10 +21,13 @@ import org.apache.doris.datasource.storage.StorageAdapter; import org.apache.doris.datasource.storage.StorageTypeId; import org.apache.doris.filesystem.properties.S3CompatibleFileSystemProperties; +import org.apache.doris.foundation.property.StoragePropertiesException; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import java.util.Map; + public class ObjectInfoAdapterTest { @Test @@ -83,4 +86,102 @@ public void testOssStageBindingCarriesEveryField() { Assertions.assertEquals("stage-sk", oss.getSecretKey()); Assertions.assertEquals("stage-token", oss.getSessionToken()); } + + @Test + public void testAzureSharedKeyWithoutTokenPreservesNativeParameters() { + for (String token : new String[] {null, ""}) { + StorageAdapter adapter = ObjectInfoAdapter.toStorageAdapter(azureObjectInfo("account-key", token)); + Map backend = adapter.getBackendConfigProperties(); + + Assertions.assertEquals(StorageTypeId.AZURE, adapter.getType()); + Assertions.assertEquals("azure", backend.get("provider")); + Assertions.assertEquals("SHARED_KEY", backend.get("AZURE_AUTH_TYPE")); + Assertions.assertEquals("account-key", backend.get("AZURE_ACCOUNT_KEY")); + Assertions.assertEquals("account", backend.get("AZURE_ACCOUNT_NAME")); + Assertions.assertEquals("https://account.blob.core.windows.net", backend.get("AZURE_ENDPOINT")); + Assertions.assertEquals("container", adapter.getOrigProps().get("azure.container")); + Assertions.assertFalse(backend.containsKey("AZURE_SAS_TOKEN")); + Assertions.assertFalse(backend.containsKey("AZURE_SAS_EXPIRY_MS")); + } + } + + @Test + public void testAzureSasReplacesSharedKeyInsteadOfCombiningCredentials() { + String token = "?sv=2024-01-01&se=2100-01-01T00%3A00%3A00Z&sig=temporary%2Bsignature"; + for (String key : new String[] {null, "", "old-account-key"}) { + ObjectInfo objectInfo = azureObjectInfo(key, token); + StorageAdapter adapter = ObjectInfoAdapter.toStorageAdapter(objectInfo); + Map backend = adapter.getBackendConfigProperties(); + + Assertions.assertEquals(StorageTypeId.AZURE, adapter.getType()); + Assertions.assertEquals("SAS", adapter.getOrigProps().get("azure.auth_type")); + Assertions.assertFalse(adapter.getOrigProps().containsKey("azure.account_key")); + Assertions.assertEquals("azure", backend.get("provider")); + Assertions.assertEquals("SAS", backend.get("AZURE_AUTH_TYPE")); + Assertions.assertEquals(token.substring(1), backend.get("AZURE_SAS_TOKEN")); + Assertions.assertEquals("4102444800000", backend.get("AZURE_SAS_EXPIRY_MS")); + Assertions.assertEquals("account", backend.get("AZURE_ACCOUNT_NAME")); + Assertions.assertEquals("container", adapter.getOrigProps().get("azure.container")); + Assertions.assertEquals("https://account.blob.core.windows.net", backend.get("AZURE_ENDPOINT")); + Assertions.assertFalse(backend.containsKey("AZURE_ACCOUNT_KEY")); + Assertions.assertFalse(backend.keySet().stream().anyMatch(name -> name.startsWith("AWS_"))); + Assertions.assertEquals(key, objectInfo.getSk()); + Assertions.assertEquals(token, objectInfo.getToken()); + } + } + + @Test + public void testAzureMalformedSasDoesNotFallBackToSharedKey() { + String token = "se=invalid-expiry&sig=must-not-be-logged"; + StoragePropertiesException failure = Assertions.assertThrows(StoragePropertiesException.class, + () -> ObjectInfoAdapter.toStorageAdapter(azureObjectInfo("account-key", token))); + + Assertions.assertEquals("Azure SAS credential has an invalid expiry", failure.getMessage()); + Assertions.assertNull(failure.getCause()); + Assertions.assertFalse(failure.getMessage().contains("must-not-be-logged")); + } + + @Test + public void testAzureBlankSuppliedSasDoesNotFallBackToSharedKey() { + IllegalArgumentException failure = Assertions.assertThrows(IllegalArgumentException.class, + () -> ObjectInfoAdapter.toStorageAdapter(azureObjectInfo("account-key", " "))); + + Assertions.assertTrue(failure.getMessage().contains("When auth_type is SAS, sas_token is required")); + } + + @Test + public void testAzureExpiredSasRejectsAccessInsteadOfFallingBackToSharedKey() { + StorageAdapter adapter = ObjectInfoAdapter.toStorageAdapter(azureObjectInfo( + "account-key", "se=2000-01-01T00%3A00%3A00Z&sig=expired")); + + Assertions.assertEquals("SAS", adapter.getOrigProps().get("azure.auth_type")); + Assertions.assertFalse(adapter.getOrigProps().containsKey("azure.account_key")); + StoragePropertiesException failure = Assertions.assertThrows(StoragePropertiesException.class, + adapter::getBackendConfigProperties); + Assertions.assertEquals("Azure SAS credential is expired", failure.getMessage()); + Assertions.assertNull(failure.getCause()); + } + + @Test + public void testObjectInfoMasksTokensWithoutChangingCredentials() { + for (Cloud.ObjectStoreInfoPB.Provider provider : new Cloud.ObjectStoreInfoPB.Provider[] { + Cloud.ObjectStoreInfoPB.Provider.AZURE, + Cloud.ObjectStoreInfoPB.Provider.OSS, + Cloud.ObjectStoreInfoPB.Provider.S3}) { + ObjectInfo objectInfo = new ObjectInfo(provider, "account", "secret-key-plain", "container", + "endpoint", "region", "prefix", null, null, null, "token-plain"); + + String rendered = objectInfo.toString(); + Assertions.assertFalse(rendered.contains("token-plain")); + Assertions.assertFalse(rendered.contains("secret-key-plain")); + Assertions.assertTrue(rendered.contains("token='******'")); + Assertions.assertEquals("token-plain", objectInfo.getToken()); + Assertions.assertEquals("secret-key-plain", objectInfo.getSk()); + } + } + + private static ObjectInfo azureObjectInfo(String key, String token) { + return new ObjectInfo(Cloud.ObjectStoreInfoPB.Provider.AZURE, "account", key, "container", + "account.blob.core.windows.net", "", "stage-prefix", null, null, null, token); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/common/util/DatasourcePrintableMapTest.java b/fe/fe-core/src/test/java/org/apache/doris/common/util/DatasourcePrintableMapTest.java index 45d68bbf7543bb..678d76428ef652 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/common/util/DatasourcePrintableMapTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/common/util/DatasourcePrintableMapTest.java @@ -17,6 +17,8 @@ package org.apache.doris.common.util; +import org.apache.doris.fs.FileSystemPluginManager; + import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -26,6 +28,25 @@ public class DatasourcePrintableMapTest { + @Test + public void testAzureWireSecretsStayMaskedIndependentlyOfInputAliases() { + FileSystemPluginManager manager = new FileSystemPluginManager(); + manager.loadBuiltins(); + Map properties = Map.of( + "AZURE_SAS_TOKEN", "sas-wire-test-secret", "AzUrE_sAs_ToKeN", "mixed-case-test-secret", + "azure.sas_token", "sas-input-test-secret", "AZURE_ACCOUNT_KEY", "shared-key-test-secret", + "AZURE_CLIENT_SECRET", "client-test-secret", "provider", "azure"); + + String rendered = new DatasourcePrintableMap<>(properties, "=", false, false, true).toString(); + + Assertions.assertFalse(rendered.contains("test-secret")); + Assertions.assertTrue(rendered.contains("provider = azure")); + for (String key : Set.of("AZURE_SAS_TOKEN", "AzUrE_sAs_ToKeN", "azure.sas_token", + "AZURE_ACCOUNT_KEY", "AZURE_CLIENT_SECRET")) { + Assertions.assertTrue(rendered.contains(key + " = " + DatasourcePrintableMap.PASSWORD_MASK)); + } + } + @Test public void testSensitiveKeysContainAliyunDLFProperties() { // Verify that SENSITIVE_KEY contains sensitive keys from AliyunDLFBaseProperties diff --git a/fe/fe-core/src/test/java/org/apache/doris/common/util/LocationPathTest.java b/fe/fe-core/src/test/java/org/apache/doris/common/util/LocationPathTest.java index 27e6843522a1a3..3c2585ac9a2fd5 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/common/util/LocationPathTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/common/util/LocationPathTest.java @@ -62,7 +62,7 @@ public class LocationPathTest { props.put("obs.secret_key", "secret_key"); props.put("fs.defaultFS", "hdfs://namenode:8020"); props.put("azure.endpoint", "https://mystorageaccount.blob.core.windows.net"); - props.put("azure.access_key", "access_key"); + props.put("azure.access_key", "mystorageaccount"); props.put("azure.secret_key", "secret_key"); props.put("broker.name", "mybroker"); @@ -283,19 +283,70 @@ public void testHdfsStorageLocationConvert() { @Test public void testOnelakeStorageLocationConvert() { + // OneLake uses its own OAuth2 binding, not an unrelated Blob SharedKey identity. + Map oneLakeAdapters = StorageAdapter.ofAll(Map.of( + "fs.azure.support", "true", + "iceberg.catalog.type", "rest", + "azure.auth_type", "OAuth2", + "azure.oauth2_account_host", "onelake.dfs.fabric.microsoft.com", + "azure.oauth2_client_id", "client-id", + "azure.oauth2_client_secret", "client-secret", + "azure.oauth2_server_uri", "https://login.microsoftonline.com/tenant/oauth2/token")) + .stream().collect(Collectors.toMap(StorageAdapter::getType, Function.identity())); + Assertions.assertTrue(oneLakeAdapters.containsKey(StorageTypeId.AZURE), + "The OneLake OAuth2 fixture must select an Azure storage binding"); String location = "abfss://1a2b3c4d-1234-5678-abcd-9876543210ef@onelake.dfs.fabric.microsoft.com/myworkspace/lakehouse/default/Files/data/test.parquet"; - LocationPath locationPath = LocationPath.ofAdapters(location, STORAGE_PROPERTIES_MAP); + LocationPath locationPath = LocationPath.ofAdapters(location, oneLakeAdapters); Assertions.assertEquals(TFileType.FILE_HDFS, locationPath.getTFileTypeForBE()); Assertions.assertEquals(FileSystemType.HDFS, locationPath.getFileSystemType()); + StorageAdapter oneLakeAdapter = locationPath.getStorageAdapter(); + Assertions.assertEquals("OAuth", oneLakeAdapter.getBackendConfigProperties( + oneLakeAdapter.resolveBackendProperties(locationPath.getNormalizedLocation())) + .get("fs.azure.account.auth.type.onelake.dfs.fabric.microsoft.com")); location = "abfs://1a2b3c4d-1234-5678-abcd-9876543210ef@onelake.dfs.fabric.microsoft.com/myworkspace/lakehouse/default/Files/data/test.parquet"; - locationPath = LocationPath.ofAdapters(location, STORAGE_PROPERTIES_MAP); + locationPath = LocationPath.ofAdapters(location, oneLakeAdapters); Assertions.assertEquals(TFileType.FILE_HDFS, locationPath.getTFileTypeForBE()); Assertions.assertEquals(FileSystemType.HDFS, locationPath.getFileSystemType()); location = "abfss://mycontainer@mystorageaccount.dfs.core.windows.net/data/2025/11/11/"; locationPath = LocationPath.ofAdapters(location, STORAGE_PROPERTIES_MAP); + // Native Azure keeps the account authority and object path; converting to s3:// would + // discard the account host needed by the BE Azure client. + Assertions.assertEquals(location, locationPath.getNormalizedLocation()); Assertions.assertEquals(TFileType.FILE_S3, locationPath.getTFileTypeForBE()); - Assertions.assertEquals(FileSystemType.S3, locationPath.getFileSystemType()); + Assertions.assertEquals(FileSystemType.AZURE, locationPath.getFileSystemType()); + + } + @Test + public void testAzureLocationRejectsAnUnrelatedAccountBinding() { + String location = "abfss://container@anotheraccount.dfs.core.windows.net/path/file.parquet"; + Assertions.assertThrows(StoragePropertiesException.class, + () -> LocationPath.ofAdapters(location, STORAGE_PROPERTIES_MAP)); + String oneLake = "abfss://workspace@onelake.dfs.fabric.microsoft.com/lakehouse/Files/file.parquet"; + Assertions.assertThrows(StoragePropertiesException.class, + () -> LocationPath.ofAdapters(oneLake, STORAGE_PROPERTIES_MAP)); + } + + @Test + public void testAzureNativeLocationKeepsAuthorityAndUsesS3ReaderFamily() { + Map azure = new HashMap<>(); + azure.put("provider", "azure"); + azure.put("azure.auth_type", "SAS"); + azure.put("AZURE_ENDPOINT", "account.blob.core.windows.net"); + azure.put("AZURE_ACCOUNT_NAME", "account"); + azure.put("azure.sas_token", "sv=2024-01-01&sig=temporary"); + azure.put("azure.sas_expiry_ms", "4102444800000"); + StorageAdapter adapter = StorageAdapter.of(azure); + Map adapters = new HashMap<>(); + adapters.put(StorageTypeId.AZURE, adapter); + + String location = "abfss://container@account.dfs.core.windows.net/path/file.parquet"; + LocationPath path = LocationPath.ofAdapters(location, adapters); + + Assertions.assertEquals(location, path.getNormalizedLocation()); + Assertions.assertEquals(TFileType.FILE_S3, path.getTFileTypeForBE()); + Assertions.assertEquals(FileSystemType.AZURE, path.getFileSystemType()); + Assertions.assertEquals("SAS", adapter.getBackendConfigProperties().get("AZURE_AUTH_TYPE")); } @Test diff --git a/fe/fe-core/src/test/java/org/apache/doris/connector/DefaultConnectorContextNormalizeUriTest.java b/fe/fe-core/src/test/java/org/apache/doris/connector/DefaultConnectorContextNormalizeUriTest.java index ce698b0ef0f7ad..dd50f84312afbf 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/connector/DefaultConnectorContextNormalizeUriTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/connector/DefaultConnectorContextNormalizeUriTest.java @@ -237,6 +237,26 @@ public void backendFileTypeVendedRestResolvesUnderEmptyStaticMap() { restCtx.getBackendFileType("oss://bkt/warehouse/db/t/data", ossVendedToken())); } + @Test + public void azureVendedPathKeepsAccountAuthorityAndUsesS3Family() { + String accountHost = "account.dfs.core.windows.net"; + Map token = Map.of( + "adls.sas-token." + accountHost, "sv=2024-01-01&sig=temporary", + "adls.sas-token-expires-at-ms." + accountHost, "4102444800000"); + DefaultConnectorContext restCtx = new DefaultConnectorContext("c", 1L); + UnaryOperator normalizer = restCtx.newStorageUriNormalizer(token); + + // The same binding contract serves data scans, Puffin/position-delete files and sink paths. + // This verifies routing and URI preservation, not remote file contents or DML execution. + for (String object : List.of("data.parquet", "data.orc", "deletion-vector.puffin", + "position-deletes.parquet", "new-data")) { + String path = "abfss://container@account.dfs.core.windows.net/table/" + object; + Assertions.assertEquals(path, restCtx.normalizeStorageUri(path, token)); + Assertions.assertEquals(path, normalizer.apply(path)); + Assertions.assertEquals(TFileType.FILE_S3.name(), restCtx.getBackendFileType(path, token)); + } + } + // ---- FIX-PERF-06: newStorageUriNormalizer hoists the (scan-invariant) token->storage-config // derivation to ONCE per scan; every application must stay byte-identical to a per-call // normalizeStorageUri(uri, token), across all four cases the per-call form covers. ---- diff --git a/fe/fe-core/src/test/java/org/apache/doris/connector/DefaultConnectorContextVendTest.java b/fe/fe-core/src/test/java/org/apache/doris/connector/DefaultConnectorContextVendTest.java index 3a282589205d09..0aaa36be8c0fc7 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/connector/DefaultConnectorContextVendTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/connector/DefaultConnectorContextVendTest.java @@ -17,7 +17,12 @@ package org.apache.doris.connector; +import org.apache.doris.datasource.storage.StorageAdapter; +import org.apache.doris.datasource.storage.StorageTypeId; import org.apache.doris.filesystem.properties.FsCacheKeys; +import org.apache.doris.foundation.property.StoragePropertiesException; +import org.apache.doris.kerberos.ExecutionAuthenticator; +import org.apache.doris.thrift.TFileType; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -27,13 +32,8 @@ import java.util.Map; /** - * FIX-REST-VENDED fe-core bridge test: pins that - * {@link DefaultConnectorContext#vendStorageCredentials} reuses the engine's - * {@code StorageProperties} normalization (the same chain legacy - * {@code AbstractVendedCredentialsProvider} runs) to turn a raw per-table OSS vended token into the - * BE-facing {@code AWS_*} storage properties. The connector cannot import that machinery, so this - * hook is the single source of truth — without it a REST native-reader table reaches BE with no - * usable credentials (403). FAILS before the fix (the method is a no-op default returning empty). + * Pins provider-owned Azure vended binding and the legacy normalization retained for other connectors. + * No test in this class creates a remote client or makes a storage request. */ public class DefaultConnectorContextVendTest { @@ -89,6 +89,157 @@ private static Map ossToken(String accessKeyId) { return token; } + @Test + public void normalizesAdlsTokenToNativeAzureProps() { + String accountHost = "account.dfs.core.windows.net"; + Map token = Map.of( + "adls.sas-token." + accountHost, "?sv=2024-01-01&sig=temporary", + "adls.sas-token-expires-at-ms." + accountHost, "4102444800000"); + + Map be = context().vendStorageCredentials(token); + + Assertions.assertEquals("azure", be.get("provider")); + Assertions.assertEquals("SAS", be.get("AZURE_AUTH_TYPE")); + Assertions.assertEquals("account", be.get("AZURE_ACCOUNT_NAME")); + Assertions.assertEquals("https://account.blob.core.windows.net", be.get("AZURE_ENDPOINT")); + Assertions.assertEquals("sv=2024-01-01&sig=temporary", be.get("AZURE_SAS_TOKEN")); + Assertions.assertEquals("4102444800000", be.get("AZURE_SAS_EXPIRY_MS")); + Assertions.assertFalse(be.keySet().stream().anyMatch(key -> key.startsWith("adls."))); + Assertions.assertFalse(be.keySet().stream().anyMatch(key -> key.startsWith("fs.azure."))); + Assertions.assertFalse(be.keySet().stream().anyMatch(key -> key.startsWith("AWS_"))); + Assertions.assertFalse(be.containsKey("fs.defaultFS")); + Assertions.assertFalse(be.containsKey("hdfs.security.authentication")); + Assertions.assertFalse(be.containsKey(FsCacheKeys.fsCacheKeyProperty("hdfs"))); + } + + @Test + public void expiredAdlsTokenIsNotSilentlyDowngraded() { + String accountHost = "account.dfs.core.windows.net"; + Map token = Map.of( + "adls.sas-token." + accountHost, "sv=2024-01-01&sig=expired", + "adls.sas-token-expires-at-ms." + accountHost, "1"); + + StoragePropertiesException failure = Assertions.assertThrows(StoragePropertiesException.class, + () -> context().vendStorageCredentials(token)); + Assertions.assertEquals("Azure SAS credential is expired", failure.getMessage()); + Assertions.assertNull(failure.getCause()); + } + + @Test + public void sasQueryExpiryIsCheckedAtBackendAccessInsteadOfBinding() { + Map token = Map.of( + "adls.sas-token.account.dfs.core.windows.net", + "sv=2024-01-01&se=2000-01-01T00%3A00%3A00Z&sig=expired", + "adls.sas-token-expires-at-ms.account.dfs.core.windows.net", "4102444800000"); + String path = "abfss://container@account.dfs.core.windows.net/table/data.parquet"; + DefaultConnectorContext context = context(); + + Assertions.assertEquals(path, context.normalizeStorageUri(path, token)); + Assertions.assertEquals(TFileType.FILE_S3.name(), context.getBackendFileType(path, token)); + StoragePropertiesException failure = Assertions.assertThrows(StoragePropertiesException.class, + () -> context.vendStorageCredentials(token)); + Assertions.assertEquals("Azure SAS credential is expired", failure.getMessage()); + } + + @Test + public void vendedSasReplacesOldAuthenticationWithoutRebindingIt() { + Map catalog = Map.of( + "azure.account_name", "account", + "azure.endpoint", "https://account.blob.core.windows.net", + "azure.account_key", "static-key", + "azure.auth_type", "SharedKey"); + Map token = new HashMap<>(); + token.put("adls.sas-token.account.dfs.core.windows.net", "sig=temporary&se=2100-01-01T00:00:00Z"); + token.put("azure.auth_type", "OAuth2"); + token.put("AZURE_AUTH_TYPE", "SharedKey"); + token.put("azure.account_key", "old-key"); + token.put("azure.oauth2_client_secret", "old-secret"); + token.put("azure.oauth2_client_id", "old-client"); + Map originalToken = new HashMap<>(token); + DefaultConnectorContext context = new DefaultConnectorContext("c", 1L, + () -> new ExecutionAuthenticator() {}, Collections::emptyMap, () -> catalog); + + Map backend = context.vendStorageCredentials(token); + + Assertions.assertEquals("SAS", backend.get("AZURE_AUTH_TYPE")); + Assertions.assertEquals("sig=temporary&se=2100-01-01T00:00:00Z", backend.get("AZURE_SAS_TOKEN")); + Assertions.assertFalse(backend.containsKey("AZURE_ACCOUNT_KEY")); + Assertions.assertFalse(backend.containsKey("AZURE_CLIENT_ID")); + Assertions.assertFalse(backend.containsKey("AZURE_CLIENT_SECRET")); + Assertions.assertEquals(originalToken, token); + Assertions.assertEquals("static-key", catalog.get("azure.account_key")); + } + + @Test + public void invalidRecognizedSasCannotFallBackToStaticSharedKey() { + Map catalog = Map.of("azure.account_name", "account", "azure.account_key", "static-key"); + StorageAdapter staticBinding = StorageAdapter.ofProvider("AZURE", catalog); + DefaultConnectorContext context = new DefaultConnectorContext("c", 1L, + () -> new ExecutionAuthenticator() {}, + () -> Map.of(StorageTypeId.AZURE, staticBinding), () -> catalog); + String path = "abfss://container@account.dfs.core.windows.net/table/data.parquet"; + Map token = new HashMap<>(); + token.put("adls.sas-token.account.dfs.core.windows.net", null); + + Assertions.assertThrows(StoragePropertiesException.class, () -> context.vendStorageCredentials(token)); + Assertions.assertThrows(StoragePropertiesException.class, () -> context.normalizeStorageUri(path, token)); + Assertions.assertThrows(StoragePropertiesException.class, + () -> context.newStorageUriNormalizer(token).apply(path)); + Assertions.assertThrows(StoragePropertiesException.class, () -> context.getBackendFileType(path, token)); + } + + @Test + public void azureVendedCredentialsRetainARealHdfsBinding() { + Map token = new HashMap<>(); + token.put("adls.sas-token.account.dfs.core.windows.net", "sig=temporary"); + token.put("fs.defaultFS", "hdfs://namenode:8020"); + token.put("hadoop.username", "mixed-user"); + DefaultConnectorContext context = context(); + String hdfsPath = "hdfs://namenode:8020/table/data.parquet"; + String azurePath = "abfss://container@account.dfs.core.windows.net/table/data.parquet"; + + Map backend = context.vendStorageCredentials(token); + + Assertions.assertEquals("SAS", backend.get("AZURE_AUTH_TYPE")); + Assertions.assertEquals("hdfs://namenode:8020", backend.get("fs.defaultFS")); + Assertions.assertEquals("mixed-user", backend.get("hadoop.username")); + Assertions.assertEquals(TFileType.FILE_HDFS.name(), context.getBackendFileType(hdfsPath, token)); + Assertions.assertEquals(TFileType.FILE_S3.name(), context.getBackendFileType(azurePath, token)); + Assertions.assertEquals(hdfsPath, context.normalizeStorageUri(hdfsPath, token)); + } + + @Test + public void unrelatedVendedPropertiesRetainEmptyLegacyResult() { + Assertions.assertTrue(context().vendStorageCredentials(Map.of("unknown.option", "value")).isEmpty()); + } + + @Test + public void refreshedSasIsReboundForEachCredentialRequest() { + DefaultConnectorContext context = context(); + String key = "adls.sas-token.account.dfs.core.windows.net"; + + Map first = context.vendStorageCredentials(Map.of(key, "sig=first")); + Map second = context.vendStorageCredentials(Map.of(key, "sig=second")); + + Assertions.assertEquals("sig=first", first.get("AZURE_SAS_TOKEN")); + Assertions.assertEquals("sig=second", second.get("AZURE_SAS_TOKEN")); + Assertions.assertEquals(first.keySet(), second.keySet()); + } + + @Test + public void normalizesS3VendedCredentialsThroughExistingPath() { + Map backend = context().vendStorageCredentials(Map.of( + "s3.endpoint", "https://s3.us-west-2.amazonaws.com", + "s3.access-key-id", "temporary-access", + "s3.secret-access-key", "temporary-secret", + "s3.session-token", "temporary-session")); + + Assertions.assertEquals("temporary-access", backend.get("AWS_ACCESS_KEY")); + Assertions.assertEquals("temporary-secret", backend.get("AWS_SECRET_KEY")); + Assertions.assertEquals("temporary-session", backend.get("AWS_TOKEN")); + Assertions.assertFalse(backend.keySet().stream().anyMatch(key -> key.startsWith("AZURE_"))); + } + @Test public void emptyOrNullInputYieldsEmpty() { // WHY: a non-REST / no-token table passes an empty map; the bridge must short-circuit to diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/AzureCatalogPropertyPersistTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/AzureCatalogPropertyPersistTest.java new file mode 100644 index 00000000000000..e9de89b747bb5a --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/AzureCatalogPropertyPersistTest.java @@ -0,0 +1,153 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource; + +import org.apache.doris.analysis.StorageBackend; +import org.apache.doris.analysis.StorageDesc; +import org.apache.doris.common.util.LocationPath; +import org.apache.doris.datasource.storage.StorageAdapter; +import org.apache.doris.datasource.storage.StorageTypeId; +import org.apache.doris.foundation.property.StoragePropertiesException; +import org.apache.doris.persist.gson.GsonUtils; +import org.apache.doris.thrift.TFileType; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.stream.Stream; + +class AzureCatalogPropertyPersistTest { + @ParameterizedTest + @MethodSource("authenticationProperties") + void authenticationRebindsAfterCatalogSerialization(Map input, Map expected) { + CatalogProperty original = new CatalogProperty(null, input); + original.setPluginDerivedStorageDefaultsSupplier(Collections::emptyMap); + Map before = original.getStorageAdaptersMap().get(StorageTypeId.AZURE) + .getBackendConfigProperties(); + + CatalogProperty restored = GsonUtils.GSON.fromJson(GsonUtils.GSON.toJson(original), CatalogProperty.class); + restored.setPluginDerivedStorageDefaultsSupplier(Collections::emptyMap); + Map after = restored.getStorageAdaptersMap().get(StorageTypeId.AZURE) + .getBackendConfigProperties(); + + Assertions.assertEquals(input, restored.getProperties()); + Assertions.assertEquals(before, after); + Assertions.assertEquals(expected, after); + String location = "abfss://container@account.dfs.core.windows.net/path/file.parquet"; + LocationPath path = LocationPath.ofAdapters(location, restored.getStorageAdaptersMap()); + Assertions.assertEquals(location, path.getNormalizedLocation()); + Assertions.assertEquals(TFileType.FILE_S3, path.getTFileTypeForBE()); + } + + @ParameterizedTest + @MethodSource("objectStoreProperties") + void storageDescriptorRebindsRawPropertiesDuringGsonPostProcessing( + Map input, Map expected) { + StorageDesc original = new StorageDesc("azure", StorageBackend.StorageType.AZURE, input); + Assertions.assertEquals(expected, original.getBackendConfigProperties()); + + StorageDesc restored = GsonUtils.GSON.fromJson(GsonUtils.GSON.toJson(original), StorageDesc.class); + + Assertions.assertEquals(input, restored.getProperties()); + Assertions.assertEquals(StorageTypeId.AZURE, restored.getStorageAdapter().getType()); + Assertions.assertEquals(expected, restored.getBackendConfigProperties()); + } + + @Test + void expiredSasCanBeRestoredButCannotSupplyExecutionCredentials() { + Map input = Map.of("azure.account_name", "account", + "azure.sas_token", "sig=expired-test-token", "azure.sas_expiry_ms", "1"); + CatalogProperty original = new CatalogProperty(null, input); + CatalogProperty restored = GsonUtils.GSON.fromJson(GsonUtils.GSON.toJson(original), CatalogProperty.class); + restored.setPluginDerivedStorageDefaultsSupplier(Collections::emptyMap); + StorageAdapter adapter = restored.getStorageAdaptersMap().get(StorageTypeId.AZURE); + Assertions.assertEquals(input, restored.getProperties()); + Assertions.assertEquals(StorageTypeId.AZURE, adapter.getType()); + StoragePropertiesException catalogError = Assertions.assertThrows(StoragePropertiesException.class, + adapter::getBackendConfigProperties); + Assertions.assertEquals("Azure SAS credential is expired", catalogError.getMessage()); + + StorageDesc descriptor = new StorageDesc("azure", StorageBackend.StorageType.AZURE, input); + StorageDesc restoredDescriptor = GsonUtils.GSON.fromJson(GsonUtils.GSON.toJson(descriptor), StorageDesc.class); + Assertions.assertEquals(input, restoredDescriptor.getProperties()); + Assertions.assertEquals(StorageTypeId.AZURE, restoredDescriptor.getStorageAdapter().getType()); + StoragePropertiesException descriptorError = Assertions.assertThrows(StoragePropertiesException.class, + restoredDescriptor::getBackendConfigProperties); + Assertions.assertEquals("Azure SAS credential is expired", descriptorError.getMessage()); + } + + @Test + void backendAuthTypeDoesNotApplyTheOAuthCatalogRestrictionToSharedKeyInput() { + StorageAdapter adapter = StorageAdapter.ofProvider("AZURE", Map.of( + "type", "hms", "azure.account_name", "account", "azure.account_key", "key", + "AZURE_AUTH_TYPE", "OAuth2")); + + Assertions.assertEquals("SHARED_KEY", adapter.getBackendConfigProperties().get("AZURE_AUTH_TYPE")); + Assertions.assertFalse(adapter.getSpiProperties().matchedProperties().containsKey("AZURE_AUTH_TYPE")); + } + + @Test + void canonicalOAuthInputRetainsTheIcebergRestRestriction() { + Map input = new HashMap<>(oauthProperties()); + input.put("type", "hms"); + + UnsupportedOperationException error = Assertions.assertThrows(UnsupportedOperationException.class, + () -> StorageAdapter.ofProvider("AZURE", input)); + + Assertions.assertEquals("OAuth2 auth type is only supported for iceberg rest catalog", error.getMessage()); + } + + private static Stream objectStoreProperties() { + return Stream.of( + Arguments.of(Map.of( + "AZURE_ENDPOINT", "https://account.blob.core.windows.net", + "AZURE_ACCOUNT_NAME", "account", "AZURE_ACCOUNT_KEY", "legacy-key", + "AZURE_CONTAINER", "container"), Map.of( + "provider", "azure", "AZURE_AUTH_TYPE", "SHARED_KEY", + "AZURE_ENDPOINT", "https://account.blob.core.windows.net", + "AZURE_ACCOUNT_NAME", "account", "AZURE_ACCOUNT_KEY", "legacy-key")), + Arguments.of(Map.of( + "azure.account_name", "account", "azure.auth_type", "SAS", + "azure.sas_token", "sig=test-signature", "azure.sas_expiry_ms", "4102444800000"), Map.of( + "provider", "azure", "AZURE_AUTH_TYPE", "SAS", + "AZURE_ENDPOINT", "https://account.blob.core.windows.net", + "AZURE_ACCOUNT_NAME", "account", "AZURE_SAS_TOKEN", "sig=test-signature", + "AZURE_SAS_EXPIRY_MS", "4102444800000"))); + } + + private static Stream authenticationProperties() { + return Stream.concat(objectStoreProperties(), Stream.of(Arguments.of(oauthProperties(), Map.of( + "provider", "azure", "AZURE_AUTH_TYPE", "OAUTH2", + "AZURE_ENDPOINT", "https://account.blob.core.windows.net", "AZURE_ACCOUNT_NAME", "account", + "AZURE_CLIENT_ID", "client-id", "AZURE_CLIENT_SECRET", "client-secret", "AZURE_TENANT_ID", "tenant", + "AZURE_OAUTH_SERVER_URI", "https://login.microsoftonline.com/tenant/oauth2/token")))); + } + + private static Map oauthProperties() { + return Map.of("type", "iceberg", "iceberg.catalog.type", "rest", "fs.azure.support", "true", + "azure.auth_type", "OAuth2", "azure.oauth2_account_host", "account.dfs.core.windows.net", + "AZURE_CLIENT_ID", "client-id", "AZURE_CLIENT_SECRET", "client-secret", "AZURE_TENANT_ID", "tenant", + "azure.oauth2_server_uri", "https://login.microsoftonline.com/tenant/oauth2/token"); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/credentials/CredentialUtilsTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/credentials/CredentialUtilsTest.java index 41a8d405d7768d..9ec05ba3d76760 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/credentials/CredentialUtilsTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/credentials/CredentialUtilsTest.java @@ -93,6 +93,19 @@ public void testFilterCloudStoragePropertiesWithMultipleCloudTypes() { Assertions.assertFalse(filtered.containsKey("hdfs.namenode")); } + @Test + public void testAdlsCredentialsAreLeftToFilesystemProviders() { + String accountHost = "account.dfs.core.windows.net"; + Map rawCredentials = new HashMap<>(); + rawCredentials.put("adls.sas-token." + accountHost, "?sv=2024-01-01&sig=temporary"); + rawCredentials.put("adls.sas-token-expires-at-ms." + accountHost, "4102444800000"); + rawCredentials.put("table.name", "ignored"); + + Assertions.assertTrue(CredentialUtils.filterCloudStorageProperties(rawCredentials).isEmpty()); + Assertions.assertEquals("?sv=2024-01-01&sig=temporary", + rawCredentials.get("adls.sas-token." + accountHost)); + } + @Test public void testFilterCloudStoragePropertiesWithEmptyInput() { Map filtered = CredentialUtils.filterCloudStorageProperties(new HashMap<>()); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/scan/FileQueryScanNodeStorageAccessTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/scan/FileQueryScanNodeStorageAccessTest.java new file mode 100644 index 00000000000000..48eb7a8708a772 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/scan/FileQueryScanNodeStorageAccessTest.java @@ -0,0 +1,192 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.scan; + +import org.apache.doris.analysis.TupleDescriptor; +import org.apache.doris.analysis.TupleId; +import org.apache.doris.catalog.TableIf; +import org.apache.doris.connector.DefaultConnectorContext; +import org.apache.doris.connector.spi.ConnectorStorageAccess; +import org.apache.doris.connector.spi.scan.ConnectorScanRange; +import org.apache.doris.datasource.split.PluginDrivenSplit; +import org.apache.doris.datasource.storage.StorageAdapter; +import org.apache.doris.kerberos.ExecutionAuthenticator; +import org.apache.doris.planner.PlanNodeId; +import org.apache.doris.planner.ScanContext; +import org.apache.doris.qe.SessionVariable; +import org.apache.doris.spi.Split; +import org.apache.doris.system.Backend; +import org.apache.doris.thrift.TFileFormatType; +import org.apache.doris.thrift.TFileRangeDesc; +import org.apache.doris.thrift.TFileScanRangeParams; +import org.apache.doris.thrift.TFileType; +import org.apache.doris.thrift.TScanRangeLocations; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.ValueSource; + +import java.lang.reflect.Method; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +/** Real provider -> engine resolver -> split -> node/range Thrift, without opening any remote file. */ +class FileQueryScanNodeStorageAccessTest { + @ParameterizedTest + @ValueSource(strings = { + "abfss://container@account.dfs.core.windows.net/table/data.parquet", + "https://account.blob.core.windows.net/container/table/data.parquet" + }) + void azureSasProducesNativeRangeAndNodeParameters(String uri) throws Exception { + DefaultConnectorContext context = new DefaultConnectorContext("c", 1L); + ConnectorStorageAccess access = context.newStorageAccessResolver(Map.of( + "adls.sas-token.account.dfs.core.windows.net", "sig=temporary&se=2100-01-01T00:00:00Z")) + .apply(uri); + TestScanNode node = new TestScanNode(); + + TFileRangeDesc range = node.translate(access); + + Assertions.assertEquals(TFileType.FILE_S3, range.getFileType()); + Assertions.assertEquals(uri, range.getPath()); + Assertions.assertEquals(16, range.getStartOffset()); + Assertions.assertEquals(32, range.getSize()); + Assertions.assertEquals(128, range.getFileSize()); + Assertions.assertFalse(range.isSetFsName()); + Assertions.assertFalse(node.getFileScanRangeParams().isSetHdfsParams()); + Assertions.assertEquals(TFileType.FILE_S3, node.getFileScanRangeParams().getFileType()); + Assertions.assertEquals(access.getBackendProperties(), node.getFileScanRangeParams().getProperties()); + Assertions.assertEquals("azure", node.getFileScanRangeParams().getProperties().get("provider")); + Assertions.assertTrue(node.getFileScanRangeParams().getProperties().keySet().stream() + .allMatch(key -> key.equals("provider") || key.startsWith("AZURE_"))); + } + + @Test + void genuineHdfsKeepsItsHadoopChannel() throws Exception { + StorageAdapter adapter = StorageAdapter.ofProvider("HDFS", Map.of("fs.defaultFS", "hdfs://namenode:8020", + "hadoop.username", "reader")); + ConnectorStorageAccess access = new DefaultConnectorContext("c", 1L, + () -> new ExecutionAuthenticator() {}, () -> Map.of(adapter.getType(), adapter)) + .newStorageAccessResolver(Collections.emptyMap()).apply("hdfs://namenode:8020/table/data.parquet"); + TestScanNode node = new TestScanNode(); + + TFileRangeDesc range = node.translate(access); + + Assertions.assertEquals(TFileType.FILE_HDFS, range.getFileType()); + Assertions.assertEquals("hdfs://namenode:8020", range.getFsName()); + Assertions.assertTrue(node.getFileScanRangeParams().isSetHdfsParams()); + Assertions.assertFalse(node.getFileScanRangeParams().isSetProperties()); + } + + @ParameterizedTest + @CsvSource({"https://example.org/data.parquet,FILE_HTTP", "s3://bucket/data.parquet,FILE_S3", + "hdfs://namenode/data.parquet,FILE_HDFS", "file:///tmp/data.parquet,FILE_LOCAL"}) + void connectorsWithoutStorageAccessRetainSchemeRouting(String path, String expectedFileType) { + ConnectorScanRange range = new ConnectorScanRange() { + @Override + public Optional getPath() { + return Optional.of(path); + } + + @Override + public Map getProperties() { + return Collections.emptyMap(); + } + }; + + Assertions.assertEquals(TFileType.valueOf(expectedFileType), new PluginDrivenSplit(range).getLocationType()); + } + + private static final class TestScanNode extends FileQueryScanNode { + TestScanNode() { + super(new PlanNodeId(0), new TupleDescriptor(new TupleId(0)), "storage-contract", + ScanContext.EMPTY, false, new SessionVariable()); + params = new TFileScanRangeParams(); + params.setFormatType(TFileFormatType.FORMAT_PARQUET); + } + + TFileRangeDesc translate(ConnectorStorageAccess access) throws Exception { + ConnectorScanRange range = new ConnectorScanRange() { + @Override + public Optional getPath() { + return Optional.of(access.getNormalizedUri()); + } + + @Override + public Optional getBackendFileType() { + return Optional.of(access.getBackendFileType()); + } + + @Override + public long getStart() { + return 16; + } + + @Override + public long getLength() { + return 32; + } + + @Override + public long getFileSize() { + return 128; + } + + @Override + public boolean isPartitionBearing() { + return true; + } + + @Override + public Map getProperties() { + return Collections.emptyMap(); + } + }; + Method translate = FileQueryScanNode.class.getDeclaredMethod("splitToScanRange", Backend.class, + Map.class, Split.class, List.class, boolean.class); + translate.setAccessible(true); + Backend backend = new Backend(1L, "127.0.0.1", 9050); + TScanRangeLocations locations = (TScanRangeLocations) translate.invoke(this, backend, + access.getBackendProperties(), new PluginDrivenSplit(range), Collections.emptyList(), true); + return locations.getScanRange().getExtScanRange().getFileScanRange().getRanges().get(0); + } + + @Override + protected TFileFormatType getFileFormatType() { + return TFileFormatType.FORMAT_PARQUET; + } + + @Override + protected List getPathPartitionKeys() { + return Collections.emptyList(); + } + + @Override + protected TableIf getTargetTable() { + throw new UnsupportedOperationException("Range transport does not access table metadata"); + } + + @Override + protected Map getLocationProperties() { + throw new UnsupportedOperationException("The resolver supplies the selected location parameters"); + } + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/storage/StorageAdapterAccessValidationTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/storage/StorageAdapterAccessValidationTest.java new file mode 100644 index 00000000000000..58b87c492fe2f0 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/storage/StorageAdapterAccessValidationTest.java @@ -0,0 +1,171 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.storage; + +import org.apache.doris.filesystem.FileSystem; +import org.apache.doris.filesystem.properties.BackendStorageProperties; +import org.apache.doris.filesystem.properties.FileSystemProperties; +import org.apache.doris.filesystem.properties.FsCacheKeys; +import org.apache.doris.filesystem.spi.FileSystemProvider; +import org.apache.doris.foundation.property.StoragePropertiesException; +import org.apache.doris.fs.FileSystemPluginManager; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.mockito.Mockito; + +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +public class StorageAdapterAccessValidationTest { + + private final FileSystemProperties properties = Mockito.mock(FileSystemProperties.class); + private final BackendStorageProperties backend = Mockito.mock(BackendStorageProperties.class); + + @AfterEach + void resetPluginManager() { + // Match the other provider-injection tests: later classes use the built-in registry. + StorageAdapter.initPluginManager(null); + } + + @Test + void rejectsInvalidCredentialsBeforeComputingBackendMap() { + StorageAdapter adapter = createTestAdapter(); + StoragePropertiesException failure = new StoragePropertiesException("Test credential is expired"); + Mockito.doThrow(failure).when(properties).validateForAccess(); + + Assertions.assertSame(failure, + Assertions.assertThrows(StoragePropertiesException.class, adapter::getBackendConfigProperties)); + Mockito.verifyNoInteractions(backend); + } + + @Test + void validatesEveryAccessWithoutRecomputingOrExposingCachedMap() { + StorageAdapter adapter = createTestAdapter(); + Map first = adapter.getBackendConfigProperties(); + Map expected = new HashMap<>(first); + first.put("caller.option", "must-not-be-cached"); + + Map second = adapter.getBackendConfigProperties(); + + Assertions.assertEquals(expected, second); + Assertions.assertNotSame(first, second); + Assertions.assertEquals("fingerprint", second.get(FsCacheKeys.fsCacheKeyProperty("test"))); + Mockito.verify(properties, Mockito.times(2)).validateForAccess(); + Mockito.verify(backend, Mockito.times(1)).toMap(); + } + + @Test + void rejectsCachedCredentialsWhenProviderReportsExpiry() { + StorageAdapter adapter = createTestAdapter(); + Assertions.assertEquals("temporary", adapter.getBackendConfigProperties().get("credential")); + StoragePropertiesException failure = new StoragePropertiesException("Test credential is expired"); + // Model the provider's clock crossing expiry after the first successful access, without sleeping. + Mockito.doThrow(failure).when(properties).validateForAccess(); + + Assertions.assertSame(failure, + Assertions.assertThrows(StoragePropertiesException.class, adapter::getBackendConfigProperties)); + Mockito.verify(properties, Mockito.times(2)).validateForAccess(); + Mockito.verify(backend, Mockito.times(1)).toMap(); + } + + @Test + void bindsExpiredAzureSasButRejectsBackendAccess() { + StorageAdapter adapter = StorageAdapter.ofProvider("AZURE", Map.of( + "azure.account_name", "account", + "azure.sas_token", "sig=temporary", + "azure.sas_expiry_ms", "1")); + + Assertions.assertEquals(StorageTypeId.AZURE, adapter.getType()); + Assertions.assertDoesNotThrow(adapter.getSpiProperties()::validate); + StoragePropertiesException failure = Assertions.assertThrows(StoragePropertiesException.class, + adapter::getBackendConfigProperties); + Assertions.assertEquals("Azure SAS credential is expired", failure.getMessage()); + Assertions.assertNull(failure.getCause()); + } + + @ParameterizedTest + @CsvSource({ + "S3, s3.us-west-2.amazonaws.com", + "OSS, oss-cn-beijing.aliyuncs.com", + "GCS, storage.googleapis.com" + }) + void preservesNonAzureObjectStoreParameters(String provider, String endpoint) { + // These providers retain their existing S3-compatible input aliases and default access validation. + StorageAdapter adapter = StorageAdapter.ofProvider(provider, Map.of( + "s3.endpoint", endpoint, + "s3.access_key", "access-key", + "s3.secret_key", "secret-key")); + + Map first = adapter.getBackendConfigProperties(); + Assertions.assertEquals(first, adapter.getBackendConfigProperties()); + Assertions.assertEquals("access-key", first.get("AWS_ACCESS_KEY")); + Assertions.assertEquals("secret-key", first.get("AWS_SECRET_KEY")); + Assertions.assertFalse(first.keySet().stream().anyMatch(key -> key.startsWith("AZURE_"))); + } + + @Test + void preservesHdfsParametersAndCacheFingerprint() { + StorageAdapter adapter = StorageAdapter.ofProvider("HDFS", Map.of( + "uri", "hdfs://namenode/data", + "hadoop.username", "test-user")); + + Map first = adapter.getBackendConfigProperties(); + Assertions.assertEquals(first, adapter.getBackendConfigProperties()); + Assertions.assertEquals(StorageTypeId.HDFS, adapter.getType()); + Assertions.assertEquals(adapter.getFsCacheFingerprint(), first.get(FsCacheKeys.fsCacheKeyProperty("hdfs"))); + Assertions.assertFalse(first.keySet().stream().anyMatch(key -> key.startsWith("AZURE_"))); + } + + private StorageAdapter createTestAdapter() { + Mockito.when(properties.providerName()).thenReturn("TEST"); + Mockito.when(properties.getSupportedSchemes()).thenReturn(Set.of("test")); + Mockito.when(properties.fsCacheFingerprint()).thenReturn("fingerprint"); + Mockito.when(properties.toBackendProperties()).thenReturn(Optional.of(backend)); + Mockito.when(backend.toMap()).thenReturn(Map.of("credential", "temporary")); + FileSystemPluginManager manager = new FileSystemPluginManager(); + manager.registerProvider(new FileSystemProvider() { + @Override + public String name() { + return "TEST"; + } + + @Override + public boolean supports(Map raw) { + return true; + } + + @Override + public FileSystemProperties bind(Map raw) { + return properties; + } + + @Override + public FileSystem create(Map raw) { + throw new UnsupportedOperationException("No client needed for access-validation tests"); + } + }); + StorageAdapter.initPluginManager(manager); + return StorageAdapter.ofProvider("TEST", Map.of()); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/storage/StorageAdapterBindingOriginTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/storage/StorageAdapterBindingOriginTest.java new file mode 100644 index 00000000000000..edb6d3a85a7451 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/storage/StorageAdapterBindingOriginTest.java @@ -0,0 +1,164 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.storage; + +import org.apache.doris.connector.iceberg.IcebergCatalogFactory; +import org.apache.doris.connector.iceberg.IcebergConnector; +import org.apache.doris.connector.spi.ConnectorContext; +import org.apache.doris.connector.spi.ConnectorStorageContext; +import org.apache.doris.filesystem.properties.StorageProperties; +import org.apache.doris.fs.FileSystemPluginManager; + +import org.apache.hadoop.conf.Configuration; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.lang.reflect.Method; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class StorageAdapterBindingOriginTest { + + private FileSystemPluginManager manager; + + @BeforeEach + void installBuiltins() { + manager = new FileSystemPluginManager(); + manager.loadBuiltins(); + StorageAdapter.initPluginManager(manager); + } + + @AfterEach + void resetPluginManager() { + StorageAdapter.initPluginManager(null); + } + + @Test + void bindingOriginDoesNotChangeWhenRawPropertiesChange() { + Map raw = new HashMap<>(Map.of( + "azure.account_name", "account", "azure.account_key", "key")); + StorageAdapter hdfs = StorageAdapter.ofAll(raw).stream() + .filter(adapter -> adapter.getType() == StorageTypeId.HDFS).findFirst().get(); + + Assertions.assertFalse(hdfs.isExplicitlyConfigured()); + raw.put("hadoop.username", "test-user"); + Assertions.assertFalse(hdfs.isExplicitlyConfigured()); + } + + @Test + void directProviderBindingIsNotAnInferredFallback() { + StorageAdapter hdfs = StorageAdapter.ofProvider("HDFS", Map.of()); + + Assertions.assertTrue(hdfs.isExplicitlyConfigured()); + Assertions.assertFalse(hdfs.getSpiProperties().isSyntheticDefault()); + } + + @Test + void explicitAndUriMatchedHdfsRemainConfigured() { + for (Map raw : List.of( + Map.of("fs.hdfs.support", "true"), + Map.of("uri", "hdfs://namenode/warehouse"), + Map.of("URI", "viewfs://mount/warehouse"))) { + List result = StorageAdapter.ofAll(raw); + + Assertions.assertEquals(1, result.size()); + Assertions.assertEquals(StorageTypeId.HDFS, result.get(0).getType()); + Assertions.assertTrue(result.get(0).isExplicitlyConfigured()); + } + } + + @Test + void mixedAzureAndHdfsBothRemainConfigured() { + List result = StorageAdapter.ofAll(Map.of( + "azure.account_name", "account", "azure.account_key", "key", + "uri", "hdfs://namenode/warehouse")); + + Assertions.assertEquals(2, result.size()); + Assertions.assertTrue(result.stream().allMatch(StorageAdapter::isExplicitlyConfigured)); + } + + @Test + void hadoopResourcesRemainInIcebergS3Configuration(@TempDir Path tmp) throws Exception { + Path xml = tmp.resolve("storage-site.xml"); + Files.writeString(xml, "" + + "fs.s3a.connection.ssl.enabledfalse" + + "fs.s3a.proxy.hostxml-proxy" + + ""); + Map raw = Map.of( + "iceberg.catalog.type", "hadoop", + "warehouse", "s3a://bucket/warehouse", + "s3.endpoint", "s3.us-east-1.amazonaws.com", + "s3.region", "us-east-1", + "s3.access_key", "access-key", + "s3.secret_key", "secret-key", + "hadoop.config.resources", xml.toString(), + "fs.s3a.proxy.host", "inline-proxy"); + List bindings = new ArrayList<>(manager.bindAll(raw)); + StorageProperties hdfs = bindings.stream() + .filter(binding -> "HDFS".equals(binding.providerName())).findFirst().get(); + + Assertions.assertFalse(hdfs.isSyntheticDefault()); + Assertions.assertEquals("false", + hdfs.toHadoopProperties().get().toHadoopConfigurationMap().get("fs.s3a.connection.ssl.enabled")); + Assertions.assertEquals(bindings, IcebergCatalogFactory.selectEffectiveStorages(bindings)); + + ConnectorContext context = new ConnectorContext() { + @Override + public String getCatalogName() { + return "storage-origin-test"; + } + + @Override + public long getCatalogId() { + return 1L; + } + + @Override + public ConnectorStorageContext getStorageContext() { + return new ConnectorStorageContext() { + @Override + public List getStorageProperties() { + return bindings; + } + }; + } + }; + try (IcebergConnector connector = new IcebergConnector(raw, context)) { + // Exercise the actual metadata configuration consumer without opening a remote catalog. + Method buildConfig = IcebergConnector.class.getDeclaredMethod("buildStorageHadoopConfig"); + buildConfig.setAccessible(true); + @SuppressWarnings("unchecked") + Map storageConfig = (Map) buildConfig.invoke(connector); + Assertions.assertEquals("false", storageConfig.get("fs.s3a.connection.ssl.enabled")); + Assertions.assertEquals("inline-proxy", storageConfig.get("fs.s3a.proxy.host")); + Assertions.assertEquals("access-key", storageConfig.get("fs.s3a.access.key")); + + Configuration configuration = IcebergCatalogFactory.buildHadoopConfiguration(raw, storageConfig); + Assertions.assertEquals("false", configuration.get("fs.s3a.connection.ssl.enabled")); + Assertions.assertEquals("inline-proxy", configuration.get("fs.s3a.proxy.host")); + Assertions.assertEquals("org.apache.hadoop.fs.s3a.S3AFileSystem", configuration.get("fs.s3a.impl")); + } + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/fs/FileSystemPluginManagerTest.java b/fe/fe-core/src/test/java/org/apache/doris/fs/FileSystemPluginManagerTest.java index 29feec06d54b2e..648bb078a916e4 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/fs/FileSystemPluginManagerTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/fs/FileSystemPluginManagerTest.java @@ -23,6 +23,7 @@ import org.apache.doris.filesystem.properties.FileSystemProperties; import org.apache.doris.filesystem.properties.StorageKind; import org.apache.doris.filesystem.spi.FileSystemProvider; +import org.apache.doris.foundation.property.StoragePropertiesException; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -31,7 +32,9 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; +import java.util.function.BiFunction; public class FileSystemPluginManagerTest { @@ -118,13 +121,203 @@ public void bindAll_returnsEmptyListWhenNoProviderSupports() { Assertions.assertTrue(bound.isEmpty()); } - // NOTE: real object-store providers (S3/OSS/COS/OBS) are runtime directory-loaded plugins - // (Env.loadPlugins), NOT on fe-core's unit-test classpath (fe-core pom: "fe-filesystem impl - // modules: runtime dependencies removed in Phase 4 P4.1"). End-to-end binding against the real - // providers is therefore covered by P1-T06 (docker / full plugin classpath), not here. + @Test + public void bindAllExceptSkipsReplacedAuthenticationBeforeBindingAndKeepsRealHdfs() { + FileSystemPluginManager manager = new FileSystemPluginManager(); + manager.loadBuiltins(); + Map raw = Map.of("azure.account_name", "account", "azure.account_key", "old-key", + "azure.auth_type", "invalid-old-authentication", "fs.defaultFS", "hdfs://namenode:8020", + "hadoop.username", "reader"); + Assertions.assertThrows(StoragePropertiesException.class, () -> manager.bindAll(raw)); + + List remaining = manager.bindAllExcept(raw, Set.of("aZuRe")); + + Assertions.assertEquals(1, remaining.size()); + Assertions.assertEquals("HDFS", remaining.get(0).providerName()); + Assertions.assertFalse(remaining.get(0).isSyntheticDefault()); + Assertions.assertEquals("reader", remaining.get(0).toBackendProperties().orElseThrow().toMap() + .get("hadoop.username")); + Assertions.assertEquals("invalid-old-authentication", raw.get("azure.auth_type")); + } + + @Test + public void bindAll_marksOnlyTheRealHdfsFallbackCreationPath() { + FileSystemPluginManager manager = new FileSystemPluginManager(); + manager.loadBuiltins(); + Map raw = Map.of("azure.account_name", "account", "azure.account_key", "key"); + + List result = manager.bindAll(raw); + + Assertions.assertEquals(2, result.size()); + Assertions.assertEquals("HDFS", result.get(0).providerName()); + Assertions.assertTrue(result.get(0).isSyntheticDefault()); + Assertions.assertEquals("AZURE", result.get(1).providerName()); + Assertions.assertFalse(result.get(1).isSyntheticDefault()); + Assertions.assertEquals(raw, result.get(0).rawProperties()); + } + + @Test + public void bindAll_keepsHdfsOnlyFallback() { + FileSystemPluginManager manager = new FileSystemPluginManager(); + manager.loadBuiltins(); + + List result = manager.bindAll(Map.of()); + + Assertions.assertEquals(1, result.size()); + Assertions.assertEquals("HDFS", result.get(0).providerName()); + Assertions.assertTrue(result.get(0).isSyntheticDefault()); + } + + @Test + public void bindAll_doesNotMarkExplicitOrUriMatchedHdfsAsSynthetic() { + FileSystemPluginManager manager = new FileSystemPluginManager(); + manager.loadBuiltins(); + for (Map raw : List.of( + Map.of("fs.hdfs.support", "true"), + Map.of("uri", "hdfs://namenode/warehouse"), + Map.of("URI", "viewfs://mount/warehouse"), + Map.of("hadoop.username", "test-user"))) { + List result = manager.bindAll(raw); + + Assertions.assertEquals(1, result.size()); + Assertions.assertEquals("HDFS", result.get(0).providerName()); + Assertions.assertFalse(result.get(0).isSyntheticDefault()); + } + } + + @Test + public void bindAll_preservesRealMixedAzureAndHdfsBindings() { + FileSystemPluginManager manager = new FileSystemPluginManager(); + manager.loadBuiltins(); + Map raw = Map.of("azure.account_name", "account", "azure.account_key", "key", + "uri", "hdfs://namenode/warehouse"); + + List result = manager.bindAll(raw); + + Assertions.assertEquals(2, result.size()); + Assertions.assertTrue(result.stream().anyMatch(binding -> "HDFS".equals(binding.providerName()))); + Assertions.assertTrue(result.stream().anyMatch(binding -> "AZURE".equals(binding.providerName()))); + Assertions.assertTrue(result.stream().noneMatch(FileSystemProperties::isSyntheticDefault)); + } + + @Test + public void bindVended_keepsTypedBindingWithoutRebindingOrDefaultHdfs() { + FileSystemPluginManager manager = new FileSystemPluginManager(); + FileSystemProperties azure = new FakeFsProps("AZURE"); + manager.registerProvider(vendedProvider("AZURE", (credentials, catalog) -> Optional.of(azure))); + manager.registerProvider(nonSupportingProvider("HDFS")); + + List result = manager.bindVended(Map.of("token", "temporary"), Map.of()).get(); + + Assertions.assertEquals(List.of(azure), result); + // Both raw binding implementations throw: neither Azure rebinding nor default HDFS is allowed. + } + + @Test + public void bindVended_declinesUnknownDialectWithoutRunningRawBinding() { + FileSystemPluginManager manager = new FileSystemPluginManager(); + manager.registerProvider(vendedProvider("AZURE", (credentials, catalog) -> Optional.empty())); + + Assertions.assertFalse(manager.bindVended(Map.of("other.token", "temporary"), Map.of()).isPresent()); + } + + @Test + public void bindVended_propagatesRecognizedCredentialFailure() { + FileSystemPluginManager manager = new FileSystemPluginManager(); + StoragePropertiesException failure = new StoragePropertiesException("Invalid provider credential"); + manager.registerProvider(vendedProvider("AZURE", (credentials, catalog) -> { + throw failure; + })); + + Assertions.assertSame(failure, Assertions.assertThrows(StoragePropertiesException.class, + () -> manager.bindVended(Map.of("token", "invalid"), Map.of()))); + } + + @Test + public void bindVended_preservesOtherProvidersAndTheirExclusivity() { + FileSystemPluginManager manager = new FileSystemPluginManager(); + FileSystemProperties azure = new FakeFsProps("AZURE"); + FileSystemProperties jfs = new FakeFsProps("JFS"); + FileSystemProperties ossHdfs = new FakeFsProps("OSS_HDFS"); + FileSystemProperties s3 = new FakeFsProps("S3"); + manager.registerProvider(vendedProvider("AZURE", (credentials, catalog) -> Optional.of(azure))); + manager.registerProvider(bindingProvider("JFS", jfs)); + manager.registerProvider(bindingProvider("HDFS", new FakeFsProps("HDFS"))); + manager.registerProvider(bindingProvider("OSS_HDFS", ossHdfs)); + manager.registerProvider(bindingProvider("OSS", new FakeFsProps("OSS"))); + manager.registerProvider(bindingProvider("S3", s3)); + + List result = manager.bindVended(Map.of("token", "temporary"), Map.of()).get(); + + Assertions.assertEquals(List.of(azure, jfs, ossHdfs, s3), result); + } + + @Test + public void bindVended_preservesExplicitFlagSuppressionOfGuessProviders() { + FileSystemPluginManager manager = new FileSystemPluginManager(); + FileSystemProperties azure = new FakeFsProps("AZURE"); + manager.registerProvider(vendedProvider("AZURE", (credentials, catalog) -> Optional.of(azure))); + manager.registerProvider(bindingProvider("S3", new FakeFsProps("S3"))); + + List result = manager.bindVended( + Map.of("token", "temporary", "fs.azure.support", "true"), Map.of()).get(); + + Assertions.assertEquals(List.of(azure), result); + } + + @Test + public void bindVended_supportsUnlistedProviderAndSuppliesCatalogContext() { + FileSystemPluginManager manager = new FileSystemPluginManager(); + FileSystemProperties custom = new FakeFsProps("CUSTOM"); + Map token = Map.of("custom.token", "temporary"); + Map connection = Map.of("custom.endpoint", "endpoint"); + manager.registerProvider(vendedProvider("CUSTOM", (credentials, catalog) -> { + Assertions.assertSame(token, credentials); + Assertions.assertSame(connection, catalog); + return Optional.of(custom); + })); + + Assertions.assertEquals(List.of(custom), manager.bindVended(token, connection).get()); + } // ---- helpers ---- + private static FileSystemProvider vendedProvider(String name, + BiFunction, Map, Optional> bindVended) { + return new FileSystemProvider() { + @Override + public String name() { + return name; + } + + @Override + public boolean supports(Map properties) { + return true; + } + + @Override + public boolean supportsGuess(Map properties) { + return true; + } + + @Override + public Optional bindVended(Map credentials, + Map catalogProperties) { + return bindVended.apply(credentials, catalogProperties); + } + + @Override + public FileSystemProperties bind(Map properties) { + throw new AssertionError("Vended binding must not be rebound through a raw property map"); + } + + @Override + public FileSystem create(Map properties) { + throw new AssertionError("Binding must not create a client"); + } + }; + } + private static FileSystemProvider bindingProvider( String name, FileSystemProperties bound) { return new FileSystemProvider() { diff --git a/fe/fe-core/src/test/java/org/apache/doris/pluginapiversion/PluginApiVersionWiringTest.java b/fe/fe-core/src/test/java/org/apache/doris/pluginapiversion/PluginApiVersionWiringTest.java index 2ae56fb673660b..2ab330f536a745 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/pluginapiversion/PluginApiVersionWiringTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/pluginapiversion/PluginApiVersionWiringTest.java @@ -112,7 +112,27 @@ public void connectorPluginWithoutOpenCsvContractIsRefused() throws IOException ConnectorPluginManager manager = new ConnectorPluginManager(); manager.loadPlugins(Collections.singletonList(connectorPluginRoot("8.0"))); Assertions.assertFalse(manager.getRegisteredTypes().contains("version_probe"), - "API 8 plugins omit the OpenCSV semantic flag and must not load on an API 9 engine"); + "API 8 plugins omit the OpenCSV semantic flag and must not load on the current engine"); + } + + @Test + public void connectorPluginWithoutLocationPrefixMatchingIsRefused() throws IOException { + ApiVersionGate gate = ApiVersionGate.forFamily("connector", ConnectorProvider.class); + Assertions.assertEquals("14.0", gate.getExpectedVersion()); + ConnectorPluginManager manager = new ConnectorPluginManager(); + + manager.loadPlugins(Collections.singletonList(connectorPluginRoot("10.0"))); + + Assertions.assertFalse(manager.getRegisteredTypes().contains("version_probe"), + "a plugin from before provider-owned location prefix matching must not be admitted"); + } + + @Test + public void connectorPluginWithoutNativeStorageBindingsIsRefused() throws IOException { + ConnectorPluginManager manager = new ConnectorPluginManager(); + manager.loadPlugins(Collections.singletonList(connectorPluginRoot("13.0"))); + Assertions.assertFalse(manager.getRegisteredTypes().contains("version_probe"), + "API 13 plugins omit native storage bindings and must not load on an API 14 engine"); } @Test @@ -150,6 +170,18 @@ public void filesystemPluginDeclaringAnotherMajorIsRefused() throws IOException "an incompatible filesystem plugin must not join the storage routing table"); } + @Test + public void filesystemPluginWithoutLocationPrefixMatchingIsRefused() throws IOException { + ApiVersionGate gate = ApiVersionGate.forFamily("filesystem", FileSystemProvider.class); + Assertions.assertEquals("7.0", gate.getExpectedVersion()); + FileSystemPluginManager manager = new FileSystemPluginManager(); + + manager.loadPlugins(Collections.singletonList(filesystemPluginRoot("6.0"))); + + Assertions.assertFalse(providerNames(manager).contains("version_probe_fs"), + "a plugin from before provider-owned location prefix matching must not join storage routing"); + } + @Test public void filesystemPluginDeclaringNothingIsRefused() throws IOException { FileSystemPluginManager manager = new FileSystemPluginManager(); From 3d76f429f063285bc681ad955dab7bf7e50e879f Mon Sep 17 00:00:00 2001 From: xylaaaaa <2392805527@qq.com> Date: Sun, 13 Sep 2026 20:01:21 +0800 Subject: [PATCH 05/10] [feature](iceberg) Route Iceberg access through native Azure bindings ### What problem does this PR solve? Related PR: #67470 Problem Summary: Keep Iceberg FileIO, scan/write planning, metadata credential selection, the Thrift expiry field and integration tests together. This is a history-only regrouping of the native Azure PR. The final seven-commit series preserves the exact code tree of azure-native-before-squash-20260913550a886487e. ### Release note None ### Check List (For Author) - Test: Manual test / Unit Test - Final tree and PR diff are verified identical to the pre-squash snapshot - Existing final-snapshot verification: 711 selected FE tests passed - Tests were not rerun for this history-only rewrite - Behavior changed: No (relative to the pre-squash PR snapshot) - Does this need documentation: No --- fe/fe-connector/fe-connector-iceberg/pom.xml | 38 ++ .../iceberg/IcebergAuthenticatedFileIO.java | 7 + .../iceberg/IcebergCatalogFactory.java | 22 +- .../connector/iceberg/IcebergConnector.java | 54 ++- .../iceberg/IcebergConnectorMetadata.java | 3 +- .../IcebergMetadataTaskProperties.java | 130 +++++++ ...tSuccessFileIOInitializationException.java | 56 +++ .../iceberg/IcebergRestFileIOClient.java | 158 ++++++++ .../iceberg/IcebergRestFileIOProperties.java | 286 +++++++++++++++ .../iceberg/IcebergScanPlanProvider.java | 338 ++++++++++++++---- .../connector/iceberg/IcebergScanRange.java | 32 +- .../iceberg/IcebergStatementScope.java | 38 ++ .../iceberg/IcebergWritePlanProvider.java | 112 +++--- .../ReauthenticatingRestSessionCatalog.java | 4 + .../IcebergMetadataTaskPropertiesTest.java | 172 +++++++++ ...ergScanPlanProviderKerberosScanIoTest.java | 30 ++ .../iceberg/IcebergScanPlanProviderTest.java | 247 ++++++++++++- .../iceberg/IcebergScanRangeTest.java | 14 + .../iceberg/IcebergStatementScopeTest.java | 77 ++++ .../IcebergStorageBindingSelectionTest.java | 151 ++++++++ .../iceberg/IcebergWritePlanProviderTest.java | 125 ++++++- ...eauthenticatingRestSessionCatalogTest.java | 57 +++ .../iceberg/RecordingConnectorContext.java | 65 +++- ...ergNativeStorageAccessIntegrationTest.java | 311 ++++++++++++++++ gensrc/thrift/PlanNodes.thrift | 3 + 25 files changed, 2359 insertions(+), 171 deletions(-) create mode 100644 fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergMetadataTaskProperties.java create mode 100644 fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergPostSuccessFileIOInitializationException.java create mode 100644 fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergRestFileIOClient.java create mode 100644 fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergRestFileIOProperties.java create mode 100644 fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergMetadataTaskPropertiesTest.java create mode 100644 fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergStorageBindingSelectionTest.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/connector/IcebergNativeStorageAccessIntegrationTest.java diff --git a/fe/fe-connector/fe-connector-iceberg/pom.xml b/fe/fe-connector/fe-connector-iceberg/pom.xml index 856594ff29f2a5..7d83e29e22a456 100644 --- a/fe/fe-connector/fe-connector-iceberg/pom.xml +++ b/fe/fe-connector/fe-connector-iceberg/pom.xml @@ -147,6 +147,44 @@ under the License. ${iceberg.version} + + + org.apache.iceberg + iceberg-azure + ${iceberg.version} + + + com.azure + azure-storage-file-datalake + + + + com.azure + azure-core-http-netty + + + + + com.azure + azure-core-http-jdk-httpclient + + + com.azure + azure-identity + + + com.azure + azure-core-http-netty + + + + + + org.apache.iceberg + iceberg-azure + ${iceberg.version} + + + com.azure + azure-storage-file-datalake + + + com.azure + azure-core-http-netty + + + + + com.azure + azure-core-http-jdk-httpclient + + + com.azure + azure-identity + + + com.azure + azure-core-http-netty + + + Databricks Unity Catalog + ^ | + |________ 表元数据、文件位置、存储凭证 _____| + | + +-- metadata / manifest + | -> Iceberg FileIO 读取并规划 + | -> 系统表等任务需要时,携带 FileIO 配置进入 BE Java scanner + | + +-- data / delete 文件 + -> FE 选择该位置对应的存储配置 + -> 生成 URI + 文件类型 + BE 访问属性 + -> BE 原生 Azure client + -> 直接访问 Azure Storage +``` + +FE 负责“访问哪一个文件、使用哪组配置”,BE 负责“用这组配置创建客户端并执行文件读写”。写入时,BE 写出文件不等于事务已成功;仍须由 Iceberg 的提交链路更新表元数据。 + +### 4.1 区分两类认证 + +| 认证对象 | 用途 | 本文需要明确的区别 | +|---|---|---| +| Databricks REST Catalog | 获取表信息、执行目录操作或提交 | REST token 不是 Azure 文件访问凭证。 | +| Azure Storage | 实际读取、写入、删除文件 | 使用 SAS、SharedKey 或本方案支持的 OAuth2 客户端密钥认证。 | + +**“Azure OAuth2 数据访问通过”不等于“Databricks REST OAuth2 登录通过”。** 它们必须分别配置和验证。 + +### 4.2 给每个文件选择一套完整配置 + +将已经解析好的存储配置称为 binding:它包含 provider 所需的账号、endpoint、认证方式等信息。请求级 resolver 根据文件 URI 选择 binding,并一次返回: + +```text +provider + 校验/归一化后的 URI + BE 文件类型 + BE 访问属性 +``` + +这样下游不会把一组配置解析出的 URI,与另一组配置的凭证拼在一起。resolver 使用 catalog 静态配置和 connector 已取得的临时凭证,**本身不向 Databricks 或 Azure 申请 SAS**。 + +新请求可以取得新凭证并建立新的配置快照;一次规划内可以复用自己的快照。快照保证配置选择一致,不保证凭证永久有效,也不等于查询执行期间会自动替换已经下发的 SAS。过期凭证必须报错,不能静默借用另一身份继续访问。 + +实现依据:[请求级配置选择](../fe/fe-core/src/main/java/org/apache/doris/connector/DefaultConnectorContext.java)及 [Azure 配置模型](../fe/fe-filesystem/fe-filesystem-azure/src/main/java/org/apache/doris/filesystem/azure/AzureFileSystemProperties.java)。 + +### 4.3 数据与元数据不能共用一份未经区分的属性 + +| 访问对象 | 执行路径 | 配置要求 | +|---|---|---| +| 原生 Azure data/delete 文件 | BE 对象存储 reader/writer + Azure SDK | 生成 `provider=azure` 和 `AZURE_*` 执行字段。 | +| Iceberg metadata/manifest | Iceberg FileIO;部分任务经 JNI 执行 | 使用 FileIO/Hadoop 能识别的认证配置,必要时携带固定的凭证有效期。 | + +当前方案复用 `FILE_S3` 作为原生对象存储的 FE/BE 传递入口,通过 `provider=azure` 选择 Azure 客户端。**这里的 `FILE_S3` 不表示用 AWS 客户端访问 Azure。** + +FileIO 也不能随意替换:例如,不能让只配置了旧静态身份的 HadoopFileIO 静默接管新下发的 Azure FileIO 凭证。具体选择与校验见 [Iceberg FileIO 属性适配](../fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergRestFileIOProperties.java)。 + +## 5. 为什么改到通用存储层,其他存储要不要接入 + +**需求来自 Azure,但“根据文件位置一致地选择存储配置”是通用问题。** 如果只在 Iceberg 中增加 Azure 特判,扫描、写入和元数据操作容易各自维护一套不一致的 URI 与凭证选择规则。 + +因此职责分成三层: + +1. **通用存储层**:组织配置快照、选择 provider、返回完整访问结果,不硬编码 Azure 认证规则。 +2. **Azure provider**:理解 Azure URI、SAS、SharedKey、OAuth2,并生成不同客户端所需的属性。 +3. **Iceberg connector**:在自己的扫描、写入和元数据生命周期中使用这些能力。 + +其他存储不需要使用 Azure 的凭证字段,也不需要为了这个需求新增一种认证方式。S3/HDFS 等已有 binding 应保持自己的行为,Azure 凭证更新不能清掉它们。 + +但通用层变化仍需做兼容性测试。旧 connector 不会因为新接口存在就自动迁移;若希望使用完整访问结果,需要显式接入 resolver。**“没有强制迁移”不等于“通用改动对旧路径没有影响”。** + +## 6. 如何判断这个开发目标完成 + +验收应围绕真实用户操作,而不是以编译通过或新增接口为终点。 + +| 验收层次 | 需要证明什么 | +|---|---| +| 目录连接 | 能通过 REST 认证发现目标库表并加载 Iceberg 元数据。 | +| 实际查询 | FE/BE 可以访问真实 Azure 文件,查询结果正确;不能只看 SHOW TABLES 或 EXPLAIN。 | +| 删除语义 | 已删除的数据不会重新出现,覆盖目标版本适用的 position delete / deletion vector。 | +| 元数据操作 | manifest 和目标系统表使用正确的 FileIO 配置,跨 JNI 后仍可访问。 | +| 原生路径 | 关闭相关缓存后,以 profile/客户端证据确认文件 I/O 确实经过原生 Azure 路径。 | +| 写入闭环 | 在允许写入的表上执行目标 DML,提交后重新读取并校验结果。 | +| 凭证边界 | 过期、账号不匹配、scope 不匹配等情况按预期失败;新凭证不混用旧身份或其他 provider 的配置。 | +| 兼容性 | 既有 SharedKey、S3/HDFS 路径和约定的升级顺序得到验证。 | + +SAS、SharedKey、OAuth2,以及 Iceberg 版本和文件格式需要分别记录结果。一个组合通过,不能代替整个矩阵。单测可以覆盖解析、路由和异常边界,但不能代替真实 Databricks/Azure 环境验收;普通外表 CI 通过也不等于专用 Azure suite 已执行。 + +真实验收需要预先准备可访问的 REST catalog、Azure 存储及相应权限,凭证通过外部配置提供,不写入测试代码。专用 suite 的环境开关、运行方法与输出基线生成方式见[验收说明](azure-native-iceberg-acceptance.md)。 + +部署时先升级 BE,再启用新 FE 的原生 SAS/OAuth2 数据访问;这些模式不能安全地交给不认识新字段的旧 BE。完整滚动升级约定也见上述验收说明。 + +## 7. 阅读入口 + +- **理解开发目标和方案**:本文。 +- **搭建验收环境并运行测试**:[Native Azure Iceberg 验收说明](azure-native-iceberg-acceptance.md)。 +- **追溯评审问题的处理**:[Azure native review 修复记录](azure-native-review-resolution.md),其中验证状态应按记录的范围理解,不作为实时 CI 状态。 + +本文不维护逐次流水线状态和完整云端测试结果;这两类记录与设计分开,避免把某次环境结果误写成长期产品承诺。