From d8896e395e7db3ee607c03df2aae6b261ba48668 Mon Sep 17 00:00:00 2001 From: comphead Date: Sun, 15 Nov 2026 09:31:39 -0800 Subject: [PATCH] feat: sign native Iceberg writes with an explicit S3A provider list and translate S3A encryption Rebased onto main after #7603 merged. Carries this PR's own changes, previously its second, fourth and sixth commits. --- .../contributor-guide/iceberg-writes.md | 18 +- .../s3-credential-provider-design.md | 18 +- docs/source/user-guide/latest/iceberg.md | 31 +- .../latest/s3-credential-providers.md | 12 +- native/core/src/cloud/s3/credential_bridge.rs | 50 +++- .../operator/CometIcebergNativeWrite.scala | 247 ++++++++++++++-- .../cloud/s3/SdkCredentialExtraction.java | 4 +- .../CometIcebergWriteDetectionSuite.scala | 274 +++++++++++++++++- .../apache/comet/IcebergReadFromS3Suite.scala | 56 +++- 9 files changed, 641 insertions(+), 69 deletions(-) diff --git a/docs/source/contributor-guide/iceberg-writes.md b/docs/source/contributor-guide/iceberg-writes.md index 941ae6532ae..884e9ffda3c 100644 --- a/docs/source/contributor-guide/iceberg-writes.md +++ b/docs/source/contributor-guide/iceberg-writes.md @@ -362,9 +362,21 @@ file in one PUT, which a store without multipart uploads needs and the native wr suffix belongs on that list only if it cannot change the written object, where it is written or who writes it, and the native writer either does not need it or has its own setting for it, such as `spark.comet.iceberg.ioTimeout` for the timeouts. Settings that can change the object, such as -encryption, ACLs, storage class, signing and the credentials provider, must keep falling back. -`IgnoredS3FileIOProperties` does the same for `S3FileIO`'s multipart, staging-directory, retry, -delete-batching and client-preload settings. +ACLs, storage class and signing, must keep falling back. Encryption and the credential settings +are admitted only where the write proto reproduces them. `hadoopS3EncryptionProperties` +translates S3A's server-side encryption into +`s3.sse.*` with S3A's precedence, where the first key that is set wins even when it is empty, and +returns `None`, which keeps the encryption keys unsupported, for anything iceberg-rust cannot +apply. `buildIcebergWriteProto` lets that translation replace the FileIO's own `s3.sse.*` +properties, which `HadoopFileIO` ignores. `usesS3ACredentialAdapter` decides when the proto names +`HadoopS3ACredentialProviderAdapter` as `s3.comet.credential.provider.class`: when the FileIO's +Hadoop configuration sets `fs.s3a.aws.credentials.provider` explicitly and the FileIO properties +name no provider class of their own. The adapter then runs that same S3A list on the executor, and +`s3aCredentialAdapterProperties` forwards the explicit `fs.s3a.*` settings and the +`hadoop.security.credential.*` and `hadoop.security.credstore.*` settings its providers read. The gate and +`buildIcebergWriteProto` call the same functions on the same configuration, so they cannot +disagree. On the FileIO side, `IgnoredS3FileIOProperties` ignores +`S3FileIO`'s multipart, staging-directory, retry, delete-batching and client-preload settings. `s3.delete-enabled` is admitted only when it is `true`, because a failed native write task deletes its files, which `S3FileIO` keeps when deletes are disabled. Keep those allowlists aligned when adding storage support. Reaching diff --git a/docs/source/contributor-guide/s3-credential-provider-design.md b/docs/source/contributor-guide/s3-credential-provider-design.md index 416e2f8e50f..3d455ed04eb 100644 --- a/docs/source/contributor-guide/s3-credential-provider-design.md +++ b/docs/source/contributor-guide/s3-credential-provider-design.md @@ -130,15 +130,15 @@ STS endpoint selection (`configure_sts_endpoint`) overrides the SDK's own endpoi `object_store::CredentialProvider` and `reqsign_core::ProvideCredential` differ in what they consume: -| Concern | Parquet (`object_store`) | Iceberg (`opendal` via `reqsign-core`) | -| ----------------------- | --------------------------------------------------------- | ------------------------------------------------------------ | -| Trait method | `get_credential() -> AwsCredential` | `provide_credential(...) -> Option` | -| Returns expiry? | No (only key/secret/token) | Yes (`expires_in: Option`) | -| Comet-side TTL wrapper? | The bridge's own reuse, until 5 minutes before the expiry | The bridge's own reuse, until 5 minutes before the expiry | -| When SPI is called | When the bridge's credential is due, or has no expiry | When the bridge's credential is due, or has no expiry | -| Vendor returns 0 expiry | Not reused; the SPI is called per request | Not reused; the bridge tells reqsign it expires in 5 minutes | - -The 5-minute fallback on the Iceberg path bounds how long reqsign reuses a credential with no reported expiry within one long-lived reader or writer. It can outlast a short-lived token, which is why a vendor that knows the expiry should report it. It is intentionally not a configuration knob. On both paths a value of `Long.MAX_VALUE` means no expiry, and a value before 2000, almost always seconds sent as milliseconds, is treated as unknown with a warning. +| Concern | Parquet (`object_store`) | Iceberg (`opendal` via `reqsign-core`) | +| ----------------------- | --------------------------------------------------------- | ------------------------------------------------------------------------------------------------ | +| Trait method | `get_credential() -> AwsCredential` | `provide_credential(...) -> Option` | +| Returns expiry? | No (only key/secret/token) | Yes (`expires_in: Option`) | +| Comet-side TTL wrapper? | The bridge's own reuse, until 5 minutes before the expiry | The bridge's own reuse, until 5 minutes before the expiry | +| When SPI is called | When the bridge's credential is due, or has no expiry | When the bridge's credential is due, or has no expiry | +| Vendor returns 0 expiry | Not reused; the SPI is called per request | Not reused; the bridge tells reqsign it expires in 5 minutes for a read and 1 minute for a write | + +The 5-minute fallback on the Iceberg path bounds how long reqsign reuses a credential with no reported expiry within one long-lived reader. It can outlast a short-lived token, which is why a vendor that knows the expiry should report it. A write uses 1 minute instead. reqsign reloads a credential with less than 2 minutes left, so a write asks the SPI for every request, as the Parquet path does. A data file stays open while its rows arrive, and an AWS SDK v1 session provider keeps handing out its current credential until a minute before it expires, so the 5-minute window could sign a file's later parts with an expired token. It is intentionally not a configuration knob. On both paths a value of `Long.MAX_VALUE` means no expiry, and a value before 2000, almost always seconds sent as milliseconds, is treated as unknown with a warning. ## Property-bag handling on the Iceberg path diff --git a/docs/source/user-guide/latest/iceberg.md b/docs/source/user-guide/latest/iceberg.md index 1cc8afd959f..9278616c652 100644 --- a/docs/source/user-guide/latest/iceberg.md +++ b/docs/source/user-guide/latest/iceberg.md @@ -542,7 +542,7 @@ A write is eligible only when ALL of the following hold: | `write.target-file-size-bytes` | any value (the two writers can choose different roll points; see accepted divergences) | | data location URI scheme | `file`, `memory`, `s3`, `s3a`, `gs`, matched case-sensitively (`S3://` falls back). `s3`, `s3a` and `gs` need a bucket in the authority (`s3://bucket/...`), so a hostless form such as `s3:/bucket/key` falls back. `gs` only when the `FileIO` opening the data location is a `GCSFileIO`; see below | | resolved `table.locationProvider()` | Iceberg's built-in `DefaultLocationProvider` | -| Hadoop S3A settings for an `s3` / `s3a` data location | `fs.s3a.access.key`, `secret.key`, `session.token`, `endpoint`, `endpoint.region`, and `path.style.access`, plus the ignored settings listed below, including their `fs.s3a.bucket..*` forms; any other effective `fs.s3a.*` setting falls back | +| Hadoop S3A settings for an `s3` / `s3a` data location | `fs.s3a.access.key`, `secret.key`, `session.token`, `endpoint`, `endpoint.region`, and `path.style.access`, plus the ignored, encryption and credential settings described below, including their `fs.s3a.bucket..*` forms; any other effective `fs.s3a.*` setting falls back | | Iceberg `FileIO` S3 settings for an `s3` / `s3a` data location | the S3 endpoint, region, static/session credentials, path-style, SSE (`none`, `s3`, `kms`, or `custom`; not `dsse-kms`), assume-role, anonymous/config-chain settings parsed by the pinned iceberg-rust version, plus Comet's credential-provider class and built-in web-identity properties. When a custom provider is configured, its vendor-owned `s3.*` / `client.*` properties are also forwarded; unsupported Iceberg-defined S3 settings still fall back | | partition spec | any, except a `void` field whose source column was dropped beside a live field (see below) | | column types | any except `uuid` (Spark plans it as a string; no Arrow cast reaches `fixed(16)`) and the v3 types `variant`, `unknown`, `timestamp_ns`, `geometry` and `geography` | @@ -607,9 +607,32 @@ default), so on a slow endpoint raise that rather than the S3A timeouts. It neve `multipart.uploads.enabled` is ignored only when it is `true`. A store without multipart uploads needs `false`, which makes S3A send each file in one PUT, but the native writer uses a multipart upload for any file over 5 MiB, so a write with `false` falls back. Settings that can change a -request still fall back: endpoint and region selection, SSL, proxies, signing, requester pays, -ACLs, storage class, checksums, headers, -encryption, access points and credentials. On the `FileIO` side, the multipart, staging-directory, +request still fall back: endpoint and region +selection, SSL, proxies, signing, requester pays, ACLs, storage class, checksums, headers and +access points. + +S3A server-side encryption is translated into iceberg-rust's `s3.sse.*` settings, with S3A's own +precedence: the data bucket's `fs.s3a.bucket..*` setting wins over the global one, +even when it is empty, which is how a bucket turns a global setting off, and +`encryption.algorithm` and `encryption.key` win over the older `server-side-encryption-algorithm` +and `server-side-encryption.key`. SSE-S3 (`AES256`), SSE-KMS (with or without a key) and SSE-C are +supported. The translated settings replace the catalog's own `s3.sse.*` properties, which +`HadoopFileIO` ignores. DSSE-KMS and client-side encryption fall back, and so do encryption +settings when a Hadoop credential store (`hadoop.security.credential.provider.path` or +`fs.s3a.security.credential.provider.path`) is configured, because the store can supply the values +instead of the configuration. + +When `aws.credentials.provider` names the S3A credential providers explicitly, iceberg-java's +writer signs with exactly that list, so the native writer takes its credentials from the same list +through `HadoopS3ACredentialProviderAdapter`, which runs S3A's own provider factory on the +executor (see [S3 credential providers](s3-credential-providers.md)). The provider list and the +assumed-role and credential-store settings it reads are then admitted. This applies only when the +catalog names no `s3.comet.credential.provider.class` of its own. A list that names an anonymous +provider, directly or through `aws.credentials.provider.mapping`, and S3A delegation tokens, +still fall back. Without an explicit list, the native writer +uses its own credential chain, as for a read. + +On the `FileIO` side, the multipart, staging-directory, retry, delete-batching and client-preload settings are ignored, and so is `s3.delete-enabled=true`. `s3.delete-enabled=false` falls back, because a failed native write task deletes the files it wrote, which `S3FileIO` keeps when deletes are disabled. Unknown explicit diff --git a/docs/source/user-guide/latest/s3-credential-providers.md b/docs/source/user-guide/latest/s3-credential-providers.md index d33ce766929..23659703ac9 100644 --- a/docs/source/user-guide/latest/s3-credential-providers.md +++ b/docs/source/user-guide/latest/s3-credential-providers.md @@ -40,7 +40,7 @@ You probably do, if any of these are true: If a native Parquet scan fails with `Unsupported credential provider: ` (for example `com.amazonaws.auth.DefaultAWSCredentialsProviderChain`), the class you named in `fs.s3a.aws.credentials.provider` is one that plain Spark/Hadoop accepts but Comet's native reader does not reimplement. Comet ships two built-in `CometS3CredentialProvider` adapters that fix this with a one-line config change; you leave your existing `fs.s3a.aws.credentials.provider` untouched. -These adapters cover the Parquet native scan path only. Enabling one is opt-in: naming it is what activates it. Note the native side forwards the `fs.s3a.*` config to `initialize()` for _any_ provider class named on the Parquet path, not just these two adapters: a vendor `CometS3CredentialProvider` that received an empty map in Comet 1.0 now receives the `fs.s3a.*` subset (including static keys), and one cached instance per distinct `fs.s3a.*` config rather than one per bucket. This is additive, but a provider that logs the map or treats an empty map as "the Parquet path" should be aware of it. +On the Parquet native scan path, enabling an adapter is opt-in: naming it is what activates it. Native Iceberg writes use `HadoopS3ACredentialProviderAdapter` without any configuration when the S3A provider list is set explicitly (see [Native Iceberg writes](#native-iceberg-writes)). Note the native side forwards the `fs.s3a.*` config to `initialize()` for _any_ provider class named on the Parquet path, not just these two adapters: a vendor `CometS3CredentialProvider` that received an empty map in Comet 1.0 now receives the `fs.s3a.*` subset (including static keys), and one cached instance per distinct `fs.s3a.*` config rather than one per bucket. This is additive, but a provider that logs the map or treats an empty map as "the Parquet path" should be aware of it. The adapters and the AWS SDK are loaded through the class loader that loaded Comet, so `hadoop-aws` and the matching AWS SDK must be visible from there — put them on the same classpath as Comet (`spark.executor.extraClassPath` / `spark.driver.extraClassPath`, or `$SPARK_HOME/jars`), not only via `--packages`. If they are only on the user-jar loader, credential resolution fails at planning with `NoClassDefFoundError` before the adapter can report anything useful. @@ -62,6 +62,12 @@ Anonymous access is the one exception. A bucket whose chain resolves to `Anonymo spark.hadoop.fs.s3a.bucket..comet.credential.provider.class= ``` +#### Native Iceberg writes + +A native Iceberg write to an `s3` or `s3a` location uses `HadoopS3ACredentialProviderAdapter` by itself when the table's `FileIO` writes through Hadoop S3A, as `HadoopFileIO` does, and `fs.s3a.aws.credentials.provider` (or the data bucket's `fs.s3a.bucket..aws.credentials.provider`) names the providers explicitly. iceberg-java's own writer signs with exactly that list, so the native writer runs the same list rather than its own credential chain. Comet forwards the `FileIO`'s explicit `fs.s3a.*` settings, and the `hadoop.security.credential.*` and `hadoop.security.credstore.*` credential-store settings Hadoop reads secrets through, to the adapter, including the catalog's `spark.sql.catalog..hadoop.*` overrides. Any other key a custom provider reads has to reach the executors through `spark.hadoop.*`. A credential without an expiry, which is every credential on the Spark 3.x builds, is fetched again for every request of a write, because a data file can stay open longer than a short-lived session credential lasts. + +A `spark.sql.catalog..s3.comet.credential.provider.class` that you set keeps its precedence: Comet then does not name the adapter, and the write falls back to iceberg-java while the S3A list is set, because the two could resolve different identities. A list that names an anonymous provider, directly or through `fs.s3a.aws.credentials.provider.mapping`, falls back too. + ### `AwsSdkCredentialProviderAdapter` Wraps a single raw AWS SDK credential-provider class that is not registered through S3A. Name the delegate in a separate key: @@ -159,7 +165,7 @@ When Comet detects IRSA (both `AWS_WEB_IDENTITY_TOKEN_FILE` and `AWS_ROLE_ARN` a **STS endpoint selection** uses the global `sts.amazonaws.com` endpoint only in the plain commercial case, and leaves everything else to the AWS SDK. The global endpoint is used when your region is in the standard commercial partition, FIPS and dual-stack are both off, no custom STS endpoint is set, and `AWS_STS_REGIONAL_ENDPOINTS` is `legacy` or unset. That matches the previous behavior, so a network that only reaches the global endpoint keeps working. In every other case the SDK's own regional endpoint is used: FIPS (`AWS_USE_FIPS_ENDPOINT`) always uses the regional FIPS endpoint since there is no global FIPS STS endpoint (an accompanying `AWS_STS_REGIONAL_ENDPOINTS=legacy` is ignored and a warning is logged), dual-stack (`AWS_USE_DUALSTACK_ENDPOINT`) uses the dual-stack regional endpoint, a custom STS endpoint (`AWS_ENDPOINT_URL_STS` or a profile) is honored, `AWS_STS_REGIONAL_ENDPOINTS=regional` stays regional, and the China, GovCloud, ISO and EUSC regions (and any region Comet does not recognize as commercial) use their own regional endpoints. -It stands aside whenever a higher-precedence credential source is configured -- a Comet bridge class or catalog static keys / `client.assume-role.arn`, static credentials in the environment (`AWS_ACCESS_KEY_ID` + `AWS_SECRET_ACCESS_KEY`), or a configured profile (`AWS_PROFILE`, or a shared credentials / config file such as `~/.aws/credentials` or `~/.aws/config`) -- and when no region is set (`AWS_REGION` / `AWS_DEFAULT_REGION`). These all rank ahead of web-identity in opendal's default chain or need its no-region fallback, so the take-over only changes the otherwise-default behavior and never switches away from an identity -- or a profile-configured STS endpoint -- you set explicitly. On an EKS/IRSA pod none of these apply, so the take-over still engages there. +It stands aside whenever a higher-precedence credential source is configured -- a Comet bridge class or catalog static keys / `client.assume-role.arn`, static credentials in the environment (`AWS_ACCESS_KEY_ID` + `AWS_SECRET_ACCESS_KEY`), or a configured profile (`AWS_PROFILE`, or a shared credentials / config file such as `~/.aws/credentials` or `~/.aws/config`) -- and when no region is set (`AWS_REGION` / `AWS_DEFAULT_REGION`). These all rank ahead of web-identity in opendal's default chain or need its no-region fallback, so the take-over only changes the otherwise-default behavior and never switches away from an identity -- or a profile-configured STS endpoint -- you set explicitly. On an EKS/IRSA pod none of these apply, so the take-over still engages there. A native Iceberg write that signs with an explicit S3A provider list (see [Native Iceberg writes](#native-iceberg-writes)) uses that list instead, as iceberg-java does. Tuning is rarely needed. Set these as catalog properties under the `s3.` prefix (the same namespace as the credential-provider SPI key): @@ -241,7 +247,7 @@ public CometS3Credentials getCredentialsForPath(CometS3CredentialContext ctx) th Spark delegation token propagation is supported on YARN and Kubernetes only. Standalone deployments need a different refresh path, typically a vendor-side service callback authenticated by long-lived state in `catalogProperties` or Hadoop conf. -Publish a real `expirationEpochMillis` when you have one. On both paths Comet reuses a credential until five minutes before that expiry and then asks you again. Comet makes at most one call at a time for each location (each bucket, for a provider without locations), and requests that arrive while it is in flight share its answer, even one Comet cannot keep. `0` means unknown: Comet does not keep the credential and asks you again for every request that does not overlap a call already in flight, and on the Iceberg path it assumes the credential lasts five minutes. `Long.MAX_VALUE` means the credential does not expire, and Comet does not keep it either. A value before 2000, almost always seconds sent as milliseconds, is treated as unknown, with a warning. If a credential can be revoked before the expiry you report, report an earlier one, or `0`. +Publish a real `expirationEpochMillis` when you have one. On both paths Comet reuses a credential until five minutes before that expiry and then asks you again. Comet makes at most one call at a time for each location (each bucket, for a provider without locations), and requests that arrive while it is in flight share its answer, even one Comet cannot keep. `0` means unknown: Comet does not keep the credential and asks you again for every request that does not overlap a call already in flight. On the Iceberg path a read assumes such a credential lasts five minutes, and a write asks you again for every request. `Long.MAX_VALUE` means the credential does not expire, and Comet does not keep it either. A value before 2000, almost always seconds sent as milliseconds, is treated as unknown, with a warning. If a credential can be revoked before the expiry you report, report an earlier one, or `0`. On the Parquet/`object_store` path, `object_store` signs a request once and sends the same signature on every retry, for up to 3 minutes by default, and it does not retry a 403. The five minutes Comet leaves before an expiry cover those retries. A credential you return with less time left than that is used for the request that asked for it and not kept. diff --git a/native/core/src/cloud/s3/credential_bridge.rs b/native/core/src/cloud/s3/credential_bridge.rs index f21b73011fb..5d5c8d3f800 100644 --- a/native/core/src/cloud/s3/credential_bridge.rs +++ b/native/core/src/cloud/s3/credential_bridge.rs @@ -44,11 +44,19 @@ use std::sync::Arc; use std::time::Duration; /// Expiry the Iceberg path assumes when the provider does not report one. It bounds how long a -/// long-lived reader or writer reuses a credential, but it can outlast a short-lived credential, -/// so a provider that knows the expiry should report it. Shared with the IRSA web-identity -/// provider (`super::web_identity`). +/// long-lived reader reuses a credential, but it can outlast a short-lived credential, so a +/// provider that knows the expiry should report it. Shared with the IRSA web-identity provider +/// (`super::web_identity`). pub(crate) const DEFAULT_EXPIRY_WHEN_UNKNOWN: Duration = Duration::from_secs(300); +/// Expiry a write assumes when the provider does not report one. reqsign reloads a credential +/// with less than two minutes left, so a write asks the provider for every request, as the +/// Parquet path does. A data file stays open while its rows arrive, and an AWS SDK v1 session +/// provider keeps handing out its current credential until a minute before it expires, so the +/// read window could sign a file's later parts with an expired token. It must stay above +/// reqsign's 10 second signing headroom. +const WRITE_EXPIRY_WHEN_UNKNOWN: Duration = Duration::from_secs(60); + /// How long before its expiry a bridge stops reusing a credential and asks the provider again. It /// matches the refresh-ahead of the other credential caches Comet keeps, and it covers object_store, /// which signs a request once and sends that signature again on every retry for up to 3 minutes by @@ -573,11 +581,12 @@ impl IcebergProvideCredential for CometS3CredentialBridge { if WARNED_MISSING_EXPIRY.set(()).is_ok() { warn!( "CometS3CredentialProvider returned credentials without expiration; \ - defaulting to {}s expiry to bound opendal caching", - DEFAULT_EXPIRY_WHEN_UNKNOWN.as_secs() + assuming {}s for reads and {}s for writes to bound opendal caching", + DEFAULT_EXPIRY_WHEN_UNKNOWN.as_secs(), + WRITE_EXPIRY_WHEN_UNKNOWN.as_secs() ); } - Some(Timestamp::now() + DEFAULT_EXPIRY_WHEN_UNKNOWN) + Some(Timestamp::now() + unknown_expiry(self.mode)) } }; @@ -590,6 +599,14 @@ impl IcebergProvideCredential for CometS3CredentialBridge { } } +/// How long the Iceberg path assumes a credential without a reported expiry lasts. +fn unknown_expiry(mode: AccessMode) -> Duration { + match mode { + AccessMode::Read => DEFAULT_EXPIRY_WHEN_UNKNOWN, + AccessMode::Write => WRITE_EXPIRY_WHEN_UNKNOWN, + } +} + fn read_required_string( env: &mut jni::Env, instance: &JObject, @@ -685,6 +702,27 @@ mod tests { } } + /// reqsign keeps a credential while it has more than two minutes left and signs with one that + /// has at least 10 seconds left. A write's assumed expiry falls between the two, so reqsign + /// asks the provider again for every request, while a read's lets it keep the credential. + #[test] + fn a_write_asks_again_for_a_credential_without_an_expiry() { + use reqsign_core::SigningCredential; + let assumed = |mode| IcebergAwsCredential { + access_key_id: "key".to_string(), + secret_access_key: "secret".to_string(), + session_token: None, + expires_in: Some(Timestamp::now() + unknown_expiry(mode)), + }; + let write = assumed(AccessMode::Write); + assert!( + !write.is_valid(), + "reqsign would keep the write's credential" + ); + assert!(write.is_valid_at(Timestamp::now() + Duration::from_secs(10))); + assert!(assumed(AccessMode::Read).is_valid()); + } + #[test] fn a_failed_fetch_is_not_kept() { let cache = CredentialCache::default(); diff --git a/spark/src/main/scala/org/apache/comet/serde/operator/CometIcebergNativeWrite.scala b/spark/src/main/scala/org/apache/comet/serde/operator/CometIcebergNativeWrite.scala index 3f38108c4d3..141c6e3c474 100644 --- a/spark/src/main/scala/org/apache/comet/serde/operator/CometIcebergNativeWrite.scala +++ b/spark/src/main/scala/org/apache/comet/serde/operator/CometIcebergNativeWrite.scala @@ -94,6 +94,7 @@ object CometIcebergNativeWrite extends CometOperatorSerde[IcebergWriteExec] { // Supported schemes whose native backend is local and needs no host. Every other supported // scheme reads its bucket from the URL host (`requireSupportedStorageScheme`). private val LocalStorageSchemes: Set[String] = Set("file", "memory") + private val S3StorageSchemes: Set[String] = Set("s3", "s3a") private val MaxSupportedFormatVersion = 3 private val ParquetWritePropertyPrefix = "write.parquet." private val CometS3CredentialProviderClassProperty = @@ -119,8 +120,9 @@ object CometIcebergNativeWrite extends CometOperatorSerde[IcebergWriteExec] { // S3 write ineligible. Spark seeds the vectored-read and syncable settings into every session, // and clusters commonly set the rest cluster-wide. The data bucket's per-bucket spellings are // admitted too. Anything that can change a request stays unsupported: endpoint and region - // selection, SSL, proxies, signing, requester pays, ACLs, storage class, checksums, headers, - // encryption and access points, and so do the credential settings. Audited against + // selection, SSL, proxies, signing, requester pays, ACLs, storage class, checksums, headers and + // access points. Encryption and credential settings are admitted only where the write proto + // reproduces them (see S3AEncryptionSuffixes and S3ACredentialSuffixes). Audited against // `org.apache.hadoop.fs.s3a.Constants` in hadoop-aws 3.3.4 and 3.4.2. private val IgnoredHadoopS3Suffixes: Set[String] = Set( // Connection pool, threads and timeouts. @@ -194,11 +196,52 @@ object CometIcebergNativeWrite extends CometOperatorSerde[IcebergWriteExec] { // native writer starts a multipart upload once a file passes 5 MiB and cannot be told not to. private val MultipartUploadsSuffix = "multipart.uploads.enabled" + // S3A server-side encryption, newer key names first. These are admitted when + // hadoopS3EncryptionProperties translates them into the `s3.sse.*` keys iceberg-rust reads. + private val S3AEncryptionAlgorithmSuffixes = + Seq("encryption.algorithm", "server-side-encryption-algorithm") + private val S3AEncryptionKeySuffixes = Seq("encryption.key", "server-side-encryption.key") + private val S3AEncryptionSuffixes: Set[String] = + (S3AEncryptionAlgorithmSuffixes ++ S3AEncryptionKeySuffixes).toSet + + // S3A credential settings. With an explicit provider list, iceberg-java's S3A writer signs with + // exactly the listed providers, which the native writer's own credential chain cannot reproduce. + // Such a write takes its credentials from that list through HadoopS3ACredentialProviderAdapter + // (see usesS3ACredentialAdapter), and these settings are admitted then. Delegation tokens stay + // unsupported, because the adapter refuses them. + private val S3ACredentialProviderSuffix = "aws.credentials.provider" + private val S3ACredentialProviderMappingSuffix = "aws.credentials.provider.mapping" + private val S3ACredentialStoreSuffix = "security.credential.provider.path" + private val S3ACredentialSuffixes: Set[String] = Set( + S3ACredentialProviderSuffix, + S3ACredentialProviderMappingSuffix, + "assumed.role.arn", + "assumed.role.session.name", + "assumed.role.session.duration", + "assumed.role.policy", + "assumed.role.external.id", + "assumed.role.sts.endpoint", + "assumed.role.sts.endpoint.region", + "assumed.role.credentials.provider", + S3ACredentialStoreSuffix) + + private val HadoopS3ACredentialAdapterClass = + "org.apache.comet.cloud.s3.HadoopS3ACredentialProviderAdapter" + + // The adapter cannot serve anonymous access, so a list that names an anonymous provider, + // directly or through the provider mapping, keeps falling back. + private val AnonymousS3ACredentialProviders: Set[String] = Set( + "org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider", + "software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider", + "com.amazonaws.auth.AnonymousAWSCredentials") + + private val HadoopCredentialStoreKey = "hadoop.security.credential.provider.path" + // Every suffix the gate can admit. Their per-bucket spellings resolve through this set even // where the runtime Hadoop version does not declare the property. private val AdmittedHadoopS3Suffixes: Set[String] = - SupportedHadoopS3Suffixes ++ IgnoredHadoopS3Suffixes + FileSystemImplSuffix + - MultipartUploadsSuffix + SupportedHadoopS3Suffixes ++ IgnoredHadoopS3Suffixes ++ S3AEncryptionSuffixes ++ + S3ACredentialSuffixes + FileSystemImplSuffix + MultipartUploadsSuffix private val FsS3aPrefix = "fs.s3a." private val FsS3aBucketPrefix = "fs.s3a.bucket." @@ -430,6 +473,8 @@ object CometIcebergNativeWrite extends CometOperatorSerde[IcebergWriteExec] { hadoopConfs: LazyHadoopConfs) { def hadoopConf: Configuration = hadoopConfs.session def s3HadoopConf: Configuration = hadoopConfs.effective + lazy val fileIOProperties: Map[String, String] = + IcebergReflection.getFileIOProperties(table).getOrElse(Map.empty) } private[comet] final class LazyHadoopConfs( @@ -488,6 +533,11 @@ object CometIcebergNativeWrite extends CometOperatorSerde[IcebergWriteExec] { } } + // A key counts as set only when a source other than Hadoop's built-in core-default.xml set it. + // Hadoop's iterator includes the many fs.s3a.* defaults loaded from there, which are library + // defaults rather than settings a user selected, and counting them would reject every ordinary + // S3 write. Site XML and programmatic or Spark sources count, and so does a user resource + // merely named `tenant-default.xml`. private def hasExplicitSource(conf: Configuration, key: String): Boolean = Option(conf.getPropertySources(key)) match { case Some(sources) if sources.nonEmpty => !sources.forall(isCoreDefaultResource) @@ -777,10 +827,11 @@ object CometIcebergNativeWrite extends CometOperatorSerde[IcebergWriteExec] { } } + private def isS3Location(location: String): Boolean = + S3StorageSchemes.contains(storageScheme(location)) + private def s3DataLocation(ctx: TriggerContext): Option[String] = - IcebergReflection - .getDataLocation(ctx.table) - .filter(location => Set("s3", "s3a").contains(storageScheme(location))) + IcebergReflection.getDataLocation(ctx.table).filter(isS3Location) private def unsupportedSettingsReason(namespace: String, keys: Seq[String]): Option[String] = keys match { @@ -795,29 +846,27 @@ object CometIcebergNativeWrite extends CometOperatorSerde[IcebergWriteExec] { * not. The data bucket must match in full: `fs.s3a.bucket.target.other.endpoint` is bucket * `target.other` and suffix `endpoint`, so it does not affect a write to `target`. Values are * deliberately never returned because this result is used in EXPLAIN fallback reasons and may - * include credentials. + * include credentials. `s3aCredentialAdapter` admits the credential settings, for a write that + * takes its credentials from S3A's provider chain (see [[usesS3ACredentialAdapter]]). */ private[comet] def unsupportedHadoopS3Settings( hadoopConf: Configuration, - targetBucket: Option[String]): Seq[String] = { + targetBucket: Option[String], + s3aCredentialAdapter: Boolean = false): Seq[String] = { + val encryption = + if (hadoopS3EncryptionProperties(hadoopConf, targetBucket).isDefined) S3AEncryptionSuffixes + else Set.empty[String] + val credentials = if (s3aCredentialAdapter) S3ACredentialSuffixes else Set.empty[String] + val translated = encryption ++ credentials val keys = hadoopConf .iterator() .asScala .map(_.getKey) .filter(_.startsWith("fs.s3a.")) - // Hadoop's iterator includes the many fs.s3a.* defaults loaded from core-default.xml. - // Those are library implementation defaults, not settings selected by the user, and - // treating them as explicit would reject every ordinary S3 write. Preserve settings from - // site XML and programmatic/Spark sources; exclude a key only when every recorded source is - // Hadoop's built-in core-default.xml resource. A user resource merely named - // `tenant-default.xml` is still explicit configuration. - .filter { key => - Option(hadoopConf.getPropertySources(key)) - .forall(sources => sources.isEmpty || !sources.forall(isCoreDefaultResource)) - } + .filter(hasExplicitSource(hadoopConf, _)) .toSeq - keys.filter(key => isUnsupportedHadoopS3Key(hadoopConf, key, targetBucket)).sorted + keys.filter(key => isUnsupportedHadoopS3Key(hadoopConf, key, targetBucket, translated)).sorted } private def isCoreDefaultResource(source: String): Boolean = @@ -843,28 +892,142 @@ object CometIcebergNativeWrite extends CometOperatorSerde[IcebergWriteExec] { private def isUnsupportedHadoopS3Key( hadoopConf: Configuration, key: String, - targetBucket: Option[String]): Boolean = + targetBucket: Option[String], + translated: Set[String]): Boolean = if (key.startsWith(FsS3aBucketPrefix)) { perBucketProperty(key) match { case Some((bucket, suffix)) => - targetBucket.contains(bucket) && !isAdmittedHadoopS3Setting(hadoopConf, key, suffix) + targetBucket.contains(bucket) && + !isAdmittedHadoopS3Setting(hadoopConf, key, suffix, translated) case None => // Preserve fail-closed behavior for a property unknown to the runtime Hadoop version. targetBucket.exists(bucket => key.startsWith(s"$FsS3aBucketPrefix$bucket.")) } } else { - !isAdmittedHadoopS3Setting(hadoopConf, key, key.stripPrefix(FsS3aPrefix)) + !isAdmittedHadoopS3Setting(hadoopConf, key, key.stripPrefix(FsS3aPrefix), translated) } private def isAdmittedHadoopS3Setting( hadoopConf: Configuration, key: String, - suffix: String): Boolean = + suffix: String, + translated: Set[String]): Boolean = SupportedHadoopS3Suffixes.contains(suffix) || IgnoredHadoopS3Suffixes.contains(suffix) || + translated.contains(suffix) || (suffix == FileSystemImplSuffix && Option(hadoopConf.getTrimmed(key)).contains(S3AFileSystemClass)) || (suffix == MultipartUploadsSuffix && "true".equalsIgnoreCase(hadoopConf.getTrimmed(key))) + // The data bucket's spellings of `suffixes`, then the global ones, in the order S3A reads them. + private def s3aKeys(bucket: Option[String], suffixes: Seq[String]): Seq[String] = + bucket.toSeq.flatMap(b => suffixes.map(s => s"$FsS3aBucketPrefix$b.$s")) ++ + suffixes.map(FsS3aPrefix + _) + + /** + * The `s3.sse.*` properties for the server-side encryption iceberg-java's S3A writer applies to + * `bucket`, or None when the native writer cannot apply the same encryption. Mirrors + * `S3AUtils.buildEncryptionSecrets`: the first key that is set wins, the bucket's before the + * global one and the newer name before the deprecated one, even when its value is empty, which + * is how a bucket turns off a global setting. The value is trimmed and matched ignoring case, + * and SSE-S3 with a key or SSE-C without one fails the S3A write, so it falls back here. A + * Hadoop credential store can override the configured values, so they fall back while one is + * configured. Client-side and dual-layer encryption have no iceberg-rust equivalent. + */ + private[comet] def hadoopS3EncryptionProperties( + hadoopConf: Configuration, + bucket: Option[String]): Option[Map[String, String]] = { + def setting(suffixes: Seq[String]): String = + s3aKeys(bucket, suffixes).iterator + .map(key => hadoopConf.get(key)) + .find(_ != null) + .fold("")(_.trim) + val algorithm = setting(S3AEncryptionAlgorithmSuffixes) + val key = setting(S3AEncryptionKeySuffixes) + val credentialStore = + (HadoopCredentialStoreKey +: s3aKeys(bucket, Seq(S3ACredentialStoreSuffix))).exists(k => + Option(hadoopConf.getTrimmed(k)).exists(_.nonEmpty)) + if (credentialStore && (algorithm.nonEmpty || key.nonEmpty)) { + None + } else { + algorithm.toUpperCase(Locale.ROOT) match { + case "" => Some(Map.empty) + case "AES256" if key.isEmpty => Some(Map("s3.sse.type" -> "s3")) + case "SSE-KMS" => + Some(Map("s3.sse.type" -> "kms") ++ Some(key).filter(_.nonEmpty).map("s3.sse.key" -> _)) + case "SSE-C" if key.nonEmpty => + sseCustomerKeyMd5(key).map(md5 => + Map("s3.sse.type" -> "custom", "s3.sse.key" -> key, "s3.sse.md5" -> md5)) + case _ => None + } + } + } + + // S3A sends SSE-C keys as base64 with the base64 MD5 of the decoded key, which iceberg-rust + // takes as `s3.sse.md5` and does not derive itself. + private def sseCustomerKeyMd5(key: String): Option[String] = + scala.util.Try(java.util.Base64.getDecoder.decode(key)).toOption.map { raw => + java.util.Base64.getEncoder + .encodeToString(java.security.MessageDigest.getInstance("MD5").digest(raw)) + } + + /** + * True when the native write must take its S3 credentials from Hadoop's S3A provider chain: + * `hadoopConf`, the table FileIO's effective Hadoop configuration, explicitly names the + * providers for `bucket`, and `fileIOProperties` name no Comet credential provider of their + * own. iceberg-java's S3A writer signs with exactly the listed providers, so the write proto + * then names HadoopS3ACredentialProviderAdapter, which builds that list with S3A's own factory + * on the executor. A FileIO without a Hadoop configuration, such as S3FileIO, has an empty one + * here. + */ + private[comet] def usesS3ACredentialAdapter( + hadoopConf: Configuration, + fileIOProperties: Map[String, String], + bucket: Option[String]): Boolean = { + // S3A copies a bucket's settings over the global ones before it builds the list. + def setting(suffix: String): Option[Seq[String]] = + s3aKeys(bucket, Seq(suffix)) + .find(hasExplicitSource(hadoopConf, _)) + .map(hadoopConf.getTrimmedStrings(_).toSeq.filter(_.nonEmpty)) + // Hadoop 3.4 resolves each name through this mapping, and fails the write on an entry that + // is not one non-empty name and value. + val mapping = setting(S3ACredentialProviderMappingSuffix) + .getOrElse(Nil) + .map(_.split("=", -1).map(_.trim)) + !fileIOProperties.get(CometS3CredentialProviderClassProperty).exists(_.trim.nonEmpty) && + mapping.forall(pair => pair.length == 2 && pair.forall(_.nonEmpty)) && + setting(S3ACredentialProviderSuffix).exists { names => + val mapped = mapping.map(pair => pair(0) -> pair(1)).toMap + names.nonEmpty && !names.exists(name => + AnonymousS3ACredentialProviders.contains(name) || + mapped.get(name).exists(AnonymousS3ACredentialProviders.contains)) + } + } + + // The keys S3A's providers read: their own `fs.s3a.*` settings, and the credential-store + // settings Hadoop's password lookup reads for them. Other `hadoop.security.*` keys, such as an + // LDAP bind password, stay out of the write proto. + private val S3ACredentialAdapterPrefixes = + Seq(FsS3aPrefix, "hadoop.security.credential.", "hadoop.security.credstore.") + + /** + * The catalog properties that route a native write's S3 credentials through + * HadoopS3ACredentialProviderAdapter. The adapter rebuilds the S3A configuration on the + * executor from that JVM's `spark.hadoop.*` settings, so it also gets the FileIO's explicit + * settings under `S3ACredentialAdapterPrefixes`, which carry the catalog's `hadoop.*` + * overrides. The native side passes the whole property bag to the adapter. It hands + * iceberg-rust none of the `fs.s3a.*` keys, and its S3 storage ignores the `hadoop.*` ones. + */ + private[comet] def s3aCredentialAdapterProperties( + hadoopConf: Configuration): Map[String, String] = + hadoopConf + .iterator() + .asScala + .map(_.getKey) + .filter(key => + S3ACredentialAdapterPrefixes.exists(key.startsWith) && hasExplicitSource(hadoopConf, key)) + .flatMap(key => Option(hadoopConf.get(key)).map(key -> _)) + .toMap + (CometS3CredentialProviderClassProperty -> HadoopS3ACredentialAdapterClass) + /** Return unsupported FileIO S3/client property names in deterministic order. */ private[comet] def unsupportedS3FileIOProperties(properties: Map[String, String]): Seq[String] = unsupportedS3FileIOProperties(properties, IcebergAwsProperties) @@ -897,15 +1060,16 @@ object CometIcebergNativeWrite extends CometOperatorSerde[IcebergWriteExec] { private val requireSupportedHadoopS3Settings: TriggerRule = ctx => s3DataLocation(ctx).flatMap { location => val dataBucket = NativeConfig.bucketForUri(new java.net.URI(location), Set.empty) + val s3aCredentials = + usesS3ACredentialAdapter(ctx.s3HadoopConf, ctx.fileIOProperties, dataBucket) unsupportedSettingsReason( "Hadoop S3A", - unsupportedHadoopS3Settings(ctx.s3HadoopConf, dataBucket)) + unsupportedHadoopS3Settings(ctx.s3HadoopConf, dataBucket, s3aCredentials)) } private val requireSupportedS3FileIOProperties: TriggerRule = ctx => s3DataLocation(ctx).flatMap { _ => - val properties = IcebergReflection.getFileIOProperties(ctx.table).getOrElse(Map.empty) - unsupportedSettingsReason("S3 FileIO", unsupportedS3FileIOProperties(properties)) + unsupportedSettingsReason("S3 FileIO", unsupportedS3FileIOProperties(ctx.fileIOProperties)) } // HadoopFileIO takes its GCS configuration from `fs.gs.*`, which is not forwarded to the @@ -1289,8 +1453,33 @@ object CometIcebergNativeWrite extends CometOperatorSerde[IcebergWriteExec] { val hadoopDerivedProperties = CometIcebergNativeScan.hadoopToIcebergS3Properties( NativeConfig.extractObjectStoreOptions(writeHadoopConf, dataUri), dataBucket) - val catalogProperties = - hadoopDerivedProperties ++ fileIOProperties + CometIcebergNativeScan.ioTimeoutProperty() + // iceberg-java writes an S3 data location through S3A, so the native write also takes the + // server-side encryption S3A applies and, when S3A's provider chain supplies the credentials, + // the adapter that runs that chain. The gate checked both. A core-default.xml value, which + // the gate does not count as a setting, is translated too, and declines here if it cannot be. + val isS3DataLocation = isS3Location(dataLocation) + val s3aEncryption = + if (isS3DataLocation) { + hadoopS3EncryptionProperties(writeHadoopConf, dataBucket).getOrElse( + throw new IllegalStateException( + "S3A server-side encryption that the native writer cannot apply")) + } else { + Map.empty[String, String] + } + val s3aCredentials = + if (isS3DataLocation && + usesS3ACredentialAdapter(writeHadoopConf, fileIOProperties, dataBucket)) { + s3aCredentialAdapterProperties(writeHadoopConf) + } else { + Map.empty[String, String] + } + // Only a FileIO with a Hadoop configuration yields S3A encryption, and such a FileIO ignores + // its own `s3.sse.*` properties, so S3A's encryption replaces them. + val storageFileIOProperties = + if (s3aEncryption.isEmpty) fileIOProperties + else fileIOProperties.filter { case (k, _) => !k.startsWith("s3.sse.") } + val catalogProperties = hadoopDerivedProperties ++ storageFileIOProperties ++ + s3aEncryption ++ s3aCredentials + CometIcebergNativeScan.ioTimeoutProperty() val common = IcebergWriteProtoTranslation.buildCommon( catalogProperties = catalogProperties, diff --git a/spark/src/main/spark-3.x/org/apache/comet/cloud/s3/SdkCredentialExtraction.java b/spark/src/main/spark-3.x/org/apache/comet/cloud/s3/SdkCredentialExtraction.java index d607b632612..c9f75b62002 100644 --- a/spark/src/main/spark-3.x/org/apache/comet/cloud/s3/SdkCredentialExtraction.java +++ b/spark/src/main/spark-3.x/org/apache/comet/cloud/s3/SdkCredentialExtraction.java @@ -52,8 +52,8 @@ static CometS3Credentials toCometCredentials(String bucket, AWSCredentials creds sessionToken = ((AWSSessionCredentials) creds).getSessionToken(); } // The v1 base interface exposes no expiration, so report 0 (unknown). Comet then asks for a - // credential on every request, and the Iceberg path assumes it lasts 5 minutes, which can be - // longer than a short-lived session credential has left. + // credential on every request. An Iceberg read assumes it lasts 5 minutes, which can be longer + // than a short-lived session credential has left, and an Iceberg write asks for every request. return new CometS3Credentials(accessKeyId, secretKey, sessionToken, 0L); } } diff --git a/spark/src/test/scala/org/apache/comet/CometIcebergWriteDetectionSuite.scala b/spark/src/test/scala/org/apache/comet/CometIcebergWriteDetectionSuite.scala index 8ca793632fe..89e1be46891 100644 --- a/spark/src/test/scala/org/apache/comet/CometIcebergWriteDetectionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometIcebergWriteDetectionSuite.scala @@ -822,8 +822,9 @@ class CometIcebergWriteDetectionSuite extends CometTestBase with CometIcebergTes hadoopConf.set("fs.s3a.bucket.target.path.style.access", "true") // Bucket `target.other` plus suffix `endpoint`, not bucket `target` plus `other.endpoint`. hadoopConf.set("fs.s3a.bucket.target.other.endpoint", "https://other.example") - hadoopConf.set("fs.s3a.bucket.target.other.encryption.algorithm", "SSE-KMS") - hadoopConf.set("fs.s3a.bucket.target.encryption.algorithm", "SSE-KMS") + // DSSE-KMS has no iceberg-rust equivalent, so it falls back for the bucket it applies to. + hadoopConf.set("fs.s3a.bucket.target.other.encryption.algorithm", "DSSE-KMS") + hadoopConf.set("fs.s3a.bucket.target.encryption.algorithm", "DSSE-KMS") assert( CometIcebergNativeWrite.unsupportedHadoopS3Settings(hadoopConf, Some("target")) == Seq( @@ -831,7 +832,7 @@ class CometIcebergWriteDetectionSuite extends CometTestBase with CometIcebergTes val longerBucket = new Configuration(false) longerBucket.set("fs.s3a.bucket.target.other.endpoint", "https://other.example") - longerBucket.set("fs.s3a.bucket.target.other.encryption.algorithm", "SSE-KMS") + longerBucket.set("fs.s3a.bucket.target.other.encryption.algorithm", "DSSE-KMS") assert( CometIcebergNativeWrite.unsupportedHadoopS3Settings(longerBucket, Some("target.other")) == Seq("fs.s3a.bucket.target.other.encryption.algorithm")) @@ -954,6 +955,254 @@ class CometIcebergWriteDetectionSuite extends CometTestBase with CometIcebergTes } } + test("S3A server-side encryption translates to the s3.sse.* keys iceberg-rust reads") { + def translate(settings: (String, String)*): Option[Map[String, String]] = { + val hadoopConf = new Configuration(false) + settings.foreach { case (k, v) => hadoopConf.set(k, v) } + CometIcebergNativeWrite.hadoopS3EncryptionProperties(hadoopConf, Some("target")) + } + val kmsKey = "arn:aws:kms:us-east-1:111122223333:key/probe" + // A base64 AES-256 key of zero bytes and the base64 MD5 of those bytes. + val customerKey = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" + val customerKeyMd5 = "cLyPS3KoaSFGi/joRB3OUQ==" + + assert(translate() == Some(Map.empty)) + assert( + translate("fs.s3a.encryption.algorithm" -> " aes256 ") == Some(Map("s3.sse.type" -> "s3"))) + assert( + translate("fs.s3a.encryption.algorithm" -> "sse-kms") == Some(Map("s3.sse.type" -> "kms"))) + assert( + translate("fs.s3a.encryption.algorithm" -> "SSE-KMS", "fs.s3a.encryption.key" -> kmsKey) == + Some(Map("s3.sse.type" -> "kms", "s3.sse.key" -> kmsKey))) + assert( + translate( + "fs.s3a.encryption.algorithm" -> "SSE-C", + "fs.s3a.encryption.key" -> customerKey) == Some( + Map( + "s3.sse.type" -> "custom", + "s3.sse.key" -> customerKey, + "s3.sse.md5" -> customerKeyMd5))) + // The deprecated names, and the data bucket's setting over the global one. + assert( + translate("fs.s3a.server-side-encryption-algorithm" -> "AES256") == Some( + Map("s3.sse.type" -> "s3"))) + assert( + translate( + "fs.s3a.encryption.algorithm" -> "SSE-KMS", + "fs.s3a.bucket.target.server-side-encryption-algorithm" -> "AES256") == Some( + Map("s3.sse.type" -> "s3"))) + assert(translate("fs.s3a.bucket.other.encryption.algorithm" -> "DSSE-KMS") == Some(Map.empty)) + // An empty setting for the data bucket turns a global one off, since NONE has no name. + assert( + translate( + "fs.s3a.encryption.algorithm" -> "SSE-KMS", + "fs.s3a.bucket.target.encryption.algorithm" -> "") == Some(Map.empty)) + // S3A ignores a key without an algorithm. + assert(translate("fs.s3a.encryption.key" -> kmsKey) == Some(Map.empty)) + + // Encryption iceberg-rust cannot apply, settings that fail the S3A write, and a credential + // store, which can supply the values in place of the configuration. + val store = "jceks://file/tmp/s3a.jceks" + Seq( + Seq("fs.s3a.encryption.algorithm" -> "DSSE-KMS"), + Seq("fs.s3a.encryption.algorithm" -> "CSE-KMS"), + Seq("fs.s3a.encryption.algorithm" -> "SSE-S3"), + Seq("fs.s3a.encryption.algorithm" -> "AES256", "fs.s3a.encryption.key" -> kmsKey), + Seq("fs.s3a.encryption.algorithm" -> "SSE-C"), + Seq("fs.s3a.encryption.algorithm" -> "SSE-C", "fs.s3a.encryption.key" -> "not base64!"), + Seq( + "fs.s3a.encryption.algorithm" -> "SSE-KMS", + "hadoop.security.credential.provider.path" -> store), + Seq( + "fs.s3a.encryption.algorithm" -> "SSE-KMS", + "fs.s3a.bucket.target.security.credential.provider.path" -> store)).foreach { settings => + assert(translate(settings: _*).isEmpty, settings) + } + } + + test("translated S3A encryption settings do not block a native write") { + val hadoopConf = new Configuration(false) + hadoopConf.set("fs.s3a.encryption.algorithm", "SSE-KMS") + hadoopConf.set("fs.s3a.bucket.target.encryption.key", "arn:aws:kms:us-east-1:1:key/k") + assert( + CometIcebergNativeWrite.unsupportedHadoopS3Settings(hadoopConf, Some("target")).isEmpty) + + // Client-side encryption for the data bucket makes every encryption setting fall back. + hadoopConf.set("fs.s3a.bucket.target.encryption.algorithm", "CSE-KMS") + assert( + CometIcebergNativeWrite.unsupportedHadoopS3Settings(hadoopConf, Some("target")) == Seq( + "fs.s3a.bucket.target.encryption.algorithm", + "fs.s3a.bucket.target.encryption.key", + "fs.s3a.encryption.algorithm")) + } + + test("S3A credential settings are admitted when the S3A provider list signs the write") { + val hadoopConf = new Configuration(false) + Seq( + "fs.s3a.aws.credentials.provider" -> + ("software.amazon.awssdk.auth.credentials.EnvironmentVariableCredentialsProvider," + + "software.amazon.awssdk.auth.credentials.WebIdentityTokenFileCredentialsProvider"), + "fs.s3a.assumed.role.arn" -> "arn:aws:iam::111122223333:role/probe", + "fs.s3a.assumed.role.session.duration" -> "30m", + "fs.s3a.security.credential.provider.path" -> "jceks://file/tmp/s3a.jceks", + "fs.s3a.bucket.target.assumed.role.arn" -> "arn:aws:iam::111122223333:role/target", + // A catalog's credential store, which Hadoop's password lookup reads for S3A's providers. + "hadoop.security.credential.provider.path" -> "jceks://file/tmp/catalog.jceks", + "hadoop.security.group.mapping.ldap.bind.password" -> "not-for-the-proto", + "dfs.replication" -> "2") + .foreach { case (k, v) => hadoopConf.set(k, v) } + assert( + CometIcebergNativeWrite.usesS3ACredentialAdapter(hadoopConf, Map.empty, Some("target"))) + // The adapter gets the settings S3A's providers read, and nothing else. + assert( + CometIcebergNativeWrite.s3aCredentialAdapterProperties(hadoopConf).keySet == Set( + "fs.s3a.aws.credentials.provider", + "fs.s3a.assumed.role.arn", + "fs.s3a.assumed.role.session.duration", + "fs.s3a.security.credential.provider.path", + "fs.s3a.bucket.target.assumed.role.arn", + "hadoop.security.credential.provider.path", + "s3.comet.credential.provider.class")) + assert( + CometIcebergNativeWrite + .unsupportedHadoopS3Settings(hadoopConf, Some("target"), s3aCredentialAdapter = true) + .isEmpty) + assert( + CometIcebergNativeWrite.unsupportedHadoopS3Settings(hadoopConf, Some("target")) == Seq( + "fs.s3a.assumed.role.arn", + "fs.s3a.assumed.role.session.duration", + "fs.s3a.aws.credentials.provider", + "fs.s3a.bucket.target.assumed.role.arn", + "fs.s3a.security.credential.provider.path")) + + // The adapter refuses S3A delegation tokens, so they still fall back. + hadoopConf.set( + "fs.s3a.delegation.token.binding", + "org.apache.hadoop.fs.s3a.auth.delegation.SessionTokenBinding") + assert( + CometIcebergNativeWrite.unsupportedHadoopS3Settings( + hadoopConf, + Some("target"), + s3aCredentialAdapter = true) == Seq("fs.s3a.delegation.token.binding")) + } + + test("the S3A credential adapter needs an explicit provider list and no Comet provider class") { + val simple = "org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider" + def uses(settings: (String, String)*)(fileIOProperties: (String, String)*): Boolean = { + val hadoopConf = new Configuration(false) + settings.foreach { case (k, v) => hadoopConf.set(k, v) } + CometIcebergNativeWrite.usesS3ACredentialAdapter( + hadoopConf, + fileIOProperties.toMap, + Some("target")) + } + + assert(uses("fs.s3a.aws.credentials.provider" -> simple)()) + // The data bucket's own list counts, another bucket's does not. + assert(uses("fs.s3a.bucket.target.aws.credentials.provider" -> simple)()) + assert(!uses("fs.s3a.bucket.other.aws.credentials.provider" -> simple)()) + // An empty list for the data bucket leaves S3A's default providers, which the adapter does + // not stand in for, so the write keeps falling back. + assert( + !uses( + "fs.s3a.aws.credentials.provider" -> simple, + "fs.s3a.bucket.target.aws.credentials.provider" -> "")()) + // The adapter cannot serve anonymous access. + val anonymous = "org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider" + assert(!uses("fs.s3a.aws.credentials.provider" -> anonymous)()) + // Hadoop 3.4 resolves a name through the provider mapping first, and fails the write on a + // malformed mapping. + val mapping = "fs.s3a.aws.credentials.provider.mapping" + assert(uses("fs.s3a.aws.credentials.provider" -> "keys", mapping -> s"keys=$simple")()) + assert(!uses("fs.s3a.aws.credentials.provider" -> "anon", mapping -> s"anon=$anonymous")()) + assert(!uses("fs.s3a.aws.credentials.provider" -> simple, mapping -> "keys")()) + // A Comet provider class in the FileIO properties keeps its precedence. + assert( + !uses("fs.s3a.aws.credentials.provider" -> simple)( + "s3.comet.credential.provider.class" -> "com.example.Provider")) + assert(!uses()()) + + // Hadoop's built-in default list is not a setting. + val coreDefault = new Configuration(false) + coreDefault.addResource( + new java.io.ByteArrayInputStream(s""" + | fs.s3a.aws.credentials.provider$simple + | + |""".stripMargin.getBytes(java.nio.charset.StandardCharsets.UTF_8)), + "core-default.xml") + assert(coreDefault.get("fs.s3a.aws.credentials.provider") == simple) + assert( + !CometIcebergNativeWrite.usesS3ACredentialAdapter(coreDefault, Map.empty, Some("target"))) + } + + test("the write proto signs through the S3A adapter and carries the translated encryption") { + val provider = "org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider" + val kmsKey = "arn:aws:kms:us-east-1:111122223333:key/probe" + // Set before the catalog initializes its FileIO, which keeps the values it saw then. + withSQLConf( + "fs.s3a.aws.credentials.provider" -> provider, + "fs.s3a.connection.maximum" -> "64", + "fs.s3a.encryption.algorithm" -> "SSE-KMS", + "fs.s3a.encryption.key" -> kmsKey, + CometConf.COMET_EXEC_LOCAL_TABLE_SCAN_ENABLED.key -> "true") { + withDetectionCatalog { _ => + // HadoopFileIO ignores its own s3.sse.* properties, so S3A's encryption replaces them. + withSQLConf( + s"spark.sql.catalog.$catalog.s3.sse.type" -> "custom", + s"spark.sql.catalog.$catalog.s3.sse.key" -> "ignored-key", + s"spark.sql.catalog.$catalog.s3.sse.md5" -> "ignored-md5") { + createTable( + "s3a_adapter", + partitionSpec = "", + properties = Some("'write.data.path'='s3a://probe-bucket/iceberg/db/s3a_adapter'")) + // Planning only: inspect the native proto without issuing any S3 requests. + val insert = spark.sessionState.sqlParser.parsePlan( + s"INSERT INTO $catalog.$ns.s3a_adapter VALUES (1, 'us', 1.0)") + val plan = + spark.sessionState.executePlan(insert, CommandExecutionMode.SKIP).executedPlan + val cometWrite = findCometWriteExec(plan) + .getOrElse(fail(s"expected CometIcebergWriteExec in:\n$plan")) + val properties = cometWrite.nativeOp.getIcebergWrite.getCommon.getCatalogPropertiesMap + assert( + properties.get("s3.comet.credential.provider.class") == + classOf[org.apache.comet.cloud.s3.HadoopS3ACredentialProviderAdapter].getName, + properties) + // The adapter reads these on the executor, with the catalog's hadoop.* overrides. + assert(properties.get("fs.s3a.aws.credentials.provider") == provider, properties) + assert(properties.get("fs.s3a.connection.maximum") == "64", properties) + assert(properties.get("s3.sse.type") == "kms", properties) + assert(properties.get("s3.sse.key") == kmsKey, properties) + assert(!properties.containsKey("s3.sse.md5"), properties) + } + } + } + } + + test("fall-back: an S3A provider list beside a Comet credential provider class") { + withTempIcebergDir { warehouseDir => + val providerCat = "s3a_provider_list_cat" + withSQLConf( + "fs.s3a.aws.credentials.provider" -> + "org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider", + s"spark.sql.catalog.$providerCat" -> "org.apache.iceberg.spark.SparkCatalog", + s"spark.sql.catalog.$providerCat.type" -> "hadoop", + s"spark.sql.catalog.$providerCat.warehouse" -> warehouseDir.getAbsolutePath, + s"spark.sql.catalog.$providerCat.s3.comet.credential.provider.class" -> + classOf[org.apache.comet.cloud.s3.MinioCometS3CredentialProvider].getName) { + spark.sql(s""" + CREATE TABLE $providerCat.$ns.provider_list (id INT, region STRING, amount DOUBLE) + USING iceberg + TBLPROPERTIES ('write.data.path'='s3a://probe-bucket/iceberg/db/provider_list') + """) + val writeExec = planInsertWriteExec(s"$providerCat.$ns.provider_list") + assertUnsupportedContains( + writeExec, + "provider_list", + "unsupported Hadoop S3A setting: fs.s3a.aws.credentials.provider") + } + } + } + test("Hadoop S3A built-in defaults are ignored but custom resources are effective") { def xml(key: String, value: String): java.io.ByteArrayInputStream = new java.io.ByteArrayInputStream(s""" @@ -963,23 +1212,23 @@ class CometIcebergWriteDetectionSuite extends CometTestBase with CometIcebergTes val key = "fs.s3a.encryption.algorithm" val coreDefaultOnly = new Configuration(false) - coreDefaultOnly.addResource(xml(key, "SSE-KMS"), "core-default.xml") - assert(coreDefaultOnly.get(key) == "SSE-KMS") + coreDefaultOnly.addResource(xml(key, "DSSE-KMS"), "core-default.xml") + assert(coreDefaultOnly.get(key) == "DSSE-KMS") assert( CometIcebergNativeWrite .unsupportedHadoopS3Settings(coreDefaultOnly, Some("target")) .isEmpty) val customDefault = new Configuration(false) - customDefault.addResource(xml(key, "SSE-KMS"), "tenant-default.xml") - assert(customDefault.get(key) == "SSE-KMS") + customDefault.addResource(xml(key, "DSSE-KMS"), "tenant-default.xml") + assert(customDefault.get(key) == "DSSE-KMS") assert( CometIcebergNativeWrite.unsupportedHadoopS3Settings(customDefault, Some("target")) == Seq( key)) val site = new Configuration(false) - site.addResource(xml(key, "SSE-KMS"), "probe-site.xml") - assert(site.get(key) == "SSE-KMS") + site.addResource(xml(key, "DSSE-KMS"), "probe-site.xml") + assert(site.get(key) == "DSSE-KMS") assert(CometIcebergNativeWrite.unsupportedHadoopS3Settings(site, Some("target")) == Seq(key)) coreDefaultOnly.set(key, "SSE-S3") @@ -996,7 +1245,7 @@ class CometIcebergWriteDetectionSuite extends CometTestBase with CometIcebergTes s"spark.sql.catalog.$catalogWithOverride" -> "org.apache.iceberg.spark.SparkCatalog", s"spark.sql.catalog.$catalogWithOverride.type" -> "hadoop", s"spark.sql.catalog.$catalogWithOverride.warehouse" -> warehouseDir.getAbsolutePath, - s"spark.sql.catalog.$catalogWithOverride.hadoop.$hadoopKey" -> "SSE-KMS") { + s"spark.sql.catalog.$catalogWithOverride.hadoop.$hadoopKey" -> "DSSE-KMS") { spark.sql(s""" CREATE TABLE $catalogWithOverride.$ns.catalog_hadoop_override ( id INT, @@ -1021,7 +1270,7 @@ class CometIcebergWriteDetectionSuite extends CometTestBase with CometIcebergTes val fileIOConf = IcebergReflection .getFileIOHadoopConf(table) .getOrElse(fail("FileIO did not expose its Hadoop configuration")) - assert(fileIOConf.get(hadoopKey) == "SSE-KMS") + assert(fileIOConf.get(hadoopKey) == "DSSE-KMS") assertUnsupportedContains( writeExec, "catalog_hadoop_override", @@ -1523,6 +1772,9 @@ class CometIcebergWriteDetectionSuite extends CometTestBase with CometIcebergTes assert(props.get("s3.endpoint") == "http://localhost:9000", props) assert(props.get("s3.access-key-id") == "probe-access-key", props) assert(props.get("s3.path-style-access") == "true", props) + // Without an S3A provider list the native writer keeps its own credential chain. + assert(!props.containsKey("s3.comet.credential.provider.class"), props) + assert(!props.containsKey("fs.s3a.access.key"), props) // The exec's string rendering must never fall through to the protobuf's toString: // Spark's argString redaction only covers Scala Maps, so the property bag -- now // carrying credential-shaped values like the access key above -- would land verbatim diff --git a/spark/src/test/scala/org/apache/comet/IcebergReadFromS3Suite.scala b/spark/src/test/scala/org/apache/comet/IcebergReadFromS3Suite.scala index 168bd0b49f9..47b0fd1bbca 100644 --- a/spark/src/test/scala/org/apache/comet/IcebergReadFromS3Suite.scala +++ b/spark/src/test/scala/org/apache/comet/IcebergReadFromS3Suite.scala @@ -20,12 +20,18 @@ package org.apache.comet import org.apache.spark.SparkConf -import org.apache.spark.sql.comet.CometIcebergNativeScanExec +import org.apache.spark.sql.comet.{CometIcebergNativeScanExec, CometIcebergWriteExec} import org.apache.spark.sql.execution.SparkPlan import org.apache.comet.iceberg.RESTCatalogHelper -class IcebergReadFromS3Suite extends CometS3TestBase with RESTCatalogHelper { +class IcebergReadFromS3Suite + extends CometS3TestBase + with RESTCatalogHelper + with CometIcebergTestBase { + + // Both base traits define it, the same way. + override protected def icebergAvailable: Boolean = super[CometIcebergTestBase].icebergAvailable override protected val testBucketName = "test-iceberg-bucket" @@ -312,4 +318,50 @@ class IcebergReadFromS3Suite extends CometS3TestBase with RESTCatalogHelper { } } } + + test("native Iceberg write signs with an explicit S3A provider list") { + assume(icebergAvailable, "Iceberg not available in classpath") + // With fs.s3a.aws.credentials.provider set, iceberg-java's S3A writer signs with exactly that + // list, and the native writer runs the same list through HadoopS3ACredentialProviderAdapter. + // The catalog's hadoop.* override reaches the adapter only through the write proto. + val providerCatalog = "s3a_provider_cat" + withSQLConf( + s"spark.sql.catalog.$providerCatalog" -> "org.apache.iceberg.spark.SparkCatalog", + s"spark.sql.catalog.$providerCatalog.type" -> "hadoop", + s"spark.sql.catalog.$providerCatalog.warehouse" -> + s"s3a://$testBucketName/warehouse-s3a-provider", + s"spark.sql.catalog.$providerCatalog.s3.endpoint" -> minioContainer.getS3URL, + s"spark.sql.catalog.$providerCatalog.s3.region" -> "us-east-1", + s"spark.sql.catalog.$providerCatalog.s3.path-style-access" -> "true", + s"spark.sql.catalog.$providerCatalog.hadoop.fs.s3a.aws.credentials.provider" -> + "org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider", + CometConf.COMET_ICEBERG_NATIVE_WRITE_ENABLED.key -> "true", + CometConf.COMET_EXEC_LOCAL_TABLE_SCAN_ENABLED.key -> "true") { + val rows = "(1, 'Alice', 10.5), (2, 'Bob', 20.3), (3, 'Charlie', 30.7)" + spark.sql( + s"CREATE TABLE $providerCatalog.db.native_write (id INT, name STRING, value DOUBLE) " + + "USING iceberg") + val plans = capturePlans(spark) { + spark.sql(s"INSERT INTO $providerCatalog.db.native_write VALUES $rows") + } + assert( + plans.exists(plan => + collectWithSubqueries(plan) { case w: CometIcebergWriteExec => w }.nonEmpty), + s"expected the native Iceberg write in:\n${plans.mkString("\n--\n")}") + + // The same rows through iceberg-java's writer, which uses the same provider list. + withSQLConf(CometConf.COMET_ICEBERG_NATIVE_WRITE_ENABLED.key -> "false") { + spark.sql( + s"CREATE TABLE $providerCatalog.db.jvm_write (id INT, name STRING, value DOUBLE) " + + "USING iceberg") + spark.sql(s"INSERT INTO $providerCatalog.db.jvm_write VALUES $rows") + } + checkAnswer( + spark.sql(s"SELECT * FROM $providerCatalog.db.native_write"), + spark.sql(s"SELECT * FROM $providerCatalog.db.jvm_write")) + + spark.sql(s"DROP TABLE $providerCatalog.db.native_write") + spark.sql(s"DROP TABLE $providerCatalog.db.jvm_write") + } + } }