Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions bindings/python/src/encryption.rs
Original file line number Diff line number Diff line change
Expand Up @@ -100,11 +100,11 @@ pub fn encode_standard_key_metadata<'py>(
if let Some(aad_prefix) = aad_prefix {
metadata = metadata.with_aad_prefix(aad_prefix);
}
if let Some(file_length) = file_length {
metadata = metadata.with_file_length(file_length);
}

Ok(PyBytes::new(py, &metadata.encode().map_err(to_py_err)?))
Ok(PyBytes::new(
py,
&metadata.encode(file_length).map_err(to_py_err)?,
))
}

pub fn register_module(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> {
Expand Down
5 changes: 2 additions & 3 deletions crates/iceberg/public-api.txt
Original file line number Diff line number Diff line change
Expand Up @@ -302,7 +302,7 @@ pub struct iceberg::encryption::EncryptionManager
impl iceberg::encryption::EncryptionManager
pub async fn iceberg::encryption::EncryptionManager::decrypt_manifest_list_key_metadata(&self, encryption_key_id: &str) -> iceberg::Result<iceberg::encryption::StandardKeyMetadata>
pub fn iceberg::encryption::EncryptionManager::encrypt(&self, raw_output: iceberg::io::OutputFile) -> iceberg::encryption::EncryptedOutputFile
pub async fn iceberg::encryption::EncryptionManager::encrypt_manifest_list_key_metadata(&self, key_metadata: &iceberg::encryption::StandardKeyMetadata) -> iceberg::Result<alloc::string::String>
pub async fn iceberg::encryption::EncryptionManager::encrypt_manifest_list_key_metadata(&self, key_metadata: &iceberg::encryption::StandardKeyMetadata, file_length: u64) -> iceberg::Result<alloc::string::String>
pub fn iceberg::encryption::EncryptionManager::generate_key_metadata(&self) -> iceberg::encryption::StandardKeyMetadata
pub fn iceberg::encryption::EncryptionManager::with_encryption_keys<F, R>(&self, f: F) -> R where F: core::ops::function::FnOnce(&std::collections::hash::map::HashMap<alloc::string::String, iceberg::spec::EncryptedKey>) -> R
impl core::fmt::Debug for iceberg::encryption::EncryptionManager
Expand Down Expand Up @@ -356,12 +356,11 @@ pub struct iceberg::encryption::StandardKeyMetadata
impl iceberg::encryption::StandardKeyMetadata
pub fn iceberg::encryption::StandardKeyMetadata::aad_prefix(&self) -> core::option::Option<&[u8]>
pub fn iceberg::encryption::StandardKeyMetadata::decode(bytes: &[u8]) -> iceberg::Result<Self>
pub fn iceberg::encryption::StandardKeyMetadata::encode(&self) -> iceberg::Result<alloc::boxed::Box<[u8]>>
pub fn iceberg::encryption::StandardKeyMetadata::encode(&self, file_length: core::option::Option<u64>) -> iceberg::Result<alloc::boxed::Box<[u8]>>
pub fn iceberg::encryption::StandardKeyMetadata::encryption_key(&self) -> &iceberg::encryption::SecureKey
pub fn iceberg::encryption::StandardKeyMetadata::file_length(&self) -> core::option::Option<u64>
pub fn iceberg::encryption::StandardKeyMetadata::try_new(encryption_key: &[u8]) -> iceberg::Result<Self>
pub fn iceberg::encryption::StandardKeyMetadata::with_aad_prefix(self, aad_prefix: &[u8]) -> Self
pub fn iceberg::encryption::StandardKeyMetadata::with_file_length(self, length: u64) -> Self
impl core::clone::Clone for iceberg::encryption::StandardKeyMetadata
pub fn iceberg::encryption::StandardKeyMetadata::clone(&self) -> iceberg::encryption::StandardKeyMetadata
impl core::cmp::Eq for iceberg::encryption::StandardKeyMetadata
Expand Down
4 changes: 1 addition & 3 deletions crates/iceberg/src/arrow/caching_delete_file_loader.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1655,9 +1655,7 @@ mod tests {
let file_metadata = output.write(Bytes::from(blob)).await.unwrap();
let encoded_key_metadata = output
.key_metadata()
.clone()
.with_file_length(file_metadata.size)
.encode()
.encode(Some(file_metadata.size))
.unwrap();

// content_offset / content_size_in_bytes are in the plaintext coordinate space, distinct
Expand Down
4 changes: 2 additions & 2 deletions crates/iceberg/src/arrow/delete_file_loader.rs
Original file line number Diff line number Diff line change
Expand Up @@ -206,7 +206,7 @@ mod tests {
let key_metadata = StandardKeyMetadata::try_new(encryption_key)
.unwrap()
.with_aad_prefix(aad_prefix)
.encode()
.encode(Some(std::fs::metadata(&del_path).unwrap().len()))
.unwrap();

let schema = Arc::new(
Expand Down Expand Up @@ -291,7 +291,7 @@ mod tests {
let key_metadata = StandardKeyMetadata::try_new(encryption_key)
.unwrap()
.with_aad_prefix(aad_prefix)
.encode()
.encode(Some(std::fs::metadata(&del_path).unwrap().len()))
.unwrap();

let schema = Arc::new(
Expand Down
4 changes: 2 additions & 2 deletions crates/iceberg/src/arrow/reader/pipeline.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1113,7 +1113,7 @@ mod tests {
let key_metadata = crate::encryption::StandardKeyMetadata::try_new(encryption_key)
.unwrap()
.with_aad_prefix(aad_prefix)
.encode()
.encode(Some(std::fs::metadata(&file_path).unwrap().len()))
.unwrap();

let reader = ArrowReaderBuilder::new(file_io, Runtime::current()).build();
Expand Down Expand Up @@ -2611,7 +2611,7 @@ mod tests {

let wrong_key_metadata = crate::encryption::StandardKeyMetadata::try_new(wrong_key)
.unwrap()
.encode()
.encode(Some(std::fs::metadata(&file_path).unwrap().len()))
.unwrap();

let reader = ArrowReaderBuilder::new(file_io, Runtime::current()).build();
Expand Down
78 changes: 52 additions & 26 deletions crates/iceberg/src/encryption/key_metadata.rs
Original file line number Diff line number Diff line change
Expand Up @@ -74,8 +74,12 @@ impl StandardKeyMetadata {
self
}

/// Sets the encrypted file length in bytes, required for AGS1 truncation protection.
pub fn with_file_length(mut self, length: u64) -> Self {
/// Sets the encrypted file length, as [`Self::decode`] would populate it.
///
/// Test-only: on the write path the length is supplied to [`Self::encode`], so production
/// code cannot construct metadata carrying a length it did not read off the wire.
#[cfg(test)]
pub(crate) fn with_file_length(mut self, length: u64) -> Self {
self.file_length = Some(length);
self
}
Expand All @@ -90,14 +94,22 @@ impl StandardKeyMetadata {
self.aad_prefix.as_deref()
}

/// Returns the encrypted file length in bytes, required for AGS1 files.
/// Returns the encrypted file length in bytes, as read off the wire by [`Self::decode`].
///
/// `None` for metadata written without one, which AGS1 reads reject.
pub fn file_length(&self) -> Option<u64> {
self.file_length
}

/// Encodes to Java-compatible format: `[0x01] [Avro binary datum]`
pub fn encode(&self) -> Result<Box<[u8]>> {
_serde::StandardKeyMetadataV1::from(self).encode()
///
/// `file_length` is the size of the file this metadata describes, as reported by
/// [`FileWrite::close`](crate::io::FileWrite::close) — the ciphertext size for AGS1
/// streams, which reads require to detect truncation. It is an argument rather than a
/// property so that omitting it has to be deliberate; pass `None` only for files whose
/// format carries its own length, such as Parquet.
pub fn encode(&self, file_length: Option<u64>) -> Result<Box<[u8]>> {
_serde::StandardKeyMetadataV1::new(self, file_length).encode()
}

/// Decodes from Java-compatible format.
Expand Down Expand Up @@ -225,15 +237,17 @@ mod _serde {
}
}

impl From<&StandardKeyMetadata> for StandardKeyMetadataV1 {
fn from(metadata: &StandardKeyMetadata) -> Self {
impl StandardKeyMetadataV1 {
/// Takes the length from the caller rather than `metadata.file_length`, so a length
/// only reaches the wire when the writer supplied one.
pub(super) fn new(metadata: &StandardKeyMetadata, file_length: Option<u64>) -> Self {
Self {
encryption_key: serde_bytes::ByteBuf::from(metadata.encryption_key.as_bytes()),
aad_prefix: metadata
.aad_prefix
.as_ref()
.map(|b| serde_bytes::ByteBuf::from(b.as_ref())),
file_length: metadata.file_length,
file_length,
}
}
}
Expand Down Expand Up @@ -267,33 +281,46 @@ mod tests {
let key = b"0123456789012345";
let aad = b"1234567890123456";

let file_length = 100_000;
let metadata = StandardKeyMetadata::try_new(key)
.unwrap()
.with_aad_prefix(aad);
let serialized = metadata.encode().unwrap();
let serialized = metadata.encode(Some(file_length)).unwrap();
let parsed = StandardKeyMetadata::decode(&serialized).unwrap();

assert_eq!(parsed.encryption_key().as_bytes(), key);
assert_eq!(parsed.aad_prefix(), Some(aad.as_slice()));
assert_eq!(parsed.file_length(), None);
assert_eq!(parsed.file_length(), Some(file_length));
}

#[test]
fn test_roundtrip_with_length() {
let key = b"0123456789012345";
let aad = b"1234567890123456";

let file_length = 100_000;
let metadata = StandardKeyMetadata::try_new(key)
fn test_encode_ignores_any_decoded_length() {
// The length on the wire is the argument, never a value carried on the struct.
let metadata = StandardKeyMetadata::try_new(b"0123456789012345")
.unwrap()
.with_aad_prefix(aad)
.with_file_length(file_length);
let serialized = metadata.encode().unwrap();
let parsed = StandardKeyMetadata::decode(&serialized).unwrap();
.with_file_length(1);

assert_eq!(parsed.encryption_key().as_bytes(), key);
assert_eq!(parsed.aad_prefix(), Some(aad.as_slice()));
assert_eq!(parsed.file_length(), Some(file_length));
assert_eq!(
StandardKeyMetadata::decode(&metadata.encode(Some(2)).unwrap())
.unwrap()
.file_length(),
Some(2)
);
assert_eq!(
StandardKeyMetadata::decode(&metadata.encode(None).unwrap())
.unwrap()
.file_length(),
None
);
}

#[test]
fn test_roundtrip_without_file_length() {
// Parquet and other self-describing formats omit it; AGS1 reads reject the result.
let metadata = StandardKeyMetadata::try_new(b"0123456789012345").unwrap();
let parsed = StandardKeyMetadata::decode(&metadata.encode(None).unwrap()).unwrap();

assert_eq!(parsed.file_length(), None);
}

#[test]
Expand All @@ -319,7 +346,7 @@ mod tests {
fn test_roundtrip_without_aad() {
let key = b"0123456789012345";
let metadata = StandardKeyMetadata::try_new(key).unwrap();
let serialized = metadata.encode().unwrap();
let serialized = metadata.encode(Some(4096)).unwrap();
let parsed = StandardKeyMetadata::decode(&serialized).unwrap();

assert_eq!(parsed.encryption_key().as_bytes(), key);
Expand Down Expand Up @@ -376,8 +403,7 @@ mod tests {
let serialized = StandardKeyMetadata::try_new(key)
.unwrap()
.with_aad_prefix(aad)
.with_file_length(file_length)
.encode()
.encode(Some(file_length))
.unwrap();

// Arbitrary junk, then bytes shaped like a further optional field appended by a
Expand Down
28 changes: 16 additions & 12 deletions crates/iceberg/src/encryption/manager.rs
Original file line number Diff line number Diff line change
Expand Up @@ -162,9 +162,13 @@ impl EncryptionManager {
///
/// Returns the `key_id` of the wrapped entry, which should be recorded on
/// the snapshot as `encryption_key_id` so readers can locate it later.
///
/// `file_length` is the written size of the manifest list, needed for AGS1 truncation
/// protection.
pub async fn encrypt_manifest_list_key_metadata(
&self,
key_metadata: &StandardKeyMetadata,
file_length: u64,
) -> Result<String> {
let kek = match self.find_active_kek()? {
Some(existing) => existing,
Expand All @@ -175,7 +179,7 @@ impl EncryptionManager {

// Use the KEK timestamp as AAD to prevent timestamp tampering attacks.
let aad = Self::kek_timestamp_aad(&kek)?;
let serialized = key_metadata.encode()?;
let serialized = key_metadata.encode(Some(file_length))?;
let wrapped_metadata = self.wrap_dek_with_kek(&serialized, &kek_bytes, Some(aad))?;

let wrapped_key = EncryptedKey::builder()
Expand Down Expand Up @@ -449,6 +453,8 @@ mod tests {
assert!(kek.properties().contains_key(KEK_CREATED_AT_PROPERTY));
}

const SAMPLE_FILE_LENGTH: u64 = 4096;

fn sample_key_metadata() -> StandardKeyMetadata {
StandardKeyMetadata::try_new(b"0123456789abcdef")
.unwrap()
Expand All @@ -461,7 +467,7 @@ mod tests {
let plaintext = sample_key_metadata();

let key_id = mgr
.encrypt_manifest_list_key_metadata(&plaintext)
.encrypt_manifest_list_key_metadata(&plaintext, SAMPLE_FILE_LENGTH)
.await
.unwrap();

Expand All @@ -472,7 +478,7 @@ mod tests {
.decrypt_manifest_list_key_metadata(&key_id)
.await
.unwrap();
assert_eq!(decrypted, plaintext);
assert_eq!(decrypted, plaintext.with_file_length(SAMPLE_FILE_LENGTH));
}

#[tokio::test]
Expand All @@ -481,7 +487,7 @@ mod tests {

// First wrap creates a new KEK + wrapped entry (2 keys)
let _id1 = mgr
.encrypt_manifest_list_key_metadata(&sample_key_metadata())
.encrypt_manifest_list_key_metadata(&sample_key_metadata(), SAMPLE_FILE_LENGTH)
.await
.unwrap();
let kek_id = mgr.with_encryption_keys(|keys| {
Expand All @@ -495,7 +501,7 @@ mod tests {

// Second wrap should reuse the existing KEK (only adds 1 new wrapped entry)
let id2 = mgr
.encrypt_manifest_list_key_metadata(&sample_key_metadata())
.encrypt_manifest_list_key_metadata(&sample_key_metadata(), SAMPLE_FILE_LENGTH)
.await
.unwrap();
let entry2 = mgr.with_encryption_keys(|keys| {
Expand Down Expand Up @@ -537,7 +543,7 @@ mod tests {

// Wrap should rotate to a new KEK since the existing one is expired
let new_entry_id = mgr
.encrypt_manifest_list_key_metadata(&sample_key_metadata())
.encrypt_manifest_list_key_metadata(&sample_key_metadata(), SAMPLE_FILE_LENGTH)
.await
.unwrap();
let entry = mgr
Expand Down Expand Up @@ -573,7 +579,7 @@ mod tests {

// First wrap caches the plaintext KEK during create_kek().
let key_id = mgr
.encrypt_manifest_list_key_metadata(&sample_key_metadata())
.encrypt_manifest_list_key_metadata(&sample_key_metadata(), SAMPLE_FILE_LENGTH)
.await
.unwrap();

Expand All @@ -590,7 +596,7 @@ mod tests {

// Wrap some metadata to get a valid encrypted entry stored on the manager
let entry_id = mgr
.encrypt_manifest_list_key_metadata(&sample_key_metadata())
.encrypt_manifest_list_key_metadata(&sample_key_metadata(), SAMPLE_FILE_LENGTH)
.await
.unwrap();

Expand Down Expand Up @@ -633,7 +639,7 @@ mod tests {

// Wrap metadata normally
let entry_id = mgr
.encrypt_manifest_list_key_metadata(&sample_key_metadata())
.encrypt_manifest_list_key_metadata(&sample_key_metadata(), SAMPLE_FILE_LENGTH)
.await
.unwrap();

Expand Down Expand Up @@ -695,9 +701,7 @@ mod tests {

let serialized_metadata = encrypted_output
.key_metadata()
.clone()
.with_file_length(file_metadata.size)
.encode()
.encode(Some(file_metadata.size))
.unwrap();

let input = io.new_input(path).unwrap();
Expand Down
3 changes: 1 addition & 2 deletions crates/iceberg/src/spec/manifest/writer.rs
Original file line number Diff line number Diff line change
Expand Up @@ -515,8 +515,7 @@ impl ManifestWriter {
.key_metadata
.map(|metadata| {
metadata
.with_file_length(file_metadata.size)
.encode()
.encode(Some(file_metadata.size))
.map(|bytes| bytes.into_vec())
})
.transpose()?;
Expand Down
20 changes: 14 additions & 6 deletions crates/iceberg/src/spec/manifest_list/manifest_file.rs
Original file line number Diff line number Diff line change
Expand Up @@ -325,9 +325,13 @@ mod test {
// returning garbage.
let wrong_key_metadata = StandardKeyMetadata::try_new(b"fedcba9876543210")
.unwrap()
.with_aad_prefix(b"test-aad-prefix!")
.with_file_length(manifest_file.manifest_length as u64);
manifest_file.key_metadata = Some(wrong_key_metadata.encode().unwrap().to_vec());
.with_aad_prefix(b"test-aad-prefix!");
manifest_file.key_metadata = Some(
wrong_key_metadata
.encode(Some(manifest_file.manifest_length as u64))
.unwrap()
.to_vec(),
);

let err = ManifestReader::new(io)
.read(&manifest_file)
Expand All @@ -351,9 +355,13 @@ mod test {
// so GCM authentication must fail even though the key is right.
let wrong_aad_metadata = StandardKeyMetadata::try_new(b"0123456789abcdef")
.unwrap()
.with_aad_prefix(b"wrong-aad-prefix")
.with_file_length(manifest_file.manifest_length as u64);
manifest_file.key_metadata = Some(wrong_aad_metadata.encode().unwrap().to_vec());
.with_aad_prefix(b"wrong-aad-prefix");
manifest_file.key_metadata = Some(
wrong_aad_metadata
.encode(Some(manifest_file.manifest_length as u64))
.unwrap()
.to_vec(),
);

let err = ManifestReader::new(io)
.read(&manifest_file)
Expand Down
4 changes: 1 addition & 3 deletions crates/iceberg/src/transaction/snapshot.rs
Original file line number Diff line number Diff line change
Expand Up @@ -498,9 +498,7 @@ impl<'a> SnapshotProducer<'a> {
self.table
.encryption_manager()
.expect("Encryption manager must be present when key metadata exists")
.encrypt_manifest_list_key_metadata(
&key_metadata.with_file_length(file_metadata.size),
)
.encrypt_manifest_list_key_metadata(&key_metadata, file_metadata.size)
.await?,
)
} else {
Expand Down
Loading
Loading