From 983aef4fbf7d4c2f5c5fb11db92e1bd6a13a1583 Mon Sep 17 00:00:00 2001 From: Xander Date: Sat, 14 Nov 2026 15:10:26 -0500 Subject: [PATCH] Add EncryptedOutputFile Writes a file, such as a manifest, as an AGS1 stream and records its key metadata with the written file length. --- pyiceberg/encryption/io.py | 60 +++++++++++++++++++++++++++++++-- tests/encryption/test_stream.py | 23 ++++++++++++- 2 files changed, 79 insertions(+), 4 deletions(-) diff --git a/pyiceberg/encryption/io.py b/pyiceberg/encryption/io.py index 1ba9dd5b7d..b8708a5b60 100644 --- a/pyiceberg/encryption/io.py +++ b/pyiceberg/encryption/io.py @@ -14,15 +14,17 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -"""Input files for metadata encrypted with AGS1 streams.""" +"""Input and output files for metadata encrypted with AGS1 streams.""" from __future__ import annotations import io +from collections.abc import Callable +from dataclasses import replace from pyiceberg.encryption.key_metadata import StandardKeyMetadata -from pyiceberg.encryption.stream import _decrypt_ags1 -from pyiceberg.io import InputFile, InputStream +from pyiceberg.encryption.stream import _decrypt_ags1, _encrypt_ags1 +from pyiceberg.io import InputFile, InputStream, OutputFile, OutputStream class EncryptedInputFile(InputFile): @@ -48,3 +50,55 @@ def open(self, seekable: bool = True) -> InputStream: def _decrypt(self) -> bytes: with self._input_file.open(seekable=False) as stream: return _decrypt_ags1(stream.read(), self._key_metadata) + + +class EncryptedOutputFile(OutputFile): + """Encrypts a file, such as a manifest, into an AGS1 stream when its output stream is closed.""" + + def __init__(self, output_file: OutputFile, key_metadata: StandardKeyMetadata) -> None: + super().__init__(output_file.location) + self._output_file = output_file + self._key_metadata = key_metadata + self._file_length: int | None = None + + @property + def key_metadata(self) -> StandardKeyMetadata: + """The key metadata to record for the file, including the length of the written file.""" + if self._file_length is None: + raise ValueError(f"Cannot get key metadata before the file is written: {self.location}") + return replace(self._key_metadata, file_length=self._file_length) + + def __len__(self) -> int: + """Return the length of the encrypted file, in bytes.""" + return self._file_length if self._file_length is not None else len(self._output_file) + + def exists(self) -> bool: + """Check whether the file exists.""" + return self._output_file.exists() + + def to_input_file(self) -> InputFile: + """Return an input file that decrypts the written file.""" + return EncryptedInputFile(self._output_file.to_input_file(), self.key_metadata) + + def create(self, overwrite: bool = False) -> OutputStream: + """Return a stream that buffers the file, then encrypts and writes it when closed.""" + return _EncryptingOutputStream(self._output_file.create(overwrite=overwrite), self._key_metadata, self._set_file_length) + + def _set_file_length(self, file_length: int) -> None: + self._file_length = file_length + + +class _EncryptingOutputStream(io.BytesIO): + def __init__(self, output_stream: OutputStream, key_metadata: StandardKeyMetadata, on_close: Callable[[int], None]) -> None: + super().__init__() + self._output_stream = output_stream + self._key_metadata = key_metadata + self._on_close = on_close + + def close(self) -> None: + if not self.closed: + encrypted = _encrypt_ags1(self.getvalue(), self._key_metadata) + with self._output_stream: + self._output_stream.write(encrypted) + self._on_close(len(encrypted)) + super().close() diff --git a/tests/encryption/test_stream.py b/tests/encryption/test_stream.py index 5211f39d19..8b42e0a7f6 100644 --- a/tests/encryption/test_stream.py +++ b/tests/encryption/test_stream.py @@ -20,7 +20,7 @@ import pytest -from pyiceberg.encryption.io import EncryptedInputFile +from pyiceberg.encryption.io import EncryptedInputFile, EncryptedOutputFile from pyiceberg.encryption.key_metadata import StandardKeyMetadata from pyiceberg.encryption.stream import ( _BLOCK_OVERHEAD, @@ -234,3 +234,24 @@ def test_encrypted_input_file_rejects_a_truncated_file(tmp_path: Path) -> None: with pytest.raises(ValueError, match="key metadata expects"): input_file.open() + + +def test_encrypted_output_file(tmp_path: Path) -> None: + path = tmp_path / "manifest.avro" + output_file = EncryptedOutputFile(PyArrowFileIO().new_output(str(path)), KEY_METADATA) + + with pytest.raises(ValueError, match="Cannot get key metadata before the file is written"): + _ = output_file.key_metadata + + with output_file.create() as stream: + stream.write(b"manifest contents") + + encrypted = path.read_bytes() + assert encrypted[:4] == b"AGS1" + assert output_file.key_metadata == replace(KEY_METADATA, file_length=len(encrypted)) + assert len(output_file) == len(encrypted) + with output_file.to_input_file().open() as stream: + assert stream.read() == b"manifest contents" + + with pytest.raises(FileExistsError): + output_file.create()