Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
60 changes: 57 additions & 3 deletions pyiceberg/encryption/io.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,15 +14,17 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
"""Input files for metadata encrypted with AGS1 streams."""
"""Input and output files for metadata encrypted with AGS1 streams."""

from __future__ import annotations

import io
from collections.abc import Callable
from dataclasses import replace

from pyiceberg.encryption.key_metadata import StandardKeyMetadata
from pyiceberg.encryption.stream import _decrypt_ags1
from pyiceberg.io import InputFile, InputStream
from pyiceberg.encryption.stream import _decrypt_ags1, _encrypt_ags1
from pyiceberg.io import InputFile, InputStream, OutputFile, OutputStream


class EncryptedInputFile(InputFile):
Expand All @@ -48,3 +50,55 @@ def open(self, seekable: bool = True) -> InputStream:
def _decrypt(self) -> bytes:
with self._input_file.open(seekable=False) as stream:
return _decrypt_ags1(stream.read(), self._key_metadata)


class EncryptedOutputFile(OutputFile):
"""Encrypts a file, such as a manifest, into an AGS1 stream when its output stream is closed."""

def __init__(self, output_file: OutputFile, key_metadata: StandardKeyMetadata) -> None:
super().__init__(output_file.location)
self._output_file = output_file
self._key_metadata = key_metadata
self._file_length: int | None = None

@property
def key_metadata(self) -> StandardKeyMetadata:
"""The key metadata to record for the file, including the length of the written file."""
if self._file_length is None:
raise ValueError(f"Cannot get key metadata before the file is written: {self.location}")
return replace(self._key_metadata, file_length=self._file_length)

def __len__(self) -> int:
"""Return the length of the encrypted file, in bytes."""
return self._file_length if self._file_length is not None else len(self._output_file)

def exists(self) -> bool:
"""Check whether the file exists."""
return self._output_file.exists()

def to_input_file(self) -> InputFile:
"""Return an input file that decrypts the written file."""
return EncryptedInputFile(self._output_file.to_input_file(), self.key_metadata)

def create(self, overwrite: bool = False) -> OutputStream:
"""Return a stream that buffers the file, then encrypts and writes it when closed."""
return _EncryptingOutputStream(self._output_file.create(overwrite=overwrite), self._key_metadata, self._set_file_length)

def _set_file_length(self, file_length: int) -> None:
self._file_length = file_length


class _EncryptingOutputStream(io.BytesIO):
def __init__(self, output_stream: OutputStream, key_metadata: StandardKeyMetadata, on_close: Callable[[int], None]) -> None:
super().__init__()
self._output_stream = output_stream
self._key_metadata = key_metadata
self._on_close = on_close

def close(self) -> None:
if not self.closed:
encrypted = _encrypt_ags1(self.getvalue(), self._key_metadata)
with self._output_stream:
self._output_stream.write(encrypted)
self._on_close(len(encrypted))
super().close()
23 changes: 22 additions & 1 deletion tests/encryption/test_stream.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@

import pytest

from pyiceberg.encryption.io import EncryptedInputFile
from pyiceberg.encryption.io import EncryptedInputFile, EncryptedOutputFile
from pyiceberg.encryption.key_metadata import StandardKeyMetadata
from pyiceberg.encryption.stream import (
_BLOCK_OVERHEAD,
Expand Down Expand Up @@ -234,3 +234,24 @@ def test_encrypted_input_file_rejects_a_truncated_file(tmp_path: Path) -> None:

with pytest.raises(ValueError, match="key metadata expects"):
input_file.open()


def test_encrypted_output_file(tmp_path: Path) -> None:
path = tmp_path / "manifest.avro"
output_file = EncryptedOutputFile(PyArrowFileIO().new_output(str(path)), KEY_METADATA)

with pytest.raises(ValueError, match="Cannot get key metadata before the file is written"):
_ = output_file.key_metadata

with output_file.create() as stream:
stream.write(b"manifest contents")

encrypted = path.read_bytes()
assert encrypted[:4] == b"AGS1"
assert output_file.key_metadata == replace(KEY_METADATA, file_length=len(encrypted))
assert len(output_file) == len(encrypted)
with output_file.to_input_file().open() as stream:
assert stream.read() == b"manifest contents"

with pytest.raises(FileExistsError):
output_file.create()
Loading