Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 27 additions & 0 deletions mkdocs/docs/api.md
Original file line number Diff line number Diff line change
Expand Up @@ -1156,6 +1156,33 @@ assert tbl.name_mapping() is not None
assert tbl.metadata.snapshots[-1].summary["abc"] == "def"
```

## Delete Files

Data files can be removed from the current snapshot by path without rewriting the remaining files. Every path must reference a live data file in the target branch.

| Parameter | Required? | Type | Description |
| ----------------------- | --------- | -------------- | ----------------------------------------------------------------------- |
| `file_paths` | ✔️ | List[str] | The full paths of the data files to remove |
| `snapshot_properties` | | Dict[str, str] | Properties to set for the new snapshot. Defaults to an empty dictionary |
| `branch` | | str | Branch to remove the files from. Defaults to `main` |

### Example

```python
# Assume an existing Iceberg table object `tbl`

file_paths = [
"s3a://warehouse/default/existing-1.parquet",
]

tbl.delete_files(file_paths=file_paths)

# The files are no longer part of the current snapshot and their rows are not scanned
```

!!! note "Concurrent changes"
If a file is no longer referenced when the delete commits, the commit fails. Reload the table and retry.

## Schema evolution

PyIceberg supports full schema evolution through the Python API. It takes care of setting the field-IDs and makes sure that only non-breaking changes are done (can be overridden).
Expand Down
69 changes: 69 additions & 0 deletions pyiceberg/table/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -1090,6 +1090,56 @@ def add_files(
for data_file in data_files:
append_files.append_data_file(data_file)

def delete_files(
self,
file_paths: list[str],
snapshot_properties: dict[str, str] = EMPTY_DICT,
branch: str | None = MAIN_BRANCH,
) -> None:
"""Remove data files from the table transaction.

Args:
file_paths: The full paths of the data files to remove.
snapshot_properties: Custom properties to be added to the snapshot summary.
branch: Branch Reference to run the delete operation.

Raises:
ValueError: If a path is duplicated, or is not referenced by the table.
"""
if not file_paths:
return

if len(file_paths) != len(set(file_paths)):
raise ValueError("File paths must be unique")

snapshot = self.table_metadata.snapshot_by_name(branch) if branch is not None else self.table_metadata.current_snapshot()
if snapshot is None:
raise ValueError("Cannot delete files from a table without a snapshot")

candidates = set(file_paths)
io = self._table.io
data_manifests = [manifest for manifest in snapshot.manifests(io) if manifest.content == ManifestContent.DATA]

def path_filter(data_file: DataFile) -> bool:
return data_file.file_path in candidates

executor = ExecutorFactory.get_or_create()
entries = chain.from_iterable(
executor.map(
lambda args: _open_manifest(*args),
[(io, manifest, path_filter, lambda _: True) for manifest in data_manifests],
)
)
data_files = {entry.data_file.file_path: entry.data_file for entry in entries}

missing = candidates - data_files.keys()
if missing:
raise ValueError(f"Cannot delete files that are not referenced by table, files: {', '.join(sorted(missing))}")

with self.update_snapshot(snapshot_properties=snapshot_properties, branch=branch).overwrite() as overwrite:
for data_file in data_files.values():
overwrite.delete_data_file(data_file)

def update_spec(self) -> UpdateSpec:
"""Create a new UpdateSpec to update the partitioning of the table.

Expand Down Expand Up @@ -1935,6 +1985,25 @@ def add_files(
branch=branch,
)

def delete_files(
self,
file_paths: list[str],
snapshot_properties: dict[str, str] = EMPTY_DICT,
branch: str | None = MAIN_BRANCH,
) -> None:
"""Shorthand API for removing data files from the table.

Args:
file_paths: The full paths of the data files to remove.
snapshot_properties: Custom properties to be added to the snapshot summary.
branch: Branch Reference to run the delete operation.

Raises:
ValueError: If a path is duplicated, or is not referenced by the table.
"""
with self.transaction() as tx:
tx.delete_files(file_paths=file_paths, snapshot_properties=snapshot_properties, branch=branch)

def update_spec(self, case_sensitive: bool = True) -> UpdateSpec:
return UpdateSpec(Transaction(self, autocommit=True), case_sensitive=case_sensitive)

Expand Down
44 changes: 44 additions & 0 deletions tests/table/test_init.py
Original file line number Diff line number Diff line change
Expand Up @@ -2595,6 +2595,50 @@ def test_overwrite_delete_data_file_on_bucket_partition(catalog: Catalog) -> Non
assert table.scan().to_arrow().num_rows == 0


def test_delete_files_removes_data_files(catalog: Catalog) -> None:
import pyarrow as pa

from pyiceberg.io.pyarrow import schema_to_pyarrow

catalog.create_namespace("default")
schema = Schema(NestedField(1, "x", IntegerType(), required=False))
table = catalog.create_table("default.delete_files", schema=schema)
table.append(pa.table({"x": [1, 2]}, schema=schema_to_pyarrow(schema)))
table.append(pa.table({"x": [3]}, schema=schema_to_pyarrow(schema)))
table = catalog.load_table("default.delete_files")

files = table.inspect.files().to_pylist()
assert len(files) == 2
file_to_delete = next(data_file["file_path"] for data_file in files if data_file["record_count"] == 1)

table.delete_files([file_to_delete], snapshot_properties={"source": "test"})

table = catalog.load_table("default.delete_files")
assert table.scan().to_arrow().to_pylist() == [{"x": 1}, {"x": 2}]
snapshot = table.current_snapshot()
assert snapshot is not None
assert snapshot.summary is not None
assert snapshot.summary.additional_properties["source"] == "test"


def test_delete_files_rejects_missing_and_duplicate_paths(catalog: Catalog) -> None:
import pyarrow as pa

from pyiceberg.io.pyarrow import schema_to_pyarrow

catalog.create_namespace("default")
schema = Schema(NestedField(1, "x", IntegerType(), required=False))
table = catalog.create_table("default.delete_files_errors", schema=schema)
table.append(pa.table({"x": [1]}, schema=schema_to_pyarrow(schema)))
file_path = next(data_file["file_path"] for data_file in table.inspect.files().to_pylist())

with pytest.raises(ValueError, match="File paths must be unique"):
table.delete_files([file_path, file_path])

with pytest.raises(ValueError, match="not referenced by table"):
table.delete_files(["file:///missing.parquet"])


def test_partial_delete_keeps_earlier_deletes_outside_the_filter(catalog: Catalog) -> None:
"""Rewriting a file applies all of its delete files, not only those whose bounds can match the delete filter."""
import pyarrow as pa
Expand Down
Loading