Skip to content

Commit 02087f2

Browse files
committed
Apply every delete file when rewriting files for a partial delete
A partial delete reads each candidate file in full and writes back the surviving rows, but planned the rewrite with the delete filter, which also prunes delete files by column bounds. Rows removed by a pruned delete file came back in the rewritten file. Plan the rewrite without pruning delete files; scans are unchanged.
1 parent e3564a5 commit 02087f2

2 files changed

Lines changed: 159 additions & 6 deletions

File tree

‎pyiceberg/table/__init__.py‎

Lines changed: 30 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -796,7 +796,20 @@ def delete(
796796
file_scan = self._scan(row_filter=delete_filter, case_sensitive=case_sensitive)
797797
if branch is not None:
798798
file_scan = file_scan.use_ref(branch)
799-
files = file_scan.plan_files()
799+
# Each file is rewritten in full, so it needs every delete file that applies to it,
800+
# not only those whose bounds can match the delete filter.
801+
snapshot = file_scan.snapshot()
802+
files = (
803+
ManifestGroupPlanner(
804+
table_metadata=self.table_metadata,
805+
io=self._table.io,
806+
row_filter=delete_filter,
807+
case_sensitive=case_sensitive,
808+
prune_delete_files=False,
809+
).plan_files(snapshot.manifests(self._table.io))
810+
if snapshot is not None
811+
else []
812+
)
800813

801814
commit_uuid = uuid.uuid4()
802815
counter = itertools.count(0)
@@ -2782,12 +2795,20 @@ def __init__(
27822795
row_filter: str | BooleanExpression = ALWAYS_TRUE,
27832796
case_sensitive: bool = True,
27842797
options: Properties = EMPTY_DICT,
2798+
prune_delete_files: bool = True,
27852799
):
2800+
"""Create a planner for a group of manifests.
2801+
2802+
Args:
2803+
prune_delete_files: Leave out delete files whose column bounds cannot match the row filter.
2804+
This is only correct when the planned data files are read with the row filter applied.
2805+
"""
27862806
self.table_metadata = table_metadata
27872807
self.io = io
27882808
self.row_filter = _parse_row_filter(row_filter)
27892809
self.case_sensitive = case_sensitive
27902810
self.options = options
2811+
self.prune_delete_files = prune_delete_files
27912812

27922813
@cached_property
27932814
def partition_filters(self) -> KeyDefaultDict[int, BooleanExpression]:
@@ -2813,7 +2834,14 @@ def plan_manifest_entries(self, manifests: Iterable[ManifestFile]) -> Iterator[l
28132834
# this filter depends on the partition spec used to write the manifest file
28142835

28152836
partition_evaluators: dict[int, Callable[[DataFile], bool]] = KeyDefaultDict(self._build_partition_evaluator)
2816-
metrics_evaluator = self._build_metrics_evaluator()
2837+
evaluate_metrics = self._build_metrics_evaluator()
2838+
metrics_evaluator = evaluate_metrics
2839+
if not self.prune_delete_files:
2840+
2841+
def keep_delete_files(data_file: DataFile) -> bool:
2842+
return data_file.content != DataFileContent.DATA or evaluate_metrics(data_file)
2843+
2844+
metrics_evaluator = keep_delete_files
28172845

28182846
min_sequence_number = _min_sequence_number(manifests)
28192847

‎tests/catalog/test_catalog_behaviors.py‎

Lines changed: 129 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -20,11 +20,14 @@
2020
"""
2121

2222
import os
23+
import struct
24+
import uuid
2325
from collections.abc import Generator
2426
from pathlib import Path
2527
from typing import Any
2628

2729
import pyarrow as pa
30+
import pyarrow.parquet as pq
2831
import pytest
2932
from pydantic_core import ValidationError
3033
from pytest_lazy_fixtures import lf
@@ -40,14 +43,25 @@
4043
TableAlreadyExistsError,
4144
)
4245
from pyiceberg.io.pyarrow import _dataframe_to_data_files, schema_to_pyarrow
46+
from pyiceberg.manifest import (
47+
DataFile,
48+
DataFileContent,
49+
FileFormat,
50+
ManifestContent,
51+
ManifestEntry,
52+
ManifestEntryStatus,
53+
ManifestWriterV2,
54+
write_manifest_list,
55+
)
4356
from pyiceberg.partitioning import UNPARTITIONED_PARTITION_SPEC, PartitionField, PartitionSpec
4457
from pyiceberg.schema import Schema
45-
from pyiceberg.table import TableProperties
46-
from pyiceberg.table.snapshots import Operation
58+
from pyiceberg.table import Table, TableProperties
59+
from pyiceberg.table.refs import MAIN_BRANCH, SnapshotRefType
60+
from pyiceberg.table.snapshots import Operation, Snapshot, Summary
4761
from pyiceberg.table.sorting import NullOrder, SortDirection, SortField, SortOrder
48-
from pyiceberg.table.update import AddSchemaUpdate, SetCurrentSchemaUpdate
62+
from pyiceberg.table.update import AddSchemaUpdate, AddSnapshotUpdate, SetCurrentSchemaUpdate, SetSnapshotRefUpdate
4963
from pyiceberg.transforms import IdentityTransform
50-
from pyiceberg.typedef import Identifier
64+
from pyiceberg.typedef import Identifier, Record
5165
from pyiceberg.types import BooleanType, DoubleType, IntegerType, LongType, NestedField, StringType
5266

5367

@@ -1409,3 +1423,114 @@ def test_record_batch_reader_schema_mismatch_writes_no_files(catalog: Catalog) -
14091423
# _append_snapshot_producer opens.
14101424
assert tbl.metadata.current_snapshot() is None
14111425
assert len(tbl.scan().to_arrow()) == 0
1426+
1427+
1428+
def _commit_position_delete_with_row(tbl: Table, data_file_path: str, pos: int, row: dict[str, Any]) -> Table:
1429+
"""Commit a position delete that stores the deleted row, which gives it bounds on the data columns."""
1430+
1431+
class DeleteManifestWriter(ManifestWriterV2):
1432+
def content(self) -> ManifestContent:
1433+
return ManifestContent.DELETES
1434+
1435+
@property
1436+
def _meta(self) -> dict[str, str]:
1437+
return {**super()._meta, "content": "deletes"}
1438+
1439+
def field(name: str, type_: pa.DataType, field_id: int) -> pa.Field:
1440+
return pa.field(name, type_, nullable=False, metadata={"PARQUET:field_id": str(field_id)})
1441+
1442+
row_type = pa.struct([field("k", pa.int64(), 1), field("v", pa.string(), 2)])
1443+
deletes = pa.Table.from_pylist(
1444+
[{"file_path": data_file_path, "pos": pos, "row": row}],
1445+
schema=pa.schema(
1446+
[
1447+
field("file_path", pa.string(), 2147483546),
1448+
field("pos", pa.int64(), 2147483545),
1449+
field("row", row_type, 2147483544),
1450+
]
1451+
),
1452+
)
1453+
delete_path = f"{tbl.location()}/data/delete-{uuid.uuid4()}.parquet"
1454+
with tbl.io.new_output(delete_path).create() as output:
1455+
pq.write_table(deletes, output)
1456+
1457+
bounds = {
1458+
2147483546: data_file_path.encode(),
1459+
2147483545: struct.pack("<q", pos),
1460+
1: struct.pack("<q", row["k"]),
1461+
2: row["v"].encode(),
1462+
}
1463+
delete_file = DataFile.from_args(
1464+
content=DataFileContent.POSITION_DELETES,
1465+
file_path=delete_path,
1466+
file_format=FileFormat.PARQUET,
1467+
partition=Record(),
1468+
record_count=1,
1469+
file_size_in_bytes=len(tbl.io.new_input(delete_path)),
1470+
lower_bounds=bounds,
1471+
upper_bounds=bounds,
1472+
)
1473+
delete_file.spec_id = tbl.spec().spec_id
1474+
1475+
parent = tbl.current_snapshot()
1476+
assert parent is not None
1477+
snapshot_id = uuid.uuid4().int >> 65
1478+
sequence_number = tbl.metadata.next_sequence_number()
1479+
with DeleteManifestWriter(
1480+
spec=tbl.spec(),
1481+
schema=tbl.schema(),
1482+
output_file=tbl.io.new_output(f"{tbl.location()}/metadata/{uuid.uuid4()}-m0.avro"),
1483+
snapshot_id=snapshot_id,
1484+
avro_compression="deflate",
1485+
) as writer:
1486+
writer.add(ManifestEntry.from_args(status=ManifestEntryStatus.ADDED, snapshot_id=snapshot_id, data_file=delete_file))
1487+
manifest_list = f"{tbl.location()}/metadata/snap-{snapshot_id}.avro"
1488+
with write_manifest_list(
1489+
format_version=2,
1490+
output_file=tbl.io.new_output(manifest_list),
1491+
snapshot_id=snapshot_id,
1492+
parent_snapshot_id=parent.snapshot_id,
1493+
sequence_number=sequence_number,
1494+
avro_compression="deflate",
1495+
) as list_writer:
1496+
list_writer.add_manifests([writer.to_manifest_file(), *parent.manifests(tbl.io)])
1497+
1498+
snapshot = Snapshot(
1499+
snapshot_id=snapshot_id,
1500+
parent_snapshot_id=parent.snapshot_id,
1501+
sequence_number=sequence_number,
1502+
manifest_list=manifest_list,
1503+
summary=Summary(Operation.DELETE),
1504+
schema_id=tbl.schema().schema_id,
1505+
)
1506+
tbl.catalog.commit_table(
1507+
tbl,
1508+
(),
1509+
(
1510+
AddSnapshotUpdate(snapshot=snapshot),
1511+
SetSnapshotRefUpdate(ref_name=MAIN_BRANCH, type=SnapshotRefType.BRANCH, snapshot_id=snapshot_id),
1512+
),
1513+
)
1514+
return tbl.catalog.load_table(tbl.name())
1515+
1516+
1517+
def test_partial_delete_keeps_earlier_deletes_outside_the_filter(catalog: Catalog) -> None:
1518+
"""Rewriting a file for a delete must apply every delete file of that file. A delete file whose
1519+
bounds cannot match the delete filter is skipped by a filtered scan, which is right for a read but
1520+
would bring its rows back in the rewritten file.
1521+
"""
1522+
catalog.create_namespace("default")
1523+
identifier = f"default.partial_delete_keeps_earlier_deletes_{catalog.name}"
1524+
iceberg_schema = Schema(NestedField(1, "k", LongType(), required=True), NestedField(2, "v", StringType(), required=True))
1525+
tbl = catalog.create_table(identifier=identifier, schema=iceberg_schema, properties={"format-version": "2"})
1526+
tbl.append(pa.Table.from_pydict({"k": [1, 2, 3], "v": ["a", "b", "c"]}, schema=schema_to_pyarrow(iceberg_schema)))
1527+
data_file_path = next(iter(tbl.scan().plan_files())).file.file_path
1528+
1529+
# Delete (2, 'b'). The delete file has bounds k = 2, so a scan for k = 1 leaves it out.
1530+
tbl = _commit_position_delete_with_row(tbl, data_file_path, pos=1, row={"k": 2, "v": "b"})
1531+
assert tbl.scan().to_arrow()["k"].to_pylist() == [1, 3]
1532+
assert [len(task.delete_files) for task in tbl.scan(row_filter="k = 1").plan_files()] == [0]
1533+
1534+
tbl.delete("k = 1")
1535+
1536+
assert tbl.scan().to_arrow()["k"].to_pylist() == [3]

0 commit comments

Comments
 (0)