|
20 | 20 | """ |
21 | 21 |
|
22 | 22 | import os |
| 23 | +import struct |
| 24 | +import uuid |
23 | 25 | from collections.abc import Generator |
24 | 26 | from pathlib import Path |
25 | 27 | from typing import Any |
26 | 28 |
|
27 | 29 | import pyarrow as pa |
| 30 | +import pyarrow.parquet as pq |
28 | 31 | import pytest |
29 | 32 | from pydantic_core import ValidationError |
30 | 33 | from pytest_lazy_fixtures import lf |
|
40 | 43 | TableAlreadyExistsError, |
41 | 44 | ) |
42 | 45 | from pyiceberg.io.pyarrow import _dataframe_to_data_files, schema_to_pyarrow |
| 46 | +from pyiceberg.manifest import ( |
| 47 | + DataFile, |
| 48 | + DataFileContent, |
| 49 | + FileFormat, |
| 50 | + ManifestContent, |
| 51 | + ManifestEntry, |
| 52 | + ManifestEntryStatus, |
| 53 | + ManifestWriterV2, |
| 54 | + write_manifest_list, |
| 55 | +) |
43 | 56 | from pyiceberg.partitioning import UNPARTITIONED_PARTITION_SPEC, PartitionField, PartitionSpec |
44 | 57 | from pyiceberg.schema import Schema |
45 | | -from pyiceberg.table import TableProperties |
46 | | -from pyiceberg.table.snapshots import Operation |
| 58 | +from pyiceberg.table import Table, TableProperties |
| 59 | +from pyiceberg.table.refs import MAIN_BRANCH, SnapshotRefType |
| 60 | +from pyiceberg.table.snapshots import Operation, Snapshot, Summary |
47 | 61 | from pyiceberg.table.sorting import NullOrder, SortDirection, SortField, SortOrder |
48 | | -from pyiceberg.table.update import AddSchemaUpdate, SetCurrentSchemaUpdate |
| 62 | +from pyiceberg.table.update import AddSchemaUpdate, AddSnapshotUpdate, SetCurrentSchemaUpdate, SetSnapshotRefUpdate |
49 | 63 | from pyiceberg.transforms import IdentityTransform |
50 | | -from pyiceberg.typedef import Identifier |
| 64 | +from pyiceberg.typedef import Identifier, Record |
51 | 65 | from pyiceberg.types import BooleanType, DoubleType, IntegerType, LongType, NestedField, StringType |
52 | 66 |
|
53 | 67 |
|
@@ -1409,3 +1423,114 @@ def test_record_batch_reader_schema_mismatch_writes_no_files(catalog: Catalog) - |
1409 | 1423 | # _append_snapshot_producer opens. |
1410 | 1424 | assert tbl.metadata.current_snapshot() is None |
1411 | 1425 | assert len(tbl.scan().to_arrow()) == 0 |
| 1426 | + |
| 1427 | + |
| 1428 | +def _commit_position_delete_with_row(tbl: Table, data_file_path: str, pos: int, row: dict[str, Any]) -> Table: |
| 1429 | + """Commit a position delete that stores the deleted row, which gives it bounds on the data columns.""" |
| 1430 | + |
| 1431 | + class DeleteManifestWriter(ManifestWriterV2): |
| 1432 | + def content(self) -> ManifestContent: |
| 1433 | + return ManifestContent.DELETES |
| 1434 | + |
| 1435 | + @property |
| 1436 | + def _meta(self) -> dict[str, str]: |
| 1437 | + return {**super()._meta, "content": "deletes"} |
| 1438 | + |
| 1439 | + def field(name: str, type_: pa.DataType, field_id: int) -> pa.Field: |
| 1440 | + return pa.field(name, type_, nullable=False, metadata={"PARQUET:field_id": str(field_id)}) |
| 1441 | + |
| 1442 | + row_type = pa.struct([field("k", pa.int64(), 1), field("v", pa.string(), 2)]) |
| 1443 | + deletes = pa.Table.from_pylist( |
| 1444 | + [{"file_path": data_file_path, "pos": pos, "row": row}], |
| 1445 | + schema=pa.schema( |
| 1446 | + [ |
| 1447 | + field("file_path", pa.string(), 2147483546), |
| 1448 | + field("pos", pa.int64(), 2147483545), |
| 1449 | + field("row", row_type, 2147483544), |
| 1450 | + ] |
| 1451 | + ), |
| 1452 | + ) |
| 1453 | + delete_path = f"{tbl.location()}/data/delete-{uuid.uuid4()}.parquet" |
| 1454 | + with tbl.io.new_output(delete_path).create() as output: |
| 1455 | + pq.write_table(deletes, output) |
| 1456 | + |
| 1457 | + bounds = { |
| 1458 | + 2147483546: data_file_path.encode(), |
| 1459 | + 2147483545: struct.pack("<q", pos), |
| 1460 | + 1: struct.pack("<q", row["k"]), |
| 1461 | + 2: row["v"].encode(), |
| 1462 | + } |
| 1463 | + delete_file = DataFile.from_args( |
| 1464 | + content=DataFileContent.POSITION_DELETES, |
| 1465 | + file_path=delete_path, |
| 1466 | + file_format=FileFormat.PARQUET, |
| 1467 | + partition=Record(), |
| 1468 | + record_count=1, |
| 1469 | + file_size_in_bytes=len(tbl.io.new_input(delete_path)), |
| 1470 | + lower_bounds=bounds, |
| 1471 | + upper_bounds=bounds, |
| 1472 | + ) |
| 1473 | + delete_file.spec_id = tbl.spec().spec_id |
| 1474 | + |
| 1475 | + parent = tbl.current_snapshot() |
| 1476 | + assert parent is not None |
| 1477 | + snapshot_id = uuid.uuid4().int >> 65 |
| 1478 | + sequence_number = tbl.metadata.next_sequence_number() |
| 1479 | + with DeleteManifestWriter( |
| 1480 | + spec=tbl.spec(), |
| 1481 | + schema=tbl.schema(), |
| 1482 | + output_file=tbl.io.new_output(f"{tbl.location()}/metadata/{uuid.uuid4()}-m0.avro"), |
| 1483 | + snapshot_id=snapshot_id, |
| 1484 | + avro_compression="deflate", |
| 1485 | + ) as writer: |
| 1486 | + writer.add(ManifestEntry.from_args(status=ManifestEntryStatus.ADDED, snapshot_id=snapshot_id, data_file=delete_file)) |
| 1487 | + manifest_list = f"{tbl.location()}/metadata/snap-{snapshot_id}.avro" |
| 1488 | + with write_manifest_list( |
| 1489 | + format_version=2, |
| 1490 | + output_file=tbl.io.new_output(manifest_list), |
| 1491 | + snapshot_id=snapshot_id, |
| 1492 | + parent_snapshot_id=parent.snapshot_id, |
| 1493 | + sequence_number=sequence_number, |
| 1494 | + avro_compression="deflate", |
| 1495 | + ) as list_writer: |
| 1496 | + list_writer.add_manifests([writer.to_manifest_file(), *parent.manifests(tbl.io)]) |
| 1497 | + |
| 1498 | + snapshot = Snapshot( |
| 1499 | + snapshot_id=snapshot_id, |
| 1500 | + parent_snapshot_id=parent.snapshot_id, |
| 1501 | + sequence_number=sequence_number, |
| 1502 | + manifest_list=manifest_list, |
| 1503 | + summary=Summary(Operation.DELETE), |
| 1504 | + schema_id=tbl.schema().schema_id, |
| 1505 | + ) |
| 1506 | + tbl.catalog.commit_table( |
| 1507 | + tbl, |
| 1508 | + (), |
| 1509 | + ( |
| 1510 | + AddSnapshotUpdate(snapshot=snapshot), |
| 1511 | + SetSnapshotRefUpdate(ref_name=MAIN_BRANCH, type=SnapshotRefType.BRANCH, snapshot_id=snapshot_id), |
| 1512 | + ), |
| 1513 | + ) |
| 1514 | + return tbl.catalog.load_table(tbl.name()) |
| 1515 | + |
| 1516 | + |
| 1517 | +def test_partial_delete_keeps_earlier_deletes_outside_the_filter(catalog: Catalog) -> None: |
| 1518 | + """Rewriting a file for a delete must apply every delete file of that file. A delete file whose |
| 1519 | + bounds cannot match the delete filter is skipped by a filtered scan, which is right for a read but |
| 1520 | + would bring its rows back in the rewritten file. |
| 1521 | + """ |
| 1522 | + catalog.create_namespace("default") |
| 1523 | + identifier = f"default.partial_delete_keeps_earlier_deletes_{catalog.name}" |
| 1524 | + iceberg_schema = Schema(NestedField(1, "k", LongType(), required=True), NestedField(2, "v", StringType(), required=True)) |
| 1525 | + tbl = catalog.create_table(identifier=identifier, schema=iceberg_schema, properties={"format-version": "2"}) |
| 1526 | + tbl.append(pa.Table.from_pydict({"k": [1, 2, 3], "v": ["a", "b", "c"]}, schema=schema_to_pyarrow(iceberg_schema))) |
| 1527 | + data_file_path = next(iter(tbl.scan().plan_files())).file.file_path |
| 1528 | + |
| 1529 | + # Delete (2, 'b'). The delete file has bounds k = 2, so a scan for k = 1 leaves it out. |
| 1530 | + tbl = _commit_position_delete_with_row(tbl, data_file_path, pos=1, row={"k": 2, "v": "b"}) |
| 1531 | + assert tbl.scan().to_arrow()["k"].to_pylist() == [1, 3] |
| 1532 | + assert [len(task.delete_files) for task in tbl.scan(row_filter="k = 1").plan_files()] == [0] |
| 1533 | + |
| 1534 | + tbl.delete("k = 1") |
| 1535 | + |
| 1536 | + assert tbl.scan().to_arrow()["k"].to_pylist() == [3] |
0 commit comments