From def6e069f0ff5515fb776b724069e8496d369572 Mon Sep 17 00:00:00 2001 From: ghoshp83 Date: Fri, 13 Nov 2026 23:34:22 +0000 Subject: [PATCH] Check every new delete file for rewritten files on a retry On a retry, the check for new deletes on the data files being replaced built its delete index with the operation's filter, so it skipped delete files whose column bounds can't match the filter. A partial delete rewrites those files in full, so it could commit over a concurrent delete outside the filter and bring the deleted rows back. Check every delete file added since the starting snapshot against the replaced data files instead. The filter is no longer used there, so drop it from the signature. --- pyiceberg/table/update/snapshot.py | 9 +-------- pyiceberg/table/update/validate.py | 7 ++----- tests/table/test_commit_retry.py | 22 ++++++++++++++++++++++ tests/table/test_validate.py | 1 - 4 files changed, 25 insertions(+), 14 deletions(-) diff --git a/pyiceberg/table/update/snapshot.py b/pyiceberg/table/update/snapshot.py index fe18fcba00..0c9831450e 100644 --- a/pyiceberg/table/update/snapshot.py +++ b/pyiceberg/table/update/snapshot.py @@ -514,14 +514,7 @@ def _validate_concurrency(self) -> None: if self._deleted_data_files: _validate_data_files_exist(table, catalog_head, self._deleted_data_files, starting_snapshot) - _validate_no_new_deletes_for_data_files( - table, - catalog_head, - conflict_detection_filter, - self._deleted_data_files, - starting_snapshot, - self._case_sensitive, - ) + _validate_no_new_deletes_for_data_files(table, catalog_head, self._deleted_data_files, starting_snapshot) def _build_partition_projection(self, spec_id: int) -> BooleanExpression: project = inclusive_projection(self.schema(), self.spec(spec_id), self._case_sensitive) diff --git a/pyiceberg/table/update/validate.py b/pyiceberg/table/update/validate.py index fa378fa0e5..66f5ddb451 100644 --- a/pyiceberg/table/update/validate.py +++ b/pyiceberg/table/update/validate.py @@ -375,26 +375,23 @@ def _validate_no_new_delete_files( def _validate_no_new_deletes_for_data_files( table: Table, starting_snapshot: Snapshot, - data_filter: BooleanExpression | None, data_files: set[DataFile], parent_snapshot: Snapshot | None, - case_sensitive: bool = True, ) -> None: """Validate no new delete files must be applied for data files that have been added to the table since a starting snapshot. Args: table: Table to validate starting_snapshot: Snapshot current at the start of the operation - data_filter: Expression used to find added data files data_files: data files to validate have no new deletes parent_snapshot: Ending snapshot on the branch being validated - case_sensitive: Whether the data filter binds column names case-sensitively """ # If there is no current state, or no files has been added if table.format_version < 2: return - deletes = _added_delete_files(table, starting_snapshot, data_filter, None, parent_snapshot, case_sensitive) + # The data files are rewritten in full, so a delete file outside the operation's filter still applies to them + deletes = _added_delete_files(table, starting_snapshot, None, None, parent_snapshot) seq_num = _starting_sequence_number(table, starting_snapshot) # Fail to any delete file found that applies to files written in or before the starting snapshot diff --git a/tests/table/test_commit_retry.py b/tests/table/test_commit_retry.py index 8b2d697a63..67653be957 100644 --- a/tests/table/test_commit_retry.py +++ b/tests/table/test_commit_retry.py @@ -636,6 +636,28 @@ def test_file_overwrite_validates_concurrent_position_delete_in_nan_partition(ca assert catalog.load_table(identifier).scan().to_arrow()["id"].to_pylist() == [2] +def test_partial_delete_validates_concurrent_delete_outside_the_filter(catalog: Catalog) -> None: + """A retried partial delete must fail when a concurrent delete file applies to a rewritten file, whatever its bounds.""" + import pyarrow as pa + + catalog.create_namespace("default") + schema = Schema(NestedField(1, "k", LongType(), required=False), NestedField(2, "v", StringType(), required=False)) + identifier = "default.concurrent_delete_outside_the_filter" + table = catalog.create_table(identifier, schema=schema, properties={"format-version": "2"}) + table.append(pa.table({"k": [1, 2, 3], "v": ["a", "b", "c"]})) + data_file = next(iter(table.scan().plan_files())).file + + deleting_table = catalog.load_table(identifier) + + # The delete file stores the deleted row (2, 'b'), so its bounds can't match k = 1. + commit_partition_position_delete(table, Record(), data_file.file_path, pos=1, row={"k": 2, "v": "b"}) + + with pytest.raises(ValidationException, match="found new delete for replace data file"): + deleting_table.delete("k = 1") + + assert catalog.load_table(identifier).scan().to_arrow()["k"].to_pylist() == [1, 3] + + def test_concurrent_overwrite_append_retries_successfully(catalog: Catalog) -> None: """Append after a concurrent overwrite should succeed via retry.""" catalog.create_namespace("default") diff --git a/tests/table/test_validate.py b/tests/table/test_validate.py index 5714f77de4..cc4f3ba3f9 100644 --- a/tests/table/test_validate.py +++ b/tests/table/test_validate.py @@ -613,7 +613,6 @@ def test_validate_no_new_delete_files_for_data_files_raises_on_conflict( _validate_no_new_deletes_for_data_files( table=table, starting_snapshot=newest_snapshot, - data_filter=None, data_files={mocked_data_file}, parent_snapshot=oldest_snapshot, )