Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 10 additions & 1 deletion pyiceberg/table/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -892,12 +892,16 @@ def upsert(
case_sensitive: bool = True,
branch: str | None = MAIN_BRANCH,
snapshot_properties: dict[str, str] = EMPTY_DICT,
difference_cols: list[str] | None = None,
) -> UpsertResult:
"""Shorthand API for performing an upsert to an iceberg table.

Args:
df: The input dataframe to upsert with the table's data.
join_cols: Columns to join on, if not provided, it will use the identifier-field-ids.
difference_cols: Columns to compare when deciding whether a matched row changed. Defaults to
every non-key column. Restricting the comparison helps when only some columns, such as a
hash column, signal a change.
when_matched_update_all: Bool indicating to update rows that are matched but require an update
due to a value in a non-key column changing
when_not_matched_insert_all: Bool indicating new rows to be inserted that do not match any
Expand Down Expand Up @@ -994,7 +998,7 @@ def upsert(
# values have actually changed. We don't want to do just a blanket overwrite for matched
# rows if the actual non-key column data hasn't changed.
# this extra step avoids unnecessary IO and writes
rows_to_update = upsert_util.get_rows_to_update(df, rows, join_cols)
rows_to_update = upsert_util.get_rows_to_update(df, rows, join_cols, difference_cols)

if len(rows_to_update) > 0:
# build the match predicate filter
Expand Down Expand Up @@ -1769,12 +1773,16 @@ def upsert(
case_sensitive: bool = True,
branch: str | None = MAIN_BRANCH,
snapshot_properties: dict[str, str] = EMPTY_DICT,
difference_cols: list[str] | None = None,
) -> UpsertResult:
"""Shorthand API for performing an upsert to an iceberg table.

Args:
df: The input dataframe to upsert with the table's data.
join_cols: Columns to join on, if not provided, it will use the identifier-field-ids.
difference_cols: Columns to compare when deciding whether a matched row changed. Defaults to
every non-key column. Restricting the comparison helps when only some columns, such as a
hash column, signal a change.
when_matched_update_all: Bool indicating to update rows that are matched but require an update
due to a value in a non-key column changing
when_not_matched_insert_all: Bool indicating new rows to be inserted that do not match any
Expand Down Expand Up @@ -1810,6 +1818,7 @@ def upsert(
return tx.upsert(
df=df,
join_cols=join_cols,
difference_cols=difference_cols,
when_matched_update_all=when_matched_update_all,
when_not_matched_insert_all=when_not_matched_insert_all,
case_sensitive=case_sensitive,
Expand Down
20 changes: 18 additions & 2 deletions pyiceberg/table/upsert_util.py
Original file line number Diff line number Diff line change
Expand Up @@ -72,17 +72,33 @@ def _values_differ(
return pc.invert(pc.or_(equal, both_null))


def get_rows_to_update(source_table: pa.Table, target_table: pa.Table, join_cols: list[str]) -> pa.Table:
def get_rows_to_update(
source_table: pa.Table, target_table: pa.Table, join_cols: list[str], difference_cols: list[str] | None = None
) -> pa.Table:
"""
Return a table with rows that need to be updated in the target table based on the join columns.

The table is joined on the identifier columns, and then checked if there are any updated rows.
Those are selected and everything is renamed correctly.

When ``difference_cols`` is set, only those columns are compared. This avoids comparing every
non-key column when the caller already knows which columns signal a change, such as a hash column.
"""
all_columns = set(source_table.column_names)
join_cols_set = set(join_cols)

non_key_cols = list(all_columns - join_cols_set)
compare_cols = non_key_cols
if difference_cols is not None:
difference_cols_set = set(difference_cols)
unknown_cols = difference_cols_set - all_columns
if unknown_cols:
raise ValueError(f"Difference columns not found in the source table: {sorted(unknown_cols)}")
compare_cols = [
column for column in source_table.column_names if column in difference_cols_set and column not in join_cols_set
]
if not compare_cols:
raise ValueError("Difference columns must include at least one non-key column")

if has_duplicate_rows(target_table, join_cols):
raise ValueError("Target table has duplicate rows, aborting upsert")
Expand Down Expand Up @@ -124,7 +140,7 @@ def get_rows_to_update(source_table: pa.Table, target_table: pa.Table, join_cols
target_indices = matching_indices[TARGET_INDEX_COLUMN_NAME]

changed: pa.Array | pa.ChunkedArray | None = None
for key in non_key_cols:
for key in compare_cols:
column_changed = _values_differ(
source_table.column(key).take(source_indices),
target_table.column(key).take(target_indices),
Expand Down
59 changes: 59 additions & 0 deletions tests/table/test_upsert.py
Original file line number Diff line number Diff line change
Expand Up @@ -391,6 +391,33 @@ def test_upsert_with_identifier_fields(catalog: Catalog) -> None:
assert [snap.summary.operation for snap in tbl.snapshots() if snap.summary is not None] == expected_operations


def test_upsert_with_difference_cols(catalog: Catalog) -> None:
identifier = "default.test_upsert_with_difference_cols"
_drop_table(catalog, identifier)

schema = Schema(
NestedField(1, "city", StringType(), required=True),
NestedField(2, "population", IntegerType(), required=True),
NestedField(3, "note", StringType(), required=False),
identifier_field_ids=[1],
)
arrow_schema = schema_to_pyarrow(schema)
tbl = catalog.create_table(identifier, schema=schema)
tbl.append(pa.Table.from_pylist([{"city": "Paris", "population": 1, "note": "a"}], schema=arrow_schema))

# population changed, which is the only compared column
df = pa.Table.from_pylist([{"city": "Paris", "population": 2, "note": "a"}], schema=arrow_schema)
result = tbl.upsert(df, difference_cols=["population"])
assert result.rows_updated == 1

# note changed, but it is not compared
df = pa.Table.from_pylist([{"city": "Paris", "population": 2, "note": "b"}], schema=arrow_schema)
result = tbl.upsert(df, difference_cols=["population"])
assert result.rows_updated == 0

assert tbl.scan().to_arrow().to_pylist() == [{"city": "Paris", "population": 2, "note": "a"}]


def test_upsert_into_empty_table(catalog: Catalog) -> None:
identifier = "default.test_upsert_into_empty_table"
_drop_table(catalog, identifier)
Expand Down Expand Up @@ -955,3 +982,35 @@ def test_get_rows_to_update_compares_nested_values() -> None:
target = pa.table({"pk": [1, 2], "value": pa.array([{"a": 1}, {"a": 3}])})

assert get_rows_to_update(source, target, ["pk"]).to_pydict() == {"pk": [2], "value": [{"a": 2}]}


def test_get_rows_to_update_only_compares_difference_cols() -> None:
source = pa.table({"pk": [1, 2], "value": [10, 21], "ignored": [1, 2]})
target = pa.table({"pk": [1, 2], "value": [10, 20], "ignored": [9, 9]})

assert get_rows_to_update(source, target, ["pk"], difference_cols=["value"]).to_pydict() == {
"pk": [2],
"value": [21],
"ignored": [2],
}


def test_get_rows_to_update_returns_nothing_when_difference_cols_are_unchanged() -> None:
source = pa.table({"pk": [1], "value": [10], "ignored": [1]})
target = pa.table({"pk": [1], "value": [10], "ignored": [2]})

assert len(get_rows_to_update(source, target, ["pk"], difference_cols=["value"])) == 0


def test_get_rows_to_update_rejects_unknown_difference_cols() -> None:
source = pa.table({"pk": [1], "value": [10]})

with pytest.raises(ValueError, match="Difference columns not found"):
get_rows_to_update(source, source, ["pk"], difference_cols=["missing"])


def test_get_rows_to_update_rejects_key_only_difference_cols() -> None:
source = pa.table({"pk": [1], "value": [10]})

with pytest.raises(ValueError, match="at least one non-key column"):
get_rows_to_update(source, source, ["pk"], difference_cols=["pk"])
Loading