Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion pyiceberg/conversions.py
Original file line number Diff line number Diff line change
Expand Up @@ -388,13 +388,17 @@ def _(_: StringType, b: bytes) -> str:

@from_bytes.register(BinaryType)
@from_bytes.register(FixedType)
@from_bytes.register(UUIDType)
@from_bytes.register(GeometryType)
@from_bytes.register(GeographyType)
def _(_: PrimitiveType, b: bytes) -> bytes:
return b


@from_bytes.register(UUIDType)
def _(_: UUIDType, b: bytes) -> uuid.UUID:
return uuid.UUID(bytes=b)


@from_bytes.register(DecimalType)
def _(primitive_type: DecimalType, buf: bytes) -> Decimal:
unscaled = int.from_bytes(buf, "big", signed=True)
Expand Down
28 changes: 14 additions & 14 deletions pyiceberg/expressions/literals.py
Original file line number Diff line number Diff line change
Expand Up @@ -165,7 +165,7 @@ def literal(value: L) -> Literal[L]:
elif isinstance(value, str):
return StringLiteral(value)
elif isinstance(value, UUID):
return UUIDLiteral(value.bytes)
return UUIDLiteral(value)
elif isinstance(value, bytes):
return BinaryLiteral(value)
elif isinstance(value, Decimal):
Expand Down Expand Up @@ -635,8 +635,8 @@ def _(self, _: TimestamptzType) -> Literal[int]:
return TimestampLiteral(timestamptz_to_micros(self.value))

@to.register(UUIDType)
def _(self, _: UUIDType) -> Literal[bytes]:
return UUIDLiteral(UUID(self.value).bytes)
def _(self, _: UUIDType) -> Literal[UUID]:
return UUIDLiteral(UUID(self.value))

@to.register(DecimalType)
def _(self, type_var: DecimalType) -> Literal[Decimal]:
Expand Down Expand Up @@ -680,34 +680,34 @@ def __repr__(self) -> str:
return f"literal({repr(self.value)})"


class UUIDLiteral(Literal[bytes]):
def __init__(self, value: bytes) -> None:
super().__init__(value, bytes)
class UUIDLiteral(Literal[UUID]):
def __init__(self, value: UUID) -> None:
super().__init__(value, UUID)

@model_serializer
def ser_model(self) -> UUID:
return UUID(bytes=self.root)
return self.root

@singledispatchmethod
def to(self, type_var: IcebergType) -> Literal: # type: ignore
raise TypeError(f"Cannot convert UUIDLiteral into {type_var}")

@to.register(UUIDType)
def _(self, _: UUIDType) -> Literal[bytes]:
def _(self, _: UUIDType) -> Literal[UUID]:
return self

@to.register(FixedType)
def _(self, type_var: FixedType) -> Literal[bytes]:
if len(type_var) == UUID_BYTES_LENGTH:
return FixedLiteral(self.value)
return FixedLiteral(self.value.bytes)
else:
raise TypeError(
f"Cannot convert UUIDLiteral into {type_var}, different length: {len(type_var)} <> {UUID_BYTES_LENGTH}"
)

@to.register(BinaryType)
def _(self, _: BinaryType) -> Literal[bytes]:
return BinaryLiteral(self.value)
return BinaryLiteral(self.value.bytes)


class FixedLiteral(Literal[bytes]):
Expand Down Expand Up @@ -736,9 +736,9 @@ def _(self, _: BinaryType) -> Literal[bytes]:
return BinaryLiteral(self.value)

@to.register(UUIDType)
def _(self, type_var: UUIDType) -> Literal[bytes]:
def _(self, type_var: UUIDType) -> Literal[UUID]:
if len(self.value) == UUID_BYTES_LENGTH:
return UUIDLiteral(self.value)
return UUIDLiteral(UUID(bytes=self.value))
else:
raise TypeError(
f"Could not convert {self.value!r} into a {type_var}, lengths differ {len(self.value)} <> {UUID_BYTES_LENGTH}"
Expand Down Expand Up @@ -771,9 +771,9 @@ def _(self, type_var: FixedType) -> Literal[bytes]:
)

@to.register(UUIDType)
def _(self, type_var: UUIDType) -> Literal[bytes]:
def _(self, type_var: UUIDType) -> Literal[UUID]:
if len(self.value) == UUID_BYTES_LENGTH:
return UUIDLiteral(self.value)
return UUIDLiteral(UUID(bytes=self.value))
else:
raise TypeError(
f"Cannot convert BinaryLiteral into {type_var}, different length: {UUID_BYTES_LENGTH} <> {len(self.value)}"
Expand Down
15 changes: 12 additions & 3 deletions pyiceberg/io/pyarrow.py
Original file line number Diff line number Diff line change
Expand Up @@ -897,10 +897,15 @@ def _literal_arrow_type(iceberg_type: IcebergType, values: Iterable[Any]) -> pa.
return visit(iceberg_type, _ConvertToArrowSchema())


def _to_arrow_value(value: Any) -> Any:
# PyArrow builds uuid values from their 16 bytes; older PyArrow releases reject a uuid.UUID
return value.bytes if isinstance(value, uuid.UUID) else value


def _convert_scalar(value: Any, iceberg_type: IcebergType) -> pa.scalar:
if not isinstance(iceberg_type, PrimitiveType):
raise ValueError(f"Expected primitive type, got: {iceberg_type}")
return pa.scalar(value=value, type=_literal_arrow_type(iceberg_type, [value]))
return pa.scalar(value=_to_arrow_value(value), type=_literal_arrow_type(iceberg_type, [value]))


class _ConvertToArrowExpression(BoundBooleanExpressionVisitor[pc.Expression]):
Expand Down Expand Up @@ -937,11 +942,15 @@ def _get_field_name(self, term: BoundTerm) -> str | tuple[str, ...]:
return term.ref().field.name

def visit_in(self, term: BoundTerm, literals: set[Any]) -> pc.Expression:
pyarrow_literals = pa.array(literals, type=_literal_arrow_type(term.ref().field.field_type, literals))
pyarrow_literals = pa.array(
[_to_arrow_value(literal) for literal in literals], type=_literal_arrow_type(term.ref().field.field_type, literals)
)
return pc.field(self._get_field_name(term)).isin(pyarrow_literals)

def visit_not_in(self, term: BoundTerm, literals: set[Any]) -> pc.Expression:
pyarrow_literals = pa.array(literals, type=_literal_arrow_type(term.ref().field.field_type, literals))
pyarrow_literals = pa.array(
[_to_arrow_value(literal) for literal in literals], type=_literal_arrow_type(term.ref().field.field_type, literals)
)
return ~pc.field(self._get_field_name(term)).isin(pyarrow_literals)

def visit_is_nan(self, term: BoundTerm) -> pc.Expression:
Expand Down
10 changes: 5 additions & 5 deletions pyiceberg/partitioning.py
Original file line number Diff line number Diff line change
Expand Up @@ -487,7 +487,7 @@ def partition_record_value(partition_field: PartitionField, value: Any, schema:
Return the Partition Record representation of the value.

The value is first converted to internal partition representation.
For example, UUID is converted to bytes[16], DateType to days since epoch, etc.
For example, UUID values become uuid.UUID, and DateType values become days since epoch.

Then the corresponding PartitionField's transform is applied to return
the final partition record value.
Expand All @@ -498,7 +498,7 @@ def partition_record_value(partition_field: PartitionField, value: Any, schema:

@singledispatch
def _to_partition_representation(type: IcebergType, value: Any) -> Any:
"""Strip the logical type into the physical type.
"""Return the internal partition representation of a value.

It can be that the value is already transformed into its physical type,
in this case it will return the original value. Keep in mind that the
Expand Down Expand Up @@ -560,13 +560,13 @@ def _(type: IcebergType, value: int | time | None) -> int | None:


@_to_partition_representation.register(UUIDType)
def _(type: IcebergType, value: uuid.UUID | int | bytes | None) -> bytes | int | None:
def _(type: IcebergType, value: uuid.UUID | int | bytes | None) -> uuid.UUID | int | None:
if value is None:
return None
elif isinstance(value, bytes):
return value # IdentityTransform
return uuid.UUID(bytes=value) # IdentityTransform
elif isinstance(value, uuid.UUID):
return value.bytes # IdentityTransform
return value # IdentityTransform
elif isinstance(value, int):
return value # BucketTransform
else:
Expand Down
12 changes: 6 additions & 6 deletions tests/expressions/test_literals.py
Original file line number Diff line number Diff line change
Expand Up @@ -411,7 +411,7 @@ def test_string_to_uuid_literal() -> None:
uuid_str = literal(str(expected))
uuid_lit = uuid_str.to(UUIDType())

assert expected.bytes == uuid_lit.value
assert expected == uuid_lit.value


def test_string_to_decimal_literal() -> None:
Expand Down Expand Up @@ -568,8 +568,8 @@ def test_binary_to_uuid() -> None:
lit = literal(test_uuid.bytes)
uuid_lit = lit.to(UUIDType())
assert uuid_lit is not None
assert lit.value == uuid_lit.value
assert uuid_lit.value == test_uuid.bytes
assert lit.value == test_uuid.bytes
assert uuid_lit.value == test_uuid


def test_incompatible_binary_to_uuid() -> None:
Expand Down Expand Up @@ -598,8 +598,8 @@ def test_fixed_to_uuid() -> None:
lit = literal(test_uuid.bytes).to(FixedType(16))
uuid_lit = lit.to(UUIDType())
assert uuid_lit is not None
assert lit.value == uuid_lit.value
assert uuid_lit.value == test_uuid.bytes
assert lit.value == test_uuid.bytes
assert uuid_lit.value == test_uuid


def test_incompatible_fixed_to_uuid() -> None:
Expand Down Expand Up @@ -1038,7 +1038,7 @@ def test_uuid_literal_initialization() -> None:
test_uuid = uuid.UUID("f79c3e09-677c-4bbd-a479-3f349cb785e7")
uuid_literal = literal(test_uuid)
assert isinstance(uuid_literal, Literal)
assert test_uuid.bytes == uuid_literal.value
assert test_uuid == uuid_literal.value


def test_uuid_to_fixed() -> None:
Expand Down
121 changes: 119 additions & 2 deletions tests/io/test_pyarrow.py
Original file line number Diff line number Diff line change
Expand Up @@ -915,6 +915,25 @@ def test_expr_not_starts_with_to_pyarrow(bound_reference: BoundReference) -> Non
)


def test_uuid_expressions_to_pyarrow() -> None:
schema = Schema(NestedField(1, "u", UUIDType(), required=False))
ref = BoundReference(schema.find_field(1), schema.accessor_for_field(1))
first, second = uuid.UUID(int=1), uuid.UUID(int=2)
literals = {literal(first), literal(second)}
# PyArrow builds uuid values from their 16 bytes, so compare with expressions built from bytes in either set order
expected_in = [
ds.field("u").isin(pa.array(values, type=pa.uuid()))
for values in ([first.bytes, second.bytes], [second.bytes, first.bytes])
]
for expression, expected in [
(BoundEqualTo(ref, literal(first)), [ds.field("u") == pa.scalar(first.bytes, type=pa.uuid())]),
(BoundIn(ref, literals), expected_in),
(BoundNotIn(ref, literals), [~candidate for candidate in expected_in]),
]:
actual = expression_to_pyarrow(expression)
assert any(actual.equals(candidate) for candidate in expected), expression


def test_and_to_pyarrow(bound_reference: BoundReference) -> None:
assert (
repr(expression_to_pyarrow(And(BoundEqualTo(bound_reference, literal("hello")), BoundIsNull(bound_reference))))
Expand Down Expand Up @@ -1571,6 +1590,53 @@ def test_identity_transform_column_projection_with_falsy_value(
)


def test_identity_transform_column_projection_uuid(tmp_path: str, catalog: InMemoryCatalog) -> None:
"""A filter on a uuid identity partition column matches the files that do not contain the column."""
schema = Schema(
NestedField(1, "other_field", StringType(), required=False),
NestedField(2, "partition_col", UUIDType(), required=False),
)
partition_spec = PartitionSpec(PartitionField(2, 1000, IdentityTransform(), "partition_col"))
catalog.create_namespace("default")
table = catalog.create_table(
"default.test_projection_partition_uuid",
schema=schema,
partition_spec=partition_spec,
properties={TableProperties.DEFAULT_NAME_MAPPING: create_mapping_from_schema(schema).model_dump_json()},
)

file_loc = f"{tmp_path}/test.parquet"
pq.write_table(pa.table([pa.array(["foo", "bar"], type=pa.string())], names=["other_field"]), file_loc)
statistics = data_file_statistics_from_parquet_metadata(
parquet_metadata=pq.read_metadata(file_loc),
stats_columns=compute_statistics_plan(table.schema(), table.metadata.properties),
parquet_column_mapping=parquet_path_to_id_mapping(table.schema()),
)
partition_value = uuid.UUID(int=1)
data_file = DataFile.from_args(
content=DataFileContent.DATA,
file_path=file_loc,
file_format=FileFormat.PARQUET,
partition=Record(partition_value),
file_size_in_bytes=os.path.getsize(file_loc),
sort_order_id=None,
spec_id=table.metadata.default_spec_id,
equality_ids=None,
key_metadata=None,
**statistics.to_serialized_dict(),
)
with table.transaction() as transaction:
with transaction.update_snapshot().overwrite() as update:
update.append_data_file(data_file)

for row_filter, expected in [
(f"partition_col = '{partition_value}'", ["foo", "bar"]),
(f"partition_col != '{partition_value}'", []),
]:
result = table.scan(row_filter=row_filter, selected_fields=("other_field",)).to_arrow()
assert result["other_field"].to_pylist() == expected, row_filter


def test_identity_transform_null_partition_value_takes_precedence_over_initial_default(tmpdir: str) -> None:
"""A null identity partition value must project as null, not as the field's initial-default."""
table_schema = Schema(
Expand Down Expand Up @@ -3518,9 +3584,9 @@ def test_determine_partitions_with_uuid_column() -> None:

partitions = list(_determine_partitions(spec, schema, arrow_table))

# The partition values are the 16 bytes of each UUID
# The partition values are uuid.UUID, as they are read back from the manifests
rows_by_partition = {p.partition_key.partition[0]: sorted(p.arrow_table_partition["id"].to_pylist()) for p in partitions}
assert rows_by_partition == {uuid.UUID(int=1).bytes: [1, 3], uuid.UUID(int=2).bytes: [2], None: [4]}
assert rows_by_partition == {uuid.UUID(int=1): [1, 3], uuid.UUID(int=2): [2], None: [4]}
# The written data keeps the uuid type
assert all(p.arrow_table_partition.schema.field("uuid").type == pa.uuid() for p in partitions)

Expand All @@ -3546,6 +3612,57 @@ def test_append_uuid_to_table_partitioned_on_uuid(catalog: InMemoryCatalog) -> N
assert {task.file.partition[0] for task in table.scan().plan_files()} == {uuid.UUID(int=1), uuid.UUID(int=2), None}


def test_scan_plans_uuid_identity_partitions(catalog: InMemoryCatalog) -> None:
schema = Schema(
NestedField(id=1, name="id", field_type=LongType(), required=False),
NestedField(id=2, name="uuid", field_type=UUIDType(), required=False),
)
spec = PartitionSpec(PartitionField(source_id=2, field_id=1000, transform=IdentityTransform(), name="uuid"))
catalog.create_namespace("default")
table = catalog.create_table("default.test_scan_uuid_partitioned", schema=schema, partition_spec=spec)
first, second, third = uuid.UUID(int=1), uuid.UUID(int=2), uuid.UUID(int=3)
table.append(
pa.table(
{
"id": pa.array([1, 2, 3], type=pa.int64()),
"uuid": pa.array([first.bytes, second.bytes, third.bytes], type=pa.uuid()),
}
)
)

# UUID partition values read from manifests must compare equal to bound UUID literals
for row_filter, expected in [
(f"uuid = '{first}'", {first}),
(f"uuid in ('{first}', '{second}')", {first, second}),
(f"uuid != '{first}'", {second, third}),
(f"uuid not in ('{first}', '{second}')", {third}),
]:
scan = table.scan(row_filter=row_filter)
assert {task.file.partition[0] for task in scan.plan_files()} == expected, row_filter
assert scan.count() == len(expected), row_filter


def test_scan_prunes_uuid_files_by_bounds(catalog: InMemoryCatalog) -> None:
schema = Schema(
NestedField(id=1, name="id", field_type=LongType(), required=False),
NestedField(id=2, name="uuid", field_type=UUIDType(), required=False),
)
catalog.create_namespace("default")
table = catalog.create_table("default.test_scan_uuid_bounds", schema=schema)
for ids in ([1, 2], [3, 4]):
values = [uuid.UUID(int=value).bytes for value in ids]
table.append(pa.table({"id": pa.array(ids, type=pa.int64()), "uuid": pa.array(values, type=pa.uuid())}))

# The lower and upper bounds of the uuid column are compared with the literal to prune files
for row_filter, expected_files in [
(f"uuid = '{uuid.UUID(int=1)}'", 1),
(f"uuid > '{uuid.UUID(int=2)}'", 1),
(f"uuid < '{uuid.UUID(int=5)}'", 2),
(f"uuid in ('{uuid.UUID(int=1)}', '{uuid.UUID(int=4)}')", 2),
]:
assert len(list(table.scan(row_filter=row_filter).plan_files())) == expected_files, row_filter


def test_append_view_types_to_partitioned_table(catalog: InMemoryCatalog) -> None:
schema = Schema(
NestedField(id=1, name="id", field_type=LongType(), required=False),
Expand Down
12 changes: 11 additions & 1 deletion tests/table/test_partitioning.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
import pytest

from pyiceberg.exceptions import ValidationError
from pyiceberg.partitioning import UNPARTITIONED_PARTITION_SPEC, PartitionField, PartitionSpec
from pyiceberg.partitioning import UNPARTITIONED_PARTITION_SPEC, PartitionField, PartitionSpec, partition_record_value
from pyiceberg.schema import Schema
from pyiceberg.transforms import (
BucketTransform,
Expand Down Expand Up @@ -220,6 +220,16 @@ def test_partition_type_missing_source_field(table_schema_simple: Schema) -> Non
)


def test_uuid_identity_partition_record_value_and_path() -> None:
schema = Schema(NestedField(field_id=1, name="u", field_type=UUIDType(), required=False))
spec = PartitionSpec(PartitionField(source_id=1, field_id=1000, transform=IdentityTransform(), name="u"))
value = UUID(int=1)
for raw in (value, value.bytes):
assert partition_record_value(spec.fields[0], raw, schema) == value, raw
# The partition path holds the uuid string, as in Java
assert spec.partition_to_path(Record(value), schema) == "u=00000000-0000-0000-0000-000000000001"


@pytest.mark.parametrize(
"source_type, value",
[
Expand Down
Loading
Loading