Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion mkdocs/docs/configuration.md
Original file line number Diff line number Diff line change
Expand Up @@ -88,8 +88,9 @@ Iceberg tables support table properties to configure table behavior.
| `write.target-file-size-bytes` | Size in bytes | 536870912 (512MB) | Target size of generated data files. Measured in uncompressed in-memory Arrow bytes, so files on disk are typically smaller after compression |
| `write.format.default` | `{parquet}` | parquet | File format for data files written by PyIceberg. Only Parquet is supported |
| `write.avro.compression-codec` | `{gzip,zstd,snappy}` | gzip | Compression codec for manifest files and manifest lists. For `snappy`, install with `pip install pyiceberg[snappy]` |
| `write.metadata.metrics.default` | `{none,counts,truncate(N),full}` | truncate(16) | Default column metrics for data files: `none` records only column sizes, `counts` adds value and null counts, `full` adds lower/upper bounds, and `truncate(N)` truncates string and binary bounds to N characters or bytes. Nested fields collect at most counts |
| `write.metadata.metrics.default` | `{none,counts,truncate(N),full}` | truncate(16) | Default column metrics for data files: `none` records only column sizes, `counts` adds value and null counts, `full` adds lower/upper bounds, and `truncate(N)` truncates string and binary bounds to N characters or bytes. Applies to nested fields as well |
| `write.metadata.metrics.column.{column-name}` | `{none,counts,truncate(N),full}` | (value of `write.metadata.metrics.default`) | Overrides `write.metadata.metrics.default` for a single column. Use the dot-separated name for nested fields |
| `write.metadata.metrics.max-inferred-column-defaults` | Integer | 100 | Maximum number of columns that get `write.metadata.metrics.default` when it is not configured explicitly. Further columns collect no metrics unless they have a `write.metadata.metrics.column` override |
| `write.summary.partition-limit` | Integer | 0 | Maximum number of changed partitions for which per-partition summaries are included in the snapshot summary. The default of 0 disables them |
| `write.metadata.previous-versions-max` | Integer | 100 | The max number of previous version metadata files to keep before deleting after commit. |
| `write.metadata.delete-after-commit.enabled` | Boolean | False | Whether to automatically delete old *tracked* metadata files after each table commit. It will retain a number of the most recent metadata files, which can be set using property `write.metadata.previous-versions-max`. |
Expand Down
73 changes: 66 additions & 7 deletions pyiceberg/io/pyarrow.py
Original file line number Diff line number Diff line change
Expand Up @@ -2565,6 +2565,48 @@ def match_metrics_mode(mode: str) -> MetricsMode:
raise ValueError(f"Unsupported metrics mode: {mode}")


def _limit_metrics_field_ids(schema: Schema, limit: int) -> set[int]:
"""Return the primitive field ids that get the default metrics mode, in schema order.

Mirrors Java's MetricsConfig.limitFieldIds: struct fields add their primitive children
before descending into nested fields, and list and map add their primitive element, key,
or value ids before descending.
"""
field_ids: set[int] = set()

def add(field_id: int) -> None:
if len(field_ids) < limit:
field_ids.add(field_id)

def visit(iceberg_type: IcebergType) -> None:
if isinstance(iceberg_type, StructType):
for field in iceberg_type.fields:
if isinstance(field.field_type, PrimitiveType):
add(field.field_id)
for field in iceberg_type.fields:
if len(field_ids) >= limit:
return
if not isinstance(field.field_type, PrimitiveType):
visit(field.field_type)
elif isinstance(iceberg_type, ListType):
if isinstance(iceberg_type.element_type, PrimitiveType):
add(iceberg_type.element_id)
else:
visit(iceberg_type.element_type)
elif isinstance(iceberg_type, MapType):
if isinstance(iceberg_type.key_type, PrimitiveType):
add(iceberg_type.key_id)
if isinstance(iceberg_type.value_type, PrimitiveType):
add(iceberg_type.value_id)
if not isinstance(iceberg_type.key_type, PrimitiveType):
visit(iceberg_type.key_type)
if not isinstance(iceberg_type.value_type, PrimitiveType):
visit(iceberg_type.value_type)

visit(schema.as_struct())
return field_ids


@dataclass(frozen=True)
class StatisticsCollector:
field_id: int
Expand All @@ -2578,15 +2620,34 @@ class PyArrowStatisticsCollector(PreOrderSchemaVisitor[list[StatisticsCollector]
_schema: Schema
_properties: dict[str, str]
_default_mode: str
_configured_default: bool
_inferred_field_ids: set[int]

def __init__(self, schema: Schema, properties: dict[str, str]):
from pyiceberg.table import TableProperties

self._schema = schema
self._properties = properties
self._configured_default = TableProperties.DEFAULT_WRITE_METRICS_MODE in self._properties
self._default_mode = self._properties.get(
TableProperties.DEFAULT_WRITE_METRICS_MODE, TableProperties.DEFAULT_WRITE_METRICS_MODE_DEFAULT
)
max_inferred = property_as_int(
properties,
TableProperties.METRICS_MAX_INFERRED_COLUMN_DEFAULTS,
TableProperties.METRICS_MAX_INFERRED_COLUMN_DEFAULTS_DEFAULT,
)
if max_inferred is None:
max_inferred = TableProperties.METRICS_MAX_INFERRED_COLUMN_DEFAULTS_DEFAULT
elif max_inferred < 0:
logger.warning(
"Invalid value for %s (negative): %s, falling back to %s",
TableProperties.METRICS_MAX_INFERRED_COLUMN_DEFAULTS,
max_inferred,
TableProperties.METRICS_MAX_INFERRED_COLUMN_DEFAULTS_DEFAULT,
)
max_inferred = TableProperties.METRICS_MAX_INFERRED_COLUMN_DEFAULTS_DEFAULT
self._inferred_field_ids = _limit_metrics_field_ids(schema, max_inferred)

def schema(
self, schema: Schema, struct_result: Callable[[], builtins.list[StatisticsCollector]]
Expand Down Expand Up @@ -2629,23 +2690,21 @@ def primitive(self, primitive: PrimitiveType) -> builtins.list[StatisticsCollect
if column_name is None:
return []

metrics_mode = match_metrics_mode(self._default_mode)

col_mode = self._properties.get(f"{TableProperties.METRICS_MODE_COLUMN_CONF_PREFIX}.{column_name}")
if col_mode:
metrics_mode = match_metrics_mode(col_mode)
elif not self._configured_default and self._field_id not in self._inferred_field_ids:
# Only the first max-inferred-column-defaults primitive fields get the default mode.
return []
else:
metrics_mode = match_metrics_mode(self._default_mode)

if (
not (isinstance(primitive, StringType) or isinstance(primitive, BinaryType))
and metrics_mode.type == MetricModeTypes.TRUNCATE
):
metrics_mode = MetricsMode(MetricModeTypes.FULL)

is_nested = column_name.find(".") >= 0

if is_nested and metrics_mode.type in [MetricModeTypes.TRUNCATE, MetricModeTypes.FULL]:
metrics_mode = MetricsMode(MetricModeTypes.COUNTS)

return [StatisticsCollector(field_id=self._field_id, iceberg_type=primitive, mode=metrics_mode, column_name=column_name)]


Expand Down
2 changes: 2 additions & 0 deletions pyiceberg/table/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -188,6 +188,8 @@ class TableProperties:
DEFAULT_WRITE_METRICS_MODE_DEFAULT = "truncate(16)"

METRICS_MODE_COLUMN_CONF_PREFIX = "write.metadata.metrics.column"
METRICS_MAX_INFERRED_COLUMN_DEFAULTS = "write.metadata.metrics.max-inferred-column-defaults"
METRICS_MAX_INFERRED_COLUMN_DEFAULTS_DEFAULT = 100

WRITE_PARTITION_SUMMARY_LIMIT = "write.summary.partition-limit"
WRITE_PARTITION_SUMMARY_LIMIT_DEFAULT = 0
Expand Down
77 changes: 69 additions & 8 deletions tests/io/test_pyarrow_stats.py
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@
)
from pyiceberg.manifest import DataFile, FileFormat
from pyiceberg.schema import Schema, pre_order_visit
from pyiceberg.table import TableProperties
from pyiceberg.table.metadata import (
TableMetadata,
TableMetadataUtil,
Expand Down Expand Up @@ -336,11 +337,11 @@ def test_bounds() -> None:
)
datafile = DataFile.from_args(**statistics.to_serialized_dict())

assert len(datafile.lower_bounds) == 2
assert len(datafile.lower_bounds) == 7
assert datafile.lower_bounds[1].decode() == "aaaaaaaaaaaaaaaa"
assert datafile.lower_bounds[2] == STRUCT_FLOAT.pack(1.69)

assert len(datafile.upper_bounds) == 2
assert len(datafile.upper_bounds) == 7
assert datafile.upper_bounds[1].decode() == "zzzzzzzzzzzzzzz{"
assert datafile.upper_bounds[2] == STRUCT_FLOAT.pack(100)

Expand Down Expand Up @@ -422,11 +423,11 @@ def test_metrics_mode_full() -> None:
assert len(datafile.null_value_counts) == 7
assert len(datafile.nan_value_counts) == 0

assert len(datafile.lower_bounds) == 2
assert len(datafile.lower_bounds) == 7
assert datafile.lower_bounds[1].decode() == "aaaaaaaaaaaaaaaaaaaa"
assert datafile.lower_bounds[2] == STRUCT_FLOAT.pack(1.69)

assert len(datafile.upper_bounds) == 2
assert len(datafile.upper_bounds) == 7
assert datafile.upper_bounds[1].decode() == "zzzzzzzzzzzzzzzzzzzz"
assert datafile.upper_bounds[2] == STRUCT_FLOAT.pack(100)

Expand All @@ -447,11 +448,11 @@ def test_metrics_mode_non_default_trunc() -> None:
assert len(datafile.null_value_counts) == 7
assert len(datafile.nan_value_counts) == 0

assert len(datafile.lower_bounds) == 2
assert len(datafile.lower_bounds) == 7
assert datafile.lower_bounds[1].decode() == "aa"
assert datafile.lower_bounds[2] == STRUCT_FLOAT.pack(1.69)

assert len(datafile.upper_bounds) == 2
assert len(datafile.upper_bounds) == 7
assert datafile.upper_bounds[1].decode() == "z{"
assert datafile.upper_bounds[2] == STRUCT_FLOAT.pack(100)

Expand All @@ -473,15 +474,75 @@ def test_column_metrics_mode() -> None:
assert len(datafile.null_value_counts) == 6
assert len(datafile.nan_value_counts) == 0

assert len(datafile.lower_bounds) == 1
assert len(datafile.lower_bounds) == 6
assert datafile.lower_bounds[2] == STRUCT_FLOAT.pack(1.69)
assert 1 not in datafile.lower_bounds

assert len(datafile.upper_bounds) == 1
assert len(datafile.upper_bounds) == 6
assert datafile.upper_bounds[2] == STRUCT_FLOAT.pack(100)
assert 1 not in datafile.upper_bounds


def test_metrics_max_inferred_column_defaults_limits_stats() -> None:
schema = Schema(
NestedField(1, "a", IntegerType(), required=False),
NestedField(2, "b", IntegerType(), required=False),
NestedField(3, "c", IntegerType(), required=False),
)

plan = compute_statistics_plan(schema, {TableProperties.METRICS_MAX_INFERRED_COLUMN_DEFAULTS: "2"})

assert set(plan) == {1, 2}


def test_metrics_max_inferred_column_defaults_keeps_column_overrides() -> None:
schema = Schema(
NestedField(1, "a", IntegerType(), required=False),
NestedField(2, "b", IntegerType(), required=False),
NestedField(3, "c", IntegerType(), required=False),
)

plan = compute_statistics_plan(
schema,
{
TableProperties.METRICS_MAX_INFERRED_COLUMN_DEFAULTS: "1",
f"{TableProperties.METRICS_MODE_COLUMN_CONF_PREFIX}.c": "full",
},
)

assert set(plan) == {1, 3}


def test_configured_metrics_default_is_not_limited() -> None:
schema = Schema(
NestedField(1, "a", IntegerType(), required=False),
NestedField(2, "b", IntegerType(), required=False),
NestedField(3, "c", IntegerType(), required=False),
)

plan = compute_statistics_plan(
schema,
{
TableProperties.DEFAULT_WRITE_METRICS_MODE: "counts",
TableProperties.METRICS_MAX_INFERRED_COLUMN_DEFAULTS: "1",
},
)

assert set(plan) == {1, 2, 3}


def test_negative_metrics_max_inferred_column_defaults_falls_back() -> None:
schema = Schema(
NestedField(1, "a", IntegerType(), required=False),
NestedField(2, "b", IntegerType(), required=False),
NestedField(3, "c", IntegerType(), required=False),
)

plan = compute_statistics_plan(schema, {TableProperties.METRICS_MAX_INFERRED_COLUMN_DEFAULTS: "-1"})

assert set(plan) == {1, 2, 3}


def construct_test_table_primitive_types() -> tuple[pq.FileMetaData, TableMetadataV1 | TableMetadataV2]:
table_metadata = {
"format-version": 2,
Expand Down
Loading