Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -26,13 +26,13 @@

/**
* Iceberg {@link ConnectorColumnHandle}, mirroring the paimon connector's {@code PaimonColumnHandle}. Carries
* the (lowercased) column name and the iceberg field id. {@code IcebergConnectorMetadata.getColumnHandles}
* the case-preserved column name and the iceberg field id. {@code IcebergConnectorMetadata.getColumnHandles}
* produces these so {@code PluginDrivenScanNode.buildColumnHandles} can hand the provider the pruned set of
* requested columns — which the field-id schema dictionary (T06) keys the {@code current_schema_id = -1}
* entry off (the CI #969249 fix: the dict's top-level names == the BE scan-slot names BY CONSTRUCTION).
*
* <p>Equality/hashCode are by name only (mirrors {@code PaimonColumnHandle}): a handle identifies a column
* by its (lowercased) name, the same key {@code allHandles.get(slot.getColumn().getName())} looks it up by.
* by its case-preserved name, the same key {@code allHandles.get(slot.getColumn().getName())} looks it up by.
*/
public class IcebergColumnHandle implements ConnectorColumnHandle {

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -118,19 +118,21 @@ static boolean isDroppedPartitionSourceColumn(ValidationException e) {

/**
* Ordered, case-preserved, de-duplicated list of the identity partition column names across <b>all</b>
* partition specs of the table (mirrors legacy {@code IcebergUtils.getIdentityPartitionColumns}). This
* is the {@code path_partition_keys} payload: it tells FE which slots are partition columns so they are
* excluded from the file-decode set (the CI #968880 double-fill guard). Non-identity transforms
* (bucket/truncate/year/month/...) are excluded.
* partition specs of the table (mirrors legacy {@code IcebergUtils.getIdentityPartitionColumns}). Names
* are resolved by field id against the query's authoritative {@code scanSchema}, so current and pinned
* reads cannot mix schema generations. This is the {@code path_partition_keys} payload: it tells FE which
* slots are partition columns so they are excluded from the file-decode set (the CI #968880 double-fill
* guard). Non-identity transforms (bucket/truncate/year/month/...) are excluded.
*/
static List<String> getIdentityPartitionColumns(Table table) {
static List<String> getIdentityPartitionColumns(Table table, Schema scanSchema) {
LinkedHashSet<String> partitionColumns = new LinkedHashSet<>();
for (PartitionSpec spec : table.specs().values()) {
for (PartitionField partitionField : spec.fields()) {
if (!partitionField.transform().isIdentity()) {
continue;
}
String columnName = table.schema().findColumnName(partitionField.sourceId());
String columnName =
topLevelSourceColumnName(scanSchema, partitionField.sourceId());
if (columnName != null) {
partitionColumns.add(columnName);
}
Expand Down Expand Up @@ -189,12 +191,8 @@ private static String toPartitionFieldName(Schema schema, PartitionSpec spec, St
* mistaken for the top-level column {@code id}: the engine can only materialize a top-level column.
*/
static String topLevelSourceColumnName(Schema schema, int sourceId) {
for (NestedField column : schema.columns()) {
if (column.fieldId() == sourceId) {
return column.name();
}
}
return null;
NestedField column = schema.asStruct().field(sourceId);
return column == null ? null : column.name();
}

/** The spec's <em>identity</em> partition over the top-level column {@code name}, or null. */
Expand Down Expand Up @@ -253,11 +251,12 @@ static boolean hasPartitionedSpec(Table table) {

/**
* Per-file map of identity partition column (case-preserved) to serialized value, skipping non-identity
* transforms and BINARY/FIXED columns (utf8 round-trip would corrupt those). Order-preserving
* (LinkedHashMap, spec field order). Mirrors legacy {@code IcebergUtils.getIdentityPartitionInfoMap}.
* transforms and BINARY/FIXED columns (utf8 round-trip would corrupt those). Keys are resolved by field id
* against {@code scanSchema}, matching {@code path_partition_keys}. Order-preserving (LinkedHashMap, spec
* field order). Mirrors legacy {@code IcebergUtils.getIdentityPartitionInfoMap}.
*/
static Map<String, String> getIdentityPartitionInfoMap(PartitionData partitionData,
PartitionSpec partitionSpec, Table table, ZoneId zone) {
PartitionSpec partitionSpec, Schema scanSchema, ZoneId zone) {
Map<String, String> partitionInfoMap = new LinkedHashMap<>();
List<NestedField> fields = partitionData.getPartitionType().asNestedType().fields();
List<PartitionField> partitionFields = partitionSpec.fields();
Expand All @@ -275,7 +274,8 @@ static Map<String, String> getIdentityPartitionInfoMap(PartitionData partitionDa
continue;
}

String columnName = table.schema().findColumnName(partitionField.sourceId());
String columnName =
topLevelSourceColumnName(scanSchema, partitionField.sourceId());

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[P2] Keep identity source lookup indexed on the per-file path

computePerFileInvariants calls this method for every distinct data file on eager and streaming scans. topLevelSourceColumnName scans all of scanSchema.columns() for each identity field, replacing the cached map lookup in Schema.findColumnName. For a 1,000-column table whose identity partition source is last, planning 1,000,000 files now performs about one billion field-ID comparisons even for a narrow projection; the scratch cache only saves repeated byte slices of the same file. Use the cached top-level lookup scanSchema.asStruct().field(sourceId), or a scan-scoped ID-to-name map, to preserve pinned names and nested-source exclusion without multiplying planning work by schema width.

if (columnName == null) {
continue;
}
Expand Down
Loading
Loading