Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions fe/check/checkstyle/suppressions.xml
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,8 @@ under the License.
<suppress files="org[\\/]apache[\\/]iceberg[\\/]DeleteFileIndex\.java" checks="[a-zA-Z0-9]*"/>
<!-- Iceberg package access is required to preserve historical schema binding in DataTableScan. -->
<suppress files="org[\\/]apache[\\/]iceberg[\\/]SchemaAwareDataTableScan\.java" checks="[a-zA-Z0-9]*"/>
<!-- Iceberg package access is required to build a StaticDataTask for REST catalog labels; other checks apply. -->
<suppress files="org[\\/]apache[\\/]iceberg[\\/]IcebergLabelsDataTask\.java" checks="ImportControl"/>

<!-- ignore gensrc/thrift/ExternalTableSchema.thrift -->
<suppress files=".*thrift/schema/external/.*" checks=".*"/>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -533,6 +533,12 @@ public ConnectorTableSchema getTableSchema(
// parse ITS schema (e.g. t$snapshots -> committed_at/snapshot_id/...). Mirrors legacy
// IcebergSysExternalTable.getSysIcebergTable + getOrCreateSchemaCacheValue; the enable.mapping.*
// flags are threaded by the shared buildTableSchema -> parseSchema (deviation 5).
if (IcebergLabelsSysTable.isLabels(iceHandle)) {
// $labels is not an iceberg metadata table: its schema is fixed and needs no table load (the
// base table's existence was already checked by getTableHandle).
return new ConnectorTableSchema(iceHandle.getTableName(),
parseSchema(IcebergLabelsSysTable.SCHEMA), "ICEBERG", Collections.emptyMap());
}
Table sysTable = loadSysTable(session, iceHandle);
return buildTableSchema(iceHandle.getTableName(), sysTable, sysTable.schema(), false);
}
Expand Down Expand Up @@ -833,6 +839,9 @@ public Map<String, ConnectorColumnHandle> getColumnHandles(
// Mirror getTableSchema: wrap the remote load in the auth context. A sys handle resolves the
// metadata-table columns (t$snapshots -> committed_at/...) so the generic scan node can look up
// its pruned sys-table slots by name; a data handle resolves the base table's columns.
if (IcebergLabelsSysTable.isLabels(iceHandle)) {
return buildColumnHandles(IcebergLabelsSysTable.SCHEMA, false);
}
Table table = iceHandle.isSystemTable() ? loadSysTable(session, iceHandle) : loadTable(session, iceHandle);
return buildColumnHandles(table.schema(), !iceHandle.isSystemTable());
}
Expand Down Expand Up @@ -1919,6 +1928,9 @@ private boolean isHmsCatalog() {
* excluding it here would be a capability regression vs master. Unlike every other entry, its scan takes
* BE's native reader rather than the JNI serialized-split path (see
* {@code IcebergScanPlanProvider.doPlanPositionDeletesSystemTableScan}).
*
* <p>A REST catalog additionally exposes {@code labels} ({@link IcebergLabelsSysTable}), which is not a
* {@code MetadataTableType}: its rows are the catalog labels returned with the table load.
*/
@Override
public List<String> listSupportedSysTables(ConnectorSession session,
Expand All @@ -1927,6 +1939,9 @@ public List<String> listSupportedSysTables(ConnectorSession session,
for (MetadataTableType type : MetadataTableType.values()) {
names.add(type.name().toLowerCase(Locale.ROOT));
}
if (supportsLabels()) {
names.add(IcebergLabelsSysTable.NAME);
}
return Collections.unmodifiableList(names);
}

Expand Down Expand Up @@ -1969,13 +1984,16 @@ public Optional<ConnectorTableHandle> getSysTableHandle(ConnectorSession session
/**
* Whether iceberg exposes a system table named {@code sysName} (case-insensitive). Mirrors
* {@code IcebergSysTable.SUPPORTED_SYS_TABLES}: every {@code MetadataTableType}, {@code POSITION_DELETES}
* included (see {@link #listSupportedSysTables}). A {@code null} name is simply not exposed (returns
* false, not NPE).
* included (see {@link #listSupportedSysTables}), plus {@code labels} for a REST catalog. A {@code null}
* name is simply not exposed (returns false, not NPE).
*/
private static boolean isSupportedSysTable(String sysName) {
private boolean isSupportedSysTable(String sysName) {
if (sysName == null) {
return false;
}
if (IcebergLabelsSysTable.NAME.equalsIgnoreCase(sysName)) {
return supportsLabels();
}
for (MetadataTableType type : MetadataTableType.values()) {
if (type.name().equalsIgnoreCase(sysName)) {
return true;
Expand All @@ -1984,6 +2002,11 @@ private static boolean isSupportedSysTable(String sysName) {
return false;
}

/** Only a REST catalog returns catalog labels with the table load, so only it exposes {@code $labels}. */
private boolean supportsLabels() {
return IcebergCatalogProperties.TYPE_REST.equals(catalogProps.getFlavor());
}

// ========== Write / Transaction (P6.3) ==========

/**
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

package org.apache.doris.connector.iceberg;

import org.apache.iceberg.Schema;
import org.apache.iceberg.types.Types;

/**
* The {@code t$labels} system table: the catalog-provided labels of an iceberg table loaded from a REST catalog
* (apache/iceberg#15750), one row per object-level or field-level label.
*
* <p>Unlike every other iceberg system table it is not a {@link org.apache.iceberg.MetadataTableType}: labels
* come from the REST load response, not from the table metadata, are not versioned with snapshots, and only a
* REST catalog returns them. Rows reflect the table load the statement read, so they follow the iceberg table
* cache like the rest of the table's state. The schema mirrors iceberg's proposed {@code LabelsTable}.
*/
final class IcebergLabelsSysTable {
static final String NAME = "labels";

static final Schema SCHEMA = new Schema(
Types.NestedField.required(1, "scope", Types.StringType.get()),
Types.NestedField.optional(2, "field_id", Types.IntegerType.get()),
Types.NestedField.optional(3, "field_name", Types.StringType.get()),
Types.NestedField.required(4, "key", Types.StringType.get()),
Types.NestedField.required(5, "value", Types.StringType.get()));

private IcebergLabelsSysTable() {
}

/** Whether {@code handle} is a {@code $labels} handle; sys handles carry the lower-cased name. */
static boolean isLabels(IcebergTableHandle handle) {
return NAME.equals(handle.getSysTableName());
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -47,12 +47,14 @@
import org.apache.iceberg.BatchScan;
import org.apache.iceberg.ContentScanTask;
import org.apache.iceberg.DataFile;
import org.apache.iceberg.DataTask;
import org.apache.iceberg.DeleteFile;
import org.apache.iceberg.DeleteFileIndex;
import org.apache.iceberg.FileContent;
import org.apache.iceberg.FileFormat;
import org.apache.iceberg.FileScanTask;
import org.apache.iceberg.HasTableOperations;
import org.apache.iceberg.IcebergLabelsDataTask;
import org.apache.iceberg.ManifestContent;
import org.apache.iceberg.ManifestFile;
import org.apache.iceberg.ManifestFiles;
Expand Down Expand Up @@ -977,6 +979,9 @@ private List<ConnectorScanRange> planSystemTableScan(IcebergTableHandle handle,
private List<ConnectorScanRange> doPlanSystemTableScan(IcebergTableHandle handle,
List<ConnectorColumnHandle> columns, Optional<ConnectorExpression> filter,
ConnectorSession session) {
if (IcebergLabelsSysTable.isLabels(handle)) {
return doPlanLabelsSystemTableScan(handle, columns, session);
}
Table metadataTable = resolveSysTable(session, handle);
if (isPositionDeletesSysTable(handle)) {
return doPlanPositionDeletesSystemTableScan(handle, metadataTable, columns, filter, session);
Expand All @@ -998,27 +1003,8 @@ private List<ConnectorScanRange> doPlanSystemTableScan(IcebergTableHandle handle
// hand from the requested columns.
List<String> projectedColumns = requestedColumnNames(columns);
if (!projectedColumns.isEmpty()) {
Schema metadataSchema = metadataTable.schema();
List<NestedField> projectedFields = new ArrayList<>(projectedColumns.size());
Set<Integer> seenFieldIds = new HashSet<>();
for (String columnName : projectedColumns) {
NestedField field = metadataSchema.findField(columnName);
if (field == null) {
throw new RuntimeException("Column " + columnName
+ " not found in iceberg system table schema " + metadataTable.name());
}
if (seenFieldIds.add(field.fieldId())) {
projectedFields.add(field);
}
}
// Iceberg returns split_offsets only when the last offset is smaller than file_size_in_bytes.
// Without that dependency, even valid offsets such as [4, 2048] are hidden as null. Append it
// after the requested columns so BE's positional reads keep their original order.
if (seenFieldIds.contains(DataFile.SPLIT_OFFSETS.fieldId())
&& seenFieldIds.add(DataFile.FILE_SIZE.fieldId())) {
projectedFields.add(metadataSchema.findField(DataFile.FILE_SIZE.fieldId()));
}
scan = scan.project(new Schema(projectedFields));
scan = scan.project(projectSystemTableSchema(
metadataTable.schema(), projectedColumns, metadataTable.name()));
}
List<ConnectorScanRange> ranges = new ArrayList<>();
try (CloseableIterable<FileScanTask> tasks = scan.planFiles()) {
Expand All @@ -1043,6 +1029,58 @@ private static boolean isPositionDeletesSysTable(IcebergTableHandle handle) {
&& MetadataTableType.POSITION_DELETES.name().equalsIgnoreCase(handle.getSysTableName());
}

/**
* The system-table schema projected to the requested columns, in the order BE lists them in
* required_fields: BE's {@code IcebergSysTableJniScanner} reads projected rows positionally. Built by hand
* because {@code scan.select(names)} would re-emit the fields in table-schema order. Any dependent metadata
* fields are appended after the requested columns.
*/
private static Schema projectSystemTableSchema(Schema schema, List<String> projectedColumns,
String tableName) {
List<NestedField> projectedFields = new ArrayList<>(projectedColumns.size());
Set<Integer> seenFieldIds = new HashSet<>();
for (String columnName : projectedColumns) {
NestedField field = schema.findField(columnName);
if (field == null) {
throw new RuntimeException("Column " + columnName
+ " not found in iceberg system table schema " + tableName);
}
if (seenFieldIds.add(field.fieldId())) {
projectedFields.add(field);
}
}
// Iceberg returns split_offsets only when the last offset is smaller than file_size_in_bytes.
// Without that dependency, even valid offsets such as [4, 2048] are hidden as null. Append it
// after the requested columns so BE's positional reads keep their original order.
if (seenFieldIds.contains(DataFile.SPLIT_OFFSETS.fieldId())
&& seenFieldIds.add(DataFile.FILE_SIZE.fieldId())) {
projectedFields.add(schema.findField(DataFile.FILE_SIZE.fieldId()));
}
return new Schema(projectedFields);
}

/**
* Plan {@code t$labels}: one serialized split whose {@code StaticDataTask} carries every catalog label of the
* base table, read by BE's JNI scanner like any other static metadata table. Labels are not versioned, so
* the handle's snapshot pin is ignored (as for {@code $snapshots}/{@code $history}); predicates are left to
* the scan node's conjuncts, the row count being bounded by the catalog response.
*/
private List<ConnectorScanRange> doPlanLabelsSystemTableScan(IcebergTableHandle handle,
List<ConnectorColumnHandle> columns, ConnectorSession session) {
Table base = resolveSysBaseTable(session, handle);
List<String> projectedColumns = requestedColumnNames(columns);
Schema projectedSchema = projectedColumns.isEmpty()
? IcebergLabelsSysTable.SCHEMA
: projectSystemTableSchema(IcebergLabelsSysTable.SCHEMA, projectedColumns,
base.name() + "$" + IcebergLabelsSysTable.NAME);
DataTask task = IcebergLabelsDataTask.of(
base, IcebergLabelsSysTable.SCHEMA, projectedSchema, SYS_TABLE_DUMMY_PATH);
return Collections.singletonList(new IcebergScanRange.Builder()
.path(SYS_TABLE_DUMMY_PATH)
.serializedSplit(IcebergSystemTableSerialization.serializeToBase64(task))
.build());
}

/**
* Plan a {@code $position_deletes} scan. Port of legacy
* {@code IcebergScanNode.doGetPositionDeletesSystemTableSplits} (upstream #65135).
Expand Down Expand Up @@ -1328,6 +1366,10 @@ private static boolean isSnapshotIndependentSystemTable(IcebergTableHandle handl
if (!handle.isSystemTable()) {
return false;
}
if (IcebergLabelsSysTable.isLabels(handle)) {
// Catalog labels come with the table load, not from a snapshot, so a table with no commit has them.
return true;
}
MetadataTableType type = MetadataTableType.from(handle.getSysTableName());
// Only metadata_log_entries has a creation row before the first data snapshot; every other metadata
// table must preserve the resolved-empty fence or a concurrent first append becomes visible.
Expand Down Expand Up @@ -3398,9 +3440,11 @@ Table wrapTableForScan(Table table) {
}
BaseTable baseTable = (BaseTable) table;
TableOperations rawOps = baseTable.operations();
// Keep the table's metrics reporter: every scan of a Kerberos catalog runs on this wrapper.
// Keep the table's metrics reporter and REST catalog labels: every scan of a Kerberos catalog runs on this
// wrapper.
return new BaseTable(new IcebergAuthenticatedTableOperations(
rawOps, new IcebergAuthenticatedFileIO(rawOps.io(), auth)), table.name(), baseTable.reporter());
rawOps, new IcebergAuthenticatedFileIO(rawOps.io(), auth)), table.name(), baseTable.reporter(),
baseTable.labels());
}

/**
Expand All @@ -3413,6 +3457,13 @@ Table wrapTableForScan(Table table) {
* task serialization) in ONE scope — no nested wrap here.
*/
private Table resolveSysTable(ConnectorSession session, IcebergTableHandle handle) {
return MetadataTableUtils.createMetadataTableInstance(
resolveSysBaseTable(session, handle),
MetadataTableType.from(handle.getSysTableName()));
}

/** The base table a system-table scan reads, shared with metadata binding; see {@link #resolveSysTable}. */
private Table resolveSysBaseTable(ConnectorSession session, IcebergTableHandle handle) {
IcebergCatalogOps ops = catalogOpsResolver.apply(session);
// Keep the raw base shared with metadata binding and ordinary scan properties. The caller already owns
// the auth scope, and avoiding a fresh load prevents system-table slots and rows crossing generations.
Expand All @@ -3428,9 +3479,7 @@ private Table resolveSysTable(ConnectorSession session, IcebergTableHandle handl
() -> ops.loadTable(handle.getDbName(), handle.getTableName())),
() -> ops.loadTable(handle.getDbName(), handle.getTableName()));
rejectServerSideScanPlanning(base, handle);
return MetadataTableUtils.createMetadataTableInstance(
base,
MetadataTableType.from(handle.getSysTableName()));
return base;
}

/** This catalog's engine-owned storage services (see {@link ConnectorContext#getStorageContext()}). */
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -292,7 +292,7 @@ private static Table snapshotReadTable(Table table, String dbName, String tableN
TableMetadata metadata = operations.current();
// Keep IO, encryption and location behavior from the raw table while pinning only metadata.
return new BaseTable(new IcebergSnapshotTableOperations(operations, metadata),
table.name(), baseTable.reporter());
table.name(), baseTable.reporter(), baseTable.labels());
}

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -314,7 +314,7 @@ private Table snapshotReadTable() {
? operations.current()
: TableMetadataParser.fromJson(snapshotMetadataLocation, snapshotMetadataJson);
return new BaseTable(new IcebergSnapshotTableOperations(operations, metadata),
table.name(), baseTable.reporter());
table.name(), baseTable.reporter(), baseTable.labels());
}

private TableLease tryBorrow() {
Expand Down
Loading
Loading