Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -125,6 +125,9 @@ public class IcebergConnectorMetadata implements ConnectorMetadata {
private static final String ICEBERG_LAST_UPDATED_SEQUENCE_NUMBER_COL = "_last_updated_sequence_number";
private static final String ICEBERG_FILE_PATH_COL = "_file";
private static final String ICEBERG_ROW_POSITION_COL = "_pos";
// The hidden DML row-id column, fe-core Column.ICEBERG_ROWID_COL (pinned by
// IcebergScanPlanProviderClassifyColumnTest through the scan provider's copy).
private static final String DORIS_ICEBERG_ROWID_COL = "__DORIS_ICEBERG_ROWID_COL__";
private static final int ICEBERG_ROW_ID_FIELD_ID = 2147483540;
private static final int ICEBERG_LAST_UPDATED_SEQUENCE_NUMBER_FIELD_ID = 2147483539;
private static final int ICEBERG_ROW_LINEAGE_MIN_VERSION = 3;
Expand Down Expand Up @@ -783,14 +786,17 @@ private static Map<String, ConnectorColumnHandle> buildColumnHandles(
rejectReservedMetadataColumns(schema);
}
Map<String, ConnectorColumnHandle> handles = new LinkedHashMap<>(
fields.size() + (appendDataFileMetadataColumns ? 2 : 0));
fields.size() + (appendDataFileMetadataColumns ? 3 : 0));
for (Types.NestedField field : fields) {
String name = field.name();
handles.put(name, new IcebergColumnHandle(name, field.fieldId()));
}
if (appendDataFileMetadataColumns) {
handles.put(ICEBERG_FILE_PATH_COL, new IcebergColumnHandle(ICEBERG_FILE_PATH_COL, -1));
handles.put(ICEBERG_ROW_POSITION_COL, new IcebergColumnHandle(ICEBERG_ROW_POSITION_COL, -1));
// Synthesized by the reader for DML and never in the schema. Its handle tells the scan provider that a
// scan produces row IDs (IcebergScanPlanProvider#rejectRowIdScanWithoutPartitionData).
handles.put(DORIS_ICEBERG_ROWID_COL, new IcebergColumnHandle(DORIS_ICEBERG_ROWID_COL, -1));
}
return handles;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -195,9 +195,12 @@ static Map<String, String> getIdentityPartitionInfoMap(PartitionData partitionDa

/**
* The serialized value for <b>every</b> partition field (identity + transform), in spec order, used for
* {@code partition_data_json}. Mirrors legacy {@code IcebergUtils.getPartitionValues}. A field whose
* value cannot be serialized (BINARY/FIXED) yields a {@code null} entry (not dropped) to keep positional
* alignment with the spec.
* {@code partition_data_json}. Mirrors legacy {@code IcebergUtils.getPartitionValues}. A non-null value whose
* type cannot be serialized (a historical field whose source column was dropped) throws
* {@link UnsupportedOperationException}: these values also identify the partition of the delete files
* BE writes for DML, so an unsupported value must never become a {@code null} entry. A null needs no type,
* so the void field a v1 table keeps for a removed partition field stays {@code null} after its source
* column is dropped.
*/
static List<String> getPartitionValues(PartitionData partitionData, PartitionSpec partitionSpec, ZoneId zone) {
List<NestedField> fields = partitionData.getPartitionType().asNestedType().fields();
Expand All @@ -206,15 +209,8 @@ static List<String> getPartitionValues(PartitionData partitionData, PartitionSpe

List<String> partitionValues = new ArrayList<>(fields.size());
for (int i = 0; i < fields.size(); i++) {
NestedField field = fields.get(i);
Object value = partitionData.get(i);
try {
partitionValues.add(serializePartitionValue(field.type(), value, zone));
} catch (UnsupportedOperationException e) {
LOG.warn("Failed to serialize Iceberg partition value for field {}: {}", field.name(),
e.getMessage());
partitionValues.add(null);
}
partitionValues.add(value == null ? null : serializePartitionValue(fields.get(i).type(), value, zone));
}
return partitionValues;
}
Expand All @@ -225,9 +221,32 @@ static List<String> getPartitionValues(PartitionData partitionData, PartitionSpe
* content is identical.
*/
static String getPartitionDataJson(PartitionData partitionData, PartitionSpec partitionSpec, ZoneId zone) {
List<String> partitionValues = getPartitionValues(partitionData, partitionSpec, zone);
return toJsonArray(getPartitionValues(partitionData, partitionSpec, zone));
}

/**
* Identifies a partition tuple {@link #getPartitionValues} cannot serialize, for the scanned-partition count
* only: a JSON array of the raw values in spec order (binary as hex), so each distinct tuple counts once, as
* legacy {@code IcebergScanNode} keyed the count by the {@link PartitionData} itself.
*/
static String getRawPartitionKey(PartitionData partitionData) {
List<String> values = new ArrayList<>(partitionData.size());
for (int i = 0; i < partitionData.size(); i++) {
Object value = partitionData.get(i);
if (value == null) {
values.add(null);
} else if (value instanceof java.nio.ByteBuffer) {
values.add(binaryHex((java.nio.ByteBuffer) value));
} else {
values.add(value.toString());
}
}
return toJsonArray(values);
}

private static String toJsonArray(List<String> values) {
try {
return JsonUtil.mapper().writeValueAsString(partitionValues);
return JsonUtil.mapper().writeValueAsString(values);
} catch (com.fasterxml.jackson.core.JsonProcessingException e) {
throw new RuntimeException("Failed to serialize iceberg partition data to JSON, error message is:"
+ e.getMessage(), e);
Expand Down Expand Up @@ -382,10 +401,7 @@ static String serializePartitionValue(Type type, Object value, ZoneId zone) {
if (value == null) {
return null;
}
java.nio.ByteBuffer bytes = ((java.nio.ByteBuffer) value).duplicate();
byte[] copy = new byte[bytes.remaining()];
bytes.get(copy);
return "0x" + java.util.HexFormat.of().formatHex(copy);
return binaryHex((java.nio.ByteBuffer) value);
case DATE:
if (value == null) {
return null;
Expand Down Expand Up @@ -430,6 +446,13 @@ static String serializePartitionValue(Type type, Object value, ZoneId zone) {
}
}

private static String binaryHex(java.nio.ByteBuffer value) {
java.nio.ByteBuffer bytes = value.duplicate();
byte[] copy = new byte[bytes.remaining()];
bytes.get(copy);
return "0x" + java.util.HexFormat.of().formatHex(copy);
}

// Canonical partition-timestamp format ("yyyy-MM-dd HH:mm:ss" with an optional micro/nano fraction),
// the form BE renders human-readable partition values in. Legacy parsed via the nereids multi-format
// DateLiteral.parseDateTime (connector-forbidden); the canonical form is the only one BE emits, so this
Expand Down Expand Up @@ -491,6 +514,9 @@ static Object parsePartitionValueFromString(String valueStr, Type icebergType, Z
case DATE:
// Iceberg date is days since epoch (1970-01-01).
return (int) LocalDate.parse(valueStr, DateTimeFormatter.ISO_LOCAL_DATE).toEpochDay();
case TIME:
// Iceberg time is microseconds since midnight, the inverse of serializePartitionValue.
return LocalTime.parse(valueStr, DateTimeFormatter.ISO_LOCAL_TIME).toNanoOfDay() / 1000;
case TIMESTAMP_NANO:
// Match the sink cast: a date-only static partition denotes midnight.
if (valueStr.length() == 10) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -441,8 +441,8 @@ public List<ConnectorScanRange> planScan(ConnectorSession session, ConnectorScan
// validate every call before a cached range list can be returned.
validateMetadataColumnReader(session, request.getColumns());
if (session == null || !session.isExternalScanTaskReuseEnabled()) {
return planScanInternal(session, icebergHandle, request.getColumns(),
request.getFilter(), request.isCountPushdown());
return rejectRowIdScanWithoutPartitionData(request.getColumns(), planScanInternal(session,
icebergHandle, request.getColumns(), request.getFilter(), request.isCountPushdown()));
}
if (icebergHandle.isSystemTable()) {
// System tables read connector metadata through their own readers; never reuse them.
Expand All @@ -469,10 +469,10 @@ public List<ConnectorScanRange> planScan(ConnectorSession session, ConnectorScan
Map<IcebergScanReuseKey, List<ConnectorScanRange>> scanReuse = session.getStatementScope()
.computeIfAbsentForPlanning(memoKey, () -> new ConcurrentHashMap<>());
IcebergScanReuseKey reuseKey = new IcebergScanReuseKey(icebergHandle, request);
return scanReuse.computeIfAbsent(reuseKey,
return rejectRowIdScanWithoutPartitionData(request.getColumns(), scanReuse.computeIfAbsent(reuseKey,
k -> Collections.unmodifiableList(planScanInternal(session,
icebergHandle, request.getColumns(), request.getFilter(),
request.isCountPushdown())));
request.isCountPushdown()))));
} catch (RuntimeException e) {
// Normal data scans and native position_deletes run on File Scanner V2. Keep the serialized JNI
// system-table route untouched because its deferred reads belong to the V1 scanner contract.
Expand Down Expand Up @@ -599,7 +599,7 @@ public ConnectorSplitSource streamSplits(ConnectorSession session, ConnectorScan
scan, session, table, filter, sliceSize);
return new IcebergStreamingSplitSource(tasks, table, formatVersion, partitioned,
orderedPartitionKeys, zone, uriNormalizer, sliceSize,
iceHandle.getRewriteFileScope(), iceHandle);
iceHandle.getRewriteFileScope(), iceHandle, requiresRowId(columns));
} catch (RuntimeException e) {
throw IcebergExceptionUtils.wrapMetadataReadFailure(iceHandle, e);
}
Expand Down Expand Up @@ -664,6 +664,7 @@ private final class IcebergStreamingSplitSource implements ConnectorSplitSource
private final long sliceSize;
private final Set<String> rewriteScope;
private final IcebergTableHandle handle;
private final boolean requiresRowId;
// Lazily opened on first hasNext() so the ctor never throws — iceberg's ParallelIterable submits
// manifest readers in tasks.iterator(), which can fail; opening it eagerly here would throw out of
// streamSplits() BEFORE the source is returned, leaking the planFiles() iterable (the engine pump's
Expand All @@ -679,7 +680,7 @@ private final class IcebergStreamingSplitSource implements ConnectorSplitSource
IcebergStreamingSplitSource(CloseableIterable<FileScanTask> tasks, Table table, int formatVersion,
boolean partitioned, List<String> orderedPartitionKeys, ZoneId zone,
UnaryOperator<String> uriNormalizer, long sliceSize, Set<String> rewriteScope,
IcebergTableHandle handle) {
IcebergTableHandle handle, boolean requiresRowId) {
this.tasks = tasks;
this.table = table;
this.formatVersion = formatVersion;
Expand All @@ -690,6 +691,7 @@ private final class IcebergStreamingSplitSource implements ConnectorSplitSource
this.sliceSize = sliceSize;
this.rewriteScope = rewriteScope;
this.handle = handle;
this.requiresRowId = requiresRowId;
}

@Override
Expand All @@ -705,6 +707,9 @@ public boolean hasNext() {
IcebergScanRange range = buildRangeForTask(iterator.next(), table, formatVersion, partitioned,
orderedPartitionKeys, zone, uriNormalizer, sliceSize, rewriteScope, null, scratch);
if (range != null) {
if (requiresRowId) {
rejectRowIdRangeWithoutPartitionData(range);
}
buffered = range;
return true;
}
Expand Down Expand Up @@ -1622,6 +1627,7 @@ private static final class PerFileScratch {
private DataFile file;
private Integer partitionSpecId;
private String partitionDataJson;
private String rawPartitionKey;
private Map<String, String> partitionValues = Collections.emptyMap();
// The table's CURRENT spec isPartitioned() — scan-invariant, memoized with the rest so the display
// gate (see IcebergScanRange#getScannedPartitionKey) costs one table.spec() per file, not per slice.
Expand Down Expand Up @@ -1676,6 +1682,7 @@ private IcebergScanRange buildRange(Table table, DataFile dataFile, FileScanTask
.formatVersion(formatVersion)
.partitionSpecId(file.partitionSpecId)
.partitionDataJson(file.partitionDataJson)
.rawPartitionKey(file.rawPartitionKey)
.firstRowId(file.firstRowId)
.lastUpdatedSequenceNumber(file.lastUpdatedSequenceNumber)
.partitionValues(file.partitionValues)
Expand All @@ -1701,13 +1708,21 @@ private PerFileScratch computePerFileInvariants(Table table, DataFile dataFile,
perFileInvariantComputeCount++;
Integer partitionSpecId = null;
String partitionDataJson = null;
String rawPartitionKey = null;
Map<String, String> partitionValues = Collections.emptyMap();
if (partitioned && dataFile.partition() instanceof PartitionData) {
PartitionData partitionData = (PartitionData) dataFile.partition();
int specId = dataFile.specId();
PartitionSpec spec = table.specs().get(specId);
partitionSpecId = specId;
partitionDataJson = IcebergPartitionUtils.getPartitionDataJson(partitionData, spec, zone);
try {
partitionDataJson = IcebergPartitionUtils.getPartitionDataJson(partitionData, spec, zone);
} catch (UnsupportedOperationException e) {
// A historical partition field whose source column was dropped has no serializable type. Only
// row IDs need the tuple, so the file stays readable and a row-ID scan is rejected per call; the
// raw values still count the file's partition in EXPLAIN partition=N/M and sql_block_rule.
rawPartitionKey = IcebergPartitionUtils.getRawPartitionKey(partitionData);
}
// Order the identity values as the path_partition_keys list (legacy getOrderedPathPartitionKeys),
// filtered to keys this file carries — so columns-from-path matches legacy ordering exactly.
Map<String, String> identityMap =
Expand Down Expand Up @@ -1747,6 +1762,7 @@ private PerFileScratch computePerFileInvariants(Table table, DataFile dataFile,
file.file = dataFile;
file.partitionSpecId = partitionSpecId;
file.partitionDataJson = partitionDataJson;
file.rawPartitionKey = rawPartitionKey;
file.partitionValues = partitionValues;
// Display-only gate (see IcebergScanRange#getScannedPartitionKey): a table whose CURRENT spec is
// unpartitioned reports no partitions, so the old-spec partitions its files still carry must not
Expand Down Expand Up @@ -2160,8 +2176,9 @@ table, predicateSchema, resolveSessionZone(session),
* The lowercased names of the requested (pruned) columns — the authoritative Doris scan slots the field-id
* dictionary keys its {@code -1} entry off (so its top-level names == the BE scan-slot names BY
* CONSTRUCTION; CI #969249). The names come straight from the {@link IcebergColumnHandle}s
* {@code IcebergConnectorMetadata.getColumnHandles} produced (already lowercased). An empty list (count-only
* scan / no column handles) makes the dictionary fall back to all top-level columns.
* {@code IcebergConnectorMetadata.getColumnHandles} produced (already lowercased), minus the synthesized
* {@code _file} / {@code _pos} / hidden row-id columns, which are not in the table schema. An empty list
* (count-only scan / no column handles) makes the dictionary fall back to all top-level columns.
*/
private static List<String> requestedLowerNames(List<ConnectorColumnHandle> columns) {
if (columns == null || columns.isEmpty()) {
Expand All @@ -2171,7 +2188,8 @@ private static List<String> requestedLowerNames(List<ConnectorColumnHandle> colu
for (ConnectorColumnHandle column : columns) {
String name = ((IcebergColumnHandle) column).getName();
if (!ICEBERG_FILE_PATH_COL.equalsIgnoreCase(name)
&& !ICEBERG_ROW_POSITION_COL.equalsIgnoreCase(name)) {
&& !ICEBERG_ROW_POSITION_COL.equalsIgnoreCase(name)
&& !DORIS_ICEBERG_ROWID_COL.equalsIgnoreCase(name)) {
names.add(name);
}
}
Expand Down Expand Up @@ -3093,6 +3111,35 @@ private static boolean sessionBool(ConnectorSession session, String key, boolean
return Boolean.parseBoolean(raw.trim());
}

private static boolean requiresRowId(List<ConnectorColumnHandle> columns) {
return columns.stream()
.filter(column -> column instanceof IcebergColumnHandle)
.anyMatch(column -> DORIS_ICEBERG_ROWID_COL.equalsIgnoreCase(((IcebergColumnHandle) column).getName()));
}

/**
* Rejects a scan that produces Iceberg row IDs (DELETE, UPDATE, MERGE) when a file's partition tuple could not
* be serialized. Row IDs carry the partition of the delete files BE writes, and a missing value must not become
* a NULL partition. Checked on every call, like {@link #validateMetadataColumnReader}, because the scan reuse
* key leaves the projection out.
*/
private static List<ConnectorScanRange> rejectRowIdScanWithoutPartitionData(
List<ConnectorColumnHandle> columns, List<ConnectorScanRange> ranges) {
if (requiresRowId(columns)) {
for (ConnectorScanRange range : ranges) {
rejectRowIdRangeWithoutPartitionData((IcebergScanRange) range);
}
}
return ranges;
}

private static void rejectRowIdRangeWithoutPartitionData(IcebergScanRange range) {
if (range.isPartitionDataUnavailable()) {
throw new DorisConnectorException("Cannot produce Iceberg row IDs with unsupported partition types in spec "
+ range.getPartitionSpecId() + ". Rewrite historical data files before DELETE or UPDATE.");
}
}

private static boolean requiresMetadataColumns(List<ConnectorColumnHandle> columns) {
return columns.stream()
.filter(column -> column instanceof IcebergColumnHandle)
Expand Down
Loading
Loading