diff --git a/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveScanPlanProvider.java b/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveScanPlanProvider.java index 0660619a399299..3571696db49e21 100644 --- a/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveScanPlanProvider.java +++ b/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveScanPlanProvider.java @@ -52,8 +52,11 @@ import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Objects; import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; import java.util.function.UnaryOperator; /** @@ -75,6 +78,7 @@ * non-transactional tables (see {@link #supportsBatchScan}) * */ + public class HiveScanPlanProvider implements ConnectorScanPlanProvider { private static final Logger LOG = LogManager.getLogger(HiveScanPlanProvider.class); @@ -97,6 +101,7 @@ public class HiveScanPlanProvider implements ConnectorScanPlanProvider { * surfaced through ConnectorSession.getSessionProperties() (VariableMgr dumps all visible vars). */ private static final String SESSION_READ_HIVE_JSON_IN_ONE_COLUMN = "read_hive_json_in_one_column"; + static final String SCAN_REUSE_NAMESPACE = "hms.scan-reuse"; /** Input format of a full-ACID (ORC) transactional Hive table; other formats are rejected. */ private static final String ORC_ACID_INPUT_FORMAT = "org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"; @@ -138,6 +143,45 @@ public boolean usesHiveParquetInt96TimeZone() { @Override public List planScan(ConnectorSession session, ConnectorScanRequest request) { HiveTableHandle hiveHandle = (HiveTableHandle) request.getTableHandle(); + if (session == null || !session.isExternalScanTaskReuseEnabled()) { + return doPlanScan(session, request); + } + if (hiveHandle.isTransactional()) { + // ACID / INSERT_ONLY reads open a per-scan read transaction with a write-id snapshot and + // a shared metastore lock; reusing the planned ranges would skip that transaction. + return doPlanScan(session, request); + } + // Statement-scoped reuse: within one statement the identical scan (same table, same + // partition set, same formats) plans once and every duplicated relation shares the result. + // The scope is NONE for offline planning and tests, in which case the loader runs on every + // call. Session variables are constant within a statement and deliberately absent. + String memoKey = SCAN_REUSE_NAMESPACE + ":" + session.getCatalogId() + ":" + session.getQueryId(); + Map> scanReuse = session.getStatementScope().computeIfAbsent( + memoKey, () -> new ConcurrentHashMap<>()); + HiveScanReuseKey reuseKey = new HiveScanReuseKey(hiveHandle); + AtomicReference> uncached = new AtomicReference<>(); + List cached = scanReuse.computeIfAbsent(reuseKey, key -> { + PlanCompleteness completeness = new PlanCompleteness(); + List planned = Collections.unmodifiableList( + doPlanScan(session, request, completeness)); + if (!completeness.isComplete()) { + // ConcurrentHashMap does not install a mapping when the loader returns null. Return this + // caller's partial result below, but let the next identical alias retry the failed directory. + uncached.set(planned); + return null; + } + return planned; + }); + return cached != null ? cached : Objects.requireNonNull(uncached.get()); + } + + private List doPlanScan(ConnectorSession session, ConnectorScanRequest request) { + return doPlanScan(session, request, null); + } + + private List doPlanScan(ConnectorSession session, ConnectorScanRequest request, + PlanCompleteness completeness) { + HiveTableHandle hiveHandle = (HiveTableHandle) request.getTableHandle(); String dbName = hiveHandle.getDbName(); String tableName = hiveHandle.getTableName(); recordPruningProfile(hiveHandle); @@ -173,7 +217,7 @@ public List planScan(ConnectorSession session, ConnectorScan HiveFileFormat partFormat = partition.fileFormat != null ? partition.fileFormat : fileFormat; listAndSplitFiles(dbName, tableName, partition, partFormat, - splittable, isLzo, targetSplitSize, fs, ranges); + splittable, isLzo, targetSplitSize, fs, ranges, completeness); } } @@ -245,6 +289,16 @@ public List planScanForPartitionBatch( ConnectorSession session, ConnectorScanRequest request, List partitionBatch) { + // Batch mode deliberately does not retain completed ranges in the statement scope: its + // purpose is to bound FE memory while splits are streamed to the coordinator. Caching every + // batch until statement close would materialize the full scan again and defeat that bound. + return doPlanScanForPartitionBatch(session, request, partitionBatch); + } + + private List doPlanScanForPartitionBatch( + ConnectorSession session, + ConnectorScanRequest request, + List partitionBatch) { HiveTableHandle hiveHandle = (HiveTableHandle) request.getTableHandle(); String dbName = hiveHandle.getDbName(); String tableName = hiveHandle.getTableName(); @@ -274,7 +328,7 @@ public List planScanForPartitionBatch( HiveFileFormat partFormat = partition.fileFormat != null ? partition.fileFormat : fileFormat; listAndSplitFiles(dbName, tableName, partition, partFormat, - splittable, isLzo, targetSplitSize, fs, ranges); + splittable, isLzo, targetSplitSize, fs, ranges, null); } return ranges; } @@ -587,7 +641,7 @@ private List convertPartitions( private void listAndSplitFiles(String dbName, String tableName, PartitionScanInfo partition, HiveFileFormat fileFormat, boolean splittable, boolean isLzo, long targetSplitSize, FileSystem fs, - List ranges) { + List ranges, PlanCompleteness completeness) { List files; try { files = fileListingCache.listDataFiles(dbName, tableName, partition.location, @@ -601,6 +655,9 @@ private void listAndSplitFiles(String dbName, String tableName, throw new DorisConnectorException( "Partition location does not exist: " + partition.location, e); } + if (completeness != null) { + completeness.markIncomplete(); + } LOG.warn("Cannot list files in partition: {}", partition.location, e); return; } @@ -617,6 +674,18 @@ private void listAndSplitFiles(String dbName, String tableName, } } + private static final class PlanCompleteness { + private boolean complete = true; + + private void markIncomplete() { + complete = false; + } + + private boolean isComplete() { + return complete; + } + } + /** * Whether a listing failure was caused by the directory not existing (a {@link FileNotFoundException} * anywhere in the cause chain), as opposed to a transient or unreadable-permission failure. Used to @@ -862,4 +931,72 @@ private static String formatNanos(long nanos) { return TimeUnit.NANOSECONDS.toMillis(nanos) + "ms"; } } + + /** + * Statement-scoped cache key for one Hive scan. + * + *

Includes every input that changes the planned split list: table identity, the file formats + * (input format / serialization lib / JSON single-column gate), the partition keys and the + * pruned partition set (each partition's location and values). ACID tables are excluded + * upstream, and session variables are statement-constant, so both stay out of the key. + */ + private static final class HiveScanReuseKey { + private final String dbName; + private final String tableName; + private final String location; + private final String inputFormat; + private final String serializationLib; + private final boolean firstColumnIsString; + private final List partitionKeyNames; + private final List prunedPartitions; + + private HiveScanReuseKey(HiveTableHandle handle) { + // Catalog and query isolation are provided by the statement-scope memo key. The table + // location identifies the data source of unpartitioned tables, whose prunedPartitions + // is null. + this.dbName = handle.getDbName(); + this.tableName = handle.getTableName(); + this.location = handle.getLocation(); + this.inputFormat = handle.getInputFormat(); + this.serializationLib = handle.getSerializationLib(); + this.firstColumnIsString = handle.isFirstColumnString(); + this.partitionKeyNames = handle.getPartitionKeyNames() == null + ? Collections.emptyList() + : Collections.unmodifiableList(new ArrayList<>(handle.getPartitionKeyNames())); + this.prunedPartitions = handle.getPrunedPartitions() == null + ? null + : Collections.unmodifiableList(new ArrayList<>(handle.getPrunedPartitions())); + } + + @Override + public boolean equals(Object object) { + if (this == object) { + return true; + } + if (!(object instanceof HiveScanReuseKey)) { + return false; + } + HiveScanReuseKey that = (HiveScanReuseKey) object; + return firstColumnIsString == that.firstColumnIsString + && Objects.equals(dbName, that.dbName) + && Objects.equals(tableName, that.tableName) + && Objects.equals(location, that.location) + && Objects.equals(inputFormat, that.inputFormat) + && Objects.equals(serializationLib, that.serializationLib) + && Objects.equals(partitionKeyNames, that.partitionKeyNames) + && Objects.equals(prunedPartitions, that.prunedPartitions); + } + + @Override + public int hashCode() { + return Objects.hash(dbName, tableName, location, + inputFormat, serializationLib, firstColumnIsString, + partitionKeyNames, prunedPartitions); + } + + @Override + public String toString() { + return "HiveScanReuseKey{table=" + dbName + "." + tableName + "}"; + } + } } diff --git a/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveScanBatchModeTest.java b/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveScanBatchModeTest.java index 2c1937e9739553..dbb39475a0710a 100644 --- a/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveScanBatchModeTest.java +++ b/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveScanBatchModeTest.java @@ -39,6 +39,7 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -78,6 +79,12 @@ public class HiveScanBatchModeTest { private static final String PARQUET_SERDE = "org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe"; + @Test + public void scanReuseNamespaceUsesConnectorType() { + String prefix = new HiveConnectorProvider().getType() + "."; + Assertions.assertTrue(HiveScanPlanProvider.SCAN_REUSE_NAMESPACE.startsWith(prefix)); + } + // ==================== supportsBatchScan: partitioned AND non-transactional ==================== @Test @@ -353,6 +360,108 @@ public void predicatePruningFailureProfileTransfersThroughTheStatementScope() { Assertions.assertTrue(provider.collectScanProfiles(session).isEmpty()); } + @Test + public void statementReuseKeepsZeroPrunedAndUnprunedScansDistinct() { + CountingLister lister = new CountingLister(); + int[] partitionListCalls = new int[1]; + HmsClient hmsClient = new FakeHmsClient() { + @Override + public List listPartitionNames(String dbName, String tableName, int maxParts) { + partitionListCalls[0]++; + return Collections.singletonList("year=2024/month=01"); + } + }; + HiveScanPlanProvider provider = provider(hmsClient, lister); + HiveTableHandle unpruned = new HiveTableHandle.Builder("db", "t", HiveTableType.HIVE) + .inputFormat(PARQUET_INPUT_FORMAT) + .serializationLib(PARQUET_SERDE) + .partitionKeyNames(PART_KEYS) + .build(); + HiveTableHandle zeroPruned = unpruned.toBuilder() + .prunedPartitions(Collections.emptyList()) + .build(); + ConnectorSession session = new ScopeSession(7L, "same-statement", new TestStatementScope()); + + List zeroRanges = provider.planScan(session, + ConnectorScanRequest.builder(zeroPruned, Collections.emptyList()).build()); + List allRanges = provider.planScan(session, + ConnectorScanRequest.builder(unpruned, Collections.emptyList()).build()); + + Assertions.assertTrue(zeroRanges.isEmpty()); + Assertions.assertEquals(1, allRanges.size(), + "an unpruned alias must not reuse a zero-pruned alias's empty ranges"); + Assertions.assertEquals(1, partitionListCalls[0]); + Assertions.assertEquals(1, lister.totalCalls); + } + + @Test + public void statementReusePlansAnIdenticalScanOnce() { + CountingLister lister = new CountingLister(); + HiveScanPlanProvider provider = provider(new FakeHmsClient(), lister); + HiveTableHandle handle = new HiveTableHandle.Builder("db", "t", HiveTableType.HIVE) + .inputFormat(PARQUET_INPUT_FORMAT) + .serializationLib(PARQUET_SERDE) + .partitionKeyNames(PART_KEYS) + .prunedPartitions(Collections.singletonList(part("year=2024/month=01"))) + .build(); + ConnectorSession session = new ScopeSession(7L, "same-statement", new TestStatementScope()); + ConnectorScanRequest request = ConnectorScanRequest.builder( + handle, Collections.emptyList()).build(); + + List first = provider.planScan(session, request); + List second = provider.planScan(session, request); + + Assertions.assertSame(first, second, "an identical scan must reuse the statement's planned range list"); + Assertions.assertEquals(1, lister.totalCalls, "the underlying Hive file listing must run once"); + } + + @Test + public void statementReuseRetriesAListingThatFailedOnce() { + FailOnceLister lister = new FailOnceLister(); + HiveScanPlanProvider provider = provider(new FakeHmsClient(), lister); + HiveTableHandle handle = new HiveTableHandle.Builder("db", "t", HiveTableType.HIVE) + .inputFormat(PARQUET_INPUT_FORMAT) + .serializationLib(PARQUET_SERDE) + .partitionKeyNames(PART_KEYS) + .prunedPartitions(Collections.singletonList(part("year=2024/month=01"))) + .build(); + ConnectorSession session = new ScopeSession(7L, "same-statement", new TestStatementScope()); + ConnectorScanRequest request = ConnectorScanRequest.builder( + handle, Collections.emptyList()).build(); + + List degraded = provider.planScan(session, request); + List recovered = provider.planScan(session, request); + List reused = provider.planScan(session, request); + + Assertions.assertTrue(degraded.isEmpty(), "the tolerated first listing failure skips its partition"); + Assertions.assertEquals(1, recovered.size(), "the next alias must retry and recover the skipped partition"); + Assertions.assertSame(recovered, reused, "only the complete retry result should enter statement reuse"); + Assertions.assertEquals(2, lister.totalCalls, + "the transient failure must not be sticky, while the successful retry should be reused"); + } + + @Test + public void missingReusePropertyDoesNotReuse() { + CountingLister lister = new CountingLister(); + HiveScanPlanProvider provider = provider(new FakeHmsClient(), lister); + HiveTableHandle handle = new HiveTableHandle.Builder("db", "t", HiveTableType.HIVE) + .inputFormat(PARQUET_INPUT_FORMAT) + .serializationLib(PARQUET_SERDE) + .partitionKeyNames(PART_KEYS) + .prunedPartitions(Collections.singletonList(part("year=2024/month=01"))) + .build(); + ConnectorSession session = new ScopeSession( + 7L, "mixed-version", new TestStatementScope(), Collections.emptyMap()); + ConnectorScanRequest request = ConnectorScanRequest.builder( + handle, Collections.emptyList()).build(); + + List first = provider.planScan(session, request); + List second = provider.planScan(session, request); + + Assertions.assertNotSame(first, second, + "an older planning FE's missing property must not enable reuse in a newer connector"); + } + // ===== object-store native read (FIX-hive-s3a: scheme normalization + canonical creds) ===== @Test @@ -540,7 +649,8 @@ public void usesHiveParquetInt96TimeZone() { Assertions.assertTrue(provider(null, new CountingLister()).usesHiveParquetInt96TimeZone()); } - private static HiveScanPlanProvider provider(HmsClient hmsClient, CountingLister lister) { + private static HiveScanPlanProvider provider( + HmsClient hmsClient, HiveFileListingCache.DirectoryLister lister) { return new HiveScanPlanProvider(hmsClient, HiveTestProperties.minimal(), new FakeConnectorContext(), new HiveReadTransactionManager(), new HiveFileListingCache(HiveTestProperties.minimal(), lister)); } @@ -565,12 +675,26 @@ public List list(String location, FileSystem fs) { } } + private static final class FailOnceLister implements HiveFileListingCache.DirectoryLister { + private int totalCalls; + + @Override + public List list(String location, FileSystem fs) { + totalCalls++; + if (totalCalls == 1) { + throw new HiveDirectoryListingException("transient listing failure", new IOException("retry")); + } + return new ArrayList<>(Collections.singletonList( + new HiveFileStatus(location + "/000000_0", 10L, 1L))); + } + } + /** * Minimal {@link HmsClient} double whose {@code getPartitions} echoes each requested name back as an * {@link HmsPartitionInfo} whose location IS the name, so the batch-scoped resolution can be asserted through * the listed locations. The rest fail loud. */ - private static final class FakeHmsClient implements HmsClient { + private static class FakeHmsClient implements HmsClient { private final List listedPartitionNames; private final String absentPartitionName; private final HmsClientException partitionFailure; diff --git a/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/ScopeSession.java b/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/ScopeSession.java index a13e5c6c40ddd3..fd08791cb6af07 100644 --- a/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/ScopeSession.java +++ b/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/ScopeSession.java @@ -35,11 +35,19 @@ final class ScopeSession implements ConnectorSession { private final long catalogId; private final String queryId; private final ConnectorStatementScope scope; + private final Map sessionProperties; ScopeSession(long catalogId, String queryId, ConnectorStatementScope scope) { + this(catalogId, queryId, scope, + Collections.singletonMap("enable_external_scan_task_reuse", "true")); + } + + ScopeSession(long catalogId, String queryId, ConnectorStatementScope scope, + Map sessionProperties) { this.catalogId = catalogId; this.queryId = queryId; this.scope = scope; + this.sessionProperties = sessionProperties; } @Override @@ -86,4 +94,9 @@ public T getProperty(String name, Class type) { public Map getCatalogProperties() { return Collections.emptyMap(); } + + @Override + public Map getSessionProperties() { + return sessionProperties; + } } diff --git a/fe/fe-connector/fe-connector-hudi/src/main/java/org/apache/doris/connector/hudi/HudiScanPlanProvider.java b/fe/fe-connector/fe-connector-hudi/src/main/java/org/apache/doris/connector/hudi/HudiScanPlanProvider.java index 66d8d21c6a500e..7e14c138a75fc2 100644 --- a/fe/fe-connector/fe-connector-hudi/src/main/java/org/apache/doris/connector/hudi/HudiScanPlanProvider.java +++ b/fe/fe-connector/fe-connector-hudi/src/main/java/org/apache/doris/connector/hudi/HudiScanPlanProvider.java @@ -19,6 +19,7 @@ import org.apache.doris.connector.spi.ConnectorContext; import org.apache.doris.connector.spi.ConnectorSession; +import org.apache.doris.connector.spi.ConnectorStatementScope; import org.apache.doris.connector.spi.ConnectorStorageContext; import org.apache.doris.connector.spi.DorisConnectorException; import org.apache.doris.connector.spi.handle.ConnectorColumnHandle; @@ -70,7 +71,10 @@ import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Objects; import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicReference; import java.util.function.Function; import java.util.function.UnaryOperator; import java.util.stream.Collectors; @@ -107,6 +111,7 @@ public class HudiScanPlanProvider implements ConnectorScanPlanProvider { // HudiScanNode.canUseNativeReader() / setScanParams (sessionVariable.isForceJniScanner()). Same key + read // path as the paimon connector's FORCE_JNI_SCANNER. Default false, so normal reads are unaffected. private static final String FORCE_JNI_SCANNER = "force_jni_scanner"; + static final String SCAN_REUSE_NAMESPACE = "hudi.scan-reuse"; // Scan-node prop carrying the base64 native-reader schema-evolution dictionary (current_schema_id + // history_schema_info). getScanNodeProperties builds it; populateScanLevelParams copies it onto the real @@ -197,14 +202,94 @@ public List planScan(ConnectorSession session, ConnectorScan } private List planScanInScope(ConnectorSession session, ConnectorScanRequest request) { + // Statement-scoped reuse: within one statement the identical scan (same table, same + // instant/incremental pin, same partition set, same table generation) plans once and every + // duplicated relation shares the result. The generation token (latest completed instant) + // fences the key against same-path table recreation: without it, alias A's ranges (stamped + // with generation-A schema_id values) could be paired with alias B's generation-B + // history_schema_info dictionary, causing BE to map files through wrong field identities. + // The scope is NONE for offline planning and tests, in which case the loader runs on every + // call. Session variables are constant within a statement and deliberately absent. + if (session == null || !session.isExternalScanTaskReuseEnabled()) { + return doPlanScan(session, request); + } + HudiStatementTable statementTable = resolveHudiTable( + session, (HudiTableHandle) request.getTableHandle()); + String memoKey = SCAN_REUSE_NAMESPACE + ":" + session.getCatalogId() + ":" + session.getQueryId(); + Map> scanReuse = session.getStatementScope().computeIfAbsent( + memoKey, () -> new ConcurrentHashMap<>()); + HudiScanReuseKey reuseKey = hudiScanReuseKey( + (HudiTableHandle) request.getTableHandle(), statementTable.generation); + AtomicReference> uncached = new AtomicReference<>(); + List cached = scanReuse.computeIfAbsent(reuseKey, key -> { + PlannedScan plannedScan = doPlanScanForReuse(session, request, statementTable); + List planned = Collections.unmodifiableList(plannedScan.ranges); + if (!plannedScan.cacheable) { + // ConcurrentHashMap does not install a mapping when the loader returns null. Return this + // caller's degraded result below, but let the next identical alias retry schema resolution. + uncached.set(planned); + return null; + } + return planned; + }); + return cached != null ? cached : Objects.requireNonNull(uncached.get()); + } + + /** + * Resolve one Hudi physical table generation for each full table identity in a statement. + * Both split planning and scan-node property construction consume the returned holder. + */ + HudiStatementTable resolveHudiTable(ConnectorSession session, HudiTableHandle hudiHandle) { + if (session != null && session.isExternalScanTaskReuseEnabled() + && session.getStatementScope() != null + && session.getStatementScope() != ConnectorStatementScope.NONE) { + String memoKey = SCAN_REUSE_NAMESPACE + ":tables:" + + session.getCatalogId() + ":" + session.getQueryId(); + Map tables = + session.getStatementScope().computeIfAbsent(memoKey, () -> new ConcurrentHashMap<>()); + HudiTableResolutionKey key = new HudiTableResolutionKey(hudiHandle); + return tables.computeIfAbsent(key, ignored -> loadHudiTable(hudiHandle)); + } + return loadHudiTable(hudiHandle); + } + + private HudiStatementTable loadHudiTable(HudiTableHandle hudiHandle) { + Configuration hadoopConf = buildHadoopConf(); + HoodieTableMetaClient metaClient = buildMetaClient(hadoopConf, hudiHandle.getBasePath()); + String generation = metaClient.getCommitsAndCompactionTimeline() + .filterCompletedInstants() + .lastInstant() + .map(HoodieInstant::requestedTime) + .orElse("none"); + return new HudiStatementTable(metaClient, generation, hadoopConf); + } + + // Package-private so the statement-reuse test can replace remote Hudi planning with a recording loader. + List doPlanScan(ConnectorSession session, ConnectorScanRequest request) { + return doPlanScan(session, request, + loadHudiTable((HudiTableHandle) request.getTableHandle())); + } + + List doPlanScan(ConnectorSession session, ConnectorScanRequest request, + HudiStatementTable statementTable) { + return doPlanScan(session, request, statementTable, null); + } + + // Package-private so statement-reuse tests can model a transient degraded schema lookup without opening + // a real Hudi table. Production planning marks the result uncacheable at each tolerated schema fallback. + PlannedScan doPlanScanForReuse(ConnectorSession session, ConnectorScanRequest request, + HudiStatementTable statementTable) { + PlanCompleteness completeness = new PlanCompleteness(); + return new PlannedScan(doPlanScan(session, request, statementTable, completeness), + completeness.isComplete()); + } + + private List doPlanScan(ConnectorSession session, ConnectorScanRequest request, + HudiStatementTable statementTable, PlanCompleteness completeness) { HudiTableHandle hudiHandle = (HudiTableHandle) request.getTableHandle(); String basePath = hudiHandle.getBasePath(); - - Configuration conf = buildHadoopConf(); - HoodieTableMetaClient metaClient = HoodieTableMetaClient.builder() - .setConf(new org.apache.hudi.storage.hadoop.HadoopStorageConfiguration(conf)) - .setBasePath(basePath) - .build(); + HoodieTableMetaClient metaClient = statementTable.metaClient; + Configuration conf = statementTable.hadoopConf; // Determine COW vs MOR from the Hudi table config (authoritative), NOT the substring-detected handle // type: an UNKNOWN detection must not silently pick the wrong read path for a COW table (detection @@ -259,6 +344,9 @@ private List planScanInScope(ConnectorSession session, Conne .map(f -> HudiTypeMapping.toHiveTypeString(f.schema())) .collect(Collectors.toList()); } catch (Exception e) { + if (completeness != null) { + completeness.markIncomplete(); + } LOG.warn("Failed to resolve Hudi schema for JNI reader, JNI splits may fail: {}", e.getMessage()); columnNames = Collections.emptyList(); @@ -276,6 +364,9 @@ private List planScanInScope(ConnectorSession session, Conne resolvedSchema = HudiSchemaUtils.resolveTableInternalSchema( new TableSchemaResolver(metaClient), avroSchema); } catch (Exception e) { + if (completeness != null) { + completeness.markIncomplete(); + } LOG.warn("Failed to resolve Hudi InternalSchema for schema_id; native reads fall back to BY_NAME: {}", e.getMessage()); } @@ -285,16 +376,7 @@ private List planScanInScope(ConnectorSession session, Conne // (base schema unresolved -> BY_NAME). A per-file resolution failure logs and returns null for that file // (BY_NAME) rather than failing the whole scan. Runs on this TCCL-pinned scan thread. final HudiSchemaUtils.ResolvedInternalSchema baseSchema = resolvedSchema; - Function schemaIdResolver = baseSchema == null ? null - : filePath -> { - try { - return HudiSchemaUtils.resolveFileInternalSchema(filePath, - baseSchema.enableSchemaEvolution, baseSchema.internalSchema, metaClient).schemaId(); - } catch (Exception e) { - LOG.warn("Failed to resolve Hudi per-file schema_id for {}: {}", filePath, e.getMessage()); - return null; - } - }; + Function schemaIdResolver = buildSchemaIdResolver(baseSchema, metaClient, completeness); String inputFormat = hudiHandle.getInputFormat(); String serdeLib = hudiHandle.getSerdeLib(); @@ -358,6 +440,54 @@ basePath, inputFormat, serdeLib, columnNames, columnTypes, partitionFieldNames(m } } + Function buildSchemaIdResolver(HudiSchemaUtils.ResolvedInternalSchema baseSchema, + HoodieTableMetaClient metaClient, PlanCompleteness completeness) { + if (baseSchema == null) { + return null; + } + return filePath -> { + try { + return resolveFileSchemaId(filePath, baseSchema, metaClient); + } catch (Exception e) { + if (completeness != null) { + completeness.markIncomplete(); + } + LOG.warn("Failed to resolve Hudi per-file schema_id for {}: {}", filePath, e.getMessage()); + return null; + } + }; + } + + // Package-private test seam for a fail-once resolver; the production implementation remains the single + // HudiSchemaUtils lookup used by every native snapshot split. + long resolveFileSchemaId(String filePath, HudiSchemaUtils.ResolvedInternalSchema baseSchema, + HoodieTableMetaClient metaClient) { + return HudiSchemaUtils.resolveFileInternalSchema(filePath, + baseSchema.enableSchemaEvolution, baseSchema.internalSchema, metaClient).schemaId(); + } + + static final class PlannedScan { + private final List ranges; + private final boolean cacheable; + + PlannedScan(List ranges, boolean cacheable) { + this.ranges = ranges; + this.cacheable = cacheable; + } + } + + static final class PlanCompleteness { + private boolean complete = true; + + void markIncomplete() { + complete = false; + } + + boolean isComplete() { + return complete; + } + } + @Override public Map getScanNodeProperties( ConnectorSession session, ConnectorTableHandle handle, @@ -418,7 +548,13 @@ public Map getScanNodeProperties( if (!isForceJniScannerEnabled(session)) { try { // Inside the executor for the same reason planScan is: the metaClient opens filesystems. - Optional dict = executor.execute(() -> schemaEvolutionDict(hudiHandle, columns)); + // When statement-scoped reuse is active, use the SAME metaClient planScan cached at the + // scope so the schema evolution dictionary reflects the same generation the scan plan + // was built against. + Optional dict = executor.execute(() -> { + HudiStatementTable statementTable = resolveHudiTable(session, hudiHandle); + return schemaEvolutionDict(statementTable, hudiHandle, columns); + }); dict.ifPresent(v -> props.put(SCHEMA_EVOLUTION_PROP, v)); } catch (Exception e) { LOG.warn("Failed to build Hudi schema-evolution dict for {}.{}; native reads fall back to BY_NAME: {}", @@ -430,9 +566,9 @@ public Map getScanNodeProperties( } /** The native-reader schema-evolution dictionary of {@code getScanNodeProperties}, or empty. */ - private Optional schemaEvolutionDict(HudiTableHandle hudiHandle, List columns) - throws Exception { - HoodieTableMetaClient metaClient = buildMetaClient(buildHadoopConf(), hudiHandle.getBasePath()); + Optional schemaEvolutionDict(HudiStatementTable statementTable, HudiTableHandle hudiHandle, + List columns) throws Exception { + HoodieTableMetaClient metaClient = statementTable.metaClient; TableSchemaResolver schemaResolver = new TableSchemaResolver(metaClient); // HD-C5b: FOR TIME AS OF over a schema-on-read table -> re-resolve the native -1 overlay from the // FULL schema AT the pinned instant. The requested column HANDLES are latest-keyed @@ -1164,4 +1300,142 @@ static Map storageHadoopConfig(ConnectorContext context) { private ConnectorStorageContext storage() { return context.getStorageContext(); } + + /** Package-private for offline unit tests of key construction. */ + static HudiScanReuseKey hudiScanReuseKey(HudiTableHandle handle, String generation) { + return new HudiScanReuseKey(handle, generation); + } + + /** One statement-stable Hudi meta client paired with the generation read from that client. */ + static final class HudiStatementTable { + private final HoodieTableMetaClient metaClient; + private final String generation; + private final Configuration hadoopConf; + + HudiStatementTable(HoodieTableMetaClient metaClient, String generation, + Configuration hadoopConf) { + this.metaClient = metaClient; + this.generation = generation; + this.hadoopConf = hadoopConf; + } + } + + /** Physical Hudi table identity used to resolve one statement-scoped meta client. */ + static final class HudiTableResolutionKey { + private final String dbName; + private final String tableName; + private final String basePath; + + HudiTableResolutionKey(HudiTableHandle handle) { + this.dbName = handle.getDbName(); + this.tableName = handle.getTableName(); + this.basePath = handle.getBasePath(); + } + + @Override + public boolean equals(Object object) { + if (this == object) { + return true; + } + if (!(object instanceof HudiTableResolutionKey)) { + return false; + } + HudiTableResolutionKey that = (HudiTableResolutionKey) object; + return Objects.equals(dbName, that.dbName) + && Objects.equals(tableName, that.tableName) + && Objects.equals(basePath, that.basePath); + } + + @Override + public int hashCode() { + return Objects.hash(dbName, tableName, basePath); + } + } + + /** + * Statement-scoped cache key for one Hudi scan. + * + *

Includes every input that changes the planned split list: table identity, the snapshot + * instant, the incremental window (begin/end instant + incremental options), the pruned + * partition set, the partition keys, the JNI metadata carriers (input format / serde), and + * the resolved table generation (latest completed instant). The generation fences the key + * against same-path table recreation: without it, alias A's ranges (stamped with + * generation-A per-file schema_id values) could be paired with alias B's generation-B + * history_schema_info dictionary, causing BE to map files through wrong field identities. + * Session variables are statement-constant and deliberately absent. + */ + static final class HudiScanReuseKey { + private final String dbName; + private final String tableName; + private final String basePath; + private final String queryInstant; + private final String beginInstant; + private final String endInstant; + private final Map incrementalParams; + private final List prunedPartitionPaths; + private final List partitionKeyNames; + private final String inputFormat; + private final String serdeLib; + private final String generation; + + // Hudi scan-planning identity is fully captured by the handle, its query/incremental + // parameters, and the resolved table generation; request-level filter, columns, and + // countPushdown do not affect split planning and are deliberately excluded from the key. + private HudiScanReuseKey(HudiTableHandle handle, String generation) { + // Catalog and query isolation are provided by the statement-scope memo key. + this.dbName = handle.getDbName(); + this.tableName = handle.getTableName(); + this.basePath = handle.getBasePath(); + this.queryInstant = handle.getQueryInstant(); + this.beginInstant = handle.getBeginInstant(); + this.endInstant = handle.getEndInstant(); + this.incrementalParams = handle.getIncrementalParams() == null + ? Collections.emptyMap() + : Collections.unmodifiableMap(new HashMap<>(handle.getIncrementalParams())); + this.prunedPartitionPaths = handle.getPrunedPartitionPaths() == null + ? null + : Collections.unmodifiableList(new ArrayList<>(handle.getPrunedPartitionPaths())); + this.partitionKeyNames = handle.getPartitionKeyNames() == null + ? Collections.emptyList() + : Collections.unmodifiableList(new ArrayList<>(handle.getPartitionKeyNames())); + this.inputFormat = handle.getInputFormat(); + this.serdeLib = handle.getSerdeLib(); + this.generation = generation; + } + + @Override + public boolean equals(Object object) { + if (this == object) { + return true; + } + if (!(object instanceof HudiScanReuseKey)) { + return false; + } + HudiScanReuseKey that = (HudiScanReuseKey) object; + return Objects.equals(dbName, that.dbName) + && Objects.equals(tableName, that.tableName) + && Objects.equals(basePath, that.basePath) + && Objects.equals(queryInstant, that.queryInstant) + && Objects.equals(beginInstant, that.beginInstant) + && Objects.equals(endInstant, that.endInstant) + && Objects.equals(incrementalParams, that.incrementalParams) + && Objects.equals(prunedPartitionPaths, that.prunedPartitionPaths) + && Objects.equals(partitionKeyNames, that.partitionKeyNames) + && Objects.equals(inputFormat, that.inputFormat) + && Objects.equals(serdeLib, that.serdeLib) + && Objects.equals(generation, that.generation); + } + + @Override + public int hashCode() { + return Objects.hash(dbName, tableName, basePath, + queryInstant, beginInstant, endInstant, incrementalParams, + prunedPartitionPaths, partitionKeyNames, inputFormat, serdeLib, generation); + } + + @Override + public String toString() { + return "HudiScanReuseKey{table=" + dbName + "." + tableName + "}"; + } + } } diff --git a/fe/fe-connector/fe-connector-hudi/src/test/java/org/apache/doris/connector/hudi/HudiScanReuseKeyTest.java b/fe/fe-connector/fe-connector-hudi/src/test/java/org/apache/doris/connector/hudi/HudiScanReuseKeyTest.java new file mode 100644 index 00000000000000..b97f7eababd9c5 --- /dev/null +++ b/fe/fe-connector/fe-connector-hudi/src/test/java/org/apache/doris/connector/hudi/HudiScanReuseKeyTest.java @@ -0,0 +1,391 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.hudi; + +import org.apache.doris.connector.spi.ConnectorSession; +import org.apache.doris.connector.spi.ConnectorStatementScope; +import org.apache.doris.connector.spi.handle.ConnectorColumnHandle; +import org.apache.doris.connector.spi.scan.ConnectorScanRange; +import org.apache.doris.connector.spi.scan.ConnectorScanRequest; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.internal.schema.InternalSchema; +import org.apache.hudi.internal.schema.Types; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Function; +import java.util.function.Supplier; + +/** Statement-scoped scan reuse key construction for Hudi (offline; no table environment needed). */ +class HudiScanReuseKeyTest { + + @Test + void scanReuseNamespaceUsesConnectorType() { + String prefix = new HudiConnectorProvider().getType() + "."; + Assertions.assertTrue(HudiScanPlanProvider.SCAN_REUSE_NAMESPACE.startsWith(prefix)); + } + + private static HudiTableHandle handle() { + return new HudiTableHandle.Builder("db", "t", "/warehouse/t", "COPY_ON_WRITE") + .inputFormat("org.apache.hudi.hadoop.HoodieParquetInputFormat") + .partitionKeyNames(Arrays.asList("year", "month")) + .prunedPartitionPaths(Arrays.asList("year=2025/month=01", "year=2025/month=02")) + .queryInstant("20250429000000000") + .build(); + } + + private static HudiScanPlanProvider.HudiScanReuseKey key(HudiTableHandle handle) { + return HudiScanPlanProvider.hudiScanReuseKey(handle, "test-generation"); + } + + @Test + void sameScanYieldsSameKey() { + Assertions.assertEquals(key(handle()), key(handle()), + "two identical handles must produce the same reuse key"); + } + + @Test + void differentQueryInstantYieldsDifferentKey() { + HudiTableHandle other = handle().toBuilder() + .queryInstant("20250430000000000") + .build(); + Assertions.assertNotEquals(key(handle()), key(other), + "a different snapshot instant must not reuse the cached ranges"); + } + + @Test + void incrementalWindowYieldsDifferentKey() { + HudiTableHandle incremental = handle().toBuilder() + .beginInstant("20250420000000000") + .endInstant("20250430000000000") + .build(); + Assertions.assertNotEquals(key(handle()), key(incremental), + "an incremental window must not reuse a snapshot scan's ranges"); + + HudiTableHandle otherWindow = incremental.toBuilder() + .endInstant("20250425000000000") + .build(); + Assertions.assertNotEquals(key(incremental), key(otherWindow), + "a different incremental window must not reuse the cached ranges"); + } + + @Test + void incrementalParamsYieldsDifferentKey() { + HudiTableHandle incrementalBase = handle().toBuilder() + .beginInstant("20250420000000000") + .endInstant("20250430000000000") + .build(); + Map params = new HashMap<>(); + params.put("hoodie.datasource.read.incr.path.glob", "/warehouse/t/*/*"); + HudiTableHandle withParams = incrementalBase.toBuilder() + .incrementalParams(params) + .build(); + Assertions.assertNotEquals(key(incrementalBase), key(withParams), + "incremental options must participate in the reuse key"); + } + + @Test + void differentPrunedPartitionsYieldDifferentKey() { + HudiTableHandle other = handle().toBuilder() + .prunedPartitionPaths(Collections.singletonList("year=2025/month=01")) + .build(); + Assertions.assertNotEquals(key(handle()), key(other), + "a different pruned partition set must not reuse the cached ranges"); + } + + @Test + void unprunedAndZeroPrunedStatesYieldDifferentKeys() { + HudiTableHandle unpruned = handle().toBuilder() + .prunedPartitionPaths(null) + .build(); + HudiTableHandle zeroPruned = handle().toBuilder() + .prunedPartitionPaths(Collections.emptyList()) + .build(); + + Assertions.assertNotEquals(key(unpruned), key(zeroPruned), + "null means enumerate all snapshot partitions while empty means the filter matched none"); + } + + @Test + void statementReuseKeepsSameInstantZeroPrunedAndUnprunedScansDistinct() { + RecordingScanProvider provider = new RecordingScanProvider(); + HudiTableHandle unpruned = handle().toBuilder() + .prunedPartitionPaths(null) + .build(); + HudiTableHandle zeroPruned = handle().toBuilder() + .prunedPartitionPaths(Collections.emptyList()) + .build(); + ConnectorSession session = new MemoSession(new MemoScope()); + + List zeroRanges = provider.planScan(session, + ConnectorScanRequest.builder( + zeroPruned, Collections.emptyList()).build()); + List allRanges = provider.planScan(session, + ConnectorScanRequest.builder( + unpruned, Collections.emptyList()).build()); + + Assertions.assertTrue(zeroRanges.isEmpty()); + Assertions.assertEquals(1, allRanges.size(), + "an unpruned time-travel alias must not reuse a zero-pruned alias's empty ranges"); + Assertions.assertEquals(2, provider.planCalls, + "the two semantic partition states must occupy separate statement-cache entries"); + } + + @Test + void statementReusePlansAnIdenticalScanOnce() { + RecordingScanProvider provider = new RecordingScanProvider(); + ConnectorSession session = new MemoSession(new MemoScope()); + ConnectorScanRequest request = ConnectorScanRequest.builder( + handle().toBuilder().prunedPartitionPaths(null).build(), + Collections.emptyList()).build(); + + List first = provider.planScan(session, request); + List second = provider.planScan(session, request); + + Assertions.assertSame(first, second, "an identical scan must reuse the statement's planned range list"); + Assertions.assertEquals(1, provider.planCalls, "the underlying Hudi planner must run once"); + } + + @Test + void statementReuseRetriesASchemaResolutionThatFailedOnce() { + RecordingScanProvider provider = new RecordingScanProvider(); + provider.failNextSchemaIdResolution = true; + ConnectorSession session = new MemoSession(new MemoScope()); + ConnectorScanRequest request = ConnectorScanRequest.builder( + handle().toBuilder().prunedPartitionPaths(null).build(), + Collections.emptyList()).build(); + + List degraded = provider.planScan(session, request); + List recovered = provider.planScan(session, request); + List reused = provider.planScan(session, request); + + Assertions.assertNotSame(degraded, recovered, + "a BY_NAME fallback caused by transient schema resolution must not enter statement reuse"); + Assertions.assertSame(recovered, reused, + "the successful schema-resolution retry should enter statement reuse"); + Assertions.assertEquals(2, provider.planCalls, + "the second alias must retry once, while the third alias reuses the recovered plan"); + } + + @Test + void statementReuseCanBeDisabled() { + RecordingScanProvider provider = new RecordingScanProvider(); + ConnectorSession session = new MemoSession(new MemoScope(), + Collections.singletonMap("enable_external_scan_task_reuse", "false")); + ConnectorScanRequest request = ConnectorScanRequest.builder( + handle().toBuilder().prunedPartitionPaths(null).build(), + Collections.emptyList()).build(); + + provider.planScan(session, request); + provider.planScan(session, request); + + Assertions.assertEquals(2, provider.planCalls, "disabled split reuse must bypass the statement memo"); + } + + @Test + void missingReusePropertyIsDisabledForMixedVersionSessions() { + RecordingScanProvider provider = new RecordingScanProvider(); + ConnectorSession session = new MemoSession(new MemoScope(), Collections.emptyMap()); + ConnectorScanRequest request = ConnectorScanRequest.builder( + handle().toBuilder().prunedPartitionPaths(null).build(), + Collections.emptyList()).build(); + + provider.planScan(session, request); + provider.planScan(session, request); + + Assertions.assertEquals(2, provider.planCalls, + "a session from an older planning FE must not implicitly enable split reuse"); + } + + @Test + void planningAndPropertiesUseTheSamePhysicalGeneration() { + RecordingScanProvider provider = new RecordingScanProvider(); + ConnectorSession session = new MemoSession(new MemoScope()); + HudiTableHandle handle = handle().toBuilder().prunedPartitionPaths(null).build(); + + provider.planScan(session, ConnectorScanRequest.builder( + handle, Collections.emptyList()).build()); + provider.getScanNodeProperties( + session, handle, Collections.emptyList(), Optional.empty()); + + Assertions.assertSame(provider.statementTable, provider.plannedTable, + "split planning must use the meta client whose generation fenced the reuse key"); + Assertions.assertSame(provider.statementTable, provider.propertiesTable, + "scan properties must use the same meta client as split planning"); + } + + @Test + void tableResolutionKeyIncludesThePhysicalBasePath() { + HudiTableHandle recreated = new HudiTableHandle.Builder( + "db", "t", "/warehouse/recreated-t", "COPY_ON_WRITE") + .inputFormat("org.apache.hudi.hadoop.HoodieParquetInputFormat") + .build(); + + Assertions.assertNotEquals( + new HudiScanPlanProvider.HudiTableResolutionKey(handle()), + new HudiScanPlanProvider.HudiTableResolutionKey(recreated), + "same logical name at a different path must resolve a different meta client"); + } + + private static final class RecordingScanProvider extends HudiScanPlanProvider { + private static final ConnectorScanRange RANGE = Collections::emptyMap; + private final HudiStatementTable statementTable = + new HudiStatementTable(null, "generation-a", new Configuration(false)); + private int planCalls; + private boolean failNextSchemaIdResolution; + private HudiStatementTable plannedTable; + private HudiStatementTable propertiesTable; + + private RecordingScanProvider() { + super(Collections.emptyMap(), null); + } + + @Override + HudiStatementTable resolveHudiTable(ConnectorSession session, HudiTableHandle handle) { + return statementTable; + } + + @Override + List doPlanScan(ConnectorSession session, ConnectorScanRequest request) { + planCalls++; + HudiTableHandle scanHandle = (HudiTableHandle) request.getTableHandle(); + return scanHandle.getPrunedPartitionPaths() == null + ? Collections.singletonList(RANGE) + : Collections.emptyList(); + } + + @Override + List doPlanScan(ConnectorSession session, ConnectorScanRequest request, + HudiStatementTable statementTable) { + plannedTable = statementTable; + return doPlanScan(session, request); + } + + @Override + PlannedScan doPlanScanForReuse(ConnectorSession session, ConnectorScanRequest request, + HudiStatementTable statementTable) { + PlanCompleteness completeness = new PlanCompleteness(); + HudiSchemaUtils.ResolvedInternalSchema baseSchema = new HudiSchemaUtils.ResolvedInternalSchema( + new InternalSchema(1L, Types.RecordType.get(Collections.emptyList())), true); + Function resolver = buildSchemaIdResolver(baseSchema, null, completeness); + resolver.apply("/warehouse/t/file.parquet"); + List ranges = doPlanScan(session, request, statementTable); + return new PlannedScan(ranges, completeness.isComplete()); + } + + @Override + long resolveFileSchemaId(String filePath, HudiSchemaUtils.ResolvedInternalSchema baseSchema, + HoodieTableMetaClient metaClient) { + if (failNextSchemaIdResolution) { + failNextSchemaIdResolution = false; + throw new IllegalStateException("transient schema lookup failure"); + } + return 1L; + } + + @Override + Optional schemaEvolutionDict(HudiStatementTable statementTable, HudiTableHandle handle, + List columns) { + propertiesTable = statementTable; + return Optional.empty(); + } + } + + private static final class MemoScope implements ConnectorStatementScope { + private final ConcurrentHashMap cache = new ConcurrentHashMap<>(); + + @Override + @SuppressWarnings("unchecked") + public T computeIfAbsent(String key, Supplier loader) { + return (T) cache.computeIfAbsent(key, ignored -> loader.get()); + } + } + + private static final class MemoSession implements ConnectorSession { + private final ConnectorStatementScope scope; + private final Map sessionProperties; + + private MemoSession(ConnectorStatementScope scope) { + this(scope, Collections.singletonMap("enable_external_scan_task_reuse", "true")); + } + + private MemoSession(ConnectorStatementScope scope, Map sessionProperties) { + this.scope = scope; + this.sessionProperties = sessionProperties; + } + + @Override + public long getCatalogId() { + return 7L; + } + + @Override + public String getQueryId() { + return "same-statement"; + } + + @Override + public ConnectorStatementScope getStatementScope() { + return scope; + } + + @Override + public String getUser() { + return "u"; + } + + @Override + public String getTimeZone() { + return "UTC"; + } + + @Override + public String getLocale() { + return "en_US"; + } + + @Override + public String getCatalogName() { + return "c"; + } + + @Override + public T getProperty(String name, Class type) { + return null; + } + + @Override + public Map getCatalogProperties() { + return Collections.emptyMap(); + } + + @Override + public Map getSessionProperties() { + return sessionProperties; + } + } +} diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergScanPlanProvider.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergScanPlanProvider.java index 6b3955d38be855..8774b6a5d7a836 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergScanPlanProvider.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergScanPlanProvider.java @@ -119,6 +119,7 @@ import java.util.function.Function; import java.util.function.Supplier; import java.util.function.UnaryOperator; +import java.util.stream.Collectors; /** * {@link ConnectorScanPlanProvider} for Iceberg tables, mirroring the paimon connector's @@ -152,6 +153,7 @@ public class IcebergScanPlanProvider implements ConnectorScanPlanProvider { private static final String ENABLE_FILE_SCANNER_V2 = "enable_file_scanner_v2"; // FIX-M3 streaming (file-count) batch gate — keys byte-identical to fe-core SessionVariable. private static final String ENABLE_EXTERNAL_TABLE_BATCH_MODE = "enable_external_table_batch_mode"; + static final String SCAN_REUSE_NAMESPACE = "iceberg.scan-reuse"; private static final String NUM_FILES_IN_BATCH_MODE = "num_files_in_batch_mode"; private static final String IGNORE_ICEBERG_DANGLING_DELETE = "ignore_iceberg_dangling_delete"; private static final long DEFAULT_NUM_FILES_IN_BATCH_MODE = 1024L; @@ -432,15 +434,49 @@ public boolean supportsFileCache() { */ @Override public List planScan(ConnectorSession session, ConnectorScanRequest request) { - IcebergTableHandle handle = (IcebergTableHandle) request.getTableHandle(); + IcebergTableHandle icebergHandle = (IcebergTableHandle) request.getTableHandle(); try { - return planScanInternal(session, handle, request.getColumns(), - request.getFilter(), request.isCountPushdown()); + // The reuse key deliberately omits projected columns because they do not change Iceberg split + // planning. Reader compatibility still depends on whether metadata columns are requested, so + // validate every call before a cached range list can be returned. + validateMetadataColumnReader(session, request.getColumns()); + if (session == null || !session.isExternalScanTaskReuseEnabled()) { + return planScanInternal(session, icebergHandle, request.getColumns(), + request.getFilter(), request.isCountPushdown()); + } + if (icebergHandle.isSystemTable()) { + // System tables read connector metadata through their own readers; never reuse them. + return planScanInternal(session, icebergHandle, request.getColumns(), + request.getFilter(), request.isCountPushdown()); + } + // Statement-scoped reuse: within one statement the identical scan (same table, same + // snapshot/ref/schema pin, same filter, same COUNT pushdown) plans once and every + // duplicated relation shares the result. The scope is NONE for offline planning and tests, + // in which case the loader runs on every call. Session variables are constant within a + // statement and deliberately absent from the key. + // + // The ranges are memoized in a map HUNG INSIDE the statement scope rather than cached + // directly: planScanInternal re-enters the scope itself (sharedTable and the v3 + // rewritableDeleteSupply are scope-backed), and a loader of the scope's + // ConcurrentHashMap must not touch that map (same-bin re-entry throws + // IllegalStateException("Recursive update"); a mid-computation resize silently drops the + // outer entry). The scope loader only constructs the memo map; planning then runs on that + // separate map, so every scope call from planScanInternal is top-level again. The memo + // key is catalog-scoped, which also isolates same-named tables across a cross-catalog + // statement. + String memoKey = SCAN_REUSE_NAMESPACE + ":" + session.getCatalogId() + ":" + session.getQueryId(); + Map> scanReuse = + session.getStatementScope().computeIfAbsent(memoKey, () -> new ConcurrentHashMap<>()); + IcebergScanReuseKey reuseKey = new IcebergScanReuseKey(icebergHandle, request); + return scanReuse.computeIfAbsent(reuseKey, + k -> Collections.unmodifiableList(planScanInternal(session, + icebergHandle, request.getColumns(), request.getFilter(), + request.isCountPushdown()))); } catch (RuntimeException e) { // Normal data scans and native position_deletes run on File Scanner V2. Keep the serialized JNI // system-table route untouched because its deferred reads belong to the V1 scanner contract. - if (!handle.isSystemTable() || isPositionDeletesSysTable(handle)) { - throw IcebergExceptionUtils.wrapMetadataReadFailure(handle, e); + if (!icebergHandle.isSystemTable() || isPositionDeletesSysTable(icebergHandle)) { + throw IcebergExceptionUtils.wrapMetadataReadFailure(icebergHandle, e); } throw e; } @@ -706,7 +742,6 @@ private List planScanInternal( Optional filter, boolean countPushdown) { IcebergTableHandle iceHandle = (IcebergTableHandle) handle; - validateMetadataColumnReader(session, columns); if (iceHandle.isResolvedEmptySnapshot() && !isSnapshotIndependentSystemTable(iceHandle)) { // Iceberg has no snapshot id that can represent "before the first commit". Returning no ranges is // the read-side MVCC fence; otherwise a refreshed Table would turn -1 into "latest" and expose a @@ -3197,4 +3232,70 @@ private Table resolveSysTable(ConnectorSession session, IcebergTableHandle handl private ConnectorStorageContext storage() { return context.getStorageContext(); } + + /** + * Statement-scoped cache key for one Iceberg scan. + * + *

Includes every input that changes the planned file list: table identity, the typed + * time-travel pin (snapshot id / ref / schema id), the rewrite-file scope, the pushed filter + * and the COUNT pushdown flag. System tables are excluded upstream, and session variables are + * statement-constant, so both stay out of the key. + */ + private static final class IcebergScanReuseKey { + private final String dbName; + private final String tableName; + private final long snapshotId; + private final String ref; + private final long schemaId; + private final List rewriteFileScope; + private final Optional filter; + private final boolean countPushdown; + + private IcebergScanReuseKey(IcebergTableHandle handle, ConnectorScanRequest request) { + this.dbName = handle.getDbName(); + this.tableName = handle.getTableName(); + this.snapshotId = handle.getSnapshotId(); + this.ref = handle.getRef(); + this.schemaId = handle.getSchemaId(); + // System tables are bypassed in planScan before this key is built, so sysTableName is + // always null here; if the system-table bypass is ever relaxed, add it back. + this.rewriteFileScope = handle.getRewriteFileScope() == null + ? Collections.emptyList() + : handle.getRewriteFileScope().stream().sorted() + .collect(Collectors.toList()); + this.filter = request.getFilter(); + this.countPushdown = request.isCountPushdown(); + } + + @Override + public boolean equals(Object object) { + if (this == object) { + return true; + } + if (!(object instanceof IcebergScanReuseKey)) { + return false; + } + IcebergScanReuseKey that = (IcebergScanReuseKey) object; + return snapshotId == that.snapshotId + && schemaId == that.schemaId + && countPushdown == that.countPushdown + && Objects.equals(dbName, that.dbName) + && Objects.equals(tableName, that.tableName) + && Objects.equals(ref, that.ref) + && Objects.equals(rewriteFileScope, that.rewriteFileScope) + && Objects.equals(filter, that.filter); + } + + @Override + public int hashCode() { + return Objects.hash(dbName, tableName, snapshotId, ref, schemaId, + rewriteFileScope, filter, countPushdown); + } + + @Override + public String toString() { + return "IcebergScanReuseKey{db=" + dbName + ", table=" + tableName + + ", snapshot=" + snapshotId + "}"; + } + } } diff --git a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergScanPlanProviderTest.java b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergScanPlanProviderTest.java index 0a5a1b3ecbca5e..42530caf9f14f0 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergScanPlanProviderTest.java +++ b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergScanPlanProviderTest.java @@ -116,6 +116,12 @@ public class IcebergScanPlanProviderTest { @TempDir Path tempDir; + @Test + public void scanReuseNamespaceUsesConnectorType() { + String prefix = new IcebergConnectorProvider().getType() + "."; + Assertions.assertTrue(IcebergScanPlanProvider.SCAN_REUSE_NAMESPACE.startsWith(prefix)); + } + private static final Schema SCHEMA = new Schema( Types.NestedField.required(1, "id", Types.IntegerType.get()), Types.NestedField.optional(2, "name", Types.StringType.get())); @@ -552,6 +558,81 @@ public void planningPassWithoutSharedScopeLoadsEachTime() { Assertions.assertEquals(2, remoteLoads, "under NONE each resolver loads (no memo)"); } + @Test + public void statementReusePlansAnIdenticalScanOnce() { + Table table = createTable("t1", SCHEMA, PartitionSpec.unpartitioned()); + table.newAppend().appendFile( + dataFile(table.spec(), "s3://b/db/t1/f1.parquet", 1024, null, null)).commit(); + IcebergScanPlanProvider provider = providerOver(table); + ConnectorSession session = new FakeScanSession("UTC", + Collections.singletonMap("enable_external_scan_task_reuse", "true")) + .withScope(new TestStatementScope()); + ConnectorScanRequest request = ConnectorScanRequest.builder( + new IcebergTableHandle("db1", "t1"), Collections.emptyList()).build(); + + List first = provider.planScan(session, request); + List second = provider.planScan(session, request); + + Assertions.assertSame(first, second, "an identical scan must reuse the statement's planned range list"); + Assertions.assertEquals(1, first.size()); + } + + @Test + public void statementReuseMatchesStructurallyEqualFilters() { + Table table = createTable("t1", SCHEMA, PartitionSpec.unpartitioned()); + table.newAppend().appendFile( + dataFile(table.spec(), "s3://b/db/t1/f1.parquet", 1024, null, null)).commit(); + IcebergScanPlanProvider provider = providerOver(table); + ConnectorSession session = new FakeScanSession("UTC", + Collections.singletonMap("enable_external_scan_task_reuse", "true")) + .withScope(new TestStatementScope()); + IcebergTableHandle handle = new IcebergTableHandle("db1", "t1"); + ConnectorScanRequest firstRequest = ConnectorScanRequest.builder(handle, Collections.emptyList()) + .filter(Optional.of(equalIdFilter(1))) + .build(); + ConnectorScanRequest secondRequest = ConnectorScanRequest.builder(handle, Collections.emptyList()) + .filter(Optional.of(equalIdFilter(1))) + .build(); + + Assertions.assertNotSame(firstRequest.getFilter().get(), secondRequest.getFilter().get()); + List first = provider.planScan(session, firstRequest); + List second = provider.planScan(session, secondRequest); + + Assertions.assertSame(first, second, + "independently built but structurally equal filters must share one statement plan"); + Assertions.assertEquals(1, first.size()); + } + + @Test + public void statementReuseStillValidatesMetadataColumnReader() { + Table table = createTable("t1", SCHEMA, PartitionSpec.unpartitioned()); + table.newAppend().appendFile( + dataFile(table.spec(), "s3://b/db/t1/f1.parquet", 1024, null, null)).commit(); + IcebergScanPlanProvider provider = providerOver(table); + ConnectorSession session = new FakeScanSession("UTC", Map.of( + "enable_external_scan_task_reuse", "true", + "force_jni_scanner", "true")) + .withScope(new TestStatementScope()); + IcebergTableHandle handle = new IcebergTableHandle("db1", "t1"); + + provider.planScan(session, ConnectorScanRequest.builder(handle, + Collections.singletonList(new IcebergColumnHandle("id", 1))).build()); + + DorisConnectorException ex = Assertions.assertThrows(DorisConnectorException.class, + () -> provider.planScan(session, ConnectorScanRequest.builder(handle, + Collections.singletonList(new IcebergColumnHandle("_file", -1))).build())); + Assertions.assertEquals( + "Iceberg metadata columns are only supported by FileScannerV2 native Parquet/ORC reader; " + + "actual reader is JNI", + ex.getMessage()); + } + + private static ConnectorExpression equalIdFilter(long value) { + return new ConnectorComparison(ConnectorComparison.Operator.EQ, + new ConnectorColumnRef("id", ConnectorType.of("INT")), + new ConnectorLiteral(ConnectorType.of("INT"), value)); + } + // --- T02 split-enumeration + predicate-pushdown tests --- @Test diff --git a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonScanPlanProvider.java b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonScanPlanProvider.java index 1d9d917743752d..a32c242e38f613 100644 --- a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonScanPlanProvider.java +++ b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonScanPlanProvider.java @@ -21,6 +21,7 @@ import org.apache.doris.connector.metastore.spi.JdbcDriverSupport; import org.apache.doris.connector.spi.ConnectorContext; import org.apache.doris.connector.spi.ConnectorSession; +import org.apache.doris.connector.spi.ConnectorStatementScope; import org.apache.doris.connector.spi.ConnectorStorageContext; import org.apache.doris.connector.spi.DorisConnectorException; import org.apache.doris.connector.spi.handle.ConnectorColumnHandle; @@ -191,6 +192,7 @@ public ConnectorColumnCategory classifyColumn(String columnName) { private static final String IGNORE_SPLIT_TYPE = "ignore_split_type"; private static final String IGNORE_SPLIT_TYPE_JNI = "IGNORE_JNI"; private static final String IGNORE_SPLIT_TYPE_NATIVE = "IGNORE_NATIVE"; + static final String SCAN_REUSE_NAMESPACE = "paimon.scan-reuse"; // FIX-NATIVE-SUBSPLIT (M-3): file-split session vars (byte-identical to SessionVariable.{FILE_SPLIT_SIZE, // MAX_INITIAL_FILE_SPLIT_SIZE, MAX_FILE_SPLIT_SIZE, MAX_INITIAL_FILE_SPLIT_NUM, MAX_FILE_SPLIT_NUM}), @@ -538,8 +540,83 @@ public void releaseReadTransaction(String queryId) { */ @Override public List planScan(ConnectorSession session, ConnectorScanRequest request) { - return planScanInternal(session, request.getTableHandle(), request.getColumns(), - request.getFilter(), request.getLimit(), request.isCountPushdown()); + PaimonTableHandle paimonHandle = (PaimonTableHandle) request.getTableHandle(); + if (session == null || !session.isExternalScanTaskReuseEnabled()) { + return planScanInternal(session, request.getTableHandle(), request.getColumns(), + request.getFilter(), request.getLimit(), request.isCountPushdown()); + } + if (paimonHandle.isSystemTable()) { + // System tables resolve their snapshot on the BE and carry deferred side effects + // (authorized file enumeration); never reuse their planned ranges. + return planScanInternal(session, request.getTableHandle(), request.getColumns(), + request.getFilter(), request.getLimit(), request.isCountPushdown()); + } + // Resolve the table ONCE at the statement scope so both the scan-planning path (here) and + // the properties path (getScanNodeProperties) observe the SAME table generation. Without + // this, a no-cache catalog or a schema change between two aliases can give alias A's ranges + // to alias B's generation-B serialized table. + Table table = resolveScanTableConsistent(session, paimonHandle); + long generation = resolvePaimonGeneration(paimonHandle, table); + // Statement-scoped reuse: within one statement the identical scan (same table, same + // branch/options pin, same generation, same projection, same filter, same limit, same COUNT + // pushdown) plans once and every duplicated relation shares the result. Session variables + // are constant within a statement and deliberately absent from the key. + String memoKey = SCAN_REUSE_NAMESPACE + ":" + session.getCatalogId() + ":" + session.getQueryId(); + Map> scanReuse = session.getStatementScope().computeIfAbsent( + memoKey, () -> new ConcurrentHashMap<>()); + PaimonScanReuseKey reuseKey = new PaimonScanReuseKey(paimonHandle, request, generation); + return scanReuse.computeIfAbsent(reuseKey, + key -> Collections.unmodifiableList(planScanInternal(session, + request.getTableHandle(), request.getColumns(), request.getFilter(), + request.getLimit(), request.isCountPushdown(), table))); + } + + private long resolvePaimonGeneration(PaimonTableHandle handle, Table table) { + Map scanOptions = effectiveScanOptions(handle); + if (PaimonScanParams.isPinnedEmptyScan(scanOptions)) { + return -1L; + } + String pinnedSnapshotId = scanOptions.get(CoreOptions.SCAN_SNAPSHOT_ID.key()); + if (pinnedSnapshotId != null) { + // The resolved handle already owns this scan's immutable data identity. Re-reading the + // table's live latest pointer here would both add remote I/O for every alias and make two + // aliases pinned to the same snapshot miss reuse when a later commit advances latest. + return Long.parseLong(pinnedSnapshotId); + } + if (catalogOps == null) { + return -1L; + } + if (context == null) { + return catalogOps.latestSnapshotId(table).orElse(-1L); + } + try { + return context.executeAuthenticated(() -> catalogOps.latestSnapshotId(table).orElse(-1L)); + } catch (RuntimeException e) { + throw e; + } catch (Exception e) { + throw new RuntimeException("Failed to resolve latest Paimon snapshot", e); + } + } + + /** Resolve one data table for each full relation identity in a statement. */ + Table resolveScanTableConsistent(ConnectorSession session, PaimonTableHandle paimonHandle) { + if (paimonHandle.isSystemTable()) { + // System-table split planning deliberately bypasses reuse. Keep its wrapper and the + // source generation retained on this handle together on the properties path too. + return resolveScanTable(paimonHandle); + } + if (session != null && session.isExternalScanTaskReuseEnabled() + && session.getStatementScope() != null + && session.getStatementScope() != ConnectorStatementScope.NONE) { + String memoKey = SCAN_REUSE_NAMESPACE + ":tables:" + + session.getCatalogId() + ":" + session.getQueryId(); + Map tables = session.getStatementScope().computeIfAbsent( + memoKey, () -> new ConcurrentHashMap<>()); + PaimonTableResolutionKey key = new PaimonTableResolutionKey( + paimonHandle, effectiveScanOptions(paimonHandle)); + return tables.computeIfAbsent(key, ignored -> resolveScanTable(paimonHandle)); + } + return resolveScanTable(paimonHandle); } /** @@ -677,6 +754,18 @@ private List planScanInternal( Optional filter, long limit, boolean countPushdown) { + return planScanInternal(session, handle, columns, filter, limit, countPushdown, + resolveScanTable((PaimonTableHandle) handle)); + } + + private List planScanInternal( + ConnectorSession session, + ConnectorTableHandle handle, + List columns, + Optional filter, + long limit, + boolean countPushdown, + Table table) { PaimonTableHandle paimonHandle = (PaimonTableHandle) handle; boolean requiresMetadataColumns = requiresMetadataColumns(columns); @@ -694,7 +783,6 @@ private List planScanInternal( // state so a commit between binding and split planning cannot appear mid-statement. return Collections.emptyList(); } - Table table = resolveScanTable(paimonHandle); Optional fileCreationTime = optionsPin ? PaimonScanParams.getPinnedFileCreationTime(pinnedOptions) : Optional.empty(); @@ -1040,7 +1128,10 @@ public Map getScanNodeProperties( Optional filter) { PaimonTableHandle paimonHandle = (PaimonTableHandle) handle; - Table table = resolveScanTable(paimonHandle); + // Keep properties on the same physical generation as the scan plan. Data tables use the + // statement memo shared with planScan; system tables keep their handle-local wrapper and + // source pair because their split planning deliberately bypasses reuse. + Table table = resolveScanTableConsistent(session, paimonHandle); Map props = new LinkedHashMap<>(); @@ -2521,4 +2612,129 @@ private static String escapeJson(String s) { private ConnectorStorageContext storage() { return context.getStorageContext(); } + + /** Full identity used to resolve one statement-scoped Paimon {@link Table}. */ + static final class PaimonTableResolutionKey { + private final String databaseName; + private final String tableName; + private final String systemTableName; + private final String branchName; + private final Map scanOptions; + + PaimonTableResolutionKey(PaimonTableHandle handle, Map effectiveScanOptions) { + this.databaseName = handle.getDatabaseName(); + this.tableName = handle.getTableName(); + this.systemTableName = handle.getSysTableName(); + this.branchName = handle.getBranchName(); + this.scanOptions = effectiveScanOptions == null + ? Collections.emptyMap() + : Collections.unmodifiableMap(new HashMap<>(effectiveScanOptions)); + } + + @Override + public boolean equals(Object object) { + if (this == object) { + return true; + } + if (!(object instanceof PaimonTableResolutionKey)) { + return false; + } + PaimonTableResolutionKey that = (PaimonTableResolutionKey) object; + return Objects.equals(databaseName, that.databaseName) + && Objects.equals(tableName, that.tableName) + && Objects.equals(systemTableName, that.systemTableName) + && Objects.equals(branchName, that.branchName) + && Objects.equals(scanOptions, that.scanOptions); + } + + @Override + public int hashCode() { + return Objects.hash(databaseName, tableName, systemTableName, branchName, scanOptions); + } + } + + /** + * Statement-scoped cache key for one Paimon scan. + * + *

Includes every input that changes the planned split list: table identity, the branch pin, + * the whole scan-options map (snapshot / tag / incremental / options pins), the resolved scan + * generation (the pinned snapshot for fixed identities, or live latest for latest-dependent + * selectors), the projected columns in order, the pushed filter, the limit and the COUNT + * pushdown flag. The generation fences latest-dependent scans against same-path table + * recreation or schema change between two aliases: without it, alias A's ranges (from + * generation A) could be paired with alias B's serialized table (from generation B). System + * tables are excluded upstream, and session variables are statement-constant, so both stay out + * of the key. + */ + private static final class PaimonScanReuseKey { + private final String databaseName; + private final String tableName; + private final String branchName; + private final Map scanOptions; + private final long generation; + private final List columnNames; + private final Optional filter; + private final long limit; + private final boolean countPushdown; + + private PaimonScanReuseKey(PaimonTableHandle handle, ConnectorScanRequest request, + long generation) { + // Catalog and query isolation are provided by the statement-scope memo key. System + // tables are bypassed in planScan before this key is built, so sysTableName is always + // null here; if the system-table bypass is ever relaxed, add it back. + this.databaseName = handle.getDatabaseName(); + this.tableName = handle.getTableName(); + this.branchName = handle.getBranchName(); + this.scanOptions = handle.getScanOptions() == null + ? Collections.emptyMap() + : Collections.unmodifiableMap(new HashMap<>(handle.getScanOptions())); + this.generation = generation; + this.columnNames = request.getColumns().stream() + .map(PaimonScanReuseKey::toPaimonColumnName) + .collect(Collectors.toList()); + this.filter = request.getFilter(); + this.limit = request.getLimit(); + this.countPushdown = request.isCountPushdown(); + } + + private static String toPaimonColumnName(ConnectorColumnHandle column) { + if (!(column instanceof PaimonColumnHandle)) { + throw new IllegalArgumentException( + "Paimon scan reuse key requires PaimonColumnHandle, got: " + column.getClass().getName()); + } + return ((PaimonColumnHandle) column).getName().toLowerCase(Locale.ROOT); + } + + @Override + public boolean equals(Object object) { + if (this == object) { + return true; + } + if (!(object instanceof PaimonScanReuseKey)) { + return false; + } + PaimonScanReuseKey that = (PaimonScanReuseKey) object; + return generation == that.generation + && limit == that.limit + && countPushdown == that.countPushdown + && Objects.equals(databaseName, that.databaseName) + && Objects.equals(tableName, that.tableName) + && Objects.equals(branchName, that.branchName) + && Objects.equals(scanOptions, that.scanOptions) + && Objects.equals(columnNames, that.columnNames) + && Objects.equals(filter, that.filter); + } + + @Override + public int hashCode() { + return Objects.hash(databaseName, tableName, branchName, + scanOptions, generation, columnNames, filter, limit, countPushdown); + } + + @Override + public String toString() { + return "PaimonScanReuseKey{table=" + databaseName + "." + tableName + + ", generation=" + generation + "}"; + } + } } diff --git a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonScanPlanProviderTest.java b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonScanPlanProviderTest.java index edf264a577d528..57416e60693220 100644 --- a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonScanPlanProviderTest.java +++ b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonScanPlanProviderTest.java @@ -19,6 +19,7 @@ import org.apache.doris.connector.spi.ConnectorContext; import org.apache.doris.connector.spi.ConnectorSession; +import org.apache.doris.connector.spi.ConnectorStatementScope; import org.apache.doris.connector.spi.ConnectorStorageContext; import org.apache.doris.connector.spi.ConnectorType; import org.apache.doris.connector.spi.DorisConnectorException; @@ -97,6 +98,9 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.OptionalLong; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Supplier; /** * Tests for {@link PaimonScanPlanProvider#resolveTable}, pinning the transient-Table reload @@ -115,6 +119,12 @@ public class PaimonScanPlanProviderTest { private static final String PAIMON_FILE_PATH_COL = "__paimon_file_path"; private static final String PAIMON_ROW_POSITION_COL = "__paimon_row_index"; + @Test + public void scanReuseNamespaceUsesConnectorType() { + String prefix = new PaimonConnectorProvider().getType() + "."; + Assertions.assertTrue(PaimonScanPlanProvider.SCAN_REUSE_NAMESPACE.startsWith(prefix)); + } + private static RowType rowType(String... columnNames) { RowType.Builder builder = RowType.builder(); for (String name : columnNames) { @@ -261,8 +271,9 @@ public void planScanEnumeratesSplitsInsideAuthScope(@TempDir Path warehouse) thr // Kerberos filesystem catalog that read runs on the PLUGIN's UGI copy, which only the plugin doAs // logs in (iceberg fourth-locus parity; iceberg CI proof: SELECT after INSERT failing SASL at the // plan-time manifest read). So planScan must wrap the enumeration in executeAuthenticated IN - // ADDITION to resolveTable's load wrap. MUTATION: dropping the planSplits wrap -> authCount stays - // 1 (load only) -> red. + // ADDITION to resolveTable's load wrap. The generation lookup is also a remote read and the fake + // below rejects it unless it runs inside authentication. MUTATION: dropping either remote-read + // wrap makes this test fail. try (Catalog catalog = new FileSystemCatalog(LocalFileIO.create(), new org.apache.paimon.fs.Path(warehouse.toUri()))) { catalog.createDatabase("db", false); @@ -286,21 +297,245 @@ public void planScanEnumeratesSplitsInsideAuthScope(@TempDir Path warehouse) thr RecordingPaimonCatalogOps ops = new RecordingPaimonCatalogOps(); ops.table = table; RecordingConnectorContext ctx = new RecordingConnectorContext(); - PaimonScanPlanProvider provider = new PaimonScanPlanProvider(PaimonCatalogProperties.of(Collections.emptyMap()), ops, ctx); + ops.latestSnapshotAuthentication = ctx::isAuthenticated; + PaimonScanPlanProvider provider = new PaimonScanPlanProvider( + PaimonCatalogProperties.of(Collections.emptyMap()), ops, ctx); PaimonTableHandle handle = new PaimonTableHandle( "db", "t", Collections.emptyList(), Collections.emptyList()); - List ranges = provider.planScan(sessionWithProps(Collections.emptyMap()), - ConnectorScanRequest.builder(handle, Collections.emptyList()) - .build()); + ConnectorSession session = sessionWithProps( + Collections.singletonMap("enable_external_scan_task_reuse", "true"), + new TestStatementScope()); + ConnectorScanRequest firstRequest = ConnectorScanRequest.builder(handle, Collections.emptyList()) + .filter(Optional.of(equalIdFilter(1))) + .build(); + ConnectorScanRequest secondRequest = ConnectorScanRequest.builder(handle, Collections.emptyList()) + .filter(Optional.of(equalIdFilter(1))) + .build(); + Assertions.assertNotSame(firstRequest.getFilter().get(), secondRequest.getFilter().get()); + List ranges = provider.planScan(session, firstRequest); + List reused = provider.planScan(session, secondRequest); Assertions.assertFalse(ranges.isEmpty(), "one committed row must plan at least one split"); - Assertions.assertEquals(2, ctx.authCount, - "planScan must run BOTH the table load (resolveTable) AND the split enumeration " - + "(scan.plan(), the remote manifest read) inside executeAuthenticated"); + Assertions.assertSame(ranges, reused, + "independently built but structurally equal filters must share one statement plan"); + Assertions.assertEquals(4, ctx.authCount, + "two scans must authenticate one memoized table load, two generation lookups, " + + "and one memoized split enumeration"); + } + } + + @Test + public void fixedSnapshotAliasesDoNotReReadLiveGeneration(@TempDir Path warehouse) throws Exception { + try (Catalog catalog = new FileSystemCatalog(LocalFileIO.create(), + new org.apache.paimon.fs.Path(warehouse.toUri()))) { + catalog.createDatabase("db", false); + Identifier id = Identifier.create("db", "fixed_reuse"); + catalog.createTable(id, Schema.newBuilder() + .column("id", DataTypes.INT()) + .option("bucket", "-1") + .build(), false); + Table table = catalog.getTable(id); + BatchWriteBuilder wb = table.newBatchWriteBuilder(); + try (BatchTableWrite write = wb.newWrite()) { + write.write(GenericRow.of(1)); + List messages = write.prepareCommit(); + try (BatchTableCommit commit = wb.newCommit()) { + commit.commit(messages); + } + } + + long pinnedSnapshotId = table.latestSnapshot().orElseThrow(AssertionError::new).id(); + RecordingPaimonCatalogOps ops = new RecordingPaimonCatalogOps(); + ops.table = table; + PaimonScanPlanProvider provider = new PaimonScanPlanProvider( + PaimonCatalogProperties.of(Collections.emptyMap()), ops); + PaimonTableHandle handle = new PaimonTableHandle( + "db", "fixed_reuse", Collections.emptyList(), Collections.emptyList()) + .withScanOptions(Collections.singletonMap( + CoreOptions.SCAN_SNAPSHOT_ID.key(), Long.toString(pinnedSnapshotId))); + ConnectorSession session = sessionWithProps( + Collections.singletonMap("enable_external_scan_task_reuse", "true"), + new TestStatementScope()); + ConnectorScanRequest request = ConnectorScanRequest.builder(handle, Collections.emptyList()).build(); + + ops.latestSnapshotId = OptionalLong.of(pinnedSnapshotId); + List first = provider.planScan(session, request); + ops.latestSnapshotId = OptionalLong.of(pinnedSnapshotId + 1); + List second = provider.planScan(session, request); + + Assertions.assertSame(first, second, + "aliases pinned to one snapshot must reuse even when live latest advances"); + Assertions.assertFalse(ops.log.contains("latestSnapshotId"), + "a fixed snapshot identity must not probe the live latest pointer"); } } + @Test + public void latestDependentIncrementalScanStillFencesOnLiveGeneration(@TempDir Path warehouse) + throws Exception { + try (Catalog catalog = new FileSystemCatalog(LocalFileIO.create(), + new org.apache.paimon.fs.Path(warehouse.toUri()))) { + catalog.createDatabase("db", false); + Identifier id = Identifier.create("db", "incremental_reuse"); + catalog.createTable(id, Schema.newBuilder() + .column("id", DataTypes.INT()) + .option("bucket", "-1") + .build(), false); + Table table = catalog.getTable(id); + BatchWriteBuilder wb = table.newBatchWriteBuilder(); + try (BatchTableWrite write = wb.newWrite()) { + write.write(GenericRow.of(1)); + List messages = write.prepareCommit(); + try (BatchTableCommit commit = wb.newCommit()) { + commit.commit(messages); + } + } + + long latestSnapshotId = table.latestSnapshot().orElseThrow(AssertionError::new).id(); + RecordingPaimonCatalogOps ops = new RecordingPaimonCatalogOps(); + ops.table = table; + PaimonScanPlanProvider provider = new PaimonScanPlanProvider( + PaimonCatalogProperties.of(Collections.emptyMap()), ops); + PaimonTableHandle handle = new PaimonTableHandle( + "db", "incremental_reuse", Collections.emptyList(), Collections.emptyList()) + .withScanOptions(Collections.singletonMap( + "incremental-between-timestamp", "0," + Long.MAX_VALUE)); + ConnectorSession session = sessionWithProps( + Collections.singletonMap("enable_external_scan_task_reuse", "true"), + new TestStatementScope()); + ConnectorScanRequest request = ConnectorScanRequest.builder(handle, Collections.emptyList()).build(); + + ops.latestSnapshotId = OptionalLong.of(latestSnapshotId); + List first = provider.planScan(session, request); + ops.latestSnapshotId = OptionalLong.of(latestSnapshotId + 1); + List second = provider.planScan(session, request); + + Assertions.assertNotSame(first, second, + "an open-ended incremental scan must not reuse across a live generation change"); + Assertions.assertEquals(2, ops.log.stream().filter("latestSnapshotId"::equals).count(), + "latest-dependent scans must retain one live generation fence per alias"); + } + } + + @Test + public void pinnedEmptyAliasesDoNotProbeLatest(@TempDir Path warehouse) throws Exception { + try (Catalog catalog = new FileSystemCatalog(LocalFileIO.create(), + new org.apache.paimon.fs.Path(warehouse.toUri()))) { + catalog.createDatabase("db", false); + Identifier id = Identifier.create("db", "empty_reuse"); + catalog.createTable(id, Schema.newBuilder() + .column("id", DataTypes.INT()) + .option("bucket", "-1") + .build(), false); + Table table = catalog.getTable(id); + RecordingPaimonCatalogOps ops = new RecordingPaimonCatalogOps(); + ops.table = table; + PaimonScanPlanProvider provider = new PaimonScanPlanProvider( + PaimonCatalogProperties.of(Collections.emptyMap()), ops); + PaimonTableHandle handle = new PaimonTableHandle( + "db", "empty_reuse", Collections.emptyList(), Collections.emptyList()) + .withScanOptions(PaimonScanParams.pinOptionsToSnapshot(Collections.emptyMap(), -1L)); + ConnectorSession session = sessionWithProps( + Collections.singletonMap("enable_external_scan_task_reuse", "true"), + new TestStatementScope()); + ConnectorScanRequest request = ConnectorScanRequest.builder(handle, Collections.emptyList()).build(); + + List first = provider.planScan(session, request); + ops.latestSnapshotId = OptionalLong.of(1L); + List second = provider.planScan(session, request); + + Assertions.assertSame(first, second, "the statement's pinned-empty plan must remain empty and reusable"); + Assertions.assertTrue(first.isEmpty()); + Assertions.assertFalse(ops.log.contains("latestSnapshotId"), + "a pinned-empty identity must not probe the live latest pointer"); + } + } + + @Test + public void statementResolutionKeepsAliasesOnOnePhysicalGeneration() { + RecordingPaimonCatalogOps ops = new RecordingPaimonCatalogOps(); + PaimonScanPlanProvider provider = new PaimonScanPlanProvider( + PaimonCatalogProperties.of(Collections.emptyMap()), ops); + FakePaimonTable generationA = new FakePaimonTable( + "generation-a", rowType("id"), Collections.emptyList(), Collections.emptyList()); + FakePaimonTable generationB = new FakePaimonTable( + "generation-b", rowType("id", "new_column"), + Collections.emptyList(), Collections.emptyList()); + PaimonTableHandle firstAlias = new PaimonTableHandle( + "db", "t", Collections.emptyList(), Collections.emptyList()); + firstAlias.setPaimonTable(generationA); + PaimonTableHandle secondAlias = new PaimonTableHandle( + "db", "t", Collections.emptyList(), Collections.emptyList()); + secondAlias.setPaimonTable(generationB); + ConnectorSession session = sessionWithProps( + Collections.singletonMap("enable_external_scan_task_reuse", "true"), + new TestStatementScope()); + + Table first = provider.resolveScanTableConsistent(session, firstAlias); + Table second = provider.resolveScanTableConsistent(session, secondAlias); + + Assertions.assertSame(generationA, first); + Assertions.assertSame(first, second, + "equal logical aliases must not mix generation-A ranges with generation-B properties"); + } + + @Test + public void systemTableResolutionKeepsWrapperAndSourceHandleLocal() { + RecordingPaimonCatalogOps ops = new RecordingPaimonCatalogOps(); + PaimonScanPlanProvider provider = new PaimonScanPlanProvider( + PaimonCatalogProperties.of(Collections.emptyMap()), ops); + FakePaimonTable wrapperA = new FakePaimonTable( + "wrapper-a", rowType("id"), Collections.emptyList(), Collections.emptyList()); + FakePaimonTable wrapperB = new FakePaimonTable( + "wrapper-b", rowType("id", "new_column"), + Collections.emptyList(), Collections.emptyList()); + FakePaimonTable sourceA = new FakePaimonTable( + "source-a", rowType("id"), Collections.emptyList(), Collections.emptyList()); + FakePaimonTable sourceB = new FakePaimonTable( + "source-b", rowType("id", "new_column"), + Collections.emptyList(), Collections.emptyList()); + PaimonTableHandle firstAlias = PaimonTableHandle.forSystemTable("db", "t", "ro", false); + firstAlias.setPaimonTable(wrapperA); + firstAlias.setSystemTableSource(sourceA); + PaimonTableHandle secondAlias = PaimonTableHandle.forSystemTable("db", "t", "ro", false); + secondAlias.setPaimonTable(wrapperB); + secondAlias.setSystemTableSource(sourceB); + ConnectorSession session = sessionWithProps( + Collections.singletonMap("enable_external_scan_task_reuse", "true"), + new TestStatementScope()); + + Table first = provider.resolveScanTableConsistent(session, firstAlias); + Table second = provider.resolveScanTableConsistent(session, secondAlias); + + Assertions.assertEquals(firstAlias, secondAlias, + "the regression requires aliases that would otherwise share the statement memo"); + Assertions.assertSame(wrapperA, first); + Assertions.assertSame(wrapperB, second, + "system-table properties must use the same handle-local generation as split planning"); + Assertions.assertSame(sourceA, firstAlias.getSystemTableSource()); + Assertions.assertSame(sourceB, secondAlias.getSystemTableSource()); + } + + @Test + public void statementTableIdentityIncludesSystemTableBranchAndOptions() { + PaimonTableHandle base = new PaimonTableHandle( + "db", "t", Collections.emptyList(), Collections.emptyList()); + PaimonTableHandle system = PaimonTableHandle.forSystemTable("db", "t", "snapshots", false); + PaimonTableHandle branch = base.withBranch("audit"); + Map pinnedOptions = Collections.singletonMap("scan.snapshot-id", "7"); + PaimonTableHandle pinned = base.withScanOptions(pinnedOptions); + + PaimonScanPlanProvider.PaimonTableResolutionKey baseKey = + new PaimonScanPlanProvider.PaimonTableResolutionKey(base, Collections.emptyMap()); + Assertions.assertNotEquals(baseKey, + new PaimonScanPlanProvider.PaimonTableResolutionKey(system, Collections.emptyMap())); + Assertions.assertNotEquals(baseKey, + new PaimonScanPlanProvider.PaimonTableResolutionKey(branch, Collections.emptyMap())); + Assertions.assertNotEquals(baseKey, + new PaimonScanPlanProvider.PaimonTableResolutionKey(pinned, pinnedOptions)); + } + @Test public void planScanPushesLimitIntoPaimonSplitPlanning(@TempDir Path warehouse) throws Exception { try (Catalog catalog = new FileSystemCatalog(LocalFileIO.create(), @@ -331,7 +566,7 @@ public void planScanPushesLimitIntoPaimonSplitPlanning(@TempDir Path warehouse) PaimonCatalogProperties.of(Collections.emptyMap()), ops); PaimonTableHandle handle = new PaimonTableHandle( "db", "limited", Collections.emptyList(), Collections.emptyList()); - ConnectorSession session = sessionWithProps(Collections.emptyMap()); + ConnectorSession session = sessionWithProps(Collections.emptyMap(), new TestStatementScope()); List unlimited = provider.planScan(session, ConnectorScanRequest.builder(handle, Collections.emptyList()).build()); @@ -671,6 +906,12 @@ public void fileCreationTimeScanDoesNotApplyLimitToDiscardedTableScan( } } + private static ConnectorExpression equalIdFilter(long value) { + return new ConnectorComparison(ConnectorComparison.Operator.EQ, + new ConnectorColumnRef("id", ConnectorType.of("INT")), + new ConnectorLiteral(ConnectorType.of("INT"), value)); + } + /** Builds a native-eligible RawFile (parquet suffix). The numeric fields are irrelevant to the * native-vs-JNI routing decision under test, only the path suffix matters. */ private static RawFile parquetRawFile(String path) { @@ -2482,7 +2723,17 @@ public void buildNativeRangesKeepsFileWholeWhenTargetNonPositive() { } private static ConnectorSession sessionWithProps(Map sessionProps) { + return sessionWithProps(sessionProps, ConnectorStatementScope.NONE); + } + + private static ConnectorSession sessionWithProps( + Map sessionProps, ConnectorStatementScope statementScope) { return new ConnectorSession() { + @Override + public ConnectorStatementScope getStatementScope() { + return statementScope; + } + @Override public String getQueryId() { return "q"; @@ -2530,6 +2781,16 @@ public Map getSessionProperties() { }; } + private static final class TestStatementScope implements ConnectorStatementScope { + private final ConcurrentHashMap cache = new ConcurrentHashMap<>(); + + @Override + @SuppressWarnings("unchecked") + public T computeIfAbsent(String key, Supplier loader) { + return (T) cache.computeIfAbsent(key, ignored -> loader.get()); + } + } + @Test public void isForceJniScannerEnabledReadsSessionProperty() { // FIX-FORCE-JNI-SCANNER (M-1): pins the EXACT session key ("force_jni_scanner", byte-identical to @@ -2557,6 +2818,16 @@ public void fileMetadataColumnsRequireFileScannerV2() { Assertions.assertTrue(PaimonScanPlanProvider.isFileScannerV2Enabled(null)); } + @Test + public void disabledScanReuseDoesNotDisableSplitTypeFiltering() { + Map sessionProperties = new HashMap<>(); + sessionProperties.put("enable_external_scan_task_reuse", "false"); + sessionProperties.put("ignore_split_type", "IGNORE_NATIVE"); + + Assertions.assertEquals("IGNORE_NATIVE", PaimonScanPlanProvider.resolveIgnoreSplitType( + sessionWithProps(sessionProperties))); + } + // --------------------------------------------------------------------- // FIX-REST-VENDED — per-table vended credentials overlaid as location.* // --------------------------------------------------------------------- diff --git a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/RecordingConnectorContext.java b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/RecordingConnectorContext.java index e858c0debab111..d0bf3610a7fd8e 100644 --- a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/RecordingConnectorContext.java +++ b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/RecordingConnectorContext.java @@ -51,6 +51,7 @@ public ConnectorStorageContext getStorageContext() { } int authCount; + int authDepth; boolean failAuth; int failAuthOnInvocation = -1; @@ -134,7 +135,16 @@ public T executeAuthenticated(Callable task) throws Exception { // Deliberately do NOT call task -> the wrapped seam call must not run. throw new RuntimeException("auth failed"); } - return task.call(); + authDepth++; + try { + return task.call(); + } finally { + authDepth--; + } + } + + boolean isAuthenticated() { + return authDepth > 0; } // A distinguishable, non-null engine filesystem. The SPI default for getFileSystem is null, so a diff --git a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/RecordingPaimonCatalogOps.java b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/RecordingPaimonCatalogOps.java index 831ea241d85da0..ce11e5ec4d71a4 100644 --- a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/RecordingPaimonCatalogOps.java +++ b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/RecordingPaimonCatalogOps.java @@ -28,6 +28,7 @@ import java.util.List; import java.util.Map; import java.util.OptionalLong; +import java.util.function.BooleanSupplier; /** * Hand-written recording fake for {@link PaimonCatalogOps} (no Mockito), mirroring the @@ -90,6 +91,7 @@ final class RecordingPaimonCatalogOps implements PaimonCatalogOps { // ---- T20 E5 MVCC seam: configurable lookup results (no real Snapshot/SnapshotManager) ---- OptionalLong latestSnapshotId = OptionalLong.empty(); + BooleanSupplier latestSnapshotAuthentication; OptionalLong snapshotIdAtOrBefore = OptionalLong.empty(); boolean snapshotExists; /** The table the metadata layer passed to the most recent MVCC seam call. */ @@ -242,6 +244,9 @@ public void dropTable(Identifier identifier, boolean ignoreIfNotExists) @Override public OptionalLong latestSnapshotId(Table table) { + if (latestSnapshotAuthentication != null && !latestSnapshotAuthentication.getAsBoolean()) { + throw new IllegalStateException("latestSnapshotId called outside authentication"); + } log.add("latestSnapshotId"); lastMvccTable = table; return latestSnapshotId; diff --git a/fe/fe-connector/fe-connector-spi/src/main/java/org/apache/doris/connector/spi/ConnectorSession.java b/fe/fe-connector/fe-connector-spi/src/main/java/org/apache/doris/connector/spi/ConnectorSession.java index 8a41604bd82d74..111e46ac7fff88 100644 --- a/fe/fe-connector/fe-connector-spi/src/main/java/org/apache/doris/connector/spi/ConnectorSession.java +++ b/fe/fe-connector/fe-connector-spi/src/main/java/org/apache/doris/connector/spi/ConnectorSession.java @@ -103,6 +103,17 @@ default Map getSessionProperties() { return java.util.Collections.emptyMap(); } + /** + * Returns whether equivalent external scan plans may be reused within the current statement. + * + *

The property must be explicitly present and {@code true}. Sessions created by an older engine or by + * offline tooling do not carry this setting and therefore keep the optimization disabled.

+ */ + default boolean isExternalScanTaskReuseEnabled() { + return "true".equalsIgnoreCase( + getSessionProperties().get("enable_external_scan_task_reuse")); + } + /** * Returns the transaction this session is currently bound to, if any. * diff --git a/fe/fe-connector/fe-connector-spi/src/test/java/org/apache/doris/connector/spi/ConnectorPluginSurfaceTest.java b/fe/fe-connector/fe-connector-spi/src/test/java/org/apache/doris/connector/spi/ConnectorPluginSurfaceTest.java index 9def858c616547..dff7baba361dee 100644 --- a/fe/fe-connector/fe-connector-spi/src/test/java/org/apache/doris/connector/spi/ConnectorPluginSurfaceTest.java +++ b/fe/fe-connector/fe-connector-spi/src/test/java/org/apache/doris/connector/spi/ConnectorPluginSurfaceTest.java @@ -85,14 +85,16 @@ public void connectorApiMajorTracksTheRecordedSurfaceChange() throws IOException Assertions.assertNotNull(in, "missing connector plugin API version resource"); version.load(in); } - // OpenCSV scan properties require major 9: inlined keys cannot fail JVM linkage on an older FE. - Assertions.assertEquals("9.0", version.getProperty("api.version")); + // ConnectorSession's external-scan-reuse policy requires major 10: an API-9 FE does not + // provide the newly added interface method to an independently built connector plugin. + Assertions.assertEquals("10.0", version.getProperty("api.version")); } /** Root entry points plus provider/handle types returned to connector plugins. */ private static final List> FROZEN_TYPES = Arrays.asList( ConnectorProvider.class, ConnectorContext.class, + ConnectorSession.class, Connector.class, ConnectorColumnHandle.class, ConnectorTableSchema.class, diff --git a/fe/fe-connector/fe-connector-spi/src/test/resources/connector-plugin-surface.txt b/fe/fe-connector/fe-connector-spi/src/test/resources/connector-plugin-surface.txt index f3d8d8cb237d25..6e4eb94c2b29f1 100644 --- a/fe/fe-connector/fe-connector-spi/src/test/resources/connector-plugin-surface.txt +++ b/fe/fe-connector/fe-connector-spi/src/test/resources/connector-plugin-surface.txt @@ -57,6 +57,22 @@ org.apache.doris.connector.spi.ConnectorProvider#supports(java.lang.String,java. org.apache.doris.connector.spi.ConnectorProvider#validateCreateTable(java.util.Map):void org.apache.doris.connector.spi.ConnectorProvider#validateProperties(java.util.Map):void org.apache.doris.connector.spi.ConnectorProvider#validatePropertiesForUpdate(java.util.Map,java.util.Map):void +org.apache.doris.connector.spi.ConnectorSession#allocateTransactionId():long +org.apache.doris.connector.spi.ConnectorSession#getCatalogId():long +org.apache.doris.connector.spi.ConnectorSession#getCatalogName():java.lang.String +org.apache.doris.connector.spi.ConnectorSession#getCatalogProperties():java.util.Map +org.apache.doris.connector.spi.ConnectorSession#getCurrentTransaction():java.util.Optional +org.apache.doris.connector.spi.ConnectorSession#getDelegatedCredential():java.util.Optional +org.apache.doris.connector.spi.ConnectorSession#getLocale():java.lang.String +org.apache.doris.connector.spi.ConnectorSession#getProperty(java.lang.String,java.lang.Class):java.lang.Object +org.apache.doris.connector.spi.ConnectorSession#getQueryId():java.lang.String +org.apache.doris.connector.spi.ConnectorSession#getSessionId():java.lang.String +org.apache.doris.connector.spi.ConnectorSession#getSessionProperties():java.util.Map +org.apache.doris.connector.spi.ConnectorSession#getStatementScope():org.apache.doris.connector.spi.ConnectorStatementScope +org.apache.doris.connector.spi.ConnectorSession#getTimeZone():java.lang.String +org.apache.doris.connector.spi.ConnectorSession#getUser():java.lang.String +org.apache.doris.connector.spi.ConnectorSession#isExternalScanTaskReuseEnabled():boolean +org.apache.doris.connector.spi.ConnectorSession#setCurrentTransaction(org.apache.doris.connector.spi.handle.ConnectorTransaction):void org.apache.doris.connector.spi.ConnectorTableSchema#equals(java.lang.Object):boolean org.apache.doris.connector.spi.ConnectorTableSchema#getColumns():java.util.List org.apache.doris.connector.spi.ConnectorTableSchema#getProperties():java.util.Map diff --git a/fe/fe-connector/pom.xml b/fe/fe-connector/pom.xml index c00ef144e591f3..a682069b38b0c8 100644 --- a/fe/fe-connector/pom.xml +++ b/fe/fe-connector/pom.xml @@ -55,7 +55,7 @@ under the License. of the latter two means bumping this property as well (and fe-extension-spi means bumping all five families). --> - 9.0 + 10.0 diff --git a/fe/fe-core/src/main/java/org/apache/doris/connector/ConnectorStatementScopeImpl.java b/fe/fe-core/src/main/java/org/apache/doris/connector/ConnectorStatementScopeImpl.java index c36ff2194fbc7d..06c3cd8a51d3cb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/connector/ConnectorStatementScopeImpl.java +++ b/fe/fe-core/src/main/java/org/apache/doris/connector/ConnectorStatementScopeImpl.java @@ -34,8 +34,11 @@ * partition-batch) reuse the single {@link org.apache.doris.connector.spi.ConnectorSession} built on the * request thread and so reach this same scope concurrently. {@code computeIfAbsent} gives every caller of * a key the same instance — required for the shared table object and for the delete supply map that scan - * and write both mutate. The loaders used by connectors do not re-enter this scope, so the map's - * single-key atomicity is safe here.

+ * and write both mutate. Loaders MUST NOT re-enter this scope: a recursive {@code computeIfAbsent} + * can throw {@code IllegalStateException} on a same-bin collision or silently drop entries during a + * mid-computation resize. Connectors that need scope-backed state during scan planning nest a + * separate connector-private memo map inside the scope, avoiding re-entry and preserving the + * String-key SPI contract.

*/ public class ConnectorStatementScopeImpl implements ConnectorStatementScope { diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/executor/TaskProcessor.java b/fe/fe-core/src/main/java/org/apache/doris/job/executor/TaskProcessor.java index 471ce07eeaec59..5a46ef13864490 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/job/executor/TaskProcessor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/job/executor/TaskProcessor.java @@ -18,6 +18,9 @@ package org.apache.doris.job.executor; import org.apache.doris.job.task.AbstractTask; +import org.apache.doris.nereids.StatementContext; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.QeProcessorImpl; import lombok.extern.log4j.Log4j2; @@ -82,6 +85,25 @@ private void runTask(AbstractTask task) { task.runTask(); } catch (Exception e) { log.warn("Execute task error, task id: {}", task.getTaskId(), e); + } finally { + closeTaskContext(); + } + } + + private void closeTaskContext() { + ConnectContext connectContext = ConnectContext.get(); + try { + if (connectContext != null) { + if (connectContext.queryId() != null) { + QeProcessorImpl.INSTANCE.unregisterQuery(connectContext.queryId()); + } + StatementContext statementContext = connectContext.getStatementContext(); + if (statementContext != null) { + statementContext.close(); + } + } + } finally { + ConnectContext.remove(); } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java index 1eab83fe576d3e..8b24e11fe31a32 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java @@ -78,6 +78,7 @@ import org.apache.doris.nereids.trees.plans.commands.UpdateMvByPartitionCommand; import org.apache.doris.nereids.trees.plans.commands.info.RefreshMTMVInfo.RefreshMode; import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.QeProcessorImpl; import org.apache.doris.qe.StmtExecutor; import org.apache.doris.rpc.RpcException; import org.apache.doris.system.SystemInfoService; @@ -357,6 +358,8 @@ public void run() throws JobException { // if status is not `RUNNING`,maybe the task was canceled, therefore, it is a normal situation LOG.info("task [{}] interruption running, because status is [{}]", getTaskId(), getStatus()); } + } finally { + closeExecutionContext(ctx); } } @@ -703,14 +706,18 @@ private IvmIncrRefreshResult executeSingleIvmAttempt(MTMVRefreshContext refreshC ivmResult = executeWithRetry(() -> { ConnectContext ivmConnectContext = MTMVPlanUtil.createMTMVContext(mtmv, MTMVPlanUtil.DISABLE_RULES_WHEN_RUN_MTMV_TASK); - setupComputeGroup(ivmConnectContext); - IvmIncrRefreshContext ivmIncrRefreshContext = new IvmIncrRefreshContext(mtmv, - ivmConnectContext, - getRefreshAuditStmt(RefreshMode.INCREMENTAL, Sets.newHashSet(needRefreshPartitions)), - this::recordQueryId, - this::registerExecutor); - mtmv.validateIvmRefreshStart(mtmvSchemaChangeVersion); - return ivmIncrRefreshManager.doRefresh(ivmIncrRefreshContext); + try { + setupComputeGroup(ivmConnectContext); + IvmIncrRefreshContext ivmIncrRefreshContext = new IvmIncrRefreshContext(mtmv, + ivmConnectContext, + getRefreshAuditStmt(RefreshMode.INCREMENTAL, Sets.newHashSet(needRefreshPartitions)), + this::recordQueryId, + this::registerExecutor); + mtmv.validateIvmRefreshStart(mtmvSchemaChangeVersion); + return ivmIncrRefreshManager.doRefresh(ivmIncrRefreshContext); + } finally { + closeExecutionContext(ivmConnectContext); + } }, "IVM refresh"); } catch (Exception e) { throw new JobException("IVM incremental refresh failed for mv=" + mtmv.getName() @@ -990,7 +997,11 @@ private IvmPlanSignature refreshPartitions(Set refreshPartitionNames, getRefreshAuditStmt(refreshMode, refreshPartitionNames), createRefreshConsumer(signatureRef)); } finally { - recordQueryId(DebugUtil.printId(mtmvCtx.queryId())); + try { + recordQueryId(DebugUtil.printId(mtmvCtx.queryId())); + } finally { + closeExecutionContext(mtmvCtx); + } } if (getStatus() == TaskStatus.CANCELED) { throw new JobException("task is CANCELED"); @@ -1022,6 +1033,19 @@ private Consumer createRefreshConsumer(AtomicReference catalogProps = new HashMap<>(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/job/executor/TaskProcessorTest.java b/fe/fe-core/src/test/java/org/apache/doris/job/executor/TaskProcessorTest.java new file mode 100644 index 00000000000000..526fa76515df07 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/job/executor/TaskProcessorTest.java @@ -0,0 +1,192 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.job.executor; + +import org.apache.doris.common.jmockit.Deencapsulation; +import org.apache.doris.common.util.DebugUtil; +import org.apache.doris.connector.spi.ConnectorStatementScope; +import org.apache.doris.job.common.TaskStatus; +import org.apache.doris.job.extensions.insert.InsertTask; +import org.apache.doris.job.extensions.mtmv.MTMVTask; +import org.apache.doris.job.task.AbstractTask; +import org.apache.doris.nereids.StatementContext; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.QeProcessorImpl; +import org.apache.doris.thrift.TUniqueId; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +public class TaskProcessorTest { + @Test + public void testReleaseStatementScopeAfterInsertTask() throws Exception { + assertReleaseStatementScope(true); + } + + @Test + public void testReleaseStatementScopeAfterMTMVTask() throws Exception { + assertReleaseStatementScope(false); + } + + @Test + public void testRemoveContextWhenStatementCloseFails() { + TaskProcessor taskProcessor = new TaskProcessor(1, 1, Thread::new); + ConnectContext connectContext = new ConnectContext(); + connectContext.setStatementContext(new StatementContext() { + @Override + public void close() { + throw new RuntimeException("injected close failure"); + } + }); + connectContext.setThreadLocalInfo(); + try { + Deencapsulation.invoke(taskProcessor, "closeTaskContext"); + Assertions.fail("closeTaskContext should propagate the close failure"); + } catch (RuntimeException e) { + Assertions.assertEquals("injected close failure", e.getMessage()); + } finally { + Assertions.assertNull(ConnectContext.get()); + taskProcessor.shutdown(); + } + } + + @Test + public void testRunQueryFinishCallbacksForTaskContext() { + TaskProcessor taskProcessor = new TaskProcessor(1, 1, Thread::new); + ConnectContext connectContext = new ConnectContext(); + connectContext.setStatementContext(new StatementContext()); + TUniqueId queryId = new TUniqueId(1L, 1L); + connectContext.setQueryId(queryId); + connectContext.setThreadLocalInfo(); + AtomicInteger callbackCount = new AtomicInteger(); + QeProcessorImpl.INSTANCE.registerQueryFinishCallback( + DebugUtil.printId(queryId), callbackCount::incrementAndGet); + try { + Deencapsulation.invoke(taskProcessor, "closeTaskContext"); + Assertions.assertEquals(1, callbackCount.get()); + Assertions.assertNull(ConnectContext.get()); + } finally { + QeProcessorImpl.INSTANCE.unregisterQuery(queryId); + ConnectContext.remove(); + taskProcessor.shutdown(); + } + } + + @Test + public void testCleanEveryMTMVExecutionContext() { + MTMVTask task = new MTMVTask(); + AtomicInteger callbackCount = new AtomicInteger(); + AtomicInteger closeCount = new AtomicInteger(); + ConnectContext taskContext = new ConnectContext(); + taskContext.setThreadLocalInfo(); + StatementContext taskStatementContext = new StatementContext(); + taskContext.setStatementContext(taskStatementContext); + taskStatementContext.getOrCreateConnectorStatementScope().computeIfAbsent( + "closeable", () -> (AutoCloseable) closeCount::incrementAndGet); + try { + for (int i = 0; i < 2; i++) { + ConnectContext executionContext = new ConnectContext(); + executionContext.setThreadLocalInfo(); + StatementContext statementContext = new StatementContext(); + executionContext.setStatementContext(statementContext); + TUniqueId queryId = new TUniqueId(2L, i + 1L); + executionContext.setQueryId(queryId); + statementContext.getOrCreateConnectorStatementScope().computeIfAbsent( + "closeable", () -> (AutoCloseable) closeCount::incrementAndGet); + QeProcessorImpl.INSTANCE.registerQueryFinishCallback( + DebugUtil.printId(queryId), callbackCount::incrementAndGet); + + Deencapsulation.invoke(task, "closeExecutionContext", executionContext); + + Assertions.assertSame(executionContext, ConnectContext.get()); + } + Assertions.assertEquals(2, callbackCount.get()); + Assertions.assertEquals(2, closeCount.get()); + + ConnectContext lastExecutionContext = ConnectContext.get(); + Deencapsulation.invoke(task, "closeExecutionContext", taskContext); + Assertions.assertSame(lastExecutionContext, ConnectContext.get()); + Assertions.assertEquals(3, closeCount.get()); + + Deencapsulation.invoke(task, "closeExecutionContext", new ConnectContext()); + Assertions.assertSame(lastExecutionContext, ConnectContext.get()); + } finally { + taskStatementContext.close(); + ConnectContext.remove(); + } + } + + private void assertReleaseStatementScope(boolean insertTask) throws Exception { + AtomicInteger closeCount = new AtomicInteger(); + CountDownLatch scopeInstalled = new CountDownLatch(1); + CountDownLatch workerReused = new CountDownLatch(1); + TaskProcessor taskProcessor = new TaskProcessor(1, 1, Thread::new); + try { + AbstractTask externalTask; + if (insertTask) { + externalTask = new InsertTask("external-insert", null, null, null) { + @Override + public void runTask() { + installStatementScope(closeCount, scopeInstalled); + } + }; + } else { + externalTask = new MTMVTask() { + @Override + public void runTask() { + installStatementScope(closeCount, scopeInstalled); + } + }; + } + externalTask.setStatus(TaskStatus.PENDING); + externalTask.setTaskId(1L); + Assertions.assertTrue(taskProcessor.addTask(externalTask)); + Assertions.assertTrue(scopeInstalled.await(10, TimeUnit.SECONDS)); + + AbstractTask nextTask = new InsertTask("next-task", null, null, null) { + @Override + public void runTask() { + Assertions.assertNull(ConnectContext.get()); + workerReused.countDown(); + } + }; + nextTask.setStatus(TaskStatus.PENDING); + nextTask.setTaskId(2L); + Assertions.assertTrue(taskProcessor.addTask(nextTask)); + + Assertions.assertTrue(workerReused.await(10, TimeUnit.SECONDS)); + Assertions.assertEquals(1, closeCount.get()); + } finally { + taskProcessor.shutdown(); + } + } + + private static void installStatementScope(AtomicInteger closeCount, CountDownLatch scopeInstalled) { + ConnectContext connectContext = new ConnectContext(); + connectContext.setThreadLocalInfo(); + StatementContext statementContext = new StatementContext(); + connectContext.setStatementContext(statementContext); + ConnectorStatementScope scope = statementContext.getOrCreateConnectorStatementScope(); + scope.computeIfAbsent("closeable", () -> (AutoCloseable) closeCount::incrementAndGet); + scopeInstalled.countDown(); + } +}