diff --git a/.claude/wf-d2-cache-recon.js b/.claude/wf-d2-cache-recon.js
new file mode 100644
index 00000000000000..a6f974839555d0
--- /dev/null
+++ b/.claude/wf-d2-cache-recon.js
@@ -0,0 +1,266 @@
+export const meta = {
+ name: 'd2-cache-recon',
+ description: 'HEAD-grounded recon for D2 connector-owned scan-side cache (retire fe-core Hive/Hudi/Iceberg caches + 4 routing gates + max-partition-time)',
+ phases: [
+ { title: 'Recon', detail: '8 dimension readers over the cache subsystem, consumers, connector templates, max-time/invalidation, Trino' },
+ { title: 'Critique', detail: '3 adversarial critics: completeness, flip-safety, scope-boundary' },
+ ],
+}
+
+const READER_SCHEMA = {
+ type: 'object',
+ additionalProperties: false,
+ properties: {
+ dimension: { type: 'string' },
+ summary: { type: 'string', description: '4-8 sentence executive summary of what you found, in plain terms' },
+ facts: {
+ type: 'array',
+ description: 'Code-grounded facts. Every one MUST cite file:line evidence verified at HEAD.',
+ items: {
+ type: 'object',
+ additionalProperties: false,
+ properties: {
+ claim: { type: 'string' },
+ evidence: { type: 'string', description: 'file:line (repo-relative). Multiple allowed.' },
+ implication: { type: 'string', description: 'why it matters for D2 / the flip' },
+ },
+ required: ['claim', 'evidence'],
+ },
+ },
+ flipImpact: {
+ type: 'array',
+ description: 'For this dimension: what silently breaks / changes / must be replaced at the atomic flip (when a type=hms catalog becomes a PluginDriven catalog and cache routing collapses to ENGINE_DEFAULT).',
+ items: { type: 'string' },
+ },
+ designOptions: {
+ type: 'array',
+ description: 'Concrete options/tradeoffs you surfaced for the design (do NOT pick one; list them with pros/cons).',
+ items: { type: 'string' },
+ },
+ openQuestions: { type: 'array', items: { type: 'string' } },
+ filesRead: { type: 'array', items: { type: 'string' } },
+ },
+ required: ['dimension', 'summary', 'facts', 'flipImpact'],
+}
+
+const CRITIC_SCHEMA = {
+ type: 'object',
+ additionalProperties: false,
+ properties: {
+ verdict: { type: 'string' },
+ gaps: {
+ type: 'array',
+ items: {
+ type: 'object',
+ additionalProperties: false,
+ properties: {
+ area: { type: 'string' },
+ whatsMissing: { type: 'string' },
+ whyItMatters: { type: 'string' },
+ severity: { type: 'string', enum: ['blocker', 'major', 'minor', 'nit'] },
+ suggestedResolution: { type: 'string' },
+ evidence: { type: 'string' },
+ },
+ required: ['area', 'whatsMissing', 'severity'],
+ },
+ },
+ confirmedRisks: {
+ type: 'array',
+ items: {
+ type: 'object',
+ additionalProperties: false,
+ properties: {
+ risk: { type: 'string' },
+ severity: { type: 'string', enum: ['blocker', 'major', 'minor', 'nit'] },
+ evidence: { type: 'string' },
+ },
+ required: ['risk', 'severity'],
+ },
+ },
+ refuted: { type: 'array', description: 'Worries you checked against HEAD and dismissed, with why.', items: { type: 'string' } },
+ },
+ required: ['verdict', 'gaps'],
+}
+
+const CTX = `
+CONTEXT — you are reconnoitering for a DESIGN DOC (do not write code, do not implement).
+
+Project: Apache Doris catalog-SPI migration. Legacy \`type=hms\` catalogs (HMSExternalCatalog/HMSExternalTable, in fe-core \`datasource/hive|hudi|iceberg\`) are being migrated to a plugin-connector model (PluginDrivenExternalCatalog/PluginDrivenExternalTable holding an fe-connector-hive connector loaded in a child-first classloader). The migration ends in ONE ATOMIC FLIP that adds "hms" to SPI_READY_TYPES and remaps GSON subtypes so every hms catalog/db/table instantiates as PluginDriven* classes instead of the legacy classes.
+
+THE D2 TASK (what this recon informs): "connector-owned scan-side cache." Today fe-core owns three engine metadata caches — HiveExternalMetaCache, HudiExternalMetaCache, IcebergExternalMetaCache — routed by ExternalMetaCacheRouteResolver.addBuiltinRoutes(): a type=hms catalog routes to HIVE+HUDI+ICEBERG caches (ExternalMetaCacheRouteResolver.java:66-70). At the flip, the catalog is no longer \`instanceof HMSExternalCatalog\`, so routing falls through to ENGINE_DEFAULT (:72-73) and these three caches SILENTLY stop being fed/invalidated for hms — no crash, no log. Decision D2 (LOCKED): the hive connector must OWN its scan-side cache (mirroring how the paimon & iceberg-native connectors own theirs today), so that routing-collapse is harmless. Then retire the three fe-core caches + the four instanceof routing gates WITH the flip set. Coupled sub-item §2.6: flipped plain-hive \`getNewestUpdateVersionOrTime\` (PluginDrivenMvccExternalTable.java:713-714) returns constant 0 (names-only listPartitions → all lastModifiedMillis -1 → filtered → orElse(0)), so hive-backed SQL-dictionary / MV auto-refresh silently never sees a newer version. D2 must let the connector surface a real max-partition modify time cheaply.
+
+HARD ARCHITECTURE RULES (from project memory — respect them in what you flag):
+- fe-core does NOT parse connector properties; storage parsing → fe-filesystem, meta parsing → fe-connector (both plugin-side).
+- fe-core depends only on fe-connector-api/-spi (maven). Plugin connector classes load in a SEPARATE child-first classloader. A connector CANNOT import fe-core datasource classes. If you see "connector references HiveExternalMetaCache", determine whether it's a real import, a comment/string, or a VENDORED same-simple-name copy in a different package — this matters a lot.
+- The generic fe-core SPI layer must stay connector-agnostic (no source-specific branches).
+- Cross-classloader calls into plugins must pin TCCL to the connector classloader.
+
+DELIVERABLE: return the READER_SCHEMA object. Every fact needs file:line evidence you actually opened and verified at HEAD (branch catalog-spi-11-hive, HEAD commit d43ba31f3b3). Distinguish legacy deletion-unit code (dies at flip, needs NO replacement) from flip-survivor code (needs a replacement path). Surface design OPTIONS with tradeoffs; do not decide.
+`
+
+phase('Recon')
+
+const readers = [
+ {
+ label: 'dim-cache-machinery',
+ prompt: `${CTX}
+
+YOUR DIMENSION = the shared fe-core cache MACHINERY (not the engine-specific caches themselves). Read and map:
+- fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java (the manager; public entry points prepareCatalog/invalidate*/removeCatalog; the CacheKey inner class; invalidateTableCache)
+- fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/ExternalMetaCache.java (interface — already known: engine()/aliases()/initCatalog/entry/invalidate*/stats/close)
+- fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/AbstractExternalMetaCache.java
+- fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/ExternalMetaCacheRegistry.java
+- fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/ExternalMetaCacheRouteResolver.java (the routing — ENGINE_DEFAULT fallback at :72-73)
+- fe/fe-core/src/main/java/org/apache/doris/connector/ExternalMetaCacheInvalidator.java
+- fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntry.java / MetaCacheEntryStats.java
+- fe/fe-core/src/main/java/org/apache/doris/datasource/doris/DorisExternalMetaCache.java (a peer engine cache — how a non-hive engine registers, for contrast)
+
+Answer: What is the engine→catalog→entry model? How do caches register with the registry and get resolved by engine name? What EXACTLY is ENGINE_DEFAULT — is there a registered "default" cache, or does registry.resolve("default") return null / throw? Trace what happens to every ExternalMetaCacheMgr public method for a flipped hms catalog (routing → ENGINE_DEFAULT). Which manager methods are called on the scan hot-path vs on REFRESH/DDL/event paths? Does the machinery itself survive the flip (it serves paimon/jdbc/doris too) or get retired? Note the CacheKey/registry stats surface.`,
+ },
+ {
+ label: 'dim-hive-cache',
+ prompt: `${CTX}
+
+YOUR DIMENSION = HiveExternalMetaCache internals (the primary cache to relocate into the hive connector).
+Read fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveExternalMetaCache.java IN FULL. Enumerate EVERY distinct cache it holds (schema cache, partition-value cache, partition cache, file cache, and any others — note the inner key classes PartitionValueCacheKey:849 / PartitionCacheKey:882 / FileCacheKey:942). For each cache: what is the key, what is the value, which loader populates it, and what remote calls (HMS thrift / filesystem listing) does the loader make. Document the four instanceof gates (:203, :209, :248, :274) — what does each gate protect and what does it do for a non-HMS catalog/table. Find how max-partition modify time / lastModifiedTime is computed here (relevant to §2.6 getNewestUpdateVersionOrTime). Note file-listing / partition-listing caching that is on the SCAN hot-path (this is the core of what the hive connector must own). Note the eager-init behavior (initCatalog).
+
+Then classify: which of these caches are SCAN-side (must be owned by the connector post-flip) vs which serve only legacy HMSExternalTable/Catalog paths that die at the flip. Surface options for WHERE the relocated cache lives (connector-internal vs the shared fe-connector-cache framework) with tradeoffs.`,
+ },
+ {
+ label: 'dim-hudi-iceberg-cache',
+ prompt: `${CTX}
+
+YOUR DIMENSION = HudiExternalMetaCache + IcebergExternalMetaCache (the other two fe-core caches routed for hms), and whether they even need a replacement.
+Read:
+- fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/HudiExternalMetaCache.java IN FULL (gate at :221 = \`!(dorisTable instanceof HMSExternalTable)\`).
+- fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java IN FULL (gates at :213 = \`!(dorisTable instanceof MTMVRelatedTableIf)\`, :234 = \`catalog instanceof HMSExternalCatalog\`).
+- fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/cache/IcebergManifestCacheLoader.java
+
+CRUCIAL NUANCE to resolve with evidence: post-flip, an iceberg-on-HMS table is served by DELEGATION to the sibling iceberg connector (which has its OWN native caches IcebergLatestSnapshotCache / IcebergManifestCache), and a hudi-on-HMS table is served by the hudi sibling connector. So does the fe-core IcebergExternalMetaCache / HudiExternalMetaCache still serve ANYTHING for a flipped hms catalog, or does it become fully dead (served by siblings)? Enumerate exactly what each caches, who feeds it, and whether the flip's sibling-delegation already covers it. Determine: can these two caches simply be RETIRED at the flip (no connector-owned replacement needed) because the sibling connectors' native caches cover it, or is there a residual fe-core-served path? Cite evidence. Note the execution plan's claim that IcebergExternalMetaCache:234 is "unreached post-flip" — verify.`,
+ },
+ {
+ label: 'dim-consumers',
+ prompt: `${CTX}
+
+YOUR DIMENSION = classify EVERY consumer of the three fe-core caches into (a) LEGACY deletion-unit → dies at flip, needs no replacement; (b) FLIP-SURVIVOR → needs a replacement path post-flip; (c) fe-core plumbing that just reroutes. For each survivor, say what it needs and where it should get it after the flip.
+
+Known consumer files (grep of HiveExternalMetaCache|HudiExternalMetaCache|IcebergExternalMetaCache, non-test):
+- fe-core LEGACY-suspect: datasource/hive/HMSExternalCatalog.java, HMSExternalTable.java, HiveDlaTable.java, source/HiveScanNode.java, datasource/hudi/HudiUtils.java, datasource/iceberg/source/IcebergScanNode.java, datasource/iceberg/IcebergUtils.java, statistics/HMSAnalysisTask.java, nereids/.../insert/HiveInsertExecutor.java, planner/HiveTableSink.java, datasource/hive/AcidUtil.java
+- fe-core SURVIVOR-suspect: catalog/RefreshManager.java, datasource/CatalogMgr.java, datasource/FilePartitionUtils.java, datasource/hive/event/MetastoreEventsProcessor.java, tablefunction/MetadataGenerator.java, nereids/rules/expression/rules/SortedPartitionRanges.java, datasource/ExternalMetaCacheMgr.java
+- CONNECTOR references (RESOLVE whether real import / comment / vendored copy): fe-connector-hive/.../HiveConnectorMetadata.java, fe-connector-hudi/.../HudiConnectorMetadata.java, fe-connector-iceberg/.../IcebergConnector.java + IcebergConnectorMetadata.java + IcebergLatestSnapshotCache.java + IcebergManifestCache.java
+
+For each file: open the actual reference line(s), state which cache/method it touches, and classify. Pay special attention to RefreshManager, CatalogMgr, FilePartitionUtils, MetadataGenerator, SortedPartitionRanges — these likely SURVIVE and drive REFRESH/invalidation/TVF; describe exactly how they invoke the cache and what the post-flip replacement must provide. For the CONNECTOR references: this is critical — a connector importing fe-core would break layering; determine the truth (package + import statement) and report it.`,
+ },
+ {
+ label: 'dim-connector-template',
+ prompt: `${CTX}
+
+YOUR DIMENSION = the CONNECTOR-OWNED cache TEMPLATE (how paimon & iceberg-native own scan-side caches today — the pattern the hive connector must copy).
+Read:
+- fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/CacheFactory.java + CacheSpec.java + MetaCacheEntry.java + MetaCacheEntryStats.java + package-info.java (the SHARED cache framework — note the memory: fe-core does NOT depend on this; it's a copied framework compiled into each plugin; Caffeine pinned to 2.9.3 lowest-common version).
+- fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergLatestSnapshotCache.java + IcebergManifestCache.java + ManifestCacheValue.java + dlf/client/DLFCachedClientPool.java
+- fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonLatestSnapshotCache.java
+- The IcebergConnector / PaimonConnector class to see where these caches are instantiated, held, and invalidated.
+
+Answer: How does a connector INSTANTIATE and HOLD a scan-side cache (per-connector field? via CacheFactory?)? How is it keyed/scoped (per-table, per-snapshot)? How is it INVALIDATED — is there an SPI the connector implements that fe-core's REFRESH TABLE / REFRESH CATALOG calls, or is it event/TTL only? CRITICALLY: how does fe-core's ExternalMetaCacheRouteResolver end up NOT routing to these connectors (paimon/iceberg-native catalogs get ENGINE_DEFAULT and it's harmless) — trace WHY it's harmless for them today, because that is exactly the state hive must reach. Note config (TTL, size, expire) — where does the connector read cache config from (catalog properties? Config?). This is the reference model for the hive connector cache; describe it precisely enough to copy.`,
+ },
+ {
+ label: 'dim-hive-connector-state',
+ prompt: `${CTX}
+
+YOUR DIMENSION = the CURRENT state of caching (or absence) inside fe-connector-hive, and what scan-side metadata the hive connector fetches per query.
+Read the fe-connector-hive metadata + scan path:
+- fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveConnectorMetadata.java (does it cache anything? where does it get schema/partitions/files?)
+- The HiveConnector class, the scan-plan provider, and the HMS client wrappers (HmsClient / ThriftHmsClient) in the hive/hms connector modules.
+- fe/fe-connector/fe-connector-metastore-hms/** and fe-connector-metastore-spi/** if present (the shared HMS resolver).
+Search the hive connector module for any existing Caffeine/cache usage, and for per-query getTableSchema / listPartitions / listPartitionNames / file-listing calls.
+
+Answer: Today (dormant, hms still legacy) does the hive connector cache ANY metadata, or does every getTableSchema/listPartitions/getTableStatistics call hit HMS thrift fresh? Enumerate the scan-side metadata reads the connector performs (schema, partition names, partition objects, file splits, column stats, table stats) and which are hot-path-per-query. Which of these does legacy HiveExternalMetaCache cache today that the connector currently re-fetches uncached? Note whether fe-connector-hive already depends on fe-connector-cache (check its pom.xml). Surface: what is the MINIMUM set of connector-owned caches needed at the flip for scan performance + correctness parity (schema? partition-name? partition-object? file-listing?), and note the TCCL/classloader considerations (filesystem listing runs plugin-side).`,
+ },
+ {
+ label: 'dim-maxtime-invalidation',
+ prompt: `${CTX}
+
+YOUR DIMENSION = (§2.6) max-partition-time / getNewestUpdateVersionOrTime, PLUS the invalidation coupling between D2 and event-pipeline Model B.
+Read:
+- fe/fe-core/src/main/java/org/apache/doris/datasource/PluginDrivenMvccExternalTable.java around :713-714 (getNewestUpdateVersionOrTime returning 0 for flipped plain-hive) AND the getTableSnapshot / beginQuerySnapshot / freshness path (freshness-kind-aware, landed 3d784673ca4).
+- How legacy computes newest-update-time: fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java getNewestUpdateVersionOrTime + any max-partition-time logic, and how HiveExternalMetaCache serves it.
+- fe/fe-core/src/main/java/org/apache/doris/dictionary/Dictionary.java + mtmv/MTMVRelatedTableIf.java (the CONSUMERS of getNewestUpdateVersionOrTime — SQL dictionary refresh + MV freshness).
+- The invalidation path: ExternalMetaCacheInvalidator, ExternalMetaCacheMgr.invalidateTableCache / invalidateDb / invalidatePartitions, RefreshManager, and how REFRESH TABLE/CATALOG + the event pipeline currently invalidate the fe-core caches.
+- The event pipeline: datasource/hive/event/MetastoreEventsProcessor.java (:116 instanceof HMSExternalCatalog) — note the planned "Connector.invalidatePartition(s)" SPI (§2.1 Model B) that pairs with D2.
+
+Answer: (1) Exactly how must the hive connector surface a cheap real max-partition modify time so flipped getNewestUpdateVersionOrTime matches legacy? What SPI shape is needed (ConnectorMvccPartitionView already exists — read fe-connector-api/.../mvcc/ConnectorMvccPartitionView.java and IcebergPartitionUtils.getNewestUpdateVersionOrTime for the iceberg precedent). (2) How is the connector-owned cache invalidated by REFRESH TABLE / REFRESH CATALOG and by metastore events — what invalidation SPI verbs does D2 need to expose, and how does that couple to event Model B (which is the NEXT task after D2)? Keep D2 and event-Model-B boundaries distinct but note the shared SPI. Surface options + tradeoffs for the max-time SPI and the invalidation SPI.`,
+ },
+ {
+ label: 'dim-trino-reference',
+ prompt: `${CTX}
+
+YOUR DIMENSION = the Trino reference architecture for connector-owned Hive metadata caching, mapped onto this design. The user explicitly wants Trino consulted for architecture-level decisions.
+From your knowledge of Trino (and any Trino-referencing comments in this codebase — grep for "Trino"/"trino" in fe-connector-hive and the metastore modules):
+- How does Trino structure Hive metastore metadata caching? (CachingHiveMetastore: the per-plugin caching decorator wrapping the raw metastore client; cache config hive.metastore-cache-ttl / metastore-refresh-interval / metastore-cache-maximum-size; per-transaction vs shared caching; the SharedHiveMetastoreCache.) Where does caching live — engine or connector? How is it invalidated (flushMetadataCache procedure, per-table/per-partition flush)?
+- How does Trino cache file listings / directory listings separately (the DirectoryLister / CachingDirectoryLister, hive.file-status-cache)?
+- How does Trino avoid a central engine-side metadata cache (the SPI is stateless Metadata calls; the connector owns all caching)?
+
+Answer: Map each Trino mechanism to a concrete recommendation for the Doris hive connector D2 design: (1) a CachingHiveMetastore-style decorator around the HMS client inside fe-connector-hive; (2) a directory/file-listing cache; (3) invalidation verbs the connector exposes to fe-core REFRESH/events. Note where Doris ALREADY diverges (Doris fe-core historically centralized the cache in HiveExternalMetaCache; the D2 move re-aligns to Trino's connector-owned model). Flag any Trino mechanism that does NOT map cleanly (e.g. Doris's GSON edit-log replay, the SPI_READY flip model, the split file-listing across classloaders). Surface options where Trino offers more than one approach.`,
+ },
+]
+
+const readerResults = await parallel(
+ readers.map(r => () => agent(r.prompt, { label: r.label, phase: 'Recon', schema: READER_SCHEMA, effort: 'high' }))
+)
+
+const valid = readerResults.filter(Boolean)
+log(`Recon done: ${valid.length}/${readers.length} dimensions returned.`)
+
+// Build a compact digest of all reader findings for the critics.
+function digest(list) {
+ return list.map(r => {
+ const facts = (r.facts || []).map(f => ` - ${f.claim} [${f.evidence}]${f.implication ? ' → ' + f.implication : ''}`).join('\n')
+ const flip = (r.flipImpact || []).map(x => ` - ${x}`).join('\n')
+ const opts = (r.designOptions || []).map(x => ` - ${x}`).join('\n')
+ const oq = (r.openQuestions || []).map(x => ` - ${x}`).join('\n')
+ return `### ${r.dimension}\nSUMMARY: ${r.summary}\nFACTS:\n${facts}\nFLIP-IMPACT:\n${flip}\nDESIGN-OPTIONS:\n${opts}\nOPEN-QUESTIONS:\n${oq}`
+ }).join('\n\n')
+}
+const allFindings = digest(valid)
+
+phase('Critique')
+
+const critics = [
+ {
+ label: 'critic-completeness',
+ prompt: `${CTX}
+
+You are a COMPLETENESS critic. Below are the combined recon findings for the D2 connector-owned-cache design. Your job: find what is MISSING or unverified that would silently break post-flip if the connector-owned cache design omits it. Adversarially hunt for: (a) a consumer of the three fe-core caches not classified (legacy-die vs survivor); (b) a cached entry (schema/partition/file/stats/max-time) with no post-flip owner; (c) a REFRESH / DDL / event / TVF / dictionary / MV path that reads the cache and would collapse to ENGINE_DEFAULT silently; (d) a scan hot-path that becomes uncached (perf regression) at the flip; (e) an invalidation verb the connector cache won't receive. For each gap, cite the evidence file:line if you can and rank severity. Also RE-VERIFY any load-bearing claim you doubt by reasoning about the cited evidence. Return CRITIC_SCHEMA.
+
+COMBINED FINDINGS:
+${allFindings}`,
+ },
+ {
+ label: 'critic-flip-safety',
+ prompt: `${CTX}
+
+You are a FLIP-SAFETY & DORMANCY critic. Below are the combined recon findings. Your job: stress-test the core D2 premise — "the hive connector owns its scan-side cache, so at the flip, cache routing collapsing to ENGINE_DEFAULT is HARMLESS, and the 3 fe-core caches + 4 gates retire cleanly." Adversarially check: (1) Can the connector-owned cache land DORMANT (inert while hms is still legacy, exactly like every prior connector step) — or does anything force it to be co-committed with the flip? What must be dormant vs must ride the flip commit? (2) Is routing-collapse actually harmless, or is there a path where ENGINE_DEFAULT routing does something WRONG (not just no-op) — e.g. registry.resolve("default") throwing, or double-caching? (3) The iceberg/hudi fe-core caches: are they truly covered by sibling delegation post-flip, or is there a residual? (4) Ordering: does D2 have hidden dependencies on event Model B (the NEXT task) or vice versa — can D2 truly land first? (5) Any TCCL/classloader hazard when the connector-owned cache does filesystem listing or HMS calls. Rank severity, cite evidence, and list worries you REFUTED. Return CRITIC_SCHEMA.
+
+COMBINED FINDINGS:
+${allFindings}`,
+ },
+ {
+ label: 'critic-scope',
+ prompt: `${CTX}
+
+You are a SCOPE-BOUNDARY critic. Below are the combined recon findings. Your job: pin down the RIGHT scope for D2 so the design doc is neither bloated nor missing a flip-blocker. Adversarially decide, with evidence: (1) What is genuinely REQUIRED for a correct+performant flip (minimum viable connector cache set: which of schema/partition-name/partition-object/file-listing/stats/max-time) vs what can be DEFERRED post-flip as a perf item. (2) Does hudi/iceberg fe-core cache RETIREMENT belong inside the D2 commit-set or does it just ride the flip's deletion (since siblings cover them)? (3) Where is the clean boundary between D2 (cache) and event Model B (invalidation source) and §2.6 (max-time) — what shared SPI sits on the seam, and should max-time + invalidation SPI land in D2 or their own steps? (4) Is the shared fe-connector-cache framework the right home, or should the hive cache be connector-internal like PaimonLatestSnapshotCache? (5) How should D2 be decomposed into independent dormant commits (propose an ordered sub-step list, mirroring how the iceberg/hudi lines were decomposed). Cite evidence, rank, and note anything the plan §2.2/§2.6 got wrong vs HEAD. Return CRITIC_SCHEMA.
+
+COMBINED FINDINGS:
+${allFindings}`,
+ },
+]
+
+const criticResults = await parallel(
+ critics.map(c => () => agent(c.prompt, { label: c.label, phase: 'Critique', schema: CRITIC_SCHEMA, effort: 'high' }))
+)
+
+return {
+ readers: valid,
+ critics: criticResults.filter(Boolean),
+}
diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
index ca20cb63990447..c5e45ab2635c4d 100644
--- a/be/src/common/config.cpp
+++ b/be/src/common/config.cpp
@@ -1540,7 +1540,10 @@ DEFINE_Validator(s3_put_token_per_second, [](int64_t config) -> bool { return co
DEFINE_mInt64(s3_put_token_limit, "0");
-DEFINE_String(trino_connector_plugin_dir, "${DORIS_HOME}/plugins/connectors");
+// The dir TrinoConnectorPluginLoader loads Trino's own plugins from, used verbatim. Keep the default
+// in sync with FE Config.trino_connector_plugin_dir: FE and BE load the same plugins and an operator
+// who leaves both untouched expects both to find them.
+DEFINE_String(trino_connector_plugin_dir, "${DORIS_HOME}/plugins/trino_plugins");
// ca_cert_file is in this path by default, Normally no modification is required
// ca cert default path is different from different OS
diff --git a/be/src/io/fs/connectivity/s3_connectivity_tester.cpp b/be/src/io/fs/connectivity/s3_connectivity_tester.cpp
index b6115f4a2f99d8..347c1b40fb2c18 100644
--- a/be/src/io/fs/connectivity/s3_connectivity_tester.cpp
+++ b/be/src/io/fs/connectivity/s3_connectivity_tester.cpp
@@ -24,6 +24,10 @@ namespace doris::io {
Status S3ConnectivityTester::test(const std::map& properties) {
auto it = properties.find(TEST_LOCATION);
+ if (it == properties.end()) {
+ return Status::InvalidArgument("Missing '{}' property in S3 connectivity test request",
+ TEST_LOCATION);
+ }
S3URI s3_uri(it->second);
RETURN_IF_ERROR(s3_uri.parse());
diff --git a/build.sh b/build.sh
index e85204d66cc943..f5866e87eb9f84 100755
--- a/build.sh
+++ b/build.sh
@@ -1042,7 +1042,11 @@ if [[ "${BUILD_FE}" -eq 1 ]]; then
mkdir -p "${DORIS_OUTPUT}/fe/conf/ssl"
mkdir -p "${DORIS_OUTPUT}/fe/plugins/jdbc_drivers/"
mkdir -p "${DORIS_OUTPUT}/fe/plugins/java_udf/"
- mkdir -p "${DORIS_OUTPUT}/fe/plugins/connectors/"
+ # Drop point for the trino-connector's own Trino plugins. Deliberately NOT the legacy
+ # plugins/connectors/: that name is still read as a fallback for deployments upgrading from
+ # <= 2.1.8, so a fresh install must not create it (an empty dir would be harmless, but the
+ # one-letter gap to the plugins/connector/ tree above is not).
+ mkdir -p "${DORIS_OUTPUT}/fe/plugins/trino_plugins/"
mkdir -p "${DORIS_OUTPUT}/fe/plugins/hadoop_conf/"
mkdir -p "${DORIS_OUTPUT}/fe/plugins/java_extensions/"
@@ -1268,7 +1272,8 @@ EOF
mkdir -p "${DORIS_OUTPUT}/be/plugins/jdbc_drivers/"
mkdir -p "${DORIS_OUTPUT}/be/plugins/java_udf/"
mkdir -p "${DORIS_OUTPUT}/be/plugins/python_udf/"
- mkdir -p "${DORIS_OUTPUT}/be/plugins/connectors/"
+ # Mirrors the FE drop point above; the BE JNI scanner loads the same Trino plugins independently.
+ mkdir -p "${DORIS_OUTPUT}/be/plugins/trino_plugins/"
mkdir -p "${DORIS_OUTPUT}/be/plugins/hadoop_conf/"
mkdir -p "${DORIS_OUTPUT}/be/plugins/java_extensions/"
cp -r -p "${DORIS_HOME}/be/src/udf/python/python_server.py" "${DORIS_OUTPUT}/be/plugins/python_udf/"
diff --git a/docker/thirdparties/hive-regression-local-env.md b/docker/thirdparties/hive-regression-local-env.md
new file mode 100644
index 00000000000000..62e82997299b60
--- /dev/null
+++ b/docker/thirdparties/hive-regression-local-env.md
@@ -0,0 +1,162 @@
+# Hive Regression Local Environment
+
+This document records the local Docker and regression configuration needed to run:
+
+```bash
+regression-test/suites/external_table_p0/hive/
+```
+
+## Required Thirdparty Components
+
+Run Hive2 and Hive3 through the thirdparty launcher:
+
+```bash
+./docker/thirdparties/run-thirdparties-docker.sh -c hive2,hive3 --hive-mode refresh
+```
+
+The Hive startup script may also start `mysql` automatically. Both
+`docker-compose/hive/hive-2x_settings.env` and
+`docker-compose/hive/hive-3x_settings.env` default `JFS_CLUSTER_META` to:
+
+```bash
+mysql://root:123456@(127.0.0.1:3316)/juicefs_meta
+```
+
+Because of that setting, `run-thirdparties-docker.sh` treats local MySQL 5.7 as
+an implicit dependency for Hive.
+
+Each Hive version starts its own compose services:
+
+- Hive2: `hive2-server`, `hive2-metastore`, `hadoop2-namenode`,
+ `hadoop2-datanode`, `hive2-metastore-postgresql`
+- Hive3: `hive3-server`, `hive3-metastore`, `hadoop3-namenode`,
+ `hadoop3-datanode`, `hive3-metastore-postgresql`
+- MySQL 5.7: required by the default JuiceFS metadata URI when using the local
+ `127.0.0.1:3316` endpoint
+
+The Hive p0 directory does not require these components by default:
+
+- `pg`
+- `iceberg`
+- `minio`
+- `hudi`
+- `kerberos`
+- `ranger`
+
+`test_external_catalog_hive.groovy` has a Ranger-specific branch guarded by
+`enableRangerTest`; if this option is not set to `true`, that branch is skipped.
+
+## Current Running Containers Observed
+
+The following relevant containers were running when checked with
+`sudo docker ps`:
+
+- Hive3 stack: `hive3-server`, `hive3-metastore`, `hadoop3-datanode`,
+ `hadoop3-namenode`, `hive3-metastore-postgresql`; all healthy
+- Hive2 stack: `hive2-server`, `hive2-metastore`, `hadoop2-datanode`,
+ `hadoop2-namenode`, `hive2-metastore-postgresql`; all healthy
+- MySQL 5.7: `mysql-doris--syt--mysql_57-1`, healthy, published as
+ `127.0.0.1:3316 -> 3306`
+
+Other Doris thirdparty containers were also running, but they are not required
+for `external_table_p0/hive/`:
+
+- PostgreSQL 14 on `5442`
+- Iceberg REST, Spark, Postgres, and MinIO
+- Oracle on `1521`
+- ClickHouse on `8123`
+- SQLServer on `1433`
+- OceanBase on `2881`
+
+The observed Hive2/Hive3 and MySQL compose projects came from another checkout
+under `/mnt/disk2/suyiteng/doris`, not from this worktree. PostgreSQL and
+Iceberg compose projects came from this worktree.
+
+## Regression Configuration
+
+The Hive p0 suite directly reads these keys from
+`regression-test/conf/regression-conf.groovy`:
+
+```groovy
+enableHiveTest
+externalEnvIp
+hive2HmsPort
+hive2HdfsPort
+hive3HmsPort
+hive3HdfsPort
+hdfsUser
+```
+
+It also uses framework helpers backed by:
+
+```groovy
+brokerName
+hdfsPasswd
+hdfsFs
+```
+
+For local Hive2/Hive3 Docker with default ports, the relevant values are:
+
+```groovy
+enableHiveTest = true
+
+hive2HmsPort = 9083
+hive2HdfsPort = 8020
+hive2ServerPort = 10000
+hive2PgPort = 5432
+
+hive3HmsPort = 9383
+hive3HdfsPort = 8320
+hive3ServerPort = 13000
+hive3PgPort = 5732
+
+hdfsUser = "doris-test"
+brokerName = "broker_name"
+hdfsPasswd = ""
+```
+
+`externalEnvIp` must be the address that Doris BE can use to reach the Hive
+Docker services. For a fully local setup this is usually:
+
+```groovy
+externalEnvIp = "127.0.0.1"
+```
+
+If BE runs in a context where `127.0.0.1` is not the Docker host, use the host
+IP selected by `run-thirdparties-docker.sh` through `IP_HOST`.
+
+## Config Gaps Noted In regression-conf.groovy.bak
+
+`regression-test/conf/regression-conf.groovy.bak` already contains the core Hive
+keys listed above. The notable follow-ups are:
+
+- `externalEnvIp` was set to `172.20.32.136`; verify this is reachable from BE,
+ or replace it with `127.0.0.1` for local host-mode Docker.
+- FE/JDBC addresses in the file used `9033`, `9022`, and `8033`; align these
+ with the current Doris worktree cluster ports before running regression.
+- `enableRangerTest` is not present. This is acceptable for the Hive p0 suite
+ because the Ranger-specific branch is guarded and skipped unless explicitly
+ enabled.
+
+## Thirdparty Script Settings
+
+Before starting Hive through `run-thirdparties-docker.sh`, set a unique
+`CONTAINER_UID` in:
+
+```bash
+docker/thirdparties/custom_settings.env
+```
+
+The default `CONTAINER_UID="doris--"` is rejected by the startup script.
+
+Hive baseline restore also depends on these settings:
+
+```bash
+s3Endpoint
+s3BucketName
+HIVE_BASELINE_VERSION
+HIVE_BASELINE_TARBALL_CACHE
+```
+
+If baseline download is unavailable, place the matching tarball manually under
+`HIVE_BASELINE_TARBALL_CACHE`.
diff --git a/fe/be-java-extensions/trino-connector-scanner/src/main/java/org/apache/doris/trinoconnector/TrinoConnectorPluginLoader.java b/fe/be-java-extensions/trino-connector-scanner/src/main/java/org/apache/doris/trinoconnector/TrinoConnectorPluginLoader.java
index 2347bd52c168ab..c6f30c7175eec8 100644
--- a/fe/be-java-extensions/trino-connector-scanner/src/main/java/org/apache/doris/trinoconnector/TrinoConnectorPluginLoader.java
+++ b/fe/be-java-extensions/trino-connector-scanner/src/main/java/org/apache/doris/trinoconnector/TrinoConnectorPluginLoader.java
@@ -40,7 +40,10 @@
public class TrinoConnectorPluginLoader {
private static final Logger LOG = LogManager.getLogger(TrinoConnectorPluginLoader.class);
- private static String pluginsDir = EnvUtils.getDorisHome() + "/plugins/connectors";
+ // Overwritten via setPluginsDir() with BE config trino_connector_plugin_dir before the plugins are
+ // loaded (see be/src/format*/**/trino_connector_jni_reader.cpp); this initializer only matters if
+ // that call is ever missed. Mirrors that config's default.
+ private static String pluginsDir = EnvUtils.getDorisHome() + "/plugins/trino_plugins";
// Suppress default constructor for noninstantiability
private TrinoConnectorPluginLoader() {
@@ -87,7 +90,7 @@ private static class TrinoConnectorPluginLoad {
TypeRegistry typeRegistry = new TypeRegistry(typeOperators, featuresConfig);
ServerPluginsProviderConfig serverPluginsProviderConfig = new ServerPluginsProviderConfig()
- .setInstalledPluginsDir(new File(checkAndReturnPluginDir()));
+ .setInstalledPluginsDir(new File(pluginsDir));
ServerPluginsProvider serverPluginsProvider = new ServerPluginsProvider(serverPluginsProviderConfig,
MoreExecutors.directExecutor());
HandleResolver handleResolver = new HandleResolver();
@@ -95,9 +98,9 @@ private static class TrinoConnectorPluginLoad {
typeRegistry, handleResolver);
trinoConnectorPluginManager.loadPlugins();
- LOG.info("TrinoConnectorPluginLoader successfully loaded plugins from: " + checkAndReturnPluginDir());
+ LOG.info("TrinoConnectorPluginLoader successfully loaded plugins from: " + pluginsDir);
} catch (Exception e) {
- LOG.warn("Failed load trino-connector plugins from " + checkAndReturnPluginDir()
+ LOG.warn("Failed load trino-connector plugins from " + pluginsDir
+ ", Exception:" + e.getMessage(), e);
}
}
@@ -118,29 +121,6 @@ public static void setPluginsDir(String pluginsDir) {
TrinoConnectorPluginLoader.pluginsDir = pluginsDir;
}
- private static String checkAndReturnPluginDir() {
- final String defaultDir = System.getenv("DORIS_HOME") + "/plugins/connectors";
- final String defaultOldDir = System.getenv("DORIS_HOME") + "/connectors";
- if (TrinoConnectorPluginLoader.pluginsDir.equals(defaultDir)) {
- // If true, which means user does not set `trino_connector_plugin_dir` and use the default one.
- // Because in 2.1.8, we change the default value of `trino_connector_plugin_dir`
- // from `DORIS_HOME/connectors` to `DORIS_HOME/plugins/connectors`,
- // so we need to check the old default dir for compatibility.
- File oldDir = new File(defaultOldDir);
- if (oldDir.exists() && oldDir.isDirectory()) {
- String[] contents = oldDir.list();
- if (contents != null && contents.length > 0) {
- // there are contents in old dir, use old one
- return defaultOldDir;
- }
- }
- return defaultDir;
- } else {
- // Return user specified dir directly.
- return TrinoConnectorPluginLoader.pluginsDir;
- }
- }
-
public static TrinoConnectorPluginManager getTrinoConnectorPluginManager() {
return TrinoConnectorPluginLoad.trinoConnectorPluginManager;
}
diff --git a/fe/fe-authentication/fe-authentication-handler/src/test/java/org/apache/doris/authentication/handler/AuthenticationPluginManagerTest.java b/fe/fe-authentication/fe-authentication-handler/src/test/java/org/apache/doris/authentication/handler/AuthenticationPluginManagerTest.java
index 6cde32bb90f0f1..260afb30a9646c 100644
--- a/fe/fe-authentication/fe-authentication-handler/src/test/java/org/apache/doris/authentication/handler/AuthenticationPluginManagerTest.java
+++ b/fe/fe-authentication/fe-authentication-handler/src/test/java/org/apache/doris/authentication/handler/AuthenticationPluginManagerTest.java
@@ -70,7 +70,6 @@ void testPluginsAutoLoaded() {
// Then
Assertions.assertNotNull(pluginNames);
Assertions.assertFalse(pluginNames.isEmpty(), "Should load at least built-in plugins");
- Assertions.assertTrue(pluginNames.contains("oidc"), "Should include oidc plugin");
Assertions.assertTrue(pluginNames.contains("password"), "Should include password plugin");
}
@@ -147,10 +146,6 @@ void testGetFactory() {
// Then
Assertions.assertTrue(factory.isPresent());
Assertions.assertEquals("password", factory.get().name());
-
- Optional oidcFactory = pluginManager.getFactory("oidc");
- Assertions.assertTrue(oidcFactory.isPresent());
- Assertions.assertEquals("oidc", oidcFactory.get().name());
}
@Test
diff --git a/fe/fe-catalog/src/main/java/org/apache/doris/catalog/Column.java b/fe/fe-catalog/src/main/java/org/apache/doris/catalog/Column.java
index aaf76f5c5da864..bc1991dc1073f9 100644
--- a/fe/fe-catalog/src/main/java/org/apache/doris/catalog/Column.java
+++ b/fe/fe-catalog/src/main/java/org/apache/doris/catalog/Column.java
@@ -182,6 +182,15 @@ public static Column generateBeforeValueColumn(Column column) {
private boolean isCompoundKey = false;
+ // Marks a connector-reserved passthrough column (e.g. iceberg v3 row-lineage _row_id /
+ // _last_updated_sequence_number). Set by ConnectorColumnConverter from the connector-declared
+ // ConnectorColumn.reservedPassthrough(); read by engine MERGE/UPDATE and sink binding so they recognize the
+ // synthetic passthrough column generically instead of string-matching source column names. NOT persisted
+ // (no @SerializedName on purpose): it is an external-table-only marker rebuilt each load from connector
+ // metadata and must never enter an internal-table schema image; on replay it stays at its default false
+ // (mirrors the runtime-only isCompoundKey / defineExpr fields).
+ private boolean reservedPassthrough = false;
+
@SerializedName(value = "hasOnUpdateDefaultValue")
private boolean hasOnUpdateDefaultValue = false;
@@ -376,6 +385,7 @@ public Column(Column column) {
this.visible = column.visible;
this.children = column.getChildren();
this.uniqueId = column.getUniqueId();
+ this.reservedPassthrough = column.reservedPassthrough;
this.defineExpr = column.getDefineExpr();
this.defineName = column.getRealDefineName();
this.hasOnUpdateDefaultValue = column.hasOnUpdateDefaultValue;
@@ -987,6 +997,14 @@ public int getUniqueId() {
return this.uniqueId;
}
+ public boolean isReservedPassthrough() {
+ return reservedPassthrough;
+ }
+
+ public void setReservedPassthrough(boolean reservedPassthrough) {
+ this.reservedPassthrough = reservedPassthrough;
+ }
+
public long getAutoIncInitValue() {
return this.autoIncInitValue;
}
diff --git a/fe/fe-common/pom.xml b/fe/fe-common/pom.xml
index 50a8fc810095ac..67ae7e2f1285d8 100644
--- a/fe/fe-common/pom.xml
+++ b/fe/fe-common/pom.xml
@@ -84,11 +84,6 @@ under the License.
org.apache.commonscommons-lang3
-
- org.apache.doris
- hive-catalog-shade
- provided
- org.roaringbitmapRoaringBitmap
diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/CatalogConfigFileUtils.java b/fe/fe-common/src/main/java/org/apache/doris/common/CatalogConfigFileUtils.java
index bce08713131140..d8e9f2e5133740 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/common/CatalogConfigFileUtils.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/common/CatalogConfigFileUtils.java
@@ -20,7 +20,6 @@
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hive.conf.HiveConf;
import java.io.File;
import java.util.function.BiConsumer;
@@ -29,14 +28,13 @@
public class CatalogConfigFileUtils {
/**
- * Generic method to load configuration files (e.g., Hadoop or Hive) from a directory.
+ * Generic method to load Hadoop-style configuration files from a directory.
*
* @param resourcesPath Comma-separated list of resource file names to be loaded.
* @param configDir Directory prefix where the configuration files reside.
- * @param configSupplier Supplier that creates a new configuration object
- * (e.g., new Configuration or new HiveConf).
+ * @param configSupplier Supplier that creates a new configuration object.
* @param addResourceMethod Method to add a resource file to the configuration object.
- * @param Type of the configuration (e.g., Configuration or HiveConf).
+ * @param Type of the configuration.
* @return A configuration object loaded with the given resource files.
* @throws IllegalArgumentException if the resourcesPath is empty or if any file does not exist.
*/
@@ -84,20 +82,4 @@ public static Configuration loadConfigurationFromHadoopConfDir(String resourcesP
Configuration::addResource
);
}
-
- /**
- * Loads a HiveConf object from a list of files under the specified config directory.
- *
- * @param resourcesPath Comma-separated list of file names to be loaded.
- * @return A HiveConf object.
- * @throws IllegalArgumentException if the input is invalid or files are missing.
- */
- public static HiveConf loadHiveConfFromHiveConfDir(String resourcesPath) {
- return loadConfigFromDir(
- resourcesPath,
- Config.hadoop_config_dir,
- HiveConf::new,
- HiveConf::addResource
- );
- }
}
diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
index 00526509177152..410c7631400448 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
@@ -2252,12 +2252,18 @@ public class Config extends ConfigBase {
public static boolean enable_fqdn_mode = false;
/**
- * If set to true, doris will try to parse the ddl of a hive view and try to execute the query
- * otherwise it will throw an AnalysisException.
+ * @deprecated No-op since the hms/iceberg SPI cutover: external views (hive, iceberg) are always served.
+ * Retained for one release so operator fe.conf that sets it still parses; will be removed later.
*/
+ @Deprecated
@ConfField(mutable = true)
public static boolean enable_query_hive_views = true;
+ /**
+ * @deprecated No-op since the iceberg SPI cutover: external views (hive, iceberg) are always served.
+ * Retained for one release so operator fe.conf that sets it still parses; will be removed later.
+ */
+ @Deprecated
@ConfField(mutable = true)
public static boolean enable_query_iceberg_views = true;
@@ -2862,9 +2868,12 @@ public class Config extends ConfigBase {
+ "multiple integration names are comma-separated"})
public static String authentication_chain = "";
+ // The dir the trino-connector catalog loads Trino's own plugins from, used verbatim. Keep the
+ // default in sync with BE config trino_connector_plugin_dir: FE and BE load the same plugins and
+ // an operator who leaves both untouched expects both to find them.
@ConfField(mutable = true, masterOnly = false, description = {
"Specify the default plugins loading path for the trino-connector catalog"})
- public static String trino_connector_plugin_dir = EnvUtils.getDorisHome() + "/plugins/connectors";
+ public static String trino_connector_plugin_dir = EnvUtils.getDorisHome() + "/plugins/trino_plugins";
@ConfField(mutable = true)
public static boolean fix_tablet_partition_id_eq_0 = false;
diff --git a/fe/fe-connector/fe-connector-api/src/main/java/org/apache/doris/connector/api/Connector.java b/fe/fe-connector/fe-connector-api/src/main/java/org/apache/doris/connector/api/Connector.java
index c7f2ff10420fbc..e1bdc83c088324 100644
--- a/fe/fe-connector/fe-connector-api/src/main/java/org/apache/doris/connector/api/Connector.java
+++ b/fe/fe-connector/fe-connector-api/src/main/java/org/apache/doris/connector/api/Connector.java
@@ -17,6 +17,8 @@
package org.apache.doris.connector.api;
+import org.apache.doris.connector.api.event.ConnectorEventSource;
+import org.apache.doris.connector.api.handle.ConnectorTableHandle;
import org.apache.doris.connector.api.handle.WriteOperation;
import org.apache.doris.connector.api.procedure.ConnectorProcedureOps;
import org.apache.doris.connector.api.scan.ConnectorScanPlanProvider;
@@ -42,11 +44,48 @@ public interface Connector extends Closeable {
/** Returns the metadata interface for the given session. */
ConnectorMetadata getMetadata(ConnectorSession session);
+ /**
+ * Whether {@code handle} is one of THIS connector's own concrete {@link ConnectorTableHandle} subclasses.
+ *
+ *
A heterogeneous gateway connector that serves several table formats through embedded sibling
+ * connectors uses this to route a foreign handle to the sibling that produced it: the sibling's concrete
+ * handle type is invisible across the plugin classloader split, so the gateway cannot {@code instanceof} it
+ * directly — it asks each sibling, and the sibling tests its OWN in-loader type. The default returns
+ * {@code false} (a connector owns no handle it did not define), so every non-gateway connector is
+ * unaffected.
+ *
+ *
fe-core NEVER calls this — it is a connector-to-sibling routing predicate only, so the engine stays
+ * format-agnostic (it discriminates handles solely by the gateway's own handle type, never by asking a
+ * connector to classify one).
+ */
+ default boolean ownsHandle(ConnectorTableHandle handle) {
+ return false;
+ }
+
/** Returns the scan plan provider for split generation. */
default ConnectorScanPlanProvider getScanPlanProvider() {
return null;
}
+ /**
+ * Returns the scan plan provider for the given table, allowing one connector to select a
+ * different provider per table.
+ *
+ *
The selection MUST happen here, at provider-acquisition time — not inside a single
+ * dispatching provider — because {@link ConnectorScanPlanProvider} has methods that do not
+ * carry the handle (e.g. {@code appendExplainInfo}) and providers are built fresh/stateless
+ * per call, so a provider returned here must already be bound to the correct backing scanner
+ * for {@code handle}. This is the seam a heterogeneous gateway connector (one catalog serving
+ * multiple table formats) overrides to delegate to per-format sub-providers by the concrete
+ * (connector-defined) handle type; the engine never inspects the format.
+ *
+ *
The default ignores {@code handle} and returns the connector-level
+ * {@link #getScanPlanProvider()}, so every single-format connector is unaffected.
+ */
+ default ConnectorScanPlanProvider getScanPlanProvider(ConnectorTableHandle handle) {
+ return getScanPlanProvider();
+ }
+
/**
* Returns the write plan provider for sink ({@code TDataSink}) generation,
* or {@code null} if this connector does not support writes.
@@ -55,6 +94,19 @@ default ConnectorWritePlanProvider getWritePlanProvider() {
return null;
}
+ /**
+ * Returns the write plan provider for the given table, allowing one connector to select a different
+ * provider per table — the write-side analogue of {@link #getScanPlanProvider(ConnectorTableHandle)}.
+ *
+ *
The default ignores {@code handle} and returns the connector-level {@link #getWritePlanProvider()}, so
+ * every single-format connector is unaffected. A heterogeneous gateway connector (one catalog serving
+ * multiple table formats) overrides this to delegate to a per-format sub-provider by the concrete
+ * (connector-defined) handle type; the engine never inspects the format.
+ */
+ default ConnectorWritePlanProvider getWritePlanProvider(ConnectorTableHandle handle) {
+ return getWritePlanProvider();
+ }
+
/**
* The write operations the engine may perform on this connector — the single admission source. Reads the
* write provider's {@link ConnectorWritePlanProvider#supportedOperations()}; no provider ⇒ empty set ⇒ all
@@ -65,12 +117,29 @@ default Set supportedWriteOperations() {
return p == null ? EnumSet.noneOf(WriteOperation.class) : p.supportedOperations();
}
+ /**
+ * Per-table view of {@link #supportedWriteOperations()}: derives from {@link #getWritePlanProvider(
+ * ConnectorTableHandle)} so a heterogeneous gateway admits the right operations for {@code handle} (e.g. an
+ * iceberg-on-HMS table admits DELETE/MERGE that the hive provider does not). The default routes through the
+ * per-handle provider, so every single-format connector is unaffected.
+ */
+ default Set supportedWriteOperations(ConnectorTableHandle handle) {
+ ConnectorWritePlanProvider p = getWritePlanProvider(handle);
+ return p == null ? EnumSet.noneOf(WriteOperation.class) : p.supportedOperations();
+ }
+
/** Null-safe view of {@link ConnectorWritePlanProvider#supportsWriteBranch()}. No provider ⇒ false. */
default boolean supportsWriteBranch() {
ConnectorWritePlanProvider p = getWritePlanProvider();
return p != null && p.supportsWriteBranch();
}
+ /** Per-table view of {@link #supportsWriteBranch()} (derives from the per-handle provider). */
+ default boolean supportsWriteBranch(ConnectorTableHandle handle) {
+ ConnectorWritePlanProvider p = getWritePlanProvider(handle);
+ return p != null && p.supportsWriteBranch();
+ }
+
/** Null-safe view of {@link ConnectorWritePlanProvider#requiresParallelWrite()}. No provider ⇒ false. */
default boolean requiresParallelWrite() {
ConnectorWritePlanProvider p = getWritePlanProvider();
@@ -89,6 +158,18 @@ default boolean requiresPartitionLocalSort() {
return p != null && p.requiresPartitionLocalSort();
}
+ /** Null-safe view of {@link ConnectorWritePlanProvider#requiresPartitionHashWrite()}. No provider ⇒ false. */
+ default boolean requiresPartitionHashWrite() {
+ ConnectorWritePlanProvider p = getWritePlanProvider();
+ return p != null && p.requiresPartitionHashWrite();
+ }
+
+ /** Per-table view of {@link #requiresPartitionHashWrite()} (derives from the per-handle provider). */
+ default boolean requiresPartitionHashWrite(ConnectorTableHandle handle) {
+ ConnectorWritePlanProvider p = getWritePlanProvider(handle);
+ return p != null && p.requiresPartitionHashWrite();
+ }
+
/**
* Null-safe view of {@link ConnectorWritePlanProvider#requiresMaterializeStaticPartitionValues()}. No
* provider ⇒ false.
@@ -98,6 +179,12 @@ default boolean requiresMaterializeStaticPartitionValues() {
return p != null && p.requiresMaterializeStaticPartitionValues();
}
+ /** Per-table view of {@link #requiresMaterializeStaticPartitionValues()} (derives from the per-handle provider). */
+ default boolean requiresMaterializeStaticPartitionValues(ConnectorTableHandle handle) {
+ ConnectorWritePlanProvider p = getWritePlanProvider(handle);
+ return p != null && p.requiresMaterializeStaticPartitionValues();
+ }
+
/**
* Returns the procedure ops for {@code ALTER TABLE EXECUTE} dispatch, or {@code null} if this
* connector exposes no table procedures. Procedure-side analogue of {@link #getWritePlanProvider()}.
@@ -106,6 +193,20 @@ default ConnectorProcedureOps getProcedureOps() {
return null;
}
+ /**
+ * Returns the procedure ops for the given table, allowing one connector to select a different set of
+ * procedures per table — the procedure-side analogue of {@link #getScanPlanProvider(
+ * ConnectorTableHandle)} / {@link #getWritePlanProvider(ConnectorTableHandle)}.
+ *
+ *
The default ignores {@code handle} and returns the connector-level {@link #getProcedureOps()}, so every
+ * single-format connector is unaffected. A heterogeneous gateway connector (one catalog serving multiple
+ * table formats) overrides this to delegate a foreign (e.g. iceberg-on-HMS) handle to a sibling connector's
+ * procedure ops by the concrete (connector-defined) handle type; the engine never inspects the format.
+ */
+ default ConnectorProcedureOps getProcedureOps(ConnectorTableHandle handle) {
+ return getProcedureOps();
+ }
+
/** Returns the set of capabilities this connector supports. */
default Set getCapabilities() {
return Collections.emptySet();
@@ -215,6 +316,38 @@ default void invalidateTable(String dbName, String tableName) {
default void invalidateAll() {
}
+ /**
+ * Invalidates the connector-side caches for every table in one database. Called by the engine on
+ * {@code REFRESH DATABASE}. The name is the REMOTE db name (as seen by the connector). Default no-op for
+ * connectors that cache nothing.
+ */
+ default void invalidateDb(String dbName) {
+ }
+
+ /**
+ * Invalidates the connector-side caches for specific partitions of a table so a subsequent read
+ * reflects the latest external state. Driven by the engine's metastore-event sync when partitions
+ * are added/dropped/altered. The names are the REMOTE db/table names and canonical partition names
+ * ({@code "col=val/.../colN=valN"}); an empty/whole-table drop is expressed by
+ * {@link #invalidateTable(String, String)}. A connector whose partition cache cannot target a single
+ * name may degrade to invalidating the whole table's partition caches (correctness-safe when the
+ * cache re-lists on miss). Default no-op for connectors that cache nothing.
+ */
+ default void invalidatePartition(String dbName, String tableName, List partitionNames) {
+ }
+
+ /**
+ * Returns this connector's incremental metadata-change source, or {@code null} if it has none.
+ * A capability-probe getter (mirrors {@link #getScanPlanProvider()} / {@link #getProcedureOps()}):
+ * the engine's single, connector-agnostic, role-aware event driver iterates catalogs and calls
+ * {@link ConnectorEventSource#pollOnce} only on connectors that expose a source, never via
+ * {@code instanceof}. The default returns {@code null}, so every connector without a metastore-event
+ * feed is unaffected.
+ */
+ default ConnectorEventSource getEventSource() {
+ return null;
+ }
+
/**
* Optional per-connector override of the catalog's schema-cache TTL (in seconds), consulted generically by
* the engine when sizing the schema meta-cache. Semantics match {@code schema.cache.ttl-second}:
diff --git a/fe/fe-connector/fe-connector-api/src/main/java/org/apache/doris/connector/api/ConnectorCapability.java b/fe/fe-connector/fe-connector-api/src/main/java/org/apache/doris/connector/api/ConnectorCapability.java
index 427d31584675f1..1e9f21416a41b6 100644
--- a/fe/fe-connector/fe-connector-api/src/main/java/org/apache/doris/connector/api/ConnectorCapability.java
+++ b/fe/fe-connector/fe-connector-api/src/main/java/org/apache/doris/connector/api/ConnectorCapability.java
@@ -137,5 +137,26 @@ public enum ConnectorCapability {
* authenticate with a single static catalog identity (every non-REST iceberg flavor, JDBC, ES, ...) must
* NOT declare it. Declared by the iceberg connector only when configured {@code iceberg.rest.session=user}.
*/
- SUPPORTS_USER_SESSION
+ SUPPORTS_USER_SESSION,
+ /**
+ * Indicates the connector exposes a metadata table (e.g. the hudi commit timeline) whose rows are read via
+ * {@link ConnectorMetadata#getMetadataTableRows}.
+ *
+ *
The {@code hudi_meta()} / TIMELINE table-valued function's plugin-driven arm delegates to the connector
+ * only when it declares this; a connector with no metadata table must NOT declare it so the TVF rejects the
+ * table with "not a hudi table". Hudi declares it connector-wide (every hudi table has a commit timeline).
+ */
+ SUPPORTS_METADATA_TABLE,
+ /**
+ * Indicates the connector's file-scan tables support {@code ANALYZE ... WITH SAMPLE} (scale-factor estimation
+ * from raw per-file byte sizes via {@link ConnectorStatisticsOps#listFileSizes}, with fe-core doing the
+ * Doris-type slot-width math).
+ *
+ *
fe-core admits sampled analyze for a plugin-driven table only when it declares this. A heterogeneous
+ * connector (hive) emits it as a PER-TABLE marker in getTableSchema for its plain-hive tables only (legacy
+ * gated on {@code dlaType==HIVE}), so iceberg/hudi-on-HMS are excluded. Connectors whose {@code doSample} is
+ * unimplemented (native iceberg/paimon, JDBC, ES) must NOT declare it so sampled analyze stays rejected at
+ * build time.
+ */
+ SUPPORTS_SAMPLE_ANALYZE
}
diff --git a/fe/fe-connector/fe-connector-api/src/main/java/org/apache/doris/connector/api/ConnectorColumn.java b/fe/fe-connector/fe-connector-api/src/main/java/org/apache/doris/connector/api/ConnectorColumn.java
index 3a2b8a42e1c6d6..6737197000cecc 100644
--- a/fe/fe-connector/fe-connector-api/src/main/java/org/apache/doris/connector/api/ConnectorColumn.java
+++ b/fe/fe-connector/fe-connector-api/src/main/java/org/apache/doris/connector/api/ConnectorColumn.java
@@ -51,6 +51,13 @@ public final class ConnectorColumn {
// _last_updated_sequence_number=2147483539, matched by field id BE-side). Defaults UNSET_UNIQUE_ID (-1),
// leaving the Doris default untouched; set via withUniqueId().
private final int uniqueId;
+ // Marks a connector-reserved passthrough column: a synthetic column whose identity the ENGINE must
+ // recognize generically (without knowing the connector's column names) — e.g. iceberg v3 row-lineage
+ // (_row_id / _last_updated_sequence_number), which fe-core MERGE/UPDATE must pass through rather than treat
+ // as user data. fe-core's ConnectorColumnConverter re-applies it via Column.setReservedPassthrough(true);
+ // engine consumers then ask Column.isReservedPassthrough() instead of string-matching source column names.
+ // Defaults false; set via reservedPassthrough().
+ private final boolean reservedPassthrough;
public ConnectorColumn(String name, ConnectorType type, String comment,
boolean nullable, String defaultValue) {
@@ -71,12 +78,13 @@ public ConnectorColumn(String name, ConnectorType type, String comment,
boolean nullable, String defaultValue, boolean isKey, boolean isAutoInc,
boolean isAggregated) {
this(name, type, comment, nullable, defaultValue, isKey, isAutoInc, isAggregated, false, true,
- UNSET_UNIQUE_ID);
+ UNSET_UNIQUE_ID, false);
}
private ConnectorColumn(String name, ConnectorType type, String comment,
boolean nullable, String defaultValue, boolean isKey, boolean isAutoInc,
- boolean isAggregated, boolean withTimeZone, boolean visible, int uniqueId) {
+ boolean isAggregated, boolean withTimeZone, boolean visible, int uniqueId,
+ boolean reservedPassthrough) {
this.name = Objects.requireNonNull(name, "name");
this.type = Objects.requireNonNull(type, "type");
this.comment = comment;
@@ -88,6 +96,7 @@ private ConnectorColumn(String name, ConnectorType type, String comment,
this.withTimeZone = withTimeZone;
this.visible = visible;
this.uniqueId = uniqueId;
+ this.reservedPassthrough = reservedPassthrough;
}
/**
@@ -97,7 +106,7 @@ private ConnectorColumn(String name, ConnectorType type, String comment,
*/
public ConnectorColumn withTimeZone() {
return new ConnectorColumn(name, type, comment, nullable, defaultValue,
- isKey, isAutoInc, isAggregated, true, visible, uniqueId);
+ isKey, isAutoInc, isAggregated, true, visible, uniqueId, reservedPassthrough);
}
/**
@@ -106,7 +115,18 @@ public ConnectorColumn withTimeZone() {
*/
public ConnectorColumn invisible() {
return new ConnectorColumn(name, type, comment, nullable, defaultValue,
- isKey, isAutoInc, isAggregated, withTimeZone, false, uniqueId);
+ isKey, isAutoInc, isAggregated, withTimeZone, false, uniqueId, reservedPassthrough);
+ }
+
+ /**
+ * Returns a copy of this column marked as a connector-reserved passthrough column. See
+ * {@link #isReservedPassthrough()}; the converter re-applies it via {@code Column.setReservedPassthrough(true)}
+ * so the engine can recognize a synthetic column (iceberg v3 row-lineage) generically, without knowing its
+ * source name.
+ */
+ public ConnectorColumn reservedPassthrough() {
+ return new ConnectorColumn(name, type, comment, nullable, defaultValue,
+ isKey, isAutoInc, isAggregated, withTimeZone, visible, uniqueId, true);
}
/**
@@ -117,7 +137,7 @@ public ConnectorColumn invisible() {
*/
public ConnectorColumn withUniqueId(int uniqueId) {
return new ConnectorColumn(name, type, comment, nullable, defaultValue,
- isKey, isAutoInc, isAggregated, withTimeZone, visible, uniqueId);
+ isKey, isAutoInc, isAggregated, withTimeZone, visible, uniqueId, reservedPassthrough);
}
public String getName() {
@@ -164,6 +184,10 @@ public int getUniqueId() {
return uniqueId;
}
+ public boolean isReservedPassthrough() {
+ return reservedPassthrough;
+ }
+
@Override
public boolean equals(Object o) {
if (this == o) {
@@ -180,6 +204,7 @@ public boolean equals(Object o) {
&& withTimeZone == that.withTimeZone
&& visible == that.visible
&& uniqueId == that.uniqueId
+ && reservedPassthrough == that.reservedPassthrough
&& name.equals(that.name)
&& type.equals(that.type)
&& Objects.equals(comment, that.comment)
@@ -189,7 +214,7 @@ public boolean equals(Object o) {
@Override
public int hashCode() {
return Objects.hash(name, type, comment, nullable, defaultValue, isKey, isAutoInc, isAggregated,
- withTimeZone, visible, uniqueId);
+ withTimeZone, visible, uniqueId, reservedPassthrough);
}
@Override
diff --git a/fe/fe-connector/fe-connector-api/src/main/java/org/apache/doris/connector/api/ConnectorColumnStatistics.java b/fe/fe-connector/fe-connector-api/src/main/java/org/apache/doris/connector/api/ConnectorColumnStatistics.java
new file mode 100644
index 00000000000000..7c413e5d313ab8
--- /dev/null
+++ b/fe/fe-connector/fe-connector-api/src/main/java/org/apache/doris/connector/api/ConnectorColumnStatistics.java
@@ -0,0 +1,101 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.connector.api;
+
+import java.util.Objects;
+
+/**
+ * Per-column statistics a connector can serve WITHOUT a table scan (e.g. hive's HMS/spark column stats),
+ * feeding the query-planner column-statistics fast path.
+ *
+ *
This carries only RAW facts. The connector does NOT compute the Doris-type-dependent
+ * {@code dataSize}/{@code avgSize} — those depend on the column's fixed slot width, which the connector
+ * cannot know without importing fe-type. fe-core derives them from {@link #getAvgSizeBytes()} (when the
+ * source knows the average value size, e.g. a hive string column's {@code avgColLen}) or the column's slot
+ * width otherwise, mirroring the {@link #getTableStatistics}-style raw/derived split. Use {@link #UNKNOWN}
+ * when statistics are unavailable.
+ */
+public final class ConnectorColumnStatistics {
+
+ /** Sentinel indicating no per-column statistics are available. */
+ public static final ConnectorColumnStatistics UNKNOWN =
+ new ConnectorColumnStatistics(-1, -1, -1, -1);
+
+ private final long rowCount;
+ private final long ndv;
+ private final long numNulls;
+ private final double avgSizeBytes;
+
+ public ConnectorColumnStatistics(long rowCount, long ndv, long numNulls, double avgSizeBytes) {
+ this.rowCount = rowCount;
+ this.ndv = ndv;
+ this.numNulls = numNulls;
+ this.avgSizeBytes = avgSizeBytes;
+ }
+
+ /** The table row count the stats are relative to (source {@code numRows}), or -1 if unknown. */
+ public long getRowCount() {
+ return rowCount;
+ }
+
+ /** Number of distinct values, or -1 if unknown. */
+ public long getNdv() {
+ return ndv;
+ }
+
+ /** Number of nulls, or -1 if unknown. */
+ public long getNumNulls() {
+ return numNulls;
+ }
+
+ /**
+ * Average per-value size in bytes when the source knows it (e.g. a hive string column's {@code avgColLen}),
+ * or {@code -1} when it does not — fe-core then falls back to the Doris column's fixed slot width.
+ */
+ public double getAvgSizeBytes() {
+ return avgSizeBytes;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (!(o instanceof ConnectorColumnStatistics)) {
+ return false;
+ }
+ ConnectorColumnStatistics that = (ConnectorColumnStatistics) o;
+ return rowCount == that.rowCount
+ && ndv == that.ndv
+ && numNulls == that.numNulls
+ && Double.compare(that.avgSizeBytes, avgSizeBytes) == 0;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(rowCount, ndv, numNulls, avgSizeBytes);
+ }
+
+ @Override
+ public String toString() {
+ return "ConnectorColumnStatistics{rowCount=" + rowCount
+ + ", ndv=" + ndv
+ + ", numNulls=" + numNulls
+ + ", avgSizeBytes=" + avgSizeBytes + "}";
+ }
+}
diff --git a/fe/fe-connector/fe-connector-api/src/main/java/org/apache/doris/connector/api/ConnectorContractValidator.java b/fe/fe-connector/fe-connector-api/src/main/java/org/apache/doris/connector/api/ConnectorContractValidator.java
index fc89fe6c56cfbd..eeacdf232b6441 100644
--- a/fe/fe-connector/fe-connector-api/src/main/java/org/apache/doris/connector/api/ConnectorContractValidator.java
+++ b/fe/fe-connector/fe-connector-api/src/main/java/org/apache/doris/connector/api/ConnectorContractValidator.java
@@ -52,5 +52,20 @@ public static void validate(Connector connector, String catalogType) {
+ "' declares requiresPartitionLocalSort without requiresParallelWrite"
+ " AND requiresFullSchemaWriteOrder");
}
+ // #4 partition-hash-write (hash without sort) likewise implies parallel write AND full-schema write
+ // order (the sink indexes partition columns by full-schema position and distributes in parallel).
+ if (connector.requiresPartitionHashWrite()
+ && !(connector.requiresParallelWrite() && connector.requiresFullSchemaWriteOrder())) {
+ throw new IllegalStateException("Connector '" + catalogType
+ + "' declares requiresPartitionHashWrite without requiresParallelWrite"
+ + " AND requiresFullSchemaWriteOrder");
+ }
+ // #5 the two hash arms are mutually exclusive: the engine checks local-sort first, so declaring both
+ // would silently ignore the hash-without-sort request. Fail loud instead.
+ if (connector.requiresPartitionLocalSort() && connector.requiresPartitionHashWrite()) {
+ throw new IllegalStateException("Connector '" + catalogType
+ + "' declares both requiresPartitionLocalSort and requiresPartitionHashWrite;"
+ + " a connector must pick at most one partition-distribution arm");
+ }
}
}
diff --git a/fe/fe-connector/fe-connector-api/src/main/java/org/apache/doris/connector/api/ConnectorMetadata.java b/fe/fe-connector/fe-connector-api/src/main/java/org/apache/doris/connector/api/ConnectorMetadata.java
index ca7880fec3be85..ae7e1e2df373a0 100644
--- a/fe/fe-connector/fe-connector-api/src/main/java/org/apache/doris/connector/api/ConnectorMetadata.java
+++ b/fe/fe-connector/fe-connector-api/src/main/java/org/apache/doris/connector/api/ConnectorMetadata.java
@@ -20,13 +20,17 @@
import org.apache.doris.connector.api.handle.ConnectorTableHandle;
import org.apache.doris.connector.api.mvcc.ConnectorMvccPartitionView;
import org.apache.doris.connector.api.mvcc.ConnectorMvccSnapshot;
+import org.apache.doris.connector.api.mvcc.ConnectorTableFreshness;
import org.apache.doris.connector.api.mvcc.ConnectorTimeTravelSpec;
+import org.apache.doris.connector.api.pushdown.ConnectorExpression;
import java.io.Closeable;
import java.io.IOException;
import java.util.Collections;
+import java.util.List;
import java.util.Map;
import java.util.Optional;
+import java.util.OptionalLong;
import java.util.Set;
/**
@@ -86,6 +90,43 @@ default Optional getMvccPartitionView(
return Optional.empty();
}
+ /**
+ * Whole-table MTMV freshness for a connector whose table-level change signal is a last-modified
+ * TIMESTAMP rather than a snapshot id (e.g. hive: {@code transient_lastDdlTime} / the max partition
+ * modify time). The generic model wraps the result in an {@code MTMVMaxTimestampSnapshot} table snapshot.
+ *
+ *
Consulted ONLY when the query-begin pin's {@link ConnectorMvccSnapshot#isLastModifiedFreshness()}
+ * is set — i.e. a last-modified connector, which the generic model reads off the pin it already holds.
+ * A snapshot-id connector (paimon/iceberg) leaves the pin flag false, so this is NEVER called for it and it
+ * pays ZERO extra metadata round-trips. And it is on the MTMV refresh path, never the scan hot path — so a
+ * partitioned last-modified connector may pay a {@code get_partitions_by_names} round-trip here (mirroring
+ * legacy, which fetched per-partition modify time only at refresh time) without regressing queries.
+ *
+ *
The default returns empty: a connector that sets the pin flag MUST override this.
+ */
+ default Optional getTableFreshness(
+ ConnectorSession session, ConnectorTableHandle handle) {
+ return Optional.empty();
+ }
+
+ /**
+ * Per-partition last-modified millis for a last-modified connector, wrapped by the generic model in an
+ * {@code MTMVTimestampSnapshot}. Fetched on the MTMV refresh path only — a last-modified connector's
+ * {@code listPartitions} is names-only on the scan hot path (its per-partition {@code lastModifiedMillis}
+ * stays {@code -1}), so the real time is fetched here on demand instead.
+ *
+ *
Consulted ONLY when the query-begin pin's {@link ConnectorMvccSnapshot#isLastModifiedFreshness()}
+ * is set (a snapshot-id connector never reaches here — zero extra round-trips). fe-core validates the
+ * partition exists in the materialized set BEFORE calling this; an {@code empty} return therefore means the
+ * partition VANISHED between the materialize and this fetch (a refresh-time race), and fe-core raises the
+ * legacy "can not find partition" error. The default returns empty: a last-modified connector MUST
+ * override it.
+ */
+ default OptionalLong getPartitionFreshnessMillis(
+ ConnectorSession session, ConnectorTableHandle handle, String partitionName) {
+ return OptionalLong.empty();
+ }
+
/**
* Resolves an explicit time-travel spec (extracted from {@code FOR TIME AS OF} /
* {@code FOR VERSION AS OF}, or the {@code @tag} / {@code @branch} / {@code @incr}
@@ -125,6 +166,27 @@ default ConnectorTableHandle applySnapshot(ConnectorSession session,
return handle; // default: connectors without time-travel ignore the pin
}
+ /**
+ * Returns extra scan-level predicates the engine MUST apply for {@code handle} at the pinned
+ * {@code snapshot} — a connector "residual predicate" the read cannot enforce by file selection alone.
+ * The canonical case is an incremental / CDC commit-time window: a rewritten base file also carries
+ * forward out-of-window rows, so a ROW-LEVEL commit-time filter is required for correctness. The engine
+ * reverse-converts each returned {@link ConnectorExpression} into its native predicate and wraps a filter
+ * over the scan, binding column references to the connector's own (visible) output columns by name.
+ *
+ *
The predicate is expressed in the connector-neutral {@link ConnectorExpression} pushdown grammar,
+ * NOT a source-specific shape — the engine NEVER discriminates by connector here; it applies whatever the
+ * connector returns. This mirrors the engine-agnostic residual-predicate model.
+ *
+ *
The default returns an EMPTY list: a connector with no synthetic scan predicate adds nothing, so the
+ * plan is byte-identical. iceberg/paimon/jdbc/... inherit this empty default; only a connector that opts in
+ * (e.g. hudi incremental read) returns a non-empty list, and only for the scans that need it.
+ */
+ default List getSyntheticScanPredicates(ConnectorSession session,
+ ConnectorTableHandle handle, ConnectorMvccSnapshot snapshot) {
+ return List.of(); // default: connectors without a residual scan predicate add nothing
+ }
+
/**
* Threads a per-group rewrite file scope into the table handle BEFORE {@code planScan}, so the
* distributed {@code rewrite_data_files} driver can scope each per-group INSERT-SELECT scan to only the
@@ -166,6 +228,22 @@ default ConnectorTableHandle applyTopnLazyMaterialization(ConnectorSession sessi
return handle; // default: connectors without column-pruned scan metadata ignore the signal
}
+ /**
+ * Engine-neutral rows for a connector metadata table (e.g. the hudi commit timeline), one row per record in
+ * the fixed column order the table-valued function declares. The TVF owns the column schema; the connector
+ * returns only the {@code String} cell values in that order (a {@code null} cell renders as SQL NULL).
+ * {@code kind} selects the metadata table (currently only {@code "timeline"}).
+ *
+ *
Default empty: a connector without a metadata table returns nothing. The plugin-driven TVF arm gates on
+ * {@link ConnectorCapability#SUPPORTS_METADATA_TABLE} before delegating here, so only a connector that
+ * declares the capability is ever asked. Connectors overriding this that read remote metadata off the
+ * planning thread must pin the TCCL to the plugin classloader themselves (fe-core does not).
+ */
+ default List> getMetadataTableRows(ConnectorSession session, ConnectorTableHandle handle,
+ String kind) {
+ return Collections.emptyList();
+ }
+
@Override
default void close() throws IOException {
}
diff --git a/fe/fe-connector/fe-connector-api/src/main/java/org/apache/doris/connector/api/ConnectorPartitionInfo.java b/fe/fe-connector/fe-connector-api/src/main/java/org/apache/doris/connector/api/ConnectorPartitionInfo.java
index fe345d0b620bce..2eb569d2642b3d 100644
--- a/fe/fe-connector/fe-connector-api/src/main/java/org/apache/doris/connector/api/ConnectorPartitionInfo.java
+++ b/fe/fe-connector/fe-connector-api/src/main/java/org/apache/doris/connector/api/ConnectorPartitionInfo.java
@@ -17,7 +17,9 @@
package org.apache.doris.connector.api;
+import java.util.ArrayList;
import java.util.Collections;
+import java.util.List;
import java.util.Map;
import java.util.Objects;
@@ -36,6 +38,27 @@ public final class ConnectorPartitionInfo {
private final long sizeBytes;
private final long lastModifiedMillis;
private final long fileCount;
+ /**
+ * Per-partition-value SQL-NULL flags, positionally aligned to the values parsed out of
+ * {@link #partitionName} (i.e. flag {@code i} describes the {@code i}-th {@code key=value} segment).
+ * Empty means "no value is NULL" — a connector that does not opt in leaves it empty and the
+ * fe-core partition-item builder treats every value as non-null (unchanged behavior). A connector
+ * that renders a genuine-NULL partition value (e.g. hive's {@code __HIVE_DEFAULT_PARTITION__} or
+ * paimon's {@code partition.default-name}) sets the corresponding flag {@code true} so fe-core builds
+ * a typed {@code NullLiteral} instead of parsing the sentinel string into the column type.
+ */
+ private final List partitionValueNullFlags;
+
+ /**
+ * The RENDERED partition values in name-segment order, positionally aligned to
+ * {@link #partitionValueNullFlags} — i.e. value {@code i} is the value of the {@code i}-th
+ * {@code key=value} segment of {@link #partitionName}, decoded exactly as fe-core's legacy name parse
+ * would produce it (so the connector supplies what fe-core used to re-parse out of the name).
+ * Empty means "not supplied": fe-core then falls back to parsing {@link #partitionName} itself
+ * (unchanged behavior). A connector that lists partitions for the MVCC partition-item path
+ * (hive/paimon/iceberg/hudi) supplies this so fe-core does not re-run the hive-style parse.
+ */
+ private final List orderedPartitionValues;
/**
* Backward-compatible constructor. Numeric stats fields are set to
@@ -48,10 +71,56 @@ public ConnectorPartitionInfo(String partitionName,
UNKNOWN, UNKNOWN, UNKNOWN, UNKNOWN);
}
+ /**
+ * Convenience constructor for a partition with unknown numeric stats but connector-supplied
+ * per-value NULL flags (e.g. hive, which lists names only).
+ */
+ public ConnectorPartitionInfo(String partitionName,
+ Map partitionValues,
+ Map properties,
+ List partitionValueNullFlags) {
+ this(partitionName, partitionValues, properties,
+ UNKNOWN, UNKNOWN, UNKNOWN, UNKNOWN, partitionValueNullFlags);
+ }
+
+ /**
+ * Convenience constructor for a partition with unknown numeric stats but connector-supplied ordered
+ * partition values (and optional per-value NULL flags). Used by the MVCC partition-item connectors
+ * (hive/paimon/iceberg/hudi) so fe-core zips values instead of re-parsing the rendered name.
+ */
+ public ConnectorPartitionInfo(String partitionName,
+ Map partitionValues,
+ Map properties,
+ List orderedPartitionValues,
+ List partitionValueNullFlags) {
+ this(partitionName, partitionValues, properties,
+ UNKNOWN, UNKNOWN, UNKNOWN, UNKNOWN, orderedPartitionValues, partitionValueNullFlags);
+ }
+
public ConnectorPartitionInfo(String partitionName,
Map partitionValues,
Map properties,
long rowCount, long sizeBytes, long lastModifiedMillis, long fileCount) {
+ this(partitionName, partitionValues, properties,
+ rowCount, sizeBytes, lastModifiedMillis, fileCount, Collections.emptyList());
+ }
+
+ public ConnectorPartitionInfo(String partitionName,
+ Map partitionValues,
+ Map properties,
+ long rowCount, long sizeBytes, long lastModifiedMillis, long fileCount,
+ List partitionValueNullFlags) {
+ this(partitionName, partitionValues, properties,
+ rowCount, sizeBytes, lastModifiedMillis, fileCount,
+ Collections.emptyList(), partitionValueNullFlags);
+ }
+
+ public ConnectorPartitionInfo(String partitionName,
+ Map partitionValues,
+ Map properties,
+ long rowCount, long sizeBytes, long lastModifiedMillis, long fileCount,
+ List orderedPartitionValues,
+ List partitionValueNullFlags) {
this.partitionName = Objects.requireNonNull(
partitionName, "partitionName");
this.partitionValues = partitionValues == null
@@ -64,6 +133,12 @@ public ConnectorPartitionInfo(String partitionName,
this.sizeBytes = sizeBytes;
this.lastModifiedMillis = lastModifiedMillis;
this.fileCount = fileCount;
+ this.orderedPartitionValues = orderedPartitionValues == null
+ ? Collections.emptyList()
+ : Collections.unmodifiableList(new ArrayList<>(orderedPartitionValues));
+ this.partitionValueNullFlags = partitionValueNullFlags == null
+ ? Collections.emptyList()
+ : Collections.unmodifiableList(new ArrayList<>(partitionValueNullFlags));
}
public String getPartitionName() {
@@ -98,6 +173,23 @@ public long getFileCount() {
return fileCount;
}
+ /**
+ * @return per-value SQL-NULL flags positionally aligned to the {@link #partitionName} value parse;
+ * empty when the connector did not opt in (no value is NULL). Unmodifiable.
+ */
+ public List getPartitionValueNullFlags() {
+ return partitionValueNullFlags;
+ }
+
+ /**
+ * @return the rendered partition values in name-segment order (aligned to
+ * {@link #getPartitionValueNullFlags()}); empty when the connector did not supply them, in which
+ * case fe-core parses {@link #getPartitionName()} itself. Unmodifiable.
+ */
+ public List getOrderedPartitionValues() {
+ return orderedPartitionValues;
+ }
+
@Override
public boolean equals(Object o) {
if (this == o) {
@@ -113,13 +205,16 @@ public boolean equals(Object o) {
&& fileCount == that.fileCount
&& partitionName.equals(that.partitionName)
&& partitionValues.equals(that.partitionValues)
- && properties.equals(that.properties);
+ && properties.equals(that.properties)
+ && orderedPartitionValues.equals(that.orderedPartitionValues)
+ && partitionValueNullFlags.equals(that.partitionValueNullFlags);
}
@Override
public int hashCode() {
return Objects.hash(partitionName, partitionValues, properties,
- rowCount, sizeBytes, lastModifiedMillis, fileCount);
+ rowCount, sizeBytes, lastModifiedMillis, fileCount,
+ orderedPartitionValues, partitionValueNullFlags);
}
@Override
@@ -128,6 +223,8 @@ public String toString() {
+ "', values=" + partitionValues
+ ", rowCount=" + rowCount
+ ", sizeBytes=" + sizeBytes
- + ", fileCount=" + fileCount + "}";
+ + ", fileCount=" + fileCount
+ + ", orderedValues=" + orderedPartitionValues
+ + ", nullFlags=" + partitionValueNullFlags + "}";
}
}
diff --git a/fe/fe-connector/fe-connector-api/src/main/java/org/apache/doris/connector/api/ConnectorStatisticsOps.java b/fe/fe-connector/fe-connector-api/src/main/java/org/apache/doris/connector/api/ConnectorStatisticsOps.java
index fb762355e53071..2e89a09d7dd881 100644
--- a/fe/fe-connector/fe-connector-api/src/main/java/org/apache/doris/connector/api/ConnectorStatisticsOps.java
+++ b/fe/fe-connector/fe-connector-api/src/main/java/org/apache/doris/connector/api/ConnectorStatisticsOps.java
@@ -19,6 +19,8 @@
import org.apache.doris.connector.api.handle.ConnectorTableHandle;
+import java.util.Collections;
+import java.util.List;
import java.util.Optional;
/**
@@ -32,4 +34,45 @@ default Optional getTableStatistics(
ConnectorTableHandle handle) {
return Optional.empty();
}
+
+ /**
+ * Returns per-column statistics the connector can serve WITHOUT a table scan — the query-planner
+ * column-statistics fast path, consulted on a stats-cache miss (fe-core's
+ * {@code ColumnStatisticsCacheLoader}). Must be cheap (a metadata read, no scan). Returns empty when
+ * unavailable, so a connector with no cheap column stats simply does not override it and fe-core falls
+ * back to a full ANALYZE. fe-core derives the Doris {@code ColumnStatistic} (dataSize / avgSize) from the
+ * returned raw facts — see {@link ConnectorColumnStatistics}.
+ */
+ default Optional getColumnStatistics(
+ ConnectorSession session,
+ ConnectorTableHandle handle,
+ String columnName) {
+ return Optional.empty();
+ }
+
+ /**
+ * Estimates the table's on-disk data size in bytes by listing its data files, for connectors that can
+ * cheaply enumerate them (e.g. hive). fe-core uses this to estimate a row count
+ * ({@code dataSize / }) when neither an exact row count nor a metastore-recorded size (from
+ * {@link #getTableStatistics}) is available — fe-core only calls it when the
+ * {@code enable_get_row_count_from_file_list} global is set. A potentially expensive remote listing, so
+ * connectors that cannot do it cheaply must NOT override it. Returns -1 when the size cannot be
+ * estimated (not supported, unlistable, empty, or any error) — the default.
+ */
+ default long estimateDataSizeByListingFiles(ConnectorSession session, ConnectorTableHandle handle) {
+ return -1;
+ }
+
+ /**
+ * Returns the RAW byte length of every data file across ALL partitions of the table (not sampled, not summed),
+ * for {@code ANALYZE ... WITH SAMPLE}: fe-core seed-shuffles and cumulates these sizes to a sample scale
+ * factor, then does the Doris-type slot-width math itself. Unlike {@link #estimateDataSizeByListingFiles} it
+ * neither partition-samples nor sums, because the sampler needs the individual file sizes. A potentially
+ * expensive full remote listing, so connectors that cannot enumerate files cheaply must NOT override it
+ * (default empty -> the sampler falls back to scale factor 1). Best-effort: an override must return empty on
+ * any listing error rather than throw (statistics must not fail a query).
+ */
+ default List listFileSizes(ConnectorSession session, ConnectorTableHandle handle) {
+ return Collections.emptyList();
+ }
}
diff --git a/fe/fe-connector/fe-connector-api/src/main/java/org/apache/doris/connector/api/ConnectorTableOps.java b/fe/fe-connector/fe-connector-api/src/main/java/org/apache/doris/connector/api/ConnectorTableOps.java
index 61000c4699cdf8..aa6d38ef669983 100644
--- a/fe/fe-connector/fe-connector-api/src/main/java/org/apache/doris/connector/api/ConnectorTableOps.java
+++ b/fe/fe-connector/fe-connector-api/src/main/java/org/apache/doris/connector/api/ConnectorTableOps.java
@@ -71,6 +71,24 @@ default Optional getSysTableHandle(ConnectorSession sessio
return Optional.empty();
}
+ /**
+ * Whether the named system table of {@code baseTableHandle} is served by the generic
+ * {@code partition_values} table-valued function (fe-core's {@code PartitionsSysTable}) rather
+ * than by a native connector scan. Default {@code false} (native, the {@link #getSysTableHandle}
+ * path).
+ *
+ *
A connector whose partitioned tables expose their partition rows through the generic
+ * partition-values TVF (e.g. hive) overrides this to return {@code true} for that sys-table name;
+ * such a name need NOT return a handle from {@link #getSysTableHandle} (the TVF path never consults
+ * it). fe-core needs the kind at discovery time (before any handle is fetched), so it cannot be
+ * inferred from an empty {@code getSysTableHandle}. {@code sysName} is the bare name (no
+ * {@code "$"}).
+ */
+ default boolean isPartitionValuesSysTable(ConnectorSession session,
+ ConnectorTableHandle baseTableHandle, String sysName) {
+ return false;
+ }
+
/** Returns the schema (columns, format, etc.) for the given table. */
default ConnectorTableSchema getTableSchema(
ConnectorSession session, ConnectorTableHandle handle) {
@@ -92,6 +110,24 @@ default ConnectorTableSchema getTableSchema(
return getTableSchema(session, handle);
}
+ /**
+ * Renders the native {@code SHOW CREATE TABLE} DDL for a table, fetching schema FRESH from the underlying
+ * metastore at call time (bypassing any connector-side table cache) so the returned statement always
+ * reflects the latest remote schema.
+ *
+ *
This is a LAZY, per-call interception point used ONLY by {@code ShowCreateTableCommand}. It intentionally
+ * does NOT participate in the {@code SUPPORTS_SHOW_CREATE_DDL} capability (which gates the engine-assembled
+ * DDL in {@code Env.getDdlStmt} for every caller, including delegated sibling tables and the HTTP schema
+ * endpoint). A connector that does not natively render its own SHOW CREATE returns {@link Optional#empty()},
+ * and the command falls through to the generic {@code Env.getDdlStmt} path unchanged.
+ *
+ * @return the full {@code CREATE TABLE} statement, or {@link Optional#empty()} to defer to the engine
+ */
+ default Optional renderShowCreateTableDdl(
+ ConnectorSession session, ConnectorTableHandle handle) {
+ return Optional.empty();
+ }
+
/** Returns a name-to-handle map for all columns of the table. */
default Map getColumnHandles(
ConnectorSession session, ConnectorTableHandle handle) {
@@ -188,6 +224,21 @@ default void renameTable(ConnectorSession session,
"RENAME TABLE not supported");
}
+ /**
+ * Truncates the table identified by {@code handle}. When {@code partitions} is non-empty only those
+ * partitions are truncated; {@code null} / empty truncates the whole table.
+ *
+ *
Connectors that support {@code TRUNCATE TABLE} override this. The default throws, matching the
+ * pre-flip behavior of the generic bridge (which had no truncate route for the SPI path).
+ *
+ * @throws DorisConnectorException if the connector does not support truncate
+ */
+ default void truncateTable(ConnectorSession session,
+ ConnectorTableHandle handle, List partitions) {
+ throw new DorisConnectorException(
+ "TRUNCATE TABLE not supported");
+ }
+
/**
* Adds a column to the table at the given position.
*
diff --git a/fe/fe-connector/fe-connector-api/src/main/java/org/apache/doris/connector/api/ConnectorTableSchema.java b/fe/fe-connector/fe-connector-api/src/main/java/org/apache/doris/connector/api/ConnectorTableSchema.java
index 8ec6aec6b99823..8a83ad1d95b488 100644
--- a/fe/fe-connector/fe-connector-api/src/main/java/org/apache/doris/connector/api/ConnectorTableSchema.java
+++ b/fe/fe-connector/fe-connector-api/src/main/java/org/apache/doris/connector/api/ConnectorTableSchema.java
@@ -17,10 +17,13 @@
package org.apache.doris.connector.api;
+import java.util.Arrays;
import java.util.Collections;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
+import java.util.Set;
/**
* Describes the schema of a connector table, including its columns
@@ -28,6 +31,16 @@
*/
public final class ConnectorTableSchema {
+ /**
+ * Common prefix for every FE-internal reserved control key below. The connector uses these keys to pass
+ * structural info to fe-core (partition columns / primary keys / SHOW CREATE render hints / per-table
+ * capabilities / distribution columns) INSIDE the same property map that also carries the source table's
+ * user-facing pass-through properties. The {@code __internal.} prefix keeps them out of the namespace a
+ * real user table property would ever use, so a source property can never be mistaken for a control key
+ * (and vice versa). These keys are FE-only — none is forwarded to BE.
+ */
+ public static final String INTERNAL_KEY_PREFIX = "__internal.";
+
/**
* Reserved property key carrying the table location string for SHOW CREATE TABLE rendering.
* Connectors emit it here (rather than as a user-facing property) so the FE renders it as the
@@ -35,7 +48,7 @@ public final class ConnectorTableSchema {
* connector's own user-facing location property (e.g. paimon's SDK {@code path} option, which
* legitimately stays in PROPERTIES).
*/
- public static final String SHOW_LOCATION_KEY = "show.location";
+ public static final String SHOW_LOCATION_KEY = INTERNAL_KEY_PREFIX + "show.location";
/**
* Reserved property key carrying the fully-rendered {@code PARTITION BY ...} clause (Doris SQL,
@@ -43,13 +56,69 @@ public final class ConnectorTableSchema {
* The connector pre-renders it (the transform-aware logic is connector-specific); the FE appends
* it verbatim and strips it from PROPERTIES.
*/
- public static final String SHOW_PARTITION_CLAUSE_KEY = "show.partition-clause";
+ public static final String SHOW_PARTITION_CLAUSE_KEY = INTERNAL_KEY_PREFIX + "show.partition-clause";
/**
* Reserved property key carrying the fully-rendered {@code ORDER BY (...)} clause for SHOW CREATE
* TABLE. The connector pre-renders it; the FE appends it verbatim and strips it from PROPERTIES.
*/
- public static final String SHOW_SORT_CLAUSE_KEY = "show.sort-clause";
+ public static final String SHOW_SORT_CLAUSE_KEY = INTERNAL_KEY_PREFIX + "show.sort-clause";
+
+ /**
+ * Reserved property key carrying a CSV of the RAW remote partition-column names (declaration order), so
+ * the generic fe-core consumer ({@code PluginDrivenExternalTable.toSchemaCacheValue}) can model the table
+ * as partitioned. Emitted by every connector that has partition columns (hive/hudi/iceberg/paimon/
+ * maxcompute). FE-only (BE receives partition columns via the separate {@code path_partition_keys} scan
+ * property); stripped from the user-facing SHOW CREATE TABLE PROPERTIES(...) block.
+ */
+ public static final String PARTITION_COLUMNS_KEY = INTERNAL_KEY_PREFIX + "partition_columns";
+
+ /**
+ * Reserved property key carrying a CSV of the table's primary-key column names (paimon). FE-only;
+ * stripped from the user-facing SHOW CREATE TABLE PROPERTIES(...) block.
+ */
+ public static final String PRIMARY_KEYS_KEY = INTERNAL_KEY_PREFIX + "primary_keys";
+
+ /**
+ * Reserved property key carrying a CSV of {@link ConnectorCapability#name()} values that THIS specific
+ * table supports, refining the connector-wide {@link Connector#getCapabilities()} set per-table.
+ *
+ *
A uniform-format connector (e.g. iceberg — every table orc/parquet) declares a scan capability for all
+ * its tables connector-wide. A heterogeneous connector (e.g. hive — orc/parquet/text/json/csv/view/hudi in
+ * one catalog) whose eligibility is per-table file-format gated cannot: Top-N lazy materialization and
+ * nested-column pruning are orc/parquet-only, and blanket-declaring them connector-wide would over-admit a
+ * text/json table (a correctness bug for nested-column pruning, which reads NULL leaves without field ids).
+ * Such a connector instead emits the capability name here, per-table, computed from that table's format.
+ *
+ *
fe-core reads it ADDITIVELY (a capability counts as supported if it is in the connector-wide set OR in
+ * this per-table list) from the already-cached schema — no remote round-trip and no file-format inspection
+ * in fe-core. Single-format connectors never emit it and are unaffected. Stripped from the user-facing
+ * SHOW CREATE TABLE PROPERTIES(...) block.
+ */
+ public static final String PER_TABLE_CAPABILITIES_KEY = INTERNAL_KEY_PREFIX + "connector.per-table-capabilities";
+
+ /**
+ * Reserved property key carrying a CSV of the table's distribution (bucketing) column names, already
+ * lowercased. A heterogeneous connector (hive) whose bucketing varies per table cannot express it as a
+ * connector-wide trait, so it emits it here per-table.
+ *
+ *
fe-core's {@code PluginDrivenExternalTable.getDistributionColumnNames()} reads it from the cached schema
+ * (no remote round-trip) so a flipped bucketed hive table picks the same NDV estimator as legacy
+ * {@code HMSExternalTable.getDistributionColumnNames} (a single bucket column selects the linear estimator in
+ * sampled analyze). A non-bucketed table omits it and connectors with no bucketing concept never emit it.
+ * Stripped from the user-facing SHOW CREATE TABLE PROPERTIES(...) block.
+ */
+ public static final String DISTRIBUTION_COLUMNS_KEY = INTERNAL_KEY_PREFIX + "connector.distribution-columns";
+
+ /**
+ * The full set of FE-internal reserved control keys. fe-core strips every member from the user-facing
+ * SHOW CREATE TABLE PROPERTIES(...) block; centralizing the set here means adding a new reserved key is a
+ * single edit that the strip picks up automatically. All members share {@link #INTERNAL_KEY_PREFIX}.
+ */
+ public static final Set RESERVED_CONTROL_KEYS = Collections.unmodifiableSet(new HashSet<>(
+ Arrays.asList(
+ PARTITION_COLUMNS_KEY, PRIMARY_KEYS_KEY, SHOW_LOCATION_KEY, SHOW_PARTITION_CLAUSE_KEY,
+ SHOW_SORT_CLAUSE_KEY, PER_TABLE_CAPABILITIES_KEY, DISTRIBUTION_COLUMNS_KEY)));
private final String tableName;
private final List columns;
diff --git a/fe/fe-connector/fe-connector-api/src/main/java/org/apache/doris/connector/api/ConnectorWriteOps.java b/fe/fe-connector/fe-connector-api/src/main/java/org/apache/doris/connector/api/ConnectorWriteOps.java
index c5e60d8d697aa4..fefacb7abaa8cd 100644
--- a/fe/fe-connector/fe-connector-api/src/main/java/org/apache/doris/connector/api/ConnectorWriteOps.java
+++ b/fe/fe-connector/fe-connector-api/src/main/java/org/apache/doris/connector/api/ConnectorWriteOps.java
@@ -73,6 +73,24 @@ default void validateStaticPartitionColumns(ConnectorSession session, ConnectorT
// default: no static-partition constraint
}
+ /**
+ * Validates that the dynamic partition-NAME list form ({@code INSERT [OVERWRITE] ... PARTITION (p1, p2)} — a
+ * list of partition column NAMES with no values, distinct from the static {@code PARTITION(col=val)} spec) is
+ * permitted on {@code handle}, throwing a {@link DorisConnectorException} with a connector-authored message
+ * otherwise. Called at analysis time, before synthesizing the write plan, so the engine rejects an
+ * unsupported statement up front (fail loud).
+ *
+ *
The default accepts everything: connectors that ignore the name-list form need not override. A connector
+ * that must reject it (e.g. hive, where {@code INSERT ... PARTITION(p1, p2)} is unsupported) MUST override
+ * this and throw, so the rejection — and its message — stay in the connector rather than being drafted by the
+ * engine. {@code partitionNames} is the list of partition column names from the PARTITION clause (the engine
+ * calls this only when the list is non-empty).
+ */
+ default void validateWritePartitionNames(ConnectorSession session, ConnectorTableHandle handle,
+ List partitionNames) {
+ // default: no partition-name-list constraint
+ }
+
// ──────────────────── TRANSACTION ────────────────────
/**
@@ -91,4 +109,19 @@ default void validateStaticPartitionColumns(ConnectorSession session, ConnectorT
default ConnectorTransaction beginTransaction(ConnectorSession session) {
throw new DorisConnectorException("Transactions not supported");
}
+
+ /**
+ * Per-table view of {@link #beginTransaction(ConnectorSession)}: opens the transaction for the connector
+ * that owns {@code handle}. The default ignores {@code handle} and returns the connector-level
+ * {@link #beginTransaction(ConnectorSession)}, so every single-format connector is unaffected.
+ *
+ *
A heterogeneous gateway (one catalog serving multiple table formats) overrides this to route a foreign
+ * handle to its sibling connector's transaction, so the session-bound transaction's concrete type matches
+ * the per-handle-selected write plan provider. The no-arg version alone would bind the gateway's own
+ * transaction and the sibling's write plan would fail to downcast it. Mirrors the per-handle
+ * {@code getWritePlanProvider(handle)} seam.
+ */
+ default ConnectorTransaction beginTransaction(ConnectorSession session, ConnectorTableHandle handle) {
+ return beginTransaction(session);
+ }
}
diff --git a/fe/fe-connector/fe-connector-api/src/main/java/org/apache/doris/connector/api/ddl/ConnectorPartitionSpec.java b/fe/fe-connector/fe-connector-api/src/main/java/org/apache/doris/connector/api/ddl/ConnectorPartitionSpec.java
index 2414661f3ed87f..8397fe271d3f84 100644
--- a/fe/fe-connector/fe-connector-api/src/main/java/org/apache/doris/connector/api/ddl/ConnectorPartitionSpec.java
+++ b/fe/fe-connector/fe-connector-api/src/main/java/org/apache/doris/connector/api/ddl/ConnectorPartitionSpec.java
@@ -46,10 +46,18 @@ public enum Style {
private final Style style;
private final List fields;
private final List initialValues;
+ private final boolean hasExplicitPartitionValues;
public ConnectorPartitionSpec(Style style,
List fields,
List initialValues) {
+ this(style, fields, initialValues, false);
+ }
+
+ public ConnectorPartitionSpec(Style style,
+ List fields,
+ List initialValues,
+ boolean hasExplicitPartitionValues) {
this.style = Objects.requireNonNull(style, "style");
this.fields = fields == null
? Collections.emptyList()
@@ -57,6 +65,7 @@ public ConnectorPartitionSpec(Style style,
this.initialValues = initialValues == null
? Collections.emptyList()
: Collections.unmodifiableList(initialValues);
+ this.hasExplicitPartitionValues = hasExplicitPartitionValues;
}
public Style getStyle() {
@@ -71,6 +80,18 @@ public List getInitialValues() {
return initialValues;
}
+ /**
+ * Whether the CREATE TABLE declared explicit partition value definitions (e.g.
+ * {@code PARTITION BY LIST(dt) (PARTITION p1 VALUES IN ('a'))}). The neutral converter does not lower
+ * those value expressions into {@link #getInitialValues()} (it stays empty), so this flag preserves the
+ * information a connector needs to reject them: Hive external tables discover partitions from the data
+ * layout and reject explicit partition values (legacy parity). Connectors that accept explicit partition
+ * definitions ignore this flag.
+ */
+ public boolean hasExplicitPartitionValues() {
+ return hasExplicitPartitionValues;
+ }
+
@Override
public boolean equals(Object o) {
if (this == o) {
@@ -81,19 +102,21 @@ public boolean equals(Object o) {
}
ConnectorPartitionSpec that = (ConnectorPartitionSpec) o;
return style == that.style
+ && hasExplicitPartitionValues == that.hasExplicitPartitionValues
&& fields.equals(that.fields)
&& initialValues.equals(that.initialValues);
}
@Override
public int hashCode() {
- return Objects.hash(style, fields, initialValues);
+ return Objects.hash(style, fields, initialValues, hasExplicitPartitionValues);
}
@Override
public String toString() {
return "ConnectorPartitionSpec{style=" + style
+ ", fields=" + fields
- + ", initialValues=" + initialValues.size() + "}";
+ + ", initialValues=" + initialValues.size()
+ + ", hasExplicitPartitionValues=" + hasExplicitPartitionValues + "}";
}
}
diff --git a/fe/fe-connector/fe-connector-api/src/main/java/org/apache/doris/connector/api/event/ConnectorEventSource.java b/fe/fe-connector/fe-connector-api/src/main/java/org/apache/doris/connector/api/event/ConnectorEventSource.java
new file mode 100644
index 00000000000000..1955ef681ed11e
--- /dev/null
+++ b/fe/fe-connector/fe-connector-api/src/main/java/org/apache/doris/connector/api/event/ConnectorEventSource.java
@@ -0,0 +1,58 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.connector.api.event;
+
+/**
+ * A connector's incremental-metadata-change source: it fetches the underlying metastore's native
+ * notification events and parses them into connector-neutral {@link MetastoreChangeDescriptor}s.
+ *
+ *
Surfaced via {@link org.apache.doris.connector.api.Connector#getEventSource()} (a
+ * capability-probe getter, NOT an {@code instanceof} — a connector without an event source returns
+ * {@code null}). The engine runs one connector-agnostic, role-aware background driver that iterates
+ * catalogs, calls {@link #pollOnce} on those whose connector exposes a source, and applies the
+ * returned descriptors to its own object graph and caches.
+ *
+ *
Engine/connector split (Trino-aligned: engine owns HA/replication, plugin owns fetch/parse).
+ * The connector owns ONLY the metastore RPC and the message deserialization/merge behind {@link
+ * #getCurrentEventId()} and {@link #pollOnce}. The engine owns everything stateful and replicated: the
+ * per-catalog cursor (passed in via {@link EventPollRequest}, stored from {@link EventPollResult}),
+ * the master/follower role, the edit-log write of the synced cursor, and the follower→master
+ * {@code REFRESH CATALOG} forward. The connector is stateless with respect to the cursor.
+ *
+ *
Classloader. The engine calls these methods under a context-classloader pin to this
+ * source's own plugin classloader, so the notification RPC and the JSON/GZIP deserialization inside
+ * resolve the plugin's bundled metastore classes. Implementations therefore need no pin of their own
+ * for the fetch/parse path.
+ */
+public interface ConnectorEventSource {
+
+ /**
+ * The metastore's current (latest) notification event id. Used by the master to cheaply decide
+ * whether there is anything new to pull before calling {@link #pollOnce}. Returns {@code -1} if the
+ * id cannot be read this cycle.
+ */
+ long getCurrentEventId();
+
+ /**
+ * Fetch and parse one batch of notification events for a catalog, returning connector-neutral
+ * descriptors plus the new cursor and an optional full-refresh signal. Never returns source-native
+ * types. See {@link EventPollRequest} / {@link EventPollResult} for the role/cursor/refresh
+ * semantics. The batch size is the connector's own concern (read from its catalog properties).
+ */
+ EventPollResult pollOnce(EventPollRequest request);
+}
diff --git a/fe/fe-connector/fe-connector-api/src/main/java/org/apache/doris/connector/api/event/EventPollRequest.java b/fe/fe-connector/fe-connector-api/src/main/java/org/apache/doris/connector/api/event/EventPollRequest.java
new file mode 100644
index 00000000000000..4662c308b22e51
--- /dev/null
+++ b/fe/fe-connector/fe-connector-api/src/main/java/org/apache/doris/connector/api/event/EventPollRequest.java
@@ -0,0 +1,66 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.connector.api.event;
+
+/**
+ * The engine-supplied input to one {@link ConnectorEventSource#pollOnce} call. The engine owns the
+ * cursor and the role; the connector owns the fetch strategy those imply.
+ *
+ *
+ *
{@link #getLastSyncedEventId()} — the last event id this FE has already applied for the
+ * catalog (the engine's per-catalog cursor; {@code -1} means "never synced" → the connector
+ * should signal a full refresh rather than replay from 0).
+ *
{@link #isMaster()} — whether this FE is the master. The master fetches the metastore's
+ * current event id and pulls new events directly; a follower pulls only up to
+ * {@link #getMasterUpperBound()} (what the master has already committed and replicated), so it
+ * never reads past the master's high-water mark.
+ *
{@link #getMasterUpperBound()} — the master's committed event id, learned by the follower via
+ * edit-log replay; ignored on the master. {@code -1} means "not yet learned" → a follower
+ * does nothing this cycle.
+ *
+ */
+public final class EventPollRequest {
+
+ private final long lastSyncedEventId;
+ private final boolean isMaster;
+ private final long masterUpperBound;
+
+ public EventPollRequest(long lastSyncedEventId, boolean isMaster, long masterUpperBound) {
+ this.lastSyncedEventId = lastSyncedEventId;
+ this.isMaster = isMaster;
+ this.masterUpperBound = masterUpperBound;
+ }
+
+ public long getLastSyncedEventId() {
+ return lastSyncedEventId;
+ }
+
+ public boolean isMaster() {
+ return isMaster;
+ }
+
+ public long getMasterUpperBound() {
+ return masterUpperBound;
+ }
+
+ @Override
+ public String toString() {
+ return "EventPollRequest{lastSyncedEventId=" + lastSyncedEventId + ", isMaster=" + isMaster
+ + ", masterUpperBound=" + masterUpperBound + '}';
+ }
+}
diff --git a/fe/fe-connector/fe-connector-api/src/main/java/org/apache/doris/connector/api/event/EventPollResult.java b/fe/fe-connector/fe-connector-api/src/main/java/org/apache/doris/connector/api/event/EventPollResult.java
new file mode 100644
index 00000000000000..9673ddc3fe44bc
--- /dev/null
+++ b/fe/fe-connector/fe-connector-api/src/main/java/org/apache/doris/connector/api/event/EventPollResult.java
@@ -0,0 +1,89 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.connector.api.event;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * The connector's answer to one {@link ConnectorEventSource#pollOnce} call.
+ *
+ *
+ *
{@link #getNewCursor()} — the event id the engine should store as its new per-catalog cursor.
+ * This is NOT always {@code lastSyncedEventId + descriptors.size()}: on a first sync or an
+ * events-gap recovery the connector advances the cursor to the metastore's current id WITHOUT
+ * emitting events (paired with {@link #isNeedsFullRefresh()}); on an empty poll it is the
+ * unchanged cursor.
+ *
{@link #getDescriptors()} — the already-merged, connector-neutral changes to apply, in order.
+ * Empty when nothing changed or when a full refresh is requested instead.
+ *
{@link #isNeedsFullRefresh()} — the connector could not produce an incremental delta (first
+ * sync, or the metastore reported its notification log was trimmed past the cursor). The engine
+ * responds per role: the master invalidates the whole catalog; a follower forwards
+ * {@code REFRESH CATALOG} to the master. The connector owns detecting the source-specific gap
+ * condition; the engine owns the HA response.
+ *
+ */
+public final class EventPollResult {
+
+ private final long newCursor;
+ private final List descriptors;
+ private final boolean needsFullRefresh;
+
+ public EventPollResult(long newCursor, List descriptors,
+ boolean needsFullRefresh) {
+ this.newCursor = newCursor;
+ this.descriptors = descriptors == null
+ ? Collections.emptyList()
+ : Collections.unmodifiableList(new ArrayList<>(descriptors));
+ this.needsFullRefresh = needsFullRefresh;
+ }
+
+ /** A result carrying incremental changes; the cursor advances to {@code newCursor}. */
+ public static EventPollResult ofChanges(long newCursor, List descriptors) {
+ return new EventPollResult(newCursor, descriptors, false);
+ }
+
+ /** A result requesting a full catalog refresh and seeding the cursor to {@code newCursor}. */
+ public static EventPollResult ofFullRefresh(long newCursor) {
+ return new EventPollResult(newCursor, Collections.emptyList(), true);
+ }
+
+ /** A no-op result: nothing changed, keep the cursor at {@code cursor}. */
+ public static EventPollResult ofNothing(long cursor) {
+ return new EventPollResult(cursor, Collections.emptyList(), false);
+ }
+
+ public long getNewCursor() {
+ return newCursor;
+ }
+
+ public List getDescriptors() {
+ return descriptors;
+ }
+
+ public boolean isNeedsFullRefresh() {
+ return needsFullRefresh;
+ }
+
+ @Override
+ public String toString() {
+ return "EventPollResult{newCursor=" + newCursor + ", descriptors=" + descriptors.size()
+ + ", needsFullRefresh=" + needsFullRefresh + '}';
+ }
+}
diff --git a/fe/fe-connector/fe-connector-api/src/main/java/org/apache/doris/connector/api/event/MetastoreChangeDescriptor.java b/fe/fe-connector/fe-connector-api/src/main/java/org/apache/doris/connector/api/event/MetastoreChangeDescriptor.java
new file mode 100644
index 00000000000000..a491c1fe58cd46
--- /dev/null
+++ b/fe/fe-connector/fe-connector-api/src/main/java/org/apache/doris/connector/api/event/MetastoreChangeDescriptor.java
@@ -0,0 +1,182 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.connector.api.event;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Objects;
+
+/**
+ * A single connector-neutral description of one metadata change the engine must apply after a
+ * metastore poll. It is the vocabulary {@link ConnectorEventSource#pollOnce} returns: the plugin
+ * fetches and parses the source's native notification events (Hive {@code NotificationEvent} +
+ * JSON/GZIP messages, iceberg/hudi-on-HMS included) and maps each to one of these, so the engine can
+ * update its catalog→db→table object graph and caches WITHOUT ever seeing a source-native type.
+ *
+ *
Type-neutrality is load-bearing. Every field is a primitive, a {@code String}, or a
+ * {@code List} — no Hive/thrift class ever crosses this boundary. The engine calls the plugin
+ * under a classloader pin; if a plugin-loaded object leaked into a field, a field access on the engine
+ * side would fail across the loader split. Keep it neutral.
+ *
+ *
Engine/connector split. The connector owns fetch+parse and emits these descriptors
+ * (already merged/deduplicated). The engine owns the structural application — register/unregister/
+ * rename in {@code CatalogMgr}, cache invalidation via {@code Connector.invalidateTable/Db/Partition},
+ * the master's edit-log write, and follower/master role handling. Trino-aligned: the plugin surfaces
+ * what changed; the engine decides how its own metadata reacts.
+ *
+ *
Partition granularity is by canonical partition NAME ({@code "col=val/.../colN=valN"}), never by
+ * raw column values — the caches are name-keyed, so carrying names avoids the values→whole-table
+ * degrade.
+ */
+public final class MetastoreChangeDescriptor {
+
+ /**
+ * The kind of change. Maps 1:1 to what the engine does when applying the descriptor. A source
+ * event that changes nothing the engine tracks (e.g. a properties-only ALTER DATABASE, or an
+ * unsupported event type) produces NO descriptor rather than a no-op {@code Op}.
+ */
+ public enum Op {
+ /** A new database exists; register it into the catalog. */
+ REGISTER_DATABASE,
+ /** A database was dropped; unregister it from the catalog. */
+ UNREGISTER_DATABASE,
+ /** A database was renamed; {@link #getDbName()} → {@link #getDbNameAfter()}. */
+ RENAME_DATABASE,
+ /** A new table exists; register it into its database. */
+ REGISTER_TABLE,
+ /** A table was dropped; unregister it from its database. */
+ UNREGISTER_TABLE,
+ /**
+ * A table was renamed OR a view was recreated; {@link #getTableName()} →
+ * {@link #getTableNameAfter()} (the after-name equals the before-name for a view recreate,
+ * which rebuilds the table object so a changed view definition takes effect).
+ */
+ RENAME_TABLE,
+ /** A table changed in place (insert / plain alter); drop its caches, keep it registered. */
+ REFRESH_TABLE,
+ /** Partitions were added; invalidate the table's partition caches so a re-list picks them up. */
+ ADD_PARTITIONS,
+ /** Partitions were dropped; invalidate the table's partition caches. */
+ DROP_PARTITIONS,
+ /** Partitions changed in place; invalidate the named partitions' caches. */
+ REFRESH_PARTITIONS
+ }
+
+ private final Op op;
+ private final String dbName;
+ private final String tableName;
+ private final String dbNameAfter;
+ private final String tableNameAfter;
+ private final List partitionNames;
+ private final long updateTime;
+ private final long eventId;
+
+ private MetastoreChangeDescriptor(Op op, String dbName, String tableName, String dbNameAfter,
+ String tableNameAfter, List partitionNames, long updateTime, long eventId) {
+ this.op = Objects.requireNonNull(op, "op");
+ this.dbName = dbName;
+ this.tableName = tableName;
+ this.dbNameAfter = dbNameAfter;
+ this.tableNameAfter = tableNameAfter;
+ this.partitionNames = partitionNames == null
+ ? Collections.emptyList()
+ : Collections.unmodifiableList(new ArrayList<>(partitionNames));
+ this.updateTime = updateTime;
+ this.eventId = eventId;
+ }
+
+ /** A database-level change ({@code REGISTER_/UNREGISTER_/RENAME_DATABASE}). */
+ public static MetastoreChangeDescriptor forDatabase(Op op, String dbName, String dbNameAfter,
+ long eventId, long updateTime) {
+ return new MetastoreChangeDescriptor(op, dbName, null, dbNameAfter, null, null, updateTime, eventId);
+ }
+
+ /** A table-level change ({@code REGISTER_/UNREGISTER_/RENAME_TABLE}, {@code REFRESH_TABLE}). */
+ public static MetastoreChangeDescriptor forTable(Op op, String dbName, String tableName,
+ String tableNameAfter, long eventId, long updateTime) {
+ return new MetastoreChangeDescriptor(op, dbName, tableName, null, tableNameAfter, null,
+ updateTime, eventId);
+ }
+
+ /**
+ * A {@link Op#RENAME_TABLE} change (also used for view-recreate, where the after-db/table equal the
+ * before-db/table). Carries both the before ({@code dbName}/{@code tableName}) and after
+ * ({@code dbNameAfter}/{@code tableNameAfter}) identities, since a Hive table rename may move the
+ * table across databases.
+ */
+ public static MetastoreChangeDescriptor forTableRename(String dbName, String tableName,
+ String dbNameAfter, String tableNameAfter, long eventId, long updateTime) {
+ return new MetastoreChangeDescriptor(Op.RENAME_TABLE, dbName, tableName, dbNameAfter,
+ tableNameAfter, null, updateTime, eventId);
+ }
+
+ /** A partition-level change ({@code ADD_/DROP_/REFRESH_PARTITIONS}). */
+ public static MetastoreChangeDescriptor forPartitions(Op op, String dbName, String tableName,
+ List partitionNames, long eventId, long updateTime) {
+ return new MetastoreChangeDescriptor(op, dbName, tableName, null, null, partitionNames,
+ updateTime, eventId);
+ }
+
+ public Op getOp() {
+ return op;
+ }
+
+ /** The remote database name (as the connector sees it). */
+ public String getDbName() {
+ return dbName;
+ }
+
+ /** The remote table name, or {@code null} for database-level ops. */
+ public String getTableName() {
+ return tableName;
+ }
+
+ /** The new database name for {@link Op#RENAME_DATABASE}, else {@code null}. */
+ public String getDbNameAfter() {
+ return dbNameAfter;
+ }
+
+ /** The new table name for {@link Op#RENAME_TABLE} (may equal {@link #getTableName()}), else {@code null}. */
+ public String getTableNameAfter() {
+ return tableNameAfter;
+ }
+
+ /** Canonical partition names for partition ops (empty for non-partition ops). */
+ public List getPartitionNames() {
+ return partitionNames;
+ }
+
+ /** The source update time in epoch millis, used as the object's freshness signal. */
+ public long getUpdateTime() {
+ return updateTime;
+ }
+
+ /** The source notification event id this descriptor was derived from. */
+ public long getEventId() {
+ return eventId;
+ }
+
+ @Override
+ public String toString() {
+ return "MetastoreChangeDescriptor{op=" + op + ", db=" + dbName + ", table=" + tableName
+ + ", dbAfter=" + dbNameAfter + ", tableAfter=" + tableNameAfter
+ + ", partitions=" + partitionNames + ", updateTime=" + updateTime
+ + ", eventId=" + eventId + '}';
+ }
+}
diff --git a/fe/fe-connector/fe-connector-api/src/main/java/org/apache/doris/connector/api/mvcc/ConnectorMvccSnapshot.java b/fe/fe-connector/fe-connector-api/src/main/java/org/apache/doris/connector/api/mvcc/ConnectorMvccSnapshot.java
index 7a16661ff84099..d89bc7cc976367 100644
--- a/fe/fe-connector/fe-connector-api/src/main/java/org/apache/doris/connector/api/mvcc/ConnectorMvccSnapshot.java
+++ b/fe/fe-connector/fe-connector-api/src/main/java/org/apache/doris/connector/api/mvcc/ConnectorMvccSnapshot.java
@@ -38,6 +38,7 @@ public final class ConnectorMvccSnapshot {
private final String description;
private final long schemaId;
private final Map properties;
+ private final boolean lastModifiedFreshness;
private ConnectorMvccSnapshot(Builder b) {
this.snapshotId = b.snapshotId;
@@ -47,6 +48,7 @@ private ConnectorMvccSnapshot(Builder b) {
this.properties = b.properties.isEmpty()
? Collections.emptyMap()
: Collections.unmodifiableMap(new HashMap<>(b.properties));
+ this.lastModifiedFreshness = b.lastModifiedFreshness;
}
/** Connector-assigned snapshot identifier (e.g. Iceberg snapshot id). */
@@ -77,6 +79,19 @@ public Map getProperties() {
return properties;
}
+ /**
+ * Whether this table's MTMV freshness is a last-modified TIMESTAMP rather than a snapshot id. When
+ * {@code true} the generic model ({@code PluginDrivenMvccExternalTable}) serves the table/partition MTMV
+ * snapshots from {@code ConnectorMetadata.getTableFreshness} / {@code getPartitionFreshnessMillis} (fetched
+ * on the MTMV refresh path only); when {@code false} (the default, e.g. paimon/iceberg — a snapshot-id
+ * connector) it keeps the snapshot-id / pin-timestamp freshness with NO extra metadata call. The flag rides
+ * on the query-begin pin so fe-core reads it off the pin it already holds — a snapshot-id connector pays
+ * zero extra round-trips.
+ */
+ public boolean isLastModifiedFreshness() {
+ return lastModifiedFreshness;
+ }
+
public static Builder builder() {
return new Builder();
}
@@ -88,12 +103,19 @@ public static final class Builder {
private String description = "";
private long schemaId = -1;
private final Map properties = new HashMap<>();
+ private boolean lastModifiedFreshness;
public Builder snapshotId(long snapshotId) {
this.snapshotId = snapshotId;
return this;
}
+ /** Marks this table's MTMV freshness as last-modified (see {@link #isLastModifiedFreshness()}). */
+ public Builder lastModifiedFreshness(boolean lastModifiedFreshness) {
+ this.lastModifiedFreshness = lastModifiedFreshness;
+ return this;
+ }
+
public Builder timestampMillis(long timestampMillis) {
this.timestampMillis = timestampMillis;
return this;
diff --git a/fe/fe-connector/fe-connector-api/src/main/java/org/apache/doris/connector/api/mvcc/ConnectorTableFreshness.java b/fe/fe-connector/fe-connector-api/src/main/java/org/apache/doris/connector/api/mvcc/ConnectorTableFreshness.java
new file mode 100644
index 00000000000000..b924e403927a95
--- /dev/null
+++ b/fe/fe-connector/fe-connector-api/src/main/java/org/apache/doris/connector/api/mvcc/ConnectorTableFreshness.java
@@ -0,0 +1,86 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.connector.api.mvcc;
+
+import java.util.Objects;
+
+/**
+ * A connector-supplied, whole-table MTMV freshness marker for a connector whose table-level change
+ * signal is a last-modified TIMESTAMP rather than a snapshot id.
+ *
+ *
Returned by {@link org.apache.doris.connector.api.ConnectorMetadata#getTableFreshness}. The
+ * generic table model ({@code PluginDrivenMvccExternalTable}) wraps it into an
+ * {@code MTMVMaxTimestampSnapshot(name, timestampMillis)} — byte-parity with legacy hive, whose table
+ * snapshot is {@code MTMVMaxTimestampSnapshot}. A connector that returns {@link java.util.Optional#empty()}
+ * (the default, e.g. paimon/iceberg) keeps the snapshot-id table snapshot unchanged.
partitioned table ⇒ {@code name} = the partition name carrying the newest modify time,
+ * {@code timestampMillis} = that max modify time (0 when there are no partitions, with
+ * {@code name} = the table name);
+ *
unpartitioned table ⇒ {@code name} = the table name, {@code timestampMillis} = the table's
+ * last-DDL time (0 when absent).
+ *
+ *
+ *
The connector computes the millis (the {@code MTMVMaxTimestampSnapshot} carries BOTH the name and
+ * the timestamp so that dropping the partition that owns the max time is detected as a change); fe-core
+ * never parses the underlying source properties.
+ */
+public final class ConnectorTableFreshness {
+
+ private final String name;
+ private final long timestampMillis;
+
+ public ConnectorTableFreshness(String name, long timestampMillis) {
+ this.name = Objects.requireNonNull(name, "name");
+ this.timestampMillis = timestampMillis;
+ }
+
+ /** The partition name owning the newest modify time (partitioned) or the table name (unpartitioned). */
+ public String getName() {
+ return name;
+ }
+
+ /** The newest modify time in epoch millis (max partition modify time, or the table last-DDL time). */
+ public long getTimestampMillis() {
+ return timestampMillis;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (!(o instanceof ConnectorTableFreshness)) {
+ return false;
+ }
+ ConnectorTableFreshness that = (ConnectorTableFreshness) o;
+ return timestampMillis == that.timestampMillis && name.equals(that.name);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(name, timestampMillis);
+ }
+
+ @Override
+ public String toString() {
+ return "ConnectorTableFreshness{name='" + name + "', timestampMillis=" + timestampMillis + "}";
+ }
+}
diff --git a/fe/fe-connector/fe-connector-api/src/main/java/org/apache/doris/connector/api/scan/ConnectorScanPlanProvider.java b/fe/fe-connector/fe-connector-api/src/main/java/org/apache/doris/connector/api/scan/ConnectorScanPlanProvider.java
index 9d7839ff8897c0..a0cb2c4e4fcf25 100644
--- a/fe/fe-connector/fe-connector-api/src/main/java/org/apache/doris/connector/api/scan/ConnectorScanPlanProvider.java
+++ b/fe/fe-connector/fe-connector-api/src/main/java/org/apache/doris/connector/api/scan/ConnectorScanPlanProvider.java
@@ -21,6 +21,7 @@
import org.apache.doris.connector.api.handle.ConnectorColumnHandle;
import org.apache.doris.connector.api.handle.ConnectorTableHandle;
import org.apache.doris.connector.api.pushdown.ConnectorExpression;
+import org.apache.doris.thrift.TFileCompressType;
import org.apache.doris.thrift.TFileScanRangeParams;
import org.apache.doris.thrift.TTableFormatFileDesc;
@@ -28,6 +29,7 @@
import java.util.List;
import java.util.Map;
import java.util.Optional;
+import java.util.OptionalLong;
/**
* Plans the set of scan ranges (splits) needed to read a connector table.
@@ -107,6 +109,23 @@ default ConnectorColumnCategory classifyColumn(String columnName) {
return ConnectorColumnCategory.DEFAULT;
}
+ /**
+ * Lets a connector adjust the file compression type the generic node inferred from the file path/extension
+ * (via {@code Util.inferFileCompressTypeByPath}) before it is shipped to BE on the scan range. The default
+ * is identity — the inferred type is used as-is.
+ *
+ *
Hive overrides this to remap {@code LZ4FRAME -> LZ4BLOCK}: hadoop/hive write {@code .lz4} files with
+ * the LZ4 block codec, not the LZ4 frame format the {@code .lz4} extension implies, so the frame
+ * inferred from the path would make BE's frame decoder fail ({@code LZ4F_getFrameInfo ERROR_frameType_unknown}).
+ * This keeps that hadoop-specific fact inside the connector; the generic node stays connector-agnostic.
+ *
+ * @param inferred the compression type the generic node inferred from the file path
+ * @return the compression type to actually send to BE (identity by default)
+ */
+ default TFileCompressType adjustFileCompressType(TFileCompressType inferred) {
+ return inferred;
+ }
+
/**
* Plans the scan for the given table, returning a list of scan ranges.
*
@@ -232,6 +251,67 @@ default boolean supportsBatchScan(ConnectorSession session, ConnectorTableHandle
return false;
}
+ /**
+ * Whether this connector's scan ranges carry meaningful byte lengths
+ * ({@link ConnectorScanRange#getLength()}) so the engine can apply {@code TABLESAMPLE} by
+ * size-weighted split selection ({@code PluginDrivenScanNode.sampleSplits}). Returning
+ * {@code false} (the default) makes {@code TABLESAMPLE} a no-op — the full table is scanned (with a
+ * warning) — matching these connectors' behavior before the SPI migration (only the legacy Hive scan
+ * node ever sampled). A connector must NOT return {@code true} unless EVERY range it plans exposes a
+ * positive, byte-proportional length: MaxCompute's default byte-size ranges and Paimon's JNI-read
+ * ranges report {@code -1}, and MaxCompute row_offset ranges report a ROW count (not bytes), so they
+ * must stay {@code false}. Mirrors {@link #supportsBatchScan}'s opt-in shape and Trino's
+ * {@code ConnectorMetadata.applySample}.
+ *
+ * @return whether split-size TABLESAMPLE is valid for this connector (default: false)
+ */
+ default boolean supportsTableSample() {
+ return false;
+ }
+
+ /**
+ * The number of DISTINCT native partitions among the just-planned scan ranges — the count the
+ * connector's SDK actually resolved after ITS full manifest/residual/transform/bucket pruning.
+ * Feeds the scan node's {@code selectedPartitionNum} (EXPLAIN {@code partition=N/M} and the
+ * {@code sql_block_rule} {@code partition_num} guard), so the reported count reflects what is
+ * really scanned rather than the engine's declared-partition-column (Nereids) prune count.
+ *
+ *
The default returns {@link OptionalLong#empty()}, so the generic node keeps its Nereids-pruned
+ * count — correct for directory-partitioned / requiredPartition-driven connectors (hive, MaxCompute),
+ * where the two coincide. A predicate-driven connector whose SDK prunes beyond the engine (Paimon
+ * manifest pruning, Iceberg hidden/transform partitioning) overrides this. Mirrors the
+ * {@link #supportsTableSample} opt-in shape; the connector downcasts its OWN range type (it produced
+ * these very ranges) to read partition identity, so the generic node stays connector-agnostic. It
+ * must never over-count (each native partition must map to exactly one identity within a scan), so it
+ * can only tighten, never loosen, the {@code partition_num} guard relative to the Nereids count.
+ *
+ * @param scanRanges the ranges this provider just returned from {@code planScan}
+ * @return the distinct scanned-partition count, or empty to keep the engine's Nereids count (default)
+ */
+ default OptionalLong scannedPartitionCount(List scanRanges) {
+ return OptionalLong.empty();
+ }
+
+ /**
+ * Connector SDK scan diagnostics harvested during the just-finished {@link #planScan} — manifest cache
+ * hit/miss, scan/planning durations, files and manifests scanned vs skipped — as connector-neutral
+ * {@link ConnectorScanProfile} groups the engine writes into the query's profile execution summary.
+ *
+ *
The default returns an empty list (connector reports nothing). A connector that wants scan
+ * diagnostics harvests them from its SDK during {@code planScan} (the paimon SDK exposes a metric
+ * registry, the iceberg SDK a metrics reporter), stashes them keyed by {@link ConnectorSession#getQueryId()},
+ * and drains them here — mirroring the per-query queryId stashes this SPI already uses (read-transaction
+ * release, rewritable-delete supply). The engine calls this immediately after {@code planScan} on the
+ * same thread, so the harvest is complete; the connector must also drop its stash on
+ * {@link #releaseReadTransaction} to reclaim any entry a thrown {@code planScan} left behind.
+ *
+ * @param session the current session (its queryId keys the connector's per-query stash)
+ * @return this scan's diagnostics, or an empty list (the default) to contribute nothing to the profile
+ */
+ default List collectScanProfiles(ConnectorSession session) {
+ return Collections.emptyList();
+ }
+
/**
* Plans the scan for a single batch of partitions (used by batch-mode scans).
*
@@ -440,4 +520,23 @@ default List getDeleteFiles(TTableFormatFileDesc tableFormatParams) {
default String getSerializedTable(Map nodeProperties) {
return null;
}
+
+ /**
+ * Releases any per-query read transaction this provider opened, called by the engine when the query
+ * finishes (via the generic query-finish callback registry). The default is a no-op: connectors that do
+ * not open a per-query read transaction (every connector except transactional/ACID hive) need not override
+ * it.
+ *
+ *
A connector that opens a metastore read transaction + shared read lock during {@link #planScan} (hive
+ * full-ACID / insert-only reads) MUST override this to commit that transaction and release the lock, else
+ * the shared read lock leaks for the metastore's lifetime. Best-effort: an implementation should log and
+ * swallow a commit failure rather than propagate (the callback registry isolates exceptions anyway).
+ * {@code queryId} is the engine query id string ({@link ConnectorSession#getQueryId()}), the same key the
+ * provider registered the transaction under.
+ *
+ * @param queryId the finishing query's id (== {@link ConnectorSession#getQueryId()})
+ */
+ default void releaseReadTransaction(String queryId) {
+ // default: no per-query read transaction to release
+ }
}
diff --git a/fe/fe-connector/fe-connector-api/src/main/java/org/apache/doris/connector/api/scan/ConnectorScanProfile.java b/fe/fe-connector/fe-connector-api/src/main/java/org/apache/doris/connector/api/scan/ConnectorScanProfile.java
new file mode 100644
index 00000000000000..034941a24261d9
--- /dev/null
+++ b/fe/fe-connector/fe-connector-api/src/main/java/org/apache/doris/connector/api/scan/ConnectorScanProfile.java
@@ -0,0 +1,66 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.connector.api.scan;
+
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+/**
+ * A connector-neutral bundle of scan diagnostics for one table scan, produced by a connector from its own
+ * SDK metrics and written verbatim into the query profile by the generic scan node
+ * (via {@link ConnectorScanPlanProvider#collectScanProfiles}).
+ *
+ *
The engine treats all three fields opaquely — it get-or-creates a profile group named
+ * {@link #getGroupName()} under the execution summary, adds a child named {@link #getScanLabel()}, and
+ * writes each {@link #getMetrics()} entry as an info string. This keeps the engine connector-agnostic:
+ * it never interprets a metric, only the connector knows what its SDK exposes (paimon manifest cache
+ * hit/miss, iceberg scanned/skipped manifests, etc.). Values are ALREADY formatted strings because the
+ * formatting (durations, byte sizes) lives with the SDK-specific harvest in the connector.
+ *
+ *
Immutable: the metrics map is copied into an unmodifiable, insertion-ordered view so the profile
+ * rendering order is stable.
+ */
+public final class ConnectorScanProfile {
+ private final String groupName;
+ private final String scanLabel;
+ private final Map metrics;
+
+ public ConnectorScanProfile(String groupName, String scanLabel, Map metrics) {
+ this.groupName = groupName;
+ this.scanLabel = scanLabel;
+ this.metrics = metrics == null
+ ? Collections.emptyMap()
+ : Collections.unmodifiableMap(new LinkedHashMap<>(metrics));
+ }
+
+ /** The profile group name (e.g. {@code "Paimon Scan Metrics"}); must match the engine's ordering key. */
+ public String getGroupName() {
+ return groupName;
+ }
+
+ /** The per-scan child label (e.g. {@code "Table Scan (db.tbl)"}). */
+ public String getScanLabel() {
+ return scanLabel;
+ }
+
+ /** The ordered metric name → already-formatted value pairs (unmodifiable). */
+ public Map getMetrics() {
+ return metrics;
+ }
+}
diff --git a/fe/fe-connector/fe-connector-api/src/main/java/org/apache/doris/connector/api/write/ConnectorWritePlanProvider.java b/fe/fe-connector/fe-connector-api/src/main/java/org/apache/doris/connector/api/write/ConnectorWritePlanProvider.java
index f313cefb0dbe4c..5de4cc93713b21 100644
--- a/fe/fe-connector/fe-connector-api/src/main/java/org/apache/doris/connector/api/write/ConnectorWritePlanProvider.java
+++ b/fe/fe-connector/fe-connector-api/src/main/java/org/apache/doris/connector/api/write/ConnectorWritePlanProvider.java
@@ -195,6 +195,22 @@ default boolean requiresPartitionLocalSort() {
return false;
}
+ /**
+ * Whether dynamic-partition writes must be hash-distributed by partition columns but not locally
+ * sorted by them. A hive-style file writer buffers a separate partition writer per partition value, so —
+ * unlike {@link #requiresPartitionLocalSort()} (MaxCompute's streaming Storage-API writer, which closes a
+ * partition writer as soon as a different partition value appears and therefore needs the rows grouped by a
+ * local sort) — the hash distribution alone keeps each partition's rows on one instance and the output file
+ * count at ~one-per-partition, with no sort cost. The engine ({@code PhysicalConnectorTableSink}) picks the
+ * matching distribution: {@code requiresPartitionLocalSort} ⇒ hash + {@code MustLocalSortOrderSpec};
+ * {@code requiresPartitionHashWrite} ⇒ hash only. A connector sets at most one of the two hash arms; a
+ * connector declaring this should also declare {@link #requiresParallelWrite()} +
+ * {@link #requiresFullSchemaWriteOrder()}. Default: no.
+ */
+ default boolean requiresPartitionHashWrite() {
+ return false;
+ }
+
/**
* Whether the connector's data files physically retain partition columns, so a static-partition write
* must materialize the PARTITION-clause literal into the data column instead of NULL-filling it (e.g.
diff --git a/fe/fe-connector/fe-connector-api/src/test/java/org/apache/doris/connector/api/ConnectorPartitionInfoTest.java b/fe/fe-connector/fe-connector-api/src/test/java/org/apache/doris/connector/api/ConnectorPartitionInfoTest.java
index fb79235d2ae73c..dc2c5d2f3afad7 100644
--- a/fe/fe-connector/fe-connector-api/src/test/java/org/apache/doris/connector/api/ConnectorPartitionInfoTest.java
+++ b/fe/fe-connector/fe-connector-api/src/test/java/org/apache/doris/connector/api/ConnectorPartitionInfoTest.java
@@ -20,6 +20,7 @@
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
+import java.util.Arrays;
import java.util.Collections;
/**
@@ -74,4 +75,117 @@ public void equalsAndHashCodeIncludeFileCount() {
// equals()/hashCode() -> a.equals(differByFileCount) -> red.
Assertions.assertNotEquals(a, differByFileCount);
}
+
+ @Test
+ public void nullFlagsCtorsCarryPerValueNullFlags() {
+ // 4-arg convenience ctor (hive: UNKNOWN stats + connector-supplied per-value NULL flags).
+ ConnectorPartitionInfo hive = new ConnectorPartitionInfo(
+ "year=__HIVE_DEFAULT_PARTITION__/month=01", Collections.emptyMap(), Collections.emptyMap(),
+ Arrays.asList(true, false));
+ // WHY: fe-core zips getPartitionValueNullFlags() index-for-index with the parsed values to decide
+ // NullLiteral vs typed literal, so the order and values must round-trip. MUTATION: dropping the
+ // flags assignment (-> empty) or reordering -> red.
+ Assertions.assertEquals(Arrays.asList(true, false), hive.getPartitionValueNullFlags());
+ Assertions.assertEquals(ConnectorPartitionInfo.UNKNOWN, hive.getRowCount());
+
+ // 8-arg ctor (paimon: real stats + NULL flags).
+ ConnectorPartitionInfo paimon = new ConnectorPartitionInfo(
+ "region=__HIVE_DEFAULT_PARTITION__", Collections.emptyMap(), Collections.emptyMap(),
+ 1L, 2L, 3L, 4L, Collections.singletonList(true));
+ Assertions.assertEquals(Collections.singletonList(true), paimon.getPartitionValueNullFlags());
+ Assertions.assertEquals(4L, paimon.getFileCount());
+ }
+
+ @Test
+ public void backwardCompatCtorsDefaultNullFlagsEmpty() {
+ // WHY: connectors that do not opt in (3-arg MaxCompute/iceberg, 7-arg hudi) must default the flags
+ // to empty so fe-core treats every value as non-null (unchanged behavior). MUTATION: defaulting to
+ // a non-empty list -> red. A null flags arg must normalize to empty (not NPE).
+ ConnectorPartitionInfo threeArg = new ConnectorPartitionInfo(
+ "p1", Collections.emptyMap(), Collections.emptyMap());
+ ConnectorPartitionInfo sevenArg = new ConnectorPartitionInfo(
+ "p1", Collections.emptyMap(), Collections.emptyMap(), 1L, 2L, 3L, 4L);
+ ConnectorPartitionInfo nullArg = new ConnectorPartitionInfo(
+ "p1", Collections.emptyMap(), Collections.emptyMap(), null);
+ Assertions.assertTrue(threeArg.getPartitionValueNullFlags().isEmpty());
+ Assertions.assertTrue(sevenArg.getPartitionValueNullFlags().isEmpty());
+ Assertions.assertTrue(nullArg.getPartitionValueNullFlags().isEmpty());
+ }
+
+ @Test
+ public void equalsAndHashCodeIncludeNullFlags() {
+ ConnectorPartitionInfo a = new ConnectorPartitionInfo(
+ "p1", Collections.emptyMap(), Collections.emptyMap(), Arrays.asList(true, false));
+ ConnectorPartitionInfo b = new ConnectorPartitionInfo(
+ "p1", Collections.emptyMap(), Collections.emptyMap(), Arrays.asList(true, false));
+ ConnectorPartitionInfo differByFlags = new ConnectorPartitionInfo(
+ "p1", Collections.emptyMap(), Collections.emptyMap(), Arrays.asList(false, false));
+
+ Assertions.assertEquals(a, b);
+ Assertions.assertEquals(a.hashCode(), b.hashCode());
+ // WHY: two partitions differing only in per-value nullness (a genuine-NULL value vs a literal
+ // value that happens to render the same string) must not compare equal. MUTATION: omitting
+ // nullFlags from equals()/hashCode() -> a.equals(differByFlags) -> red.
+ Assertions.assertNotEquals(a, differByFlags);
+ }
+
+ @Test
+ public void orderedValuesCtorsCarryValuesAlignedToNullFlags() {
+ // 5-arg convenience ctor (hive/iceberg: UNKNOWN stats + connector-supplied ordered values [+ flags]).
+ ConnectorPartitionInfo hive = new ConnectorPartitionInfo(
+ "nation=cn/city=__HIVE_DEFAULT_PARTITION__", Collections.emptyMap(), Collections.emptyMap(),
+ Arrays.asList("cn", "__HIVE_DEFAULT_PARTITION__"), Arrays.asList(false, true));
+ // WHY: fe-core zips getOrderedPartitionValues() index-for-index with types + nullFlags INSTEAD of
+ // re-parsing the rendered name, so the ordered values must round-trip exactly and stay aligned to the
+ // flags. MUTATION: dropping the ordered-values assignment (-> empty, fe-core falls back to the name
+ // parse) or reordering -> red.
+ Assertions.assertEquals(Arrays.asList("cn", "__HIVE_DEFAULT_PARTITION__"), hive.getOrderedPartitionValues());
+ Assertions.assertEquals(Arrays.asList(false, true), hive.getPartitionValueNullFlags());
+ Assertions.assertEquals(ConnectorPartitionInfo.UNKNOWN, hive.getRowCount());
+
+ // 9-arg full ctor (paimon/hudi: real stats + ordered values + flags).
+ ConnectorPartitionInfo paimon = new ConnectorPartitionInfo(
+ "region=us", Collections.emptyMap(), Collections.emptyMap(),
+ 1L, 2L, 3L, 4L, Collections.singletonList("us"), Collections.singletonList(false));
+ Assertions.assertEquals(Collections.singletonList("us"), paimon.getOrderedPartitionValues());
+ Assertions.assertEquals(4L, paimon.getFileCount());
+ }
+
+ @Test
+ public void backwardCompatCtorsDefaultOrderedValuesEmpty() {
+ // WHY: ctors predating the ordered-values field (3-arg, 7-arg, 8-arg nullFlags, 4-arg) must default
+ // ordered values to empty so fe-core falls back to parsing the rendered name (unchanged behavior).
+ // MUTATION: defaulting to a non-empty list -> fe-core skips the parse with garbage -> red. A null arg
+ // must normalize to empty (not NPE).
+ ConnectorPartitionInfo threeArg = new ConnectorPartitionInfo(
+ "p1", Collections.emptyMap(), Collections.emptyMap());
+ ConnectorPartitionInfo eightArgFlags = new ConnectorPartitionInfo(
+ "p1", Collections.emptyMap(), Collections.emptyMap(), 1L, 2L, 3L, 4L,
+ Collections.singletonList(true));
+ ConnectorPartitionInfo nullArg = new ConnectorPartitionInfo(
+ "p1", Collections.emptyMap(), Collections.emptyMap(), null, null);
+ Assertions.assertTrue(threeArg.getOrderedPartitionValues().isEmpty());
+ Assertions.assertTrue(eightArgFlags.getOrderedPartitionValues().isEmpty());
+ Assertions.assertTrue(nullArg.getOrderedPartitionValues().isEmpty());
+ Assertions.assertTrue(nullArg.getPartitionValueNullFlags().isEmpty());
+ }
+
+ @Test
+ public void equalsAndHashCodeIncludeOrderedValues() {
+ ConnectorPartitionInfo a = new ConnectorPartitionInfo(
+ "p1", Collections.emptyMap(), Collections.emptyMap(),
+ Arrays.asList("cn", "bj"), Collections.emptyList());
+ ConnectorPartitionInfo b = new ConnectorPartitionInfo(
+ "p1", Collections.emptyMap(), Collections.emptyMap(),
+ Arrays.asList("cn", "bj"), Collections.emptyList());
+ ConnectorPartitionInfo differByValues = new ConnectorPartitionInfo(
+ "p1", Collections.emptyMap(), Collections.emptyMap(),
+ Arrays.asList("cn", "sh"), Collections.emptyList());
+
+ Assertions.assertEquals(a, b);
+ Assertions.assertEquals(a.hashCode(), b.hashCode());
+ // WHY: two partitions with the same rendered name but different ordered values must not compare equal.
+ // MUTATION: omitting orderedPartitionValues from equals()/hashCode() -> a.equals(differByValues) -> red.
+ Assertions.assertNotEquals(a, differByValues);
+ }
}
diff --git a/fe/fe-connector/fe-connector-api/src/test/java/org/apache/doris/connector/api/ConnectorScanProviderSelectionTest.java b/fe/fe-connector/fe-connector-api/src/test/java/org/apache/doris/connector/api/ConnectorScanProviderSelectionTest.java
new file mode 100644
index 00000000000000..fe0c66e23dc43c
--- /dev/null
+++ b/fe/fe-connector/fe-connector-api/src/test/java/org/apache/doris/connector/api/ConnectorScanProviderSelectionTest.java
@@ -0,0 +1,154 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.connector.api;
+
+import org.apache.doris.connector.api.handle.ConnectorColumnHandle;
+import org.apache.doris.connector.api.handle.ConnectorTableHandle;
+import org.apache.doris.connector.api.pushdown.ConnectorExpression;
+import org.apache.doris.connector.api.scan.ConnectorScanPlanProvider;
+import org.apache.doris.connector.api.scan.ConnectorScanRange;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Optional;
+
+/**
+ * Pins the per-table scan-provider selection seam
+ * {@link Connector#getScanPlanProvider(ConnectorTableHandle)}.
+ *
+ *
WHY this matters (Rule 9): after the hive/hms cut-over a single catalog is heterogeneous
+ * (plain-hive + iceberg-on-HMS + hudi-on-HMS under one gateway connector). The engine must pick a
+ * DIFFERENT scan provider per table without ever inspecting the format itself. The selection must happen
+ * here — at provider-acquisition time — rather than inside a single dispatching provider, because
+ * {@link ConnectorScanPlanProvider} has handle-less methods (e.g. {@code appendExplainInfo}) and providers
+ * are built fresh/stateless per call, so a returned provider must already be bound to the right backing
+ * scanner for the handle. The default must stay inert for every single-format connector.
+ */
+public class ConnectorScanProviderSelectionTest {
+
+ /** A scan provider identified by name so tests can assert WHICH provider was selected. */
+ private static final class NamedProvider implements ConnectorScanPlanProvider {
+ private final String name;
+
+ private NamedProvider(String name) {
+ this.name = name;
+ }
+
+ @Override
+ public List planScan(ConnectorSession session, ConnectorTableHandle handle,
+ List columns, Optional filter) {
+ return Collections.emptyList();
+ }
+
+ @Override
+ public String toString() {
+ return name;
+ }
+ }
+
+ /** Bare connector implementing only the abstract methods; nothing scan-related is overridden. */
+ private abstract static class FakeConnector implements Connector {
+ @Override
+ public ConnectorMetadata getMetadata(ConnectorSession session) {
+ return null;
+ }
+
+ @Override
+ public void close() {
+ }
+ }
+
+ private static ConnectorTableHandle handle() {
+ return new ConnectorTableHandle() {
+ };
+ }
+
+ @Test
+ public void defaultDelegatesToNoArgProvider() {
+ // A single-format connector overrides only the no-arg getter. The per-handle default must delegate
+ // to it (NOT return null), so every existing connector routes unchanged after the seam is added.
+ // MUTATION: making the default return null instead of getScanPlanProvider() -> non-null assert red.
+ NamedProvider only = new NamedProvider("only");
+ Connector connector = new FakeConnector() {
+ @Override
+ public ConnectorScanPlanProvider getScanPlanProvider() {
+ return only;
+ }
+ };
+
+ Assertions.assertSame(only, connector.getScanPlanProvider(handle()),
+ "the per-handle default must delegate to the connector-level no-arg provider");
+ }
+
+ @Test
+ public void defaultReturnsNullWhenConnectorHasNoScanCapability() {
+ // A connector with no scan capability (no-arg default returns null) must keep returning null through
+ // the per-handle seam, preserving the null-tolerant scan path in PluginDrivenScanNode.
+ Connector connector = new FakeConnector() {
+ };
+
+ Assertions.assertNull(connector.getScanPlanProvider(handle()),
+ "with no scan provider at all the per-handle seam stays null");
+ }
+
+ @Test
+ public void overrideSelectsProviderPerHandle() {
+ // A heterogeneous gateway overrides the per-handle seam and returns a DIFFERENT provider per table,
+ // and must NOT fall back to the no-arg provider once it has an answer for the handle.
+ // MUTATION: keying the override on the no-arg getter (ignoring the handle) -> per-handle assert red.
+ ConnectorTableHandle icebergHandle = handle();
+ ConnectorTableHandle hiveHandle = handle();
+ NamedProvider icebergProvider = new NamedProvider("iceberg");
+ NamedProvider hiveProvider = new NamedProvider("hive");
+ NamedProvider fallback = new NamedProvider("fallback");
+ Connector gateway = new FakeConnector() {
+ @Override
+ public ConnectorScanPlanProvider getScanPlanProvider() {
+ return fallback;
+ }
+
+ @Override
+ public ConnectorScanPlanProvider getScanPlanProvider(ConnectorTableHandle handle) {
+ return handle == icebergHandle ? icebergProvider : hiveProvider;
+ }
+ };
+
+ Assertions.assertSame(icebergProvider, gateway.getScanPlanProvider(icebergHandle),
+ "gateway routes the iceberg-on-HMS handle to the iceberg provider");
+ Assertions.assertSame(hiveProvider, gateway.getScanPlanProvider(hiveHandle),
+ "gateway routes the plain-hive handle to the hive provider");
+ Assertions.assertNotSame(fallback, gateway.getScanPlanProvider(icebergHandle),
+ "an overriding gateway does not fall back to the connector-level no-arg provider");
+ }
+
+ @Test
+ public void ownsHandleDefaultsFalse() {
+ // A connector owns no handle it did not define. The default must be false so a gateway asking each
+ // embedded sibling "is this foreign handle yours?" gets a negative from every connector that did not
+ // produce it, and only the producing sibling (which overrides ownsHandle) claims it.
+ // MUTATION: flipping the default to true -> a gateway would route a foreign handle to the WRONG sibling.
+ Connector connector = new FakeConnector() {
+ };
+
+ Assertions.assertFalse(connector.ownsHandle(handle()),
+ "the ownsHandle default must be false (a connector owns no handle it did not define)");
+ }
+}
diff --git a/fe/fe-connector/fe-connector-api/src/test/java/org/apache/doris/connector/api/procedure/ConnectorProcedureOpsDefaultsTest.java b/fe/fe-connector/fe-connector-api/src/test/java/org/apache/doris/connector/api/procedure/ConnectorProcedureOpsDefaultsTest.java
index b7c432003a822a..af45adba1980c5 100644
--- a/fe/fe-connector/fe-connector-api/src/test/java/org/apache/doris/connector/api/procedure/ConnectorProcedureOpsDefaultsTest.java
+++ b/fe/fe-connector/fe-connector-api/src/test/java/org/apache/doris/connector/api/procedure/ConnectorProcedureOpsDefaultsTest.java
@@ -74,6 +74,72 @@ public void getProcedureOpsDefaultsToNull() {
+ "stay behaviorally unchanged)");
}
+ @Test
+ public void getProcedureOpsPerHandleDefaultsToNoArg() {
+ // A single-format connector overrides only the no-arg getter; the per-handle default must delegate to it
+ // (NOT return null), so every existing connector routes ALTER TABLE EXECUTE unchanged after the seam is
+ // added. MUTATION: making the default return null instead of getProcedureOps() -> non-null assert red.
+ ConnectorProcedureOps only = new BareProcedureOps();
+ Connector connector = new Connector() {
+ @Override
+ public ConnectorMetadata getMetadata(ConnectorSession session) {
+ return null;
+ }
+
+ @Override
+ public ConnectorProcedureOps getProcedureOps() {
+ return only;
+ }
+ };
+
+ Assertions.assertSame(only, connector.getProcedureOps(handle()),
+ "the per-handle default must delegate to the connector-level no-arg procedure ops");
+ }
+
+ @Test
+ public void getProcedureOpsPerHandleStaysNullWhenConnectorHasNoProcedures() {
+ // A connector with no procedures (no-arg default returns null) must keep returning null through the
+ // per-handle seam, so ALTER TABLE EXECUTE is never dispatched to it.
+ Assertions.assertNull(new BareConnector().getProcedureOps(handle()),
+ "with no procedure ops at all the per-handle seam stays null");
+ }
+
+ @Test
+ public void getProcedureOpsPerHandleOverrideSelectsPerHandle() {
+ // A heterogeneous gateway overrides the per-handle seam and returns the SIBLING's ops for a foreign
+ // handle while a plain (hive) handle keeps the connector-level null, and must NOT fall back to the no-arg
+ // getter once it has a per-handle answer. MUTATION: keying the override on the no-arg getter (ignoring
+ // the handle) -> the foreign-handle assert reads null -> red.
+ ConnectorTableHandle foreignHandle = handle();
+ ConnectorProcedureOps siblingOps = new BareProcedureOps();
+ Connector gateway = new Connector() {
+ @Override
+ public ConnectorMetadata getMetadata(ConnectorSession session) {
+ return null;
+ }
+
+ @Override
+ public ConnectorProcedureOps getProcedureOps() {
+ return null;
+ }
+
+ @Override
+ public ConnectorProcedureOps getProcedureOps(ConnectorTableHandle handle) {
+ return handle == foreignHandle ? siblingOps : getProcedureOps();
+ }
+ };
+
+ Assertions.assertSame(siblingOps, gateway.getProcedureOps(foreignHandle),
+ "a gateway routes the foreign (iceberg-on-HMS) handle to the sibling's procedure ops");
+ Assertions.assertNull(gateway.getProcedureOps(handle()),
+ "a non-foreign (plain-hive) handle keeps the connector-level null (no procedures)");
+ }
+
+ private static ConnectorTableHandle handle() {
+ return new ConnectorTableHandle() {
+ };
+ }
+
@Test
public void getExecutionModeDefaultsToSingleCall() {
// A connector that declares only synchronous procedures inherits SINGLE_CALL for every name, so the
diff --git a/fe/fe-connector/fe-connector-api/src/test/java/org/apache/doris/connector/api/scan/ConnectorScanPlanProviderCompressTypeTest.java b/fe/fe-connector/fe-connector-api/src/test/java/org/apache/doris/connector/api/scan/ConnectorScanPlanProviderCompressTypeTest.java
new file mode 100644
index 00000000000000..1862224b47f69f
--- /dev/null
+++ b/fe/fe-connector/fe-connector-api/src/test/java/org/apache/doris/connector/api/scan/ConnectorScanPlanProviderCompressTypeTest.java
@@ -0,0 +1,61 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.connector.api.scan;
+
+import org.apache.doris.connector.api.ConnectorSession;
+import org.apache.doris.connector.api.handle.ConnectorColumnHandle;
+import org.apache.doris.connector.api.handle.ConnectorTableHandle;
+import org.apache.doris.connector.api.pushdown.ConnectorExpression;
+import org.apache.doris.thrift.TFileCompressType;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Optional;
+
+/**
+ * Guards the additive {@code adjustFileCompressType} SPI default on {@link ConnectorScanPlanProvider}.
+ *
+ *
WHY: the default MUST be identity so no connector's inferred file compression type is silently altered
+ * — only a connector that explicitly opts in (hive/hudi remap {@code LZ4FRAME -> LZ4BLOCK}) changes it. If the
+ * default ever stopped being identity, every non-opting connector's reads would ship a different compress type
+ * to BE. This is the zero-break guard for es/jdbc/paimon/iceberg/trino/maxcompute.
+ */
+public class ConnectorScanPlanProviderCompressTypeTest {
+
+ /** Bare provider: only the abstract 4-arg planScan implemented; everything else inherits SPI defaults. */
+ private static final class BareProvider implements ConnectorScanPlanProvider {
+ @Override
+ public List planScan(ConnectorSession session, ConnectorTableHandle handle,
+ List columns, Optional filter) {
+ return Collections.emptyList();
+ }
+ }
+
+ @Test
+ public void testAdjustFileCompressTypeDefaultsToIdentity() {
+ ConnectorScanPlanProvider provider = new BareProvider();
+ // The default must NOT touch any type — including LZ4FRAME, the one hive/hudi opt in to remap.
+ for (TFileCompressType type : TFileCompressType.values()) {
+ Assertions.assertEquals(type, provider.adjustFileCompressType(type),
+ "default adjustFileCompressType must be identity for " + type);
+ }
+ }
+}
diff --git a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/CacheSpec.java b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/CacheSpec.java
index c950cafaa3ce16..47cd0596b46096 100644
--- a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/CacheSpec.java
+++ b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/CacheSpec.java
@@ -168,6 +168,17 @@ public static String metaCacheKeyPrefix(String engine) {
return META_CACHE_PREFIX + engine + ".";
}
+ /**
+ * Build the standard external meta cache TTL key for one engine+entry.
+ * Example: {@code meta.cache.hive.file.ttl-second}.
+ *
+ *
Used to translate a legacy catalog TTL knob (e.g. {@code file.meta.cache.ttl-second}) into the
+ * namespaced key a cache actually reads, via {@link #applyCompatibilityMap}.
+ */
+ public static String metaCacheTtlKey(String engine, String entryName) {
+ return META_CACHE_PREFIX + engine + "." + entryName + KEY_TTL_SECOND;
+ }
+
/**
* Returns true when the given property key belongs to one engine's meta cache namespace.
*/
diff --git a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCacheEntry.java b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCacheEntry.java
index 0e38e11a91899a..425697ef9b1098 100644
--- a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCacheEntry.java
+++ b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCacheEntry.java
@@ -157,6 +157,45 @@ public void put(K key, V value) {
data.put(key, value);
}
+ /**
+ * The current invalidation generation. Capture this BEFORE a slow external load, then hand it to
+ * {@link #putIfNotInvalidatedSince} so a {@code flush}/{@code invalidate*} that raced the load does not
+ * get its clear silently undone by a stale write-back. Mirrors the guard the manual-miss-load path
+ * ({@link #getWithManualLoad}) applies around its own put; exposed so a caller that does its OWN bulk
+ * external read (e.g. a decorator batching a multi-key RPC and putting each result under its own key)
+ * can reuse the same generation guard instead of an unguarded {@link #put}.
+ */
+ public long invalidationGeneration() {
+ return invalidateGeneration.get();
+ }
+
+ /**
+ * Generation-guarded put: caches {@code (key, value)} only if no invalidation has happened since
+ * {@code generation} was captured (before the caller's external load). If a {@code flush}/{@code
+ * invalidate*} raced the load — bumping the generation either before the put (skip) or between the put
+ * and the recheck (drop only the value we wrote, via {@link #removeLoadedValue}) — the stale value is
+ * NOT left cached, exactly as {@link #getWithManualLoad} does for its single-key load. Additive: the
+ * existing {@link #put} is unchanged; a disabled entry is a no-op.
+ */
+ public void putIfNotInvalidatedSince(long generation, K key, V value) {
+ if (!effectiveEnabled) {
+ return;
+ }
+ synchronized (loadLock(key)) {
+ // A racing flush already bumped the generation before we could put: skip so a stale pre-flush
+ // value is not re-cached (mirrors getWithManualLoad's pre-put guard).
+ if (generation != invalidateGeneration.get()) {
+ return;
+ }
+ data.put(key, value);
+ // A flush landing between the check and the put: drop only the value we just wrote, keeping any
+ // newer replacement intact (mirrors getWithManualLoad's post-put guard).
+ if (generation != invalidateGeneration.get()) {
+ removeLoadedValue(key, value);
+ }
+ }
+ }
+
public void invalidateKey(K key) {
invalidateGeneration.incrementAndGet();
if (data.asMap().remove(key) != null) {
diff --git a/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/MetaCacheEntryTest.java b/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/MetaCacheEntryTest.java
index 8782bf9f1439d9..8284f4ae1ed672 100644
--- a/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/MetaCacheEntryTest.java
+++ b/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/MetaCacheEntryTest.java
@@ -212,6 +212,37 @@ public void manualMissLoadDeduplicatesConcurrentSameKey() throws Exception {
}
}
+ @Test
+ public void putIfNotInvalidatedSinceHonorsGenerationGuard() {
+ ExecutorService refreshExecutor = Executors.newSingleThreadExecutor();
+ try {
+ // Contextual + manual-miss entry, mirroring the hive partition-object cache that uses the guarded put.
+ CacheSpec cacheSpec = CacheSpec.of(true, CacheSpec.CACHE_NO_TTL, 10L);
+ MetaCacheEntry entry = new MetaCacheEntry<>(
+ "test", null, cacheSpec, refreshExecutor, false, true, 0L, true);
+
+ // No invalidation since the captured generation -> the guarded put caches normally.
+ long g1 = entry.invalidationGeneration();
+ entry.putIfNotInvalidatedSince(g1, "a", 1);
+ Assertions.assertEquals(Integer.valueOf(1), entry.getIfPresent("a"));
+
+ // An invalidation between the capture and the put (the flush-races-an-in-flight-load case) must make
+ // the put a no-op, so a stale pre-invalidation value is NOT re-cached to the TTL.
+ long g2 = entry.invalidationGeneration();
+ entry.invalidateAll(); // bumps the generation, as a racing flush / REFRESH would
+ entry.putIfNotInvalidatedSince(g2, "b", 2);
+ // MUTATION: an unguarded put (the pre-R3 raw put) leaves "b" cached here -> getIfPresent == 2 -> red.
+ Assertions.assertNull(entry.getIfPresent("b"), "a stale-generation put must not re-cache the value");
+
+ // A generation captured AFTER the invalidation puts normally again (the guard is not a one-shot latch).
+ long g3 = entry.invalidationGeneration();
+ entry.putIfNotInvalidatedSince(g3, "c", 3);
+ Assertions.assertEquals(Integer.valueOf(3), entry.getIfPresent("c"));
+ } finally {
+ refreshExecutor.shutdownNow();
+ }
+ }
+
@SuppressWarnings("unchecked")
private static LoadingCache