Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
19 commits
Select commit Hold shift + click to select a range
721d33e
[improvement](fe) Use typed keys with structural equals for scan reuse
924060929 Aug 6, 2026
854378f
[improvement](fe) Keep scan plan reuse keys backward compatible
924060929 Aug 10, 2026
296287b
[fix](filescan) Preserve zero-pruned scan identity
924060929 Aug 12, 2026
7443e98
[test](filescan) Cover statement scan reuse behavior
924060929 Aug 13, 2026
3c8bcb2
[test](filescan) Cover structural filter reuse
924060929 Aug 13, 2026
7ef7db2
[fix](fe) Release job task statement contexts
924060929 Aug 14, 2026
7dcaf72
[fix](fe) Close every MTMV execution context
924060929 Aug 14, 2026
9240b31
[fix](fe) Preserve MTMV refresh snapshot consistency
924060929 Aug 14, 2026
bec6c88
[fix](filescan) Add external split reuse switch
924060929 Aug 14, 2026
9b9d499
[fix](filescan) Fix scan reuse compatibility and cleanup
924060929 Aug 20, 2026
36c6898
[fix](filescan) Guard scan reuse namespaces
924060929 Aug 25, 2026
5169757
[fix](filescan) Quiesce asynchronous split producers
924060929 Aug 25, 2026
429f6f6
[fix](filescan) Bound asynchronous split cancellation
924060929 Aug 25, 2026
6db239b
[fix](filescan) Keep scan reuse changes scoped
924060929 Aug 25, 2026
f2cf559
[fix](filescan) Close task scan resources
924060929 Aug 25, 2026
1f4109d
[refactor](fe) Centralize external scan reuse session policy
924060929 Aug 31, 2026
705054b
[fix](filescan) Fence scan reuse to one table generation
924060929 Sep 15, 2026
98f451d
[fix](filescan) Avoid caching degraded external scan plans
924060929 Sep 20, 2026
616b309
[fix](paimon) Reuse fixed scan identities without live probes
924060929 Sep 20, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -52,8 +52,11 @@
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicReference;
import java.util.function.UnaryOperator;

/**
Expand All @@ -75,6 +78,7 @@
* non-transactional tables (see {@link #supportsBatchScan})</li>
* </ul>
*/

public class HiveScanPlanProvider implements ConnectorScanPlanProvider {

private static final Logger LOG = LogManager.getLogger(HiveScanPlanProvider.class);
Expand All @@ -97,6 +101,7 @@ public class HiveScanPlanProvider implements ConnectorScanPlanProvider {
* surfaced through ConnectorSession.getSessionProperties() (VariableMgr dumps all visible vars).
*/
private static final String SESSION_READ_HIVE_JSON_IN_ONE_COLUMN = "read_hive_json_in_one_column";
static final String SCAN_REUSE_NAMESPACE = "hms.scan-reuse";

/** Input format of a full-ACID (ORC) transactional Hive table; other formats are rejected. */
private static final String ORC_ACID_INPUT_FORMAT = "org.apache.hadoop.hive.ql.io.orc.OrcInputFormat";
Expand Down Expand Up @@ -138,6 +143,45 @@ public boolean usesHiveParquetInt96TimeZone() {
@Override
public List<ConnectorScanRange> planScan(ConnectorSession session, ConnectorScanRequest request) {
HiveTableHandle hiveHandle = (HiveTableHandle) request.getTableHandle();
if (session == null || !session.isExternalScanTaskReuseEnabled()) {
return doPlanScan(session, request);
}
if (hiveHandle.isTransactional()) {
// ACID / INSERT_ONLY reads open a per-scan read transaction with a write-id snapshot and
// a shared metastore lock; reusing the planned ranges would skip that transaction.
return doPlanScan(session, request);
}
// Statement-scoped reuse: within one statement the identical scan (same table, same
// partition set, same formats) plans once and every duplicated relation shares the result.
// The scope is NONE for offline planning and tests, in which case the loader runs on every
// call. Session variables are constant within a statement and deliberately absent.
String memoKey = SCAN_REUSE_NAMESPACE + ":" + session.getCatalogId() + ":" + session.getQueryId();
Map<HiveScanReuseKey, List<ConnectorScanRange>> scanReuse = session.getStatementScope().computeIfAbsent(
memoKey, () -> new ConcurrentHashMap<>());
HiveScanReuseKey reuseKey = new HiveScanReuseKey(hiveHandle);
AtomicReference<List<ConnectorScanRange>> uncached = new AtomicReference<>();
List<ConnectorScanRange> cached = scanReuse.computeIfAbsent(reuseKey, key -> {
PlanCompleteness completeness = new PlanCompleteness();
List<ConnectorScanRange> planned = Collections.unmodifiableList(
doPlanScan(session, request, completeness));
if (!completeness.isComplete()) {
// ConcurrentHashMap does not install a mapping when the loader returns null. Return this
// caller's partial result below, but let the next identical alias retry the failed directory.
uncached.set(planned);
return null;
}
return planned;
});
return cached != null ? cached : Objects.requireNonNull(uncached.get());
}

private List<ConnectorScanRange> doPlanScan(ConnectorSession session, ConnectorScanRequest request) {
return doPlanScan(session, request, null);
}

private List<ConnectorScanRange> doPlanScan(ConnectorSession session, ConnectorScanRequest request,
PlanCompleteness completeness) {
HiveTableHandle hiveHandle = (HiveTableHandle) request.getTableHandle();
String dbName = hiveHandle.getDbName();
String tableName = hiveHandle.getTableName();
recordPruningProfile(hiveHandle);
Expand Down Expand Up @@ -173,7 +217,7 @@ public List<ConnectorScanRange> planScan(ConnectorSession session, ConnectorScan
HiveFileFormat partFormat = partition.fileFormat != null
? partition.fileFormat : fileFormat;
listAndSplitFiles(dbName, tableName, partition, partFormat,
splittable, isLzo, targetSplitSize, fs, ranges);
splittable, isLzo, targetSplitSize, fs, ranges, completeness);
}
}

Expand Down Expand Up @@ -245,6 +289,16 @@ public List<ConnectorScanRange> planScanForPartitionBatch(
ConnectorSession session,
ConnectorScanRequest request,
List<String> partitionBatch) {
// Batch mode deliberately does not retain completed ranges in the statement scope: its
// purpose is to bound FE memory while splits are streamed to the coordinator. Caching every
// batch until statement close would materialize the full scan again and defeat that bound.
return doPlanScanForPartitionBatch(session, request, partitionBatch);
}

private List<ConnectorScanRange> doPlanScanForPartitionBatch(
ConnectorSession session,
ConnectorScanRequest request,
List<String> partitionBatch) {
HiveTableHandle hiveHandle = (HiveTableHandle) request.getTableHandle();
String dbName = hiveHandle.getDbName();
String tableName = hiveHandle.getTableName();
Expand Down Expand Up @@ -274,7 +328,7 @@ public List<ConnectorScanRange> planScanForPartitionBatch(
HiveFileFormat partFormat = partition.fileFormat != null
? partition.fileFormat : fileFormat;
listAndSplitFiles(dbName, tableName, partition, partFormat,
splittable, isLzo, targetSplitSize, fs, ranges);
splittable, isLzo, targetSplitSize, fs, ranges, null);
}
return ranges;
}
Expand Down Expand Up @@ -587,7 +641,7 @@ private List<PartitionScanInfo> convertPartitions(
private void listAndSplitFiles(String dbName, String tableName,
PartitionScanInfo partition, HiveFileFormat fileFormat,
boolean splittable, boolean isLzo, long targetSplitSize, FileSystem fs,
List<ConnectorScanRange> ranges) {
List<ConnectorScanRange> ranges, PlanCompleteness completeness) {
List<HiveFileStatus> files;
try {
files = fileListingCache.listDataFiles(dbName, tableName, partition.location,
Expand All @@ -601,6 +655,9 @@ private void listAndSplitFiles(String dbName, String tableName,
throw new DorisConnectorException(
"Partition location does not exist: " + partition.location, e);
}
if (completeness != null) {
completeness.markIncomplete();
}
LOG.warn("Cannot list files in partition: {}", partition.location, e);
return;
}
Expand All @@ -617,6 +674,18 @@ private void listAndSplitFiles(String dbName, String tableName,
}
}

private static final class PlanCompleteness {
private boolean complete = true;

private void markIncomplete() {
complete = false;
}

private boolean isComplete() {
return complete;
}
}

/**
* Whether a listing failure was caused by the directory not existing (a {@link FileNotFoundException}
* anywhere in the cause chain), as opposed to a transient or unreadable-permission failure. Used to
Expand Down Expand Up @@ -862,4 +931,72 @@ private static String formatNanos(long nanos) {
return TimeUnit.NANOSECONDS.toMillis(nanos) + "ms";
}
}

/**
* Statement-scoped cache key for one Hive scan.
*
* <p>Includes every input that changes the planned split list: table identity, the file formats
* (input format / serialization lib / JSON single-column gate), the partition keys and the
* pruned partition set (each partition's location and values). ACID tables are excluded
* upstream, and session variables are statement-constant, so both stay out of the key.
*/
private static final class HiveScanReuseKey {
private final String dbName;
private final String tableName;
private final String location;
private final String inputFormat;
private final String serializationLib;
private final boolean firstColumnIsString;
private final List<String> partitionKeyNames;
private final List<HmsPartitionInfo> prunedPartitions;

private HiveScanReuseKey(HiveTableHandle handle) {
// Catalog and query isolation are provided by the statement-scope memo key. The table
// location identifies the data source of unpartitioned tables, whose prunedPartitions
// is null.
this.dbName = handle.getDbName();
this.tableName = handle.getTableName();
this.location = handle.getLocation();
this.inputFormat = handle.getInputFormat();
this.serializationLib = handle.getSerializationLib();
this.firstColumnIsString = handle.isFirstColumnString();
this.partitionKeyNames = handle.getPartitionKeyNames() == null
? Collections.emptyList()
: Collections.unmodifiableList(new ArrayList<>(handle.getPartitionKeyNames()));
this.prunedPartitions = handle.getPrunedPartitions() == null
Comment thread
924060929 marked this conversation as resolved.
Resolved
? null
: Collections.unmodifiableList(new ArrayList<>(handle.getPrunedPartitions()));
Comment thread
924060929 marked this conversation as resolved.
Resolved
}

@Override
public boolean equals(Object object) {
if (this == object) {
return true;
}
if (!(object instanceof HiveScanReuseKey)) {
return false;
}
HiveScanReuseKey that = (HiveScanReuseKey) object;
return firstColumnIsString == that.firstColumnIsString
&& Objects.equals(dbName, that.dbName)
&& Objects.equals(tableName, that.tableName)
&& Objects.equals(location, that.location)
&& Objects.equals(inputFormat, that.inputFormat)
&& Objects.equals(serializationLib, that.serializationLib)
&& Objects.equals(partitionKeyNames, that.partitionKeyNames)
&& Objects.equals(prunedPartitions, that.prunedPartitions);
}

@Override
public int hashCode() {
return Objects.hash(dbName, tableName, location,
inputFormat, serializationLib, firstColumnIsString,
partitionKeyNames, prunedPartitions);
}

@Override
public String toString() {
return "HiveScanReuseKey{table=" + dbName + "." + tableName + "}";
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
Expand Down Expand Up @@ -78,6 +79,12 @@ public class HiveScanBatchModeTest {
private static final String PARQUET_SERDE =
"org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe";

@Test
public void scanReuseNamespaceUsesConnectorType() {
String prefix = new HiveConnectorProvider().getType() + ".";
Assertions.assertTrue(HiveScanPlanProvider.SCAN_REUSE_NAMESPACE.startsWith(prefix));
}

// ==================== supportsBatchScan: partitioned AND non-transactional ====================

@Test
Expand Down Expand Up @@ -353,6 +360,108 @@ public void predicatePruningFailureProfileTransfersThroughTheStatementScope() {
Assertions.assertTrue(provider.collectScanProfiles(session).isEmpty());
}

@Test
public void statementReuseKeepsZeroPrunedAndUnprunedScansDistinct() {
CountingLister lister = new CountingLister();
int[] partitionListCalls = new int[1];
HmsClient hmsClient = new FakeHmsClient() {
@Override
public List<String> listPartitionNames(String dbName, String tableName, int maxParts) {
partitionListCalls[0]++;
return Collections.singletonList("year=2024/month=01");
}
};
HiveScanPlanProvider provider = provider(hmsClient, lister);
HiveTableHandle unpruned = new HiveTableHandle.Builder("db", "t", HiveTableType.HIVE)
.inputFormat(PARQUET_INPUT_FORMAT)
.serializationLib(PARQUET_SERDE)
.partitionKeyNames(PART_KEYS)
.build();
HiveTableHandle zeroPruned = unpruned.toBuilder()
.prunedPartitions(Collections.emptyList())
.build();
ConnectorSession session = new ScopeSession(7L, "same-statement", new TestStatementScope());

List<ConnectorScanRange> zeroRanges = provider.planScan(session,
ConnectorScanRequest.builder(zeroPruned, Collections.<ConnectorColumnHandle>emptyList()).build());
List<ConnectorScanRange> allRanges = provider.planScan(session,
ConnectorScanRequest.builder(unpruned, Collections.<ConnectorColumnHandle>emptyList()).build());

Assertions.assertTrue(zeroRanges.isEmpty());
Assertions.assertEquals(1, allRanges.size(),
"an unpruned alias must not reuse a zero-pruned alias's empty ranges");
Assertions.assertEquals(1, partitionListCalls[0]);
Assertions.assertEquals(1, lister.totalCalls);
}

@Test
public void statementReusePlansAnIdenticalScanOnce() {
CountingLister lister = new CountingLister();
HiveScanPlanProvider provider = provider(new FakeHmsClient(), lister);
HiveTableHandle handle = new HiveTableHandle.Builder("db", "t", HiveTableType.HIVE)
.inputFormat(PARQUET_INPUT_FORMAT)
.serializationLib(PARQUET_SERDE)
.partitionKeyNames(PART_KEYS)
.prunedPartitions(Collections.singletonList(part("year=2024/month=01")))
.build();
ConnectorSession session = new ScopeSession(7L, "same-statement", new TestStatementScope());
ConnectorScanRequest request = ConnectorScanRequest.builder(
handle, Collections.<ConnectorColumnHandle>emptyList()).build();

List<ConnectorScanRange> first = provider.planScan(session, request);
List<ConnectorScanRange> second = provider.planScan(session, request);

Assertions.assertSame(first, second, "an identical scan must reuse the statement's planned range list");
Assertions.assertEquals(1, lister.totalCalls, "the underlying Hive file listing must run once");
}

@Test
public void statementReuseRetriesAListingThatFailedOnce() {
FailOnceLister lister = new FailOnceLister();
HiveScanPlanProvider provider = provider(new FakeHmsClient(), lister);
HiveTableHandle handle = new HiveTableHandle.Builder("db", "t", HiveTableType.HIVE)
.inputFormat(PARQUET_INPUT_FORMAT)
.serializationLib(PARQUET_SERDE)
.partitionKeyNames(PART_KEYS)
.prunedPartitions(Collections.singletonList(part("year=2024/month=01")))
.build();
ConnectorSession session = new ScopeSession(7L, "same-statement", new TestStatementScope());
ConnectorScanRequest request = ConnectorScanRequest.builder(
handle, Collections.<ConnectorColumnHandle>emptyList()).build();

List<ConnectorScanRange> degraded = provider.planScan(session, request);
List<ConnectorScanRange> recovered = provider.planScan(session, request);
List<ConnectorScanRange> reused = provider.planScan(session, request);

Assertions.assertTrue(degraded.isEmpty(), "the tolerated first listing failure skips its partition");
Assertions.assertEquals(1, recovered.size(), "the next alias must retry and recover the skipped partition");
Assertions.assertSame(recovered, reused, "only the complete retry result should enter statement reuse");
Assertions.assertEquals(2, lister.totalCalls,
"the transient failure must not be sticky, while the successful retry should be reused");
}

@Test
public void missingReusePropertyDoesNotReuse() {
CountingLister lister = new CountingLister();
HiveScanPlanProvider provider = provider(new FakeHmsClient(), lister);
HiveTableHandle handle = new HiveTableHandle.Builder("db", "t", HiveTableType.HIVE)
.inputFormat(PARQUET_INPUT_FORMAT)
.serializationLib(PARQUET_SERDE)
.partitionKeyNames(PART_KEYS)
.prunedPartitions(Collections.singletonList(part("year=2024/month=01")))
.build();
ConnectorSession session = new ScopeSession(
7L, "mixed-version", new TestStatementScope(), Collections.emptyMap());
ConnectorScanRequest request = ConnectorScanRequest.builder(
handle, Collections.<ConnectorColumnHandle>emptyList()).build();

List<ConnectorScanRange> first = provider.planScan(session, request);
List<ConnectorScanRange> second = provider.planScan(session, request);

Assertions.assertNotSame(first, second,
"an older planning FE's missing property must not enable reuse in a newer connector");
}

// ===== object-store native read (FIX-hive-s3a: scheme normalization + canonical creds) =====

@Test
Expand Down Expand Up @@ -540,7 +649,8 @@ public void usesHiveParquetInt96TimeZone() {
Assertions.assertTrue(provider(null, new CountingLister()).usesHiveParquetInt96TimeZone());
}

private static HiveScanPlanProvider provider(HmsClient hmsClient, CountingLister lister) {
private static HiveScanPlanProvider provider(
HmsClient hmsClient, HiveFileListingCache.DirectoryLister lister) {
return new HiveScanPlanProvider(hmsClient, HiveTestProperties.minimal(), new FakeConnectorContext(),
new HiveReadTransactionManager(), new HiveFileListingCache(HiveTestProperties.minimal(), lister));
}
Expand All @@ -565,12 +675,26 @@ public List<HiveFileStatus> list(String location, FileSystem fs) {
}
}

private static final class FailOnceLister implements HiveFileListingCache.DirectoryLister {
private int totalCalls;

@Override
public List<HiveFileStatus> list(String location, FileSystem fs) {
totalCalls++;
if (totalCalls == 1) {
throw new HiveDirectoryListingException("transient listing failure", new IOException("retry"));
}
return new ArrayList<>(Collections.singletonList(
new HiveFileStatus(location + "/000000_0", 10L, 1L)));
}
}

/**
* Minimal {@link HmsClient} double whose {@code getPartitions} echoes each requested name back as an
* {@link HmsPartitionInfo} whose location IS the name, so the batch-scoped resolution can be asserted through
* the listed locations. The rest fail loud.
*/
private static final class FakeHmsClient implements HmsClient {
private static class FakeHmsClient implements HmsClient {
private final List<String> listedPartitionNames;
private final String absentPartitionName;
private final HmsClientException partitionFailure;
Expand Down
Loading
Loading