Repository navigation
[test](fe) Cover what keeps external scan plans apart and what never reuses them #68848
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -26,6 +26,7 @@ | |
| import org.apache.doris.connector.hms.HmsTableInfo; | ||
| import org.apache.doris.connector.spi.ConnectorSession; | ||
| import org.apache.doris.connector.spi.ConnectorStatementScope; | ||
| import org.apache.doris.connector.spi.DorisConnectorException; | ||
| import org.apache.doris.connector.spi.handle.ConnectorColumnHandle; | ||
| import org.apache.doris.connector.spi.scan.ConnectorScanPlanProvider; | ||
| import org.apache.doris.connector.spi.scan.ConnectorScanProfile; | ||
|
|
@@ -44,9 +45,11 @@ | |
| import java.util.Arrays; | ||
| import java.util.Collections; | ||
| import java.util.HashMap; | ||
| import java.util.IdentityHashMap; | ||
| import java.util.List; | ||
| import java.util.Map; | ||
| import java.util.Optional; | ||
| import java.util.Set; | ||
| import java.util.function.Supplier; | ||
|
|
||
| /** | ||
|
|
@@ -78,6 +81,10 @@ public class HiveScanBatchModeTest { | |
| "org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat"; | ||
| private static final String PARQUET_SERDE = | ||
| "org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe"; | ||
| private static final String TEXT_INPUT_FORMAT = "org.apache.hadoop.mapred.TextInputFormat"; | ||
| private static final String LZO_TEXT_INPUT_FORMAT = "com.hadoop.mapred.DeprecatedLzoTextInputFormat"; | ||
| private static final String LAZY_SIMPLE_SERDE = "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"; | ||
| private static final String OPEN_CSV_SERDE = "org.apache.hadoop.hive.serde2.OpenCSVSerde"; | ||
|
|
||
| @Test | ||
| public void scanReuseNamespaceUsesConnectorType() { | ||
|
|
@@ -462,6 +469,128 @@ public void missingReusePropertyDoesNotReuse() { | |
| "an older planning FE's missing property must not enable reuse in a newer connector"); | ||
| } | ||
|
|
||
| @Test | ||
| public void statementReuseKeepsScansWithDifferentInputsApart() { | ||
| // Separately built but equal handles share one plan; a scan that differs in any input the planned ranges | ||
| // depend on plans on its own (the ranges carry the partition files and the BE file format). | ||
| // MUTATION: dropping a fact from HiveScanReuseKey (or the catalog id from the memo key) makes that scan | ||
| // return an earlier plan -> red. | ||
| HiveScanPlanProvider provider = provider(new FakeHmsClient(), new CountingLister()); | ||
| TestStatementScope scope = new TestStatementScope(); | ||
| ConnectorSession session = new ScopeSession(7L, "same-statement", scope); | ||
| HiveTableHandle text = textHandle("t", LAZY_SIMPLE_SERDE, part("year=2024/month=01")); | ||
|
|
||
| List<ConnectorScanRange> plain = provider.planScan(session, scanOf(text)); | ||
| Assertions.assertEquals(1, plain.size()); | ||
| Assertions.assertSame(plain, provider.planScan(session, | ||
| scanOf(textHandle("t", LAZY_SIMPLE_SERDE, part("year=2024/month=01")))), | ||
| "separately built but equal handles must share one plan"); | ||
| List<ConnectorScanRange> otherPartition = provider.planScan(session, | ||
| scanOf(textHandle("t", LAZY_SIMPLE_SERDE, part("year=2024/month=02")))); | ||
| List<ConnectorScanRange> otherTable = provider.planScan(session, | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. [P2] Cover same-catalog database identity too. |
||
| scanOf(textHandle("t2", LAZY_SIMPLE_SERDE, part("year=2024/month=01")))); | ||
| List<ConnectorScanRange> csv = provider.planScan(session, | ||
| scanOf(textHandle("t", OPEN_CSV_SERDE, part("year=2024/month=01")))); | ||
| List<ConnectorScanRange> stringFirstColumn = provider.planScan(session, | ||
| scanOf(text.toBuilder().firstColumnIsString(true).build())); | ||
| List<ConnectorScanRange> parquet = provider.planScan(session, scanOf(text.toBuilder() | ||
| .inputFormat(PARQUET_INPUT_FORMAT) | ||
| .serializationLib(PARQUET_SERDE) | ||
| .build())); | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. [P2] Exercise the Hive input-format key independently. This
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Fixed in 6ebef6b. The test now also plans the same partition with |
||
| // Same serde as the plain text scan; only the input format differs, and an LZO text table reads *.lzo files. | ||
| List<ConnectorScanRange> lzo = provider.planScan(session, | ||
| scanOf(text.toBuilder().inputFormat(LZO_TEXT_INPUT_FORMAT).build())); | ||
| Assertions.assertTrue(lzo.isEmpty(), "an LZO text scan skips the non-.lzo file the plain text scan reads"); | ||
| List<ConnectorScanRange> otherCatalog = provider.planScan( | ||
| new ScopeSession(8L, "same-statement", scope), scanOf(text)); | ||
| HiveTableHandle unpartitioned = new HiveTableHandle.Builder("db", "u", HiveTableType.HIVE) | ||
| .inputFormat(PARQUET_INPUT_FORMAT) | ||
| .serializationLib(PARQUET_SERDE) | ||
| .partitionKeyNames(Collections.emptyList()) | ||
| .location("hdfs://nn/warehouse/u") | ||
| .build(); | ||
| List<ConnectorScanRange> atLocation = provider.planScan(session, scanOf(unpartitioned)); | ||
| List<ConnectorScanRange> moved = provider.planScan(session, | ||
| scanOf(unpartitioned.toBuilder().location("hdfs://nn/warehouse/u_moved").build())); | ||
| Assertions.assertEquals("hdfs://nn/warehouse/u_moved/000000_0", | ||
| ((HiveScanRange) moved.get(0)).getPath().orElse(null), | ||
| "an unpartitioned table is identified by its location"); | ||
|
|
||
| List<List<ConnectorScanRange>> plans = Arrays.asList(plain, otherPartition, otherTable, csv, | ||
| stringFirstColumn, parquet, lzo, otherCatalog, atLocation, moved); | ||
| Set<List<ConnectorScanRange>> distinct = Collections.newSetFromMap(new IdentityHashMap<>()); | ||
| distinct.addAll(plans); | ||
| Assertions.assertEquals(plans.size(), distinct.size(), "scans with different inputs must not share a plan"); | ||
| } | ||
|
|
||
| @Test | ||
| public void partitionBatchPlanningNeverUsesStatementReuse() { | ||
| // Large partitioned scans stream their splits per batch on background threads; the batch path must not | ||
| // retain plans in the statement memo. MUTATION: routing planScanForPartitionBatch through the memo -> | ||
| // the second batch reuses the first plan -> red. | ||
| HiveScanPlanProvider provider = provider(new FakeHmsClient(), new CountingLister()); | ||
| TestStatementScope scope = new TestStatementScope(); | ||
| ConnectorSession session = new ScopeSession(7L, "same-statement", scope); | ||
| HiveTableHandle handle = textHandle("t", LAZY_SIMPLE_SERDE, part("year=2024/month=01")); | ||
| List<String> batch = Collections.singletonList("year=2024/month=01"); | ||
|
|
||
| List<ConnectorScanRange> first = provider.planScanForPartitionBatch(session, scanOf(handle), batch); | ||
| List<ConnectorScanRange> second = provider.planScanForPartitionBatch(session, scanOf(handle), batch); | ||
|
|
||
| Assertions.assertEquals(1, first.size()); | ||
| Assertions.assertNotSame(first, second); | ||
| Assertions.assertFalse(scope.contains(HiveScanPlanProvider.SCAN_REUSE_NAMESPACE + ":7:same-statement"), | ||
| "partition batches must not be planned through the statement reuse memo"); | ||
| } | ||
|
|
||
| @Test | ||
| public void transactionalScanNeverUsesStatementReuse() { | ||
| // Each transactional scan opens its own read transaction pinning a write-id snapshot; a reused plan would | ||
| // skip it. The fake metastore refuses the transaction, so planning stops right after asking for it. | ||
| // MUTATION: dropping the transactional bypass in planScan -> the scan installs the statement reuse memo | ||
| // before planning -> red. | ||
| int[] requestedTransactions = new int[1]; | ||
| HmsClient hmsClient = new FakeHmsClient() { | ||
| @Override | ||
| public long openTxn(String user) { | ||
| requestedTransactions[0]++; | ||
| throw new UnsupportedOperationException("no read transactions in this test"); | ||
| } | ||
| }; | ||
| HiveScanPlanProvider provider = provider(hmsClient, new CountingLister()); | ||
| Map<String, String> insertOnly = new HashMap<>(); | ||
| insertOnly.put("transactional", "true"); | ||
| insertOnly.put("transactional_properties", "insert_only"); | ||
| HiveTableHandle handle = new HiveTableHandle.Builder("db", "t", HiveTableType.HIVE) | ||
| .inputFormat(PARQUET_INPUT_FORMAT) | ||
| .serializationLib(PARQUET_SERDE) | ||
| .partitionKeyNames(PART_KEYS) | ||
| .tableParameters(insertOnly) | ||
| .prunedPartitions(Collections.singletonList(part("year=2024/month=01"))) | ||
| .build(); | ||
| TestStatementScope scope = new TestStatementScope(); | ||
| ConnectorSession session = new ScopeSession(7L, "same-statement", scope); | ||
|
|
||
| Assertions.assertThrows(DorisConnectorException.class, () -> provider.planScan(session, scanOf(handle))); | ||
|
|
||
| Assertions.assertEquals(1, requestedTransactions[0], "a transactional scan asks for its read transaction"); | ||
| Assertions.assertFalse(scope.contains(HiveScanPlanProvider.SCAN_REUSE_NAMESPACE + ":7:same-statement"), | ||
| "a transactional scan must not be planned through the statement reuse memo"); | ||
| } | ||
|
|
||
| private static HiveTableHandle textHandle(String tableName, String serde, HmsPartitionInfo partition) { | ||
| return new HiveTableHandle.Builder("db", tableName, HiveTableType.HIVE) | ||
| .inputFormat(TEXT_INPUT_FORMAT) | ||
| .serializationLib(serde) | ||
| .partitionKeyNames(PART_KEYS) | ||
| .prunedPartitions(Collections.singletonList(partition)) | ||
| .build(); | ||
| } | ||
|
|
||
| private static ConnectorScanRequest scanOf(HiveTableHandle handle) { | ||
| return ConnectorScanRequest.builder(handle, Collections.<ConnectorColumnHandle>emptyList()).build(); | ||
| } | ||
|
|
||
| // ===== object-store native read (FIX-hive-s3a: scheme normalization + canonical creds) ===== | ||
|
|
||
| @Test | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
[P2] Exercise partition metadata independently in this reuse test. Every partitioned
thandle usesPART_KEYS,part(name)has empty HMS values, andotherPartitionchanges location. Dropping key order fromHiveScanReuseKey, or values fromHmsPartitionInfoequality/hash, would leave these assertions green while reused ranges could carry the wrong BE partition columns. Add nonempty-value pairs that vary key order and, separately, values at one location; assert their range partition maps.