From 27a37490741e5bb1b7e273e71962671bc2e7e1f8 Mon Sep 17 00:00:00 2001 From: comphead Date: Tue, 13 Oct 2026 12:36:13 -0700 Subject: [PATCH 1/2] test: tag the Azure ABFS suites with AzureTest AbfsAuthParitySuite, AbfsAuthResolverSuite and AbfsScanFallbackSuite resolve ABFS authentication through the hadoop-azure on the test classpath, so they pass only when that hadoop-azure matches the Hadoop the build runs on. A build against a different Hadoop line, for example a vendor Hadoop with no matching hadoop-azure pin, had no way to skip them: the root pom hard-codes tagsToExclude, so -DtagsToExclude on the command line was ignored. Add an org.apache.comet.tags.AzureTest tag, annotate the three suites with it, and append a tagsToExclude property (empty by default) to the scalatest-maven-plugin exclusions so -DtagsToExclude=org.apache.comet.tags.AzureTest skips them. CI keeps running them, since every Spark profile pins a matching hadoop-azure. --- docs/source/contributor-guide/development.md | 5 +++ pom.xml | 5 ++- .../java/org/apache/comet/tags/AzureTest.java | 37 +++++++++++++++++++ .../objectstore/AbfsAuthParitySuite.scala | 2 + .../objectstore/AbfsAuthResolverSuite.scala | 2 + .../objectstore/AbfsScanFallbackSuite.scala | 2 + 6 files changed, 52 insertions(+), 1 deletion(-) create mode 100644 spark/src/test/java/org/apache/comet/tags/AzureTest.java diff --git a/docs/source/contributor-guide/development.md b/docs/source/contributor-guide/development.md index ea4a65e3089..f55e957d903 100644 --- a/docs/source/contributor-guide/development.md +++ b/docs/source/contributor-guide/development.md @@ -606,6 +606,11 @@ still assert while recording, so a recorded file cannot contradict what hadoop-a file is written only when every case ran. Review the diff of the regenerated files before committing them: each line that changes is a behaviour change on that hadoop-azure release. +`AbfsAuthParitySuite`, `AbfsAuthResolverSuite` and `AbfsScanFallbackSuite` are tagged +`org.apache.comet.tags.AzureTest`. They pass only when the test-scoped hadoop-azure matches the +Hadoop the build runs on. A build against another Hadoop, where no matching hadoop-azure is +pinned, can skip them with `-DtagsToExclude=org.apache.comet.tags.AzureTest`. + ## Benchmark There's a `make` command to run micro benchmarks in the repo. For diff --git a/pom.xml b/pom.xml index 2a737c9c20c..4bd65dc53e3 100644 --- a/pom.xml +++ b/pom.xml @@ -123,6 +123,9 @@ under the License. reflection only). Each Spark profile pins it to its Hadoop line; override with -Dhadoop-azure.version to test another release. --> 3.4.2 + + ${project.basedir}/../native/target/debug darwin x86_64 @@ -1124,7 +1127,7 @@ under the License. SparkTestSuite.txt D - org.apache.comet.IntegrationTestSuite + org.apache.comet.IntegrationTestSuite,${tagsToExclude} file:src/test/resources/log4j2.properties diff --git a/spark/src/test/java/org/apache/comet/tags/AzureTest.java b/spark/src/test/java/org/apache/comet/tags/AzureTest.java new file mode 100644 index 00000000000..00619f98d8f --- /dev/null +++ b/spark/src/test/java/org/apache/comet/tags/AzureTest.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.comet.tags; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +import org.scalatest.TagAnnotation; + +/** + * Marks suites that resolve Azure ABFS authentication through the hadoop-azure on the test + * classpath. They pass only when that hadoop-azure matches the Hadoop the build runs on, so a build + * against another Hadoop can skip them with -DtagsToExclude=org.apache.comet.tags.AzureTest. + */ +@TagAnnotation +@Retention(RetentionPolicy.RUNTIME) +@Target({ElementType.METHOD, ElementType.TYPE}) +public @interface AzureTest {} diff --git a/spark/src/test/scala/org/apache/comet/objectstore/AbfsAuthParitySuite.scala b/spark/src/test/scala/org/apache/comet/objectstore/AbfsAuthParitySuite.scala index 3d95864130a..1491195faec 100644 --- a/spark/src/test/scala/org/apache/comet/objectstore/AbfsAuthParitySuite.scala +++ b/spark/src/test/scala/org/apache/comet/objectstore/AbfsAuthParitySuite.scala @@ -43,6 +43,7 @@ import com.fasterxml.jackson.databind.{JsonNode, ObjectMapper} import org.apache.comet.objectstore.AbfsAuthParityCases._ import org.apache.comet.objectstore.AbfsAuthResolver._ import org.apache.comet.objectstore.AbfsReflection.Handles +import org.apache.comet.tags.AzureTest /** * Pins the resolver's behaviour per hadoop-azure version and cross-checks it against the real @@ -67,6 +68,7 @@ import org.apache.comet.objectstore.AbfsReflection.Handles * assert while recording, so a recorded file never contradicts the hadoop-azure source. The file * is written only when every case ran, so a filtered run cannot leave a partial file behind. */ +@AzureTest class AbfsAuthParitySuite extends AnyFunSuite with Matchers with BeforeAndAfterAll { import AbfsAuthParitySuite._ diff --git a/spark/src/test/scala/org/apache/comet/objectstore/AbfsAuthResolverSuite.scala b/spark/src/test/scala/org/apache/comet/objectstore/AbfsAuthResolverSuite.scala index 4d69c2e01a3..eefb3ee07f2 100644 --- a/spark/src/test/scala/org/apache/comet/objectstore/AbfsAuthResolverSuite.scala +++ b/spark/src/test/scala/org/apache/comet/objectstore/AbfsAuthResolverSuite.scala @@ -35,12 +35,14 @@ import org.apache.hadoop.fs.azurebfs.services.KeyProvider import org.apache.comet.objectstore.AbfsAuthResolver._ import org.apache.comet.objectstore.AbfsReflection.Handles +import org.apache.comet.tags.AzureTest /** * Runs the resolver against the hadoop-azure on the test classpath (3.3.4 on the spark-3.x * profiles, 3.4.1, 3.4.2 and 3.5.0 on spark-4.0, 4.1 and 4.2). Cases that depend on a feature one * line lacks are gated on the capability probes rather than on a version number. */ +@AzureTest class AbfsAuthResolverSuite extends AnyFunSuite with Matchers { private val account = "myacct.dfs.core.windows.net" diff --git a/spark/src/test/scala/org/apache/comet/objectstore/AbfsScanFallbackSuite.scala b/spark/src/test/scala/org/apache/comet/objectstore/AbfsScanFallbackSuite.scala index 016491d8af8..38a6e3bd40b 100644 --- a/spark/src/test/scala/org/apache/comet/objectstore/AbfsScanFallbackSuite.scala +++ b/spark/src/test/scala/org/apache/comet/objectstore/AbfsScanFallbackSuite.scala @@ -31,12 +31,14 @@ import org.apache.spark.sql.types.{IntegerType, StructType} import org.apache.comet.{CometConf, ExtendedExplainInfo} import org.apache.comet.hadoop.fs.FakeAbfsFileSystem +import org.apache.comet.tags.AzureTest /** * Plans scans over `abfss://` paths served by [[FakeAbfsFileSystem]], with authentication * resolved offline by the hadoop-azure on the test classpath. A scan Comet declines runs on * Spark's own scan against the local files. */ +@AzureTest class AbfsScanFallbackSuite extends CometTestBase { private val multipleAuthoritiesReason = "more than one container or account" From 8f09f151e979cdbb3dde0ac274d5118ff583bfe8 Mon Sep 17 00:00:00 2001 From: comphead Date: Wed, 14 Oct 2026 10:19:25 -0700 Subject: [PATCH 2/2] test: tag only the AzureBlobFileSystem cross-check with AzureTest On a mismatched classpath only the FileSystem.newInstance cross-check in AbfsAuthParitySuite fails. AzureBlobFileSystem links against Hadoop classes an older Hadoop lacks, while the resolver reaches nothing newer than AbfsConfiguration, so the resolver cases, AbfsAuthResolverSuite and AbfsScanFallbackSuite still pass there. Tag just that test, with a ScalaTest Tag of the same name, so a build that passes -DtagsToExclude=org.apache.comet.tags.AzureTest keeps running the resolver cases. --- docs/source/contributor-guide/development.md | 9 ++++---- .../objectstore/AbfsAuthParitySuite.scala | 5 +++-- .../objectstore/AbfsAuthResolverSuite.scala | 2 -- .../objectstore/AbfsScanFallbackSuite.scala | 2 -- .../org/apache/comet/tags/AzureTest.scala} | 21 +++++++------------ 5 files changed, 15 insertions(+), 24 deletions(-) rename spark/src/test/{java/org/apache/comet/tags/AzureTest.java => scala/org/apache/comet/tags/AzureTest.scala} (60%) diff --git a/docs/source/contributor-guide/development.md b/docs/source/contributor-guide/development.md index f55e957d903..eb12723bdd8 100644 --- a/docs/source/contributor-guide/development.md +++ b/docs/source/contributor-guide/development.md @@ -606,10 +606,11 @@ still assert while recording, so a recorded file cannot contradict what hadoop-a file is written only when every case ran. Review the diff of the regenerated files before committing them: each line that changes is a behaviour change on that hadoop-azure release. -`AbfsAuthParitySuite`, `AbfsAuthResolverSuite` and `AbfsScanFallbackSuite` are tagged -`org.apache.comet.tags.AzureTest`. They pass only when the test-scoped hadoop-azure matches the -Hadoop the build runs on. A build against another Hadoop, where no matching hadoop-azure is -pinned, can skip them with `-DtagsToExclude=org.apache.comet.tags.AzureTest`. +The FileSystem cross-check in `AbfsAuthParitySuite` is tagged `org.apache.comet.tags.AzureTest`. +It initializes a real `AzureBlobFileSystem`, so it passes only when the test-scoped hadoop-azure +matches the Hadoop the build runs on. A build against another Hadoop, where no matching +hadoop-azure is pinned, can skip it with `-DtagsToExclude=org.apache.comet.tags.AzureTest` and +still run the resolver cases. ## Benchmark diff --git a/spark/src/test/scala/org/apache/comet/objectstore/AbfsAuthParitySuite.scala b/spark/src/test/scala/org/apache/comet/objectstore/AbfsAuthParitySuite.scala index 1491195faec..eeca4d0fb99 100644 --- a/spark/src/test/scala/org/apache/comet/objectstore/AbfsAuthParitySuite.scala +++ b/spark/src/test/scala/org/apache/comet/objectstore/AbfsAuthParitySuite.scala @@ -68,7 +68,6 @@ import org.apache.comet.tags.AzureTest * assert while recording, so a recorded file never contradicts the hadoop-azure source. The file * is written only when every case ran, so a filtered run cannot leave a partial file behind. */ -@AzureTest class AbfsAuthParitySuite extends AnyFunSuite with Matchers with BeforeAndAfterAll { import AbfsAuthParitySuite._ @@ -186,7 +185,9 @@ class AbfsAuthParitySuite extends AnyFunSuite with Matchers with BeforeAndAfterA } } - test("FileSystem.newInstance agrees with the resolver on every case") { + // Tagged AzureTest: AzureBlobFileSystem links against Hadoop classes that only a matching + // Hadoop has, while the resolver cases above need nothing newer than AbfsConfiguration. + test("FileSystem.newInstance agrees with the resolver on every case", AzureTest) { val timings = mutable.ArrayBuffer[(String, Long)]() val disagreements = cases.flatMap { c => val outcome = resolve(hadoopConf(c), c.uri) diff --git a/spark/src/test/scala/org/apache/comet/objectstore/AbfsAuthResolverSuite.scala b/spark/src/test/scala/org/apache/comet/objectstore/AbfsAuthResolverSuite.scala index eefb3ee07f2..4d69c2e01a3 100644 --- a/spark/src/test/scala/org/apache/comet/objectstore/AbfsAuthResolverSuite.scala +++ b/spark/src/test/scala/org/apache/comet/objectstore/AbfsAuthResolverSuite.scala @@ -35,14 +35,12 @@ import org.apache.hadoop.fs.azurebfs.services.KeyProvider import org.apache.comet.objectstore.AbfsAuthResolver._ import org.apache.comet.objectstore.AbfsReflection.Handles -import org.apache.comet.tags.AzureTest /** * Runs the resolver against the hadoop-azure on the test classpath (3.3.4 on the spark-3.x * profiles, 3.4.1, 3.4.2 and 3.5.0 on spark-4.0, 4.1 and 4.2). Cases that depend on a feature one * line lacks are gated on the capability probes rather than on a version number. */ -@AzureTest class AbfsAuthResolverSuite extends AnyFunSuite with Matchers { private val account = "myacct.dfs.core.windows.net" diff --git a/spark/src/test/scala/org/apache/comet/objectstore/AbfsScanFallbackSuite.scala b/spark/src/test/scala/org/apache/comet/objectstore/AbfsScanFallbackSuite.scala index 38a6e3bd40b..016491d8af8 100644 --- a/spark/src/test/scala/org/apache/comet/objectstore/AbfsScanFallbackSuite.scala +++ b/spark/src/test/scala/org/apache/comet/objectstore/AbfsScanFallbackSuite.scala @@ -31,14 +31,12 @@ import org.apache.spark.sql.types.{IntegerType, StructType} import org.apache.comet.{CometConf, ExtendedExplainInfo} import org.apache.comet.hadoop.fs.FakeAbfsFileSystem -import org.apache.comet.tags.AzureTest /** * Plans scans over `abfss://` paths served by [[FakeAbfsFileSystem]], with authentication * resolved offline by the hadoop-azure on the test classpath. A scan Comet declines runs on * Spark's own scan against the local files. */ -@AzureTest class AbfsScanFallbackSuite extends CometTestBase { private val multipleAuthoritiesReason = "more than one container or account" diff --git a/spark/src/test/java/org/apache/comet/tags/AzureTest.java b/spark/src/test/scala/org/apache/comet/tags/AzureTest.scala similarity index 60% rename from spark/src/test/java/org/apache/comet/tags/AzureTest.java rename to spark/src/test/scala/org/apache/comet/tags/AzureTest.scala index 00619f98d8f..aacf5b87e4a 100644 --- a/spark/src/test/java/org/apache/comet/tags/AzureTest.java +++ b/spark/src/test/scala/org/apache/comet/tags/AzureTest.scala @@ -17,21 +17,14 @@ * under the License. */ -package org.apache.comet.tags; +package org.apache.comet.tags -import java.lang.annotation.ElementType; -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; -import java.lang.annotation.Target; - -import org.scalatest.TagAnnotation; +import org.scalatest.Tag /** - * Marks suites that resolve Azure ABFS authentication through the hadoop-azure on the test - * classpath. They pass only when that hadoop-azure matches the Hadoop the build runs on, so a build - * against another Hadoop can skip them with -DtagsToExclude=org.apache.comet.tags.AzureTest. + * Marks tests that initialize a real `AzureBlobFileSystem` from the hadoop-azure on the test + * classpath. They pass only when that hadoop-azure matches the Hadoop the build runs on, so a + * build against another Hadoop can skip them with + * `-DtagsToExclude=org.apache.comet.tags.AzureTest`. */ -@TagAnnotation -@Retention(RetentionPolicy.RUNTIME) -@Target({ElementType.METHOD, ElementType.TYPE}) -public @interface AzureTest {} +object AzureTest extends Tag("org.apache.comet.tags.AzureTest")