diff --git a/docs/source/contributor-guide/development.md b/docs/source/contributor-guide/development.md index ea4a65e3089..eb12723bdd8 100644 --- a/docs/source/contributor-guide/development.md +++ b/docs/source/contributor-guide/development.md @@ -606,6 +606,12 @@ still assert while recording, so a recorded file cannot contradict what hadoop-a file is written only when every case ran. Review the diff of the regenerated files before committing them: each line that changes is a behaviour change on that hadoop-azure release. +The FileSystem cross-check in `AbfsAuthParitySuite` is tagged `org.apache.comet.tags.AzureTest`. +It initializes a real `AzureBlobFileSystem`, so it passes only when the test-scoped hadoop-azure +matches the Hadoop the build runs on. A build against another Hadoop, where no matching +hadoop-azure is pinned, can skip it with `-DtagsToExclude=org.apache.comet.tags.AzureTest` and +still run the resolver cases. + ## Benchmark There's a `make` command to run micro benchmarks in the repo. For diff --git a/pom.xml b/pom.xml index 2a737c9c20c..4bd65dc53e3 100644 --- a/pom.xml +++ b/pom.xml @@ -123,6 +123,9 @@ under the License. reflection only). Each Spark profile pins it to its Hadoop line; override with -Dhadoop-azure.version to test another release. --> 3.4.2 + + ${project.basedir}/../native/target/debug darwin x86_64 @@ -1124,7 +1127,7 @@ under the License. SparkTestSuite.txt D - org.apache.comet.IntegrationTestSuite + org.apache.comet.IntegrationTestSuite,${tagsToExclude} file:src/test/resources/log4j2.properties diff --git a/spark/src/test/scala/org/apache/comet/objectstore/AbfsAuthParitySuite.scala b/spark/src/test/scala/org/apache/comet/objectstore/AbfsAuthParitySuite.scala index 3d95864130a..eeca4d0fb99 100644 --- a/spark/src/test/scala/org/apache/comet/objectstore/AbfsAuthParitySuite.scala +++ b/spark/src/test/scala/org/apache/comet/objectstore/AbfsAuthParitySuite.scala @@ -43,6 +43,7 @@ import com.fasterxml.jackson.databind.{JsonNode, ObjectMapper} import org.apache.comet.objectstore.AbfsAuthParityCases._ import org.apache.comet.objectstore.AbfsAuthResolver._ import org.apache.comet.objectstore.AbfsReflection.Handles +import org.apache.comet.tags.AzureTest /** * Pins the resolver's behaviour per hadoop-azure version and cross-checks it against the real @@ -184,7 +185,9 @@ class AbfsAuthParitySuite extends AnyFunSuite with Matchers with BeforeAndAfterA } } - test("FileSystem.newInstance agrees with the resolver on every case") { + // Tagged AzureTest: AzureBlobFileSystem links against Hadoop classes that only a matching + // Hadoop has, while the resolver cases above need nothing newer than AbfsConfiguration. + test("FileSystem.newInstance agrees with the resolver on every case", AzureTest) { val timings = mutable.ArrayBuffer[(String, Long)]() val disagreements = cases.flatMap { c => val outcome = resolve(hadoopConf(c), c.uri) diff --git a/spark/src/test/scala/org/apache/comet/tags/AzureTest.scala b/spark/src/test/scala/org/apache/comet/tags/AzureTest.scala new file mode 100644 index 00000000000..aacf5b87e4a --- /dev/null +++ b/spark/src/test/scala/org/apache/comet/tags/AzureTest.scala @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.comet.tags + +import org.scalatest.Tag + +/** + * Marks tests that initialize a real `AzureBlobFileSystem` from the hadoop-azure on the test + * classpath. They pass only when that hadoop-azure matches the Hadoop the build runs on, so a + * build against another Hadoop can skip them with + * `-DtagsToExclude=org.apache.comet.tags.AzureTest`. + */ +object AzureTest extends Tag("org.apache.comet.tags.AzureTest")