Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions .asf.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,9 @@ github:
main:
required_pull_request_reviews:
required_approving_review_count: 1
required_status_checks:
contexts:
- "CI Required"
branch-0.1:
required_pull_request_reviews:
required_approving_review_count: 1
Expand Down
51 changes: 48 additions & 3 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ name: CI
# in their own subgroup so they never cancel the real commit run; opened and
# synchronize both map to `commit` so a new push still supersedes its predecessor.
concurrency:
group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}-${{ github.event.action == 'labeled' && github.event.label.name || 'commit' }}
group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}-${{ github.event_name }}-${{ github.event.action == 'labeled' && github.event.label.name || 'commit' }}
cancel-in-progress: true

on:
Expand All @@ -37,6 +37,8 @@ on:
branches:
- main
workflow_dispatch:
merge_group:
types: [checks_requested]

jobs:
# ---------------------------------------------------------------------------
Expand Down Expand Up @@ -98,6 +100,9 @@ jobs:
- name: Check CI config invariants
run: python3 dev/ci/check-ci-config.py

- name: Check CI gate and queue routing
run: python3 -m unittest discover -s dev/ci -p 'test_ci_*.py'

- name: Install actionlint
run: |
curl -sSfL https://raw.githubusercontent.com/rhysd/actionlint/main/scripts/download-actionlint.bash | bash
Expand Down Expand Up @@ -150,6 +155,8 @@ jobs:
PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }}
PUSH_BEFORE: ${{ github.event.before }}
PUSH_AFTER: ${{ github.sha }}
QUEUE_BASE_SHA: ${{ github.event.merge_group.base_sha }}
QUEUE_HEAD_SHA: ${{ github.event.merge_group.head_sha }}
run: |
set -euo pipefail
: > changed_files.txt
Expand All @@ -160,13 +167,19 @@ jobs:
:
elif [[ "$EVENT_NAME" == "pull_request" ]]; then
git diff --name-only "$PR_BASE_SHA"..."$PR_HEAD_SHA" > changed_files.txt
else
elif [[ "$EVENT_NAME" == "merge_group" ]]; then
# Include every queued change relative to the current base branch.
git diff --name-only "${QUEUE_BASE_SHA:?}".."${QUEUE_HEAD_SHA:?}" > changed_files.txt
elif [[ "$EVENT_NAME" == "push" ]]; then
# push to main; first push to a branch has all-zero before sha
if [[ "$PUSH_BEFORE" =~ ^0+$ ]]; then
git ls-tree -r --name-only "$PUSH_AFTER" > changed_files.txt
else
git diff --name-only "$PUSH_BEFORE".."$PUSH_AFTER" > changed_files.txt
fi
else
echo "Unsupported CI event: $EVENT_NAME" >&2
exit 1
fi
echo "Changed files:"
cat changed_files.txt
Expand Down Expand Up @@ -296,7 +309,7 @@ jobs:
iceberg_1_11:
name: Iceberg Spark SQL Tests (Iceberg 1.11)
needs: changes
# Runs on every PR: Iceberg 1.11 is our only Spark 4.1 Iceberg coverage.
# Runs on PRs, main, and the merge queue; this is our Spark 4.1 Iceberg coverage.
if: needs.changes.outputs.iceberg_1_11 == 'true'
uses: ./.github/workflows/iceberg_spark_test_reusable.yml
with:
Expand All @@ -305,3 +318,35 @@ jobs:
spark-short: '4.1'
spark-full: '4.1.3'
java: 17

ci-required:
# Label/manual runs must never overwrite the canonical PR/queue verdict.
name: ${{ (github.event.action == 'labeled' || github.event_name == 'workflow_dispatch') && 'CI Optional' || 'CI Required' }}
if: ${{ always() }}
needs:
- preflight
- changes
- pr_build_linux
- pr_build_macos
- pr_benchmark_check
- docs
- spark_3_4
- spark_3_5
- spark_4_0
- spark_4_1
- iceberg_1_8
- iceberg_1_9
- iceberg_1_10
- iceberg_1_11
runs-on: ubuntu-slim
timeout-minutes: 5
permissions:
contents: read
steps:
- name: Check CI jobs
shell: bash
env:
NEEDS: ${{ toJSON(needs) }}
run: |
jq -r 'to_entries[] | "\(.key): \(.value.result)"' <<< "$NEEDS"
jq -e 'all(.[]; .result == "success" or .result == "skipped")' <<< "$NEEDS"
25 changes: 14 additions & 11 deletions dev/ci/compute-changes.py
Original file line number Diff line number Diff line change
Expand Up @@ -302,27 +302,28 @@
#
# "pr" every pull request
# "push" push to main
# "queue" merge_group/checks_requested
# "label:<name>" a pull request carrying that label
#
# workflow_dispatch always runs everything, so it is not listed. "pr" and
# "label:" are mutually exclusive -- a job is either unconditional on pull
# requests or opt-in, never both -- and check-ci-config.py rejects a job that
# lists both rather than letting the label quietly win.
POLICY = {
"build_linux": ["pr", "push"],
"build_macos": ["pr", "push"],
"benchmark": ["pr", "push"],
"build_linux": ["pr", "push", "queue"],
"build_macos": ["pr", "push", "queue"],
"benchmark": ["pr", "push", "queue"],
# docs deploys to asf-site, so it must not run from a pull request.
"docs": ["push"],
"spark_3_4": ["push", "label:run-spark-3.4-tests"],
"spark_3_5": ["pr", "push"],
"spark_4_0": ["push", "label:run-spark-4.0-tests"],
"spark_4_1": ["pr", "push"],
"iceberg_1_8": ["push", "label:run-iceberg-tests"],
"iceberg_1_9": ["push", "label:run-iceberg-tests"],
"iceberg_1_10": ["push", "label:run-iceberg-tests"],
"spark_3_4": ["push", "queue", "label:run-spark-3.4-tests"],
"spark_3_5": ["pr", "push", "queue"],
"spark_4_0": ["push", "queue", "label:run-spark-4.0-tests"],
"spark_4_1": ["pr", "push", "queue"],
"iceberg_1_8": ["push", "queue", "label:run-iceberg-tests"],
"iceberg_1_9": ["push", "queue", "label:run-iceberg-tests"],
"iceberg_1_10": ["push", "queue", "label:run-iceberg-tests"],
# Iceberg 1.11 is our only Spark 4.1 Iceberg coverage, so it is not opt-in.
"iceberg_1_11": ["pr", "push"],
"iceberg_1_11": ["pr", "push", "queue"],
}


Expand All @@ -342,6 +343,8 @@ def event_allows(job, event):

if name == "workflow_dispatch":
return True
if name == "merge_group":
return event.get("action") == "checks_requested" and "queue" in tiers
if name == "push":
return "push" in tiers
if name != "pull_request":
Expand Down
129 changes: 129 additions & 0 deletions dev/ci/test_ci_queue.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,129 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.

import importlib.util
import json
import os
from pathlib import Path
import re
import subprocess
import textwrap
import unittest

ROOT = Path(__file__).resolve().parents[2]


def load(name):
spec = importlib.util.spec_from_file_location(name, ROOT / "dev/ci" / f"{name}.py")
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module


routing = load("compute-changes")
PR_JOBS = {"build_linux", "build_macos", "benchmark", "spark_3_5", "spark_4_1", "iceberg_1_11"}
OTHER_VERSIONS = {"spark_3_4", "spark_4_0", "iceberg_1_8", "iceberg_1_9", "iceberg_1_10"}
QUEUE_JOBS = PR_JOBS | OTHER_VERSIONS


class QueuePolicyTest(unittest.TestCase):
def event(self, name, action="", labels=(), label=""):
return {"name": name, "action": action, "labels": list(labels),
"label": label}

def allowed(self, event):
return {job for job in routing.POLICY if routing.event_allows(job, event)}

def test_queue_and_push_run_full_coverage_without_reducing_prs(self):
self.assertEqual(self.allowed(self.event("pull_request", "synchronize")), PR_JOBS)
self.assertEqual(self.allowed(self.event("push")), QUEUE_JOBS | {"docs"})
self.assertEqual(self.allowed(self.event("merge_group", "checks_requested")), QUEUE_JOBS)

def test_unknown_event_and_queue_action_cannot_launch_jobs(self):
for event in (self.event("schedule"), self.event("merge_group", "destroyed")):
self.assertEqual(self.allowed(event), set())

def test_label_opt_in_does_not_rerun_primary(self):
for label, expected in (("run-spark-3.4-tests", {"spark_3_4"}),
("run-spark-4.0-tests", {"spark_4_0"}),
("run-iceberg-tests", {"iceberg_1_8", "iceberg_1_9",
"iceberg_1_10"})):
with self.subTest(label=label):
self.assertEqual(self.allowed(self.event("pull_request", "labeled", [label], label)), expected)
self.assertEqual(self.allowed(self.event("pull_request", "synchronize", [label])), PR_JOBS | expected)
self.assertEqual(self.allowed(self.event("pull_request", "labeled", [label, "dependencies"], "dependencies")), set())

def test_docs_and_combined_queue_changes(self):
event = self.event("merge_group", "checks_requested")
self.assertFalse(any(routing.compute(["docs/source/user-guide/overview.md"], event).values()))
# The queue diff can include different integration edits from two PRs.
jobs = routing.compute(["dev/diffs/3.5.9.diff", "dev/diffs/iceberg/1.11.0.diff"], event)
self.assertEqual({k for k, v in jobs.items() if v}, {
"spark_3_5", "iceberg_1_8", "iceberg_1_9", "iceberg_1_10", "iceberg_1_11"})
jobs = routing.compute(["native/core/src/lib.rs"], event)
self.assertEqual({k for k, v in jobs.items() if v}, QUEUE_JOBS - {"benchmark"})

def test_manual_run_exercises_all_coverage(self):
event = self.event("workflow_dispatch")
self.assertEqual(self.allowed(event), QUEUE_JOBS | {"docs"})


class RequiredGateTest(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.workflow = (ROOT / ".github/workflows/ci.yml").read_text()
cls.blocks = dict(re.findall(
r"^ ([\w-]+):\n(.*?)(?=^ [\w-]+:|\Z)",
cls.workflow.split("\njobs:\n", 1)[1], re.M | re.S))
cls.required = cls.blocks["ci-required"]
# Exercise the actual shell step, not a separate copy of its predicate.
cls.script = textwrap.dedent(cls.required.split(" run: |\n", 1)[1])

def run_gate(self, results):
needs = {job: {"result": result} for job, result in results.items()}
return subprocess.run(
["bash", "-e", "-o", "pipefail", "-c", self.script],
env={**os.environ, "NEEDS": json.dumps(needs)},
capture_output=True, text=True).returncode

def test_success_and_skipped_jobs_pass_without_a_job_registry(self):
self.assertEqual(self.run_gate({"preflight": "success", "changes": "success",
"new-test-job": "success", "docs": "skipped"}), 0)
self.assertEqual(self.run_gate({"optional-job": "skipped"}), 0)

def test_failed_cancelled_or_missing_result_blocks(self):
for job in ("preflight", "changes", "new-test-job"):
for result in ("failure", "cancelled", None):
with self.subTest(job=job, result=result):
results = {"preflight": "success", "changes": "success",
"new-test-job": "success", "docs": "skipped"}
results[job] = result
self.assertNotEqual(self.run_gate(results), 0)

def test_workflow_wiring_covers_every_job(self):
dependencies = set(re.findall(r"^ - ([\w-]+)$", self.required, re.M))
self.assertEqual(dependencies, set(self.blocks) - {"ci-required"})
self.assertIn("if: ${{ always() }}", self.required)
self.assertIn("github.event.action == 'labeled'", self.required)
self.assertIn("'CI Optional' || 'CI Required'", self.required)
self.assertIn(" merge_group:\n types: [checks_requested]", self.workflow)
self.assertIn('"${QUEUE_BASE_SHA:?}".."${QUEUE_HEAD_SHA:?}"', self.workflow)
self.assertIn(' - "CI Required"', (ROOT / ".asf.yaml").read_text())


if __name__ == "__main__":
unittest.main()
Loading