From 1f6dc805250415d330d900af77b8749e254c76d5 Mon Sep 17 00:00:00 2001 From: Alexandre Teixeira <111787685+alteixeira20@users.noreply.github.com> Date: Fri, 3 Jul 2026 20:17:28 +0100 Subject: [PATCH] ci: add focused test guidance signal (#4982) * ci: add focused test guidance signal * ci: diff focused guidance from merge base --- .github/scripts/focused_test_guidance.py | 124 +++++++++++++++++++ .github/workflows/ci.yml | 54 +++++++++ tests/test_focused_test_guidance.py | 148 +++++++++++++++++++++++ 3 files changed, 326 insertions(+) create mode 100644 .github/scripts/focused_test_guidance.py create mode 100644 tests/test_focused_test_guidance.py diff --git a/.github/scripts/focused_test_guidance.py b/.github/scripts/focused_test_guidance.py new file mode 100644 index 000000000..1426d35fa --- /dev/null +++ b/.github/scripts/focused_test_guidance.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python3 +"""Report focused pytest guidance for changed paths under tests/.""" + +from __future__ import annotations + +import argparse +import os +import shlex +import subprocess +import sys +from collections.abc import Iterable +from pathlib import PurePosixPath + + +def parse_paths(raw_paths: bytes) -> list[str]: + """Decode the NUL-delimited output of ``git diff --name-only -z``.""" + return [os.fsdecode(path) for path in raw_paths.split(b"\0") if path] + + +def changed_paths_from_merge_base(base_sha: str, head_sha: str) -> list[str]: + """Return changed ``tests/`` paths using GitHub PR three-dot semantics. + + GitHub PR changed files are based on the merge base and the PR head, not a + direct endpoint diff between the current base branch tip and the PR head. + Using the direct endpoint diff can include files changed only on the base + branch when the PR branch is stale. + """ + merge_base = subprocess.check_output( + ["git", "merge-base", base_sha, head_sha], + stderr=subprocess.DEVNULL, + ).strip() + raw_paths = subprocess.check_output( + [ + "git", + "diff", + "--name-only", + "--diff-filter=ACMRT", + "-z", + os.fsdecode(merge_base), + head_sha, + "--", + "tests/", + ], + ) + return parse_paths(raw_paths) + + +def select_test_paths(paths: Iterable[str]) -> list[str]: + """Return unique, repository-relative paths contained by tests/.""" + selected: set[str] = set() + for raw_path in paths: + path = PurePosixPath(raw_path) + if path.is_absolute() or ".." in path.parts: + continue + parts = tuple(part for part in path.parts if part != ".") + if len(parts) >= 2 and parts[0] == "tests": + selected.add(PurePosixPath(*parts).as_posix()) + return sorted(selected) + + +def is_pytest_file(path: str) -> bool: + """Return whether a changed path follows this repository's pytest naming.""" + name = PurePosixPath(path).name + return name.endswith(".py") and ( + name.startswith("test_") or name.endswith("_test.py") + ) + + +def pytest_command(paths: Iterable[str]) -> str: + """Build a copyable pytest command for changed runnable test files.""" + command = ["python3", "-m", "pytest", "-q", *paths] + return shlex.join(command) + + +def format_report(paths: Iterable[str]) -> str: + """Format focused guidance for CI logs and the workflow summary.""" + changed_paths = select_test_paths(paths) + runnable_paths = [path for path in changed_paths if is_pytest_file(path)] + lines = ["## Focused test guidance (report-only)", ""] + if not changed_paths: + lines.append("No changed paths under `tests/`.") + else: + lines.extend(["Changed paths under `tests/`:", ""]) + lines.extend(f"- `{path}`" for path in changed_paths) + lines.extend(["", "Suggested focused validation:", ""]) + if runnable_paths: + lines.append(f"```sh\n{pytest_command(runnable_paths)}\n```") + else: + lines.append("No directly runnable pytest files changed.") + lines.extend( + [ + "", + "This guidance does not infer tests from source changes. " + "Existing blocking CI remains the source of truth.", + ] + ) + return "\n".join(lines) + + +def _parse_args(argv: list[str]) -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Report focused pytest guidance for changed tests/ paths.", + ) + parser.add_argument("--base-sha", help="Pull request base commit SHA.") + parser.add_argument("--head-sha", help="Pull request head commit SHA.") + return parser.parse_args(argv) + + +def main(argv: list[str] | None = None) -> int: + args = _parse_args(sys.argv[1:] if argv is None else argv) + if bool(args.base_sha) != bool(args.head_sha): + raise SystemExit("--base-sha and --head-sha must be provided together") + + if args.base_sha and args.head_sha: + paths = changed_paths_from_merge_base(args.base_sha, args.head_sha) + else: + paths = parse_paths(sys.stdin.buffer.read()) + + print(format_report(paths)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 787bd9dea..f7d3659e8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,6 +15,60 @@ concurrency: cancel-in-progress: true jobs: + focused-test-guidance: + name: Focused test guidance (report-only) + if: github.event_name == 'pull_request' + runs-on: ubuntu-latest + continue-on-error: true + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + fetch-depth: 0 + persist-credentials: false + - name: Report changed test paths + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + run: | + report_file="$RUNNER_TEMP/focused-test-guidance.md" + publish_report() { + cat "$report_file" + if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then + cat "$report_file" >> "$GITHUB_STEP_SUMMARY" || true + fi + return 0 + } + + report_unavailable() { + { + printf '%s\n\n' '## Focused test guidance unavailable (report-only)' + printf '%s\n\n' "$1" + printf '%s\n' 'Existing blocking CI remains the source of truth.' + } > "$report_file" + publish_report + exit 0 + } + + if [ -z "$BASE_SHA" ] || [ -z "$HEAD_SHA" ]; then + report_unavailable "Pull request base/head metadata is missing." + fi + + if ! git cat-file -e "${BASE_SHA}^{commit}" 2>/dev/null; then + report_unavailable "The pull request base commit is unavailable locally." + fi + + if ! git cat-file -e "${HEAD_SHA}^{commit}" 2>/dev/null; then + report_unavailable "The pull request head commit is unavailable locally." + fi + + if ! python3 .github/scripts/focused_test_guidance.py \ + --base-sha "$BASE_SHA" \ + --head-sha "$HEAD_SHA" > "$report_file"; then + report_unavailable "The focused test guidance helper could not produce a report." + fi + + publish_report + python-syntax: name: Python syntax (compileall) runs-on: ubuntu-latest diff --git a/tests/test_focused_test_guidance.py b/tests/test_focused_test_guidance.py new file mode 100644 index 000000000..adf46f9e4 --- /dev/null +++ b/tests/test_focused_test_guidance.py @@ -0,0 +1,148 @@ +"""Tests for the report-only changed-test guidance helper.""" + +from __future__ import annotations + +import importlib.util +import subprocess +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +SCRIPT_PATH = ROOT / ".github" / "scripts" / "focused_test_guidance.py" + + +def _load_helper(): + spec = importlib.util.spec_from_file_location("focused_test_guidance", SCRIPT_PATH) + assert spec is not None + assert spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +guidance = _load_helper() + + +def test_parse_paths_supports_nul_delimited_git_output(): + raw_paths = b"tests/test_alpha.py\0tests/path with spaces/test_beta.py\0" + + assert guidance.parse_paths(raw_paths) == [ + "tests/test_alpha.py", + "tests/path with spaces/test_beta.py", + ] + + +def test_select_test_paths_ignores_paths_outside_tests(): + paths = [ + "src/test_alpha.py", + "tests/test_beta.py", + "./tests/unit/example_test.py", + "tests/../src/test_gamma.py", + ] + + assert guidance.select_test_paths(paths) == [ + "tests/test_beta.py", + "tests/unit/example_test.py", + ] + + +def test_select_test_paths_deduplicates_paths(): + paths = ["tests/test_alpha.py", "./tests/test_alpha.py"] + + assert guidance.select_test_paths(paths) == ["tests/test_alpha.py"] + + +def test_format_report_builds_command_for_changed_python_test(): + report = guidance.format_report(["tests/test_beta.py"]) + + assert "- `tests/test_beta.py`" in report + assert "python3 -m pytest -q tests/test_beta.py" in report + assert "does not infer tests from source changes" in report + assert "Existing blocking CI remains the source of truth" in report + + +def test_format_report_lists_changed_non_python_test_path_without_command(): + report = guidance.format_report(["tests/README.md"]) + + assert "- `tests/README.md`" in report + assert "No directly runnable pytest files changed." in report + assert "python3 -m pytest" not in report + + +def test_format_report_ignores_src_path(): + report = guidance.format_report(["src/test_ignored.py"]) + + assert "src/test_ignored.py" not in report + assert "No changed paths under `tests/`." in report + + +def test_format_report_shell_quotes_path_with_spaces(): + report = guidance.format_report(["tests/path with spaces/test_beta.py"]) + + assert "- `tests/path with spaces/test_beta.py`" in report + assert ( + "python3 -m pytest -q 'tests/path with spaces/test_beta.py'" + ) in report + + +def test_format_report_handles_no_changed_test_paths(): + report = guidance.format_report([]) + + assert "No changed paths under `tests/`." in report + assert "No directly runnable pytest files changed." in report + +def _git(repo: Path, *args: str) -> str: + return subprocess.check_output(["git", *args], cwd=repo, text=True).strip() + + +def _write(path: Path, content: str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") + + +def test_changed_paths_from_merge_base_excludes_base_only_test_changes(tmp_path, monkeypatch): + repo = tmp_path / "repo" + repo.mkdir() + + _git(repo, "init") + _git(repo, "config", "user.email", "ci@example.test") + _git(repo, "config", "user.name", "CI Test") + + _write(repo / "tests/test_shared.py", "def test_shared():\n assert True\n") + _git(repo, "add", "tests/test_shared.py") + _git(repo, "commit", "-m", "base") + ancestor = _git(repo, "rev-parse", "HEAD") + + _git(repo, "checkout", "-b", "feature") + _write(repo / "tests/test_pr_delta.py", "def test_pr_delta():\n assert True\n") + _git(repo, "add", "tests/test_pr_delta.py") + _git(repo, "commit", "-m", "add pr test") + head_sha = _git(repo, "rev-parse", "HEAD") + + _git(repo, "checkout", "-b", "dev", ancestor) + _write(repo / "tests/test_shared.py", "def test_shared():\n assert 1 == 1\n") + _git(repo, "add", "tests/test_shared.py") + _git(repo, "commit", "-m", "base-only test change") + base_sha = _git(repo, "rev-parse", "HEAD") + + endpoint_paths = guidance.parse_paths( + subprocess.check_output( + [ + "git", + "diff", + "--name-only", + "--diff-filter=ACMRT", + "-z", + base_sha, + head_sha, + "--", + "tests/", + ], + cwd=repo, + ) + ) + assert "tests/test_shared.py" in endpoint_paths + + monkeypatch.chdir(repo) + assert guidance.changed_paths_from_merge_base(base_sha, head_sha) == [ + "tests/test_pr_delta.py" + ]