From 84fe3c1caf9ce4f200e4fae9c54f00ce038d32df Mon Sep 17 00:00:00 2001 From: scarreras Date: Sat, 21 Nov 2026 07:47:59 -0300 Subject: [PATCH 1/6] fix: emit individual JUnit cases for subtests Related to backblaze-labs/quality-keeper#612. Preserve each subtest outcome without changing execution or relaxing report validation. --- .../+junit-subtest-counts.infrastructure.md | 1 + doc/source/contributing.rst | 9 ++ test/conftest.py | 2 +- test/junit_subtests.py | 61 ++++++++++++ test/static/test_junit_subtests.py | 98 +++++++++++++++++++ 5 files changed, 170 insertions(+), 1 deletion(-) create mode 100644 changelog.d/+junit-subtest-counts.infrastructure.md create mode 100644 test/junit_subtests.py create mode 100644 test/static/test_junit_subtests.py diff --git a/changelog.d/+junit-subtest-counts.infrastructure.md b/changelog.d/+junit-subtest-counts.infrastructure.md new file mode 100644 index 00000000..a4963d13 --- /dev/null +++ b/changelog.d/+junit-subtest-counts.infrastructure.md @@ -0,0 +1 @@ +Emit individual JUnit cases for pytest and unittest subtests so report totals match their case entries, including failures and skips. diff --git a/doc/source/contributing.rst b/doc/source/contributing.rst index 74676259..ecae4ac9 100644 --- a/doc/source/contributing.rst +++ b/doc/source/contributing.rst @@ -155,6 +155,15 @@ To run tests matching a keyword expression:: $ nox -s unit-3.10 -- -k keyword +JUnit reports include each subtest as its own case, as well as its parent test. +The repository's ``test.junit_subtests`` compatibility plugin supplies those cases +because pytest 9's reporter counts subtest outcomes without emitting separate +case elements. It copies reports only for JUnit serialization; assertions, test +selection, terminal outcomes and exit codes keep their normal pytest behavior. +The plugin uses pytest's internal reporter API, so dependency updates should run +``pytest test/static/test_junit_subtests.py`` (serial and xdist regression cases). +Remove the compatibility layer only after upstream reporting passes those checks. + Where to put a new test ======================= diff --git a/test/conftest.py b/test/conftest.py index cd17e2a4..e5ea5ba4 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -11,7 +11,7 @@ import pytest -pytest_plugins = ['b2sdk.v3.testing'] +pytest_plugins = ['b2sdk.v3.testing', 'test.junit_subtests'] def pytest_configure(config): diff --git a/test/junit_subtests.py b/test/junit_subtests.py new file mode 100644 index 00000000..7c084561 --- /dev/null +++ b/test/junit_subtests.py @@ -0,0 +1,61 @@ +###################################################################### +# +# File: test/junit_subtests.py +# +# Copyright 2026 Backblaze Inc. All Rights Reserved. +# +# License https://www.backblaze.com/using_b2_code.html +# +###################################################################### +"""Keep pytest 9's JUnit subtest counts consistent with its case elements. + +Only the XML reporter sees copied subtest reports with distinct identities. +Other plugins, test execution and pytest's terminal/exit outcomes are unchanged. +This compatibility layer deliberately uses pytest's internal reporter API; +its subprocess regressions must pass when upgrading pytest. +""" + +from __future__ import annotations + +from copy import copy +from itertools import count + +import pytest +from _pytest.junitxml import LogXML, xml_key +from _pytest.subtests import SubtestReport + + +class SubtestLogXML(LogXML): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._subtest_ids = count(1) + + @pytest.hookimpl(trylast=True) + def pytest_runtest_logreport(self, report): + if not isinstance(report, SubtestReport): + return super().pytest_runtest_logreport(report) + report = copy(report) + report.nodeid += f'::subtest-{next(self._subtest_ids)}' + super().pytest_runtest_logreport(report) + # Subtests have a call report, but no later teardown to close the case. + self.finalize(report) + self.open_reports = [entry for entry in self.open_reports if entry is not report] + + +@pytest.hookimpl(trylast=True) +def pytest_configure(config): + original = config.stash.get(xml_key, None) + if original is None: + return # No --junitxml, or an xdist worker: leave reporting untouched. + replacement = SubtestLogXML( + original.logfile, + original.prefix, + original.suite_name, + original.logging, + original.report_duration, + original.family, + original.log_passing_tests, + ) + config.pluginmanager.unregister(original) + config.stash[xml_key] = replacement + config.pluginmanager.register(replacement) diff --git a/test/static/test_junit_subtests.py b/test/static/test_junit_subtests.py new file mode 100644 index 00000000..ffa452f4 --- /dev/null +++ b/test/static/test_junit_subtests.py @@ -0,0 +1,98 @@ +###################################################################### +# +# File: test/static/test_junit_subtests.py +# +# Copyright 2026 Backblaze Inc. All Rights Reserved. +# +# License https://www.backblaze.com/using_b2_code.html +# +###################################################################### +"""Exercise real pytest reports, including subtest outcomes and xdist delivery.""" + +from __future__ import annotations + +import os +import subprocess +import sys +import textwrap +from pathlib import Path +from xml.etree import ElementTree as ET + +import pytest + +ROOT = Path(__file__).resolve().parents[2] + + +@pytest.mark.parametrize('workers', [0, 2]) +@pytest.mark.parametrize('outcome', ['passed', 'failed', 'skipped']) +def test_junit_subtest_totals_match_individual_cases(tmp_path, workers, outcome): + # Use the repository's plugin declarations, without loading its SDK fixtures + # into the child fixture directory or inheriting external pytest options. + declarations = (ROOT / 'test/conftest.py').read_text() + line = next(line for line in declarations.splitlines() if line.startswith('pytest_plugins =')) + (tmp_path / 'conftest.py').write_text(line + '\n') + (tmp_path / 'pytest.ini').write_text('[pytest]\n') + (tmp_path / 'test_sample.py').write_text( + textwrap.dedent(f""" + import unittest + import pytest + + def check(i): + if i == 1 and {outcome!r} == 'failed': + assert False, 'deliberate subtest failure' + if i == 1 and {outcome!r} == 'skipped': + pytest.skip('deliberate subtest skip') + + class TestUnittest(unittest.TestCase): + def test_subtests(self): + for i in range(2): + with self.subTest(i=i): + check(i) + + def test_pytest(subtests): + for i in range(2): + with subtests.test(i=i): + check(i) + + def test_pass(): + assert True + + @pytest.mark.skip(reason='deliberate regular skip') + def test_skip(): + pass + + @pytest.mark.xfail(strict=True, reason='deliberate expected failure') + def test_xfail(): + assert False + """) + ) + env = { + key: value + for key, value in os.environ.items() + if not key.startswith(('B2_', 'PYTEST_', 'COV_CORE_', 'COVERAGE_')) + } + env['PYTHONPATH'] = str(ROOT) + completed = subprocess.run( + [sys.executable, '-m', 'pytest', '-n', str(workers), '--junitxml=result.xml', '-q'], + cwd=tmp_path, + env=env, + capture_output=True, + text=True, + timeout=90, + ) + assert completed.returncode == (1 if outcome == 'failed' else 0), ( + completed.stdout + completed.stderr + ) + suite = ET.parse(tmp_path / 'result.xml').getroot().find('testsuite') + cases = suite.findall('testcase') + assert int(suite.get('tests')) == len(cases), completed.stdout + # Two parents, four independently reported subtests, and three plain tests. + assert len(cases) == 9 + for tag, attribute in [('failure', 'failures'), ('error', 'errors'), ('skipped', 'skipped')]: + assert int(suite.get(attribute)) == sum(case.find(tag) is not None for case in cases) + assert all( + sum(child.tag in ('failure', 'error', 'skipped') for child in case) <= 1 for case in cases + ) + assert int(suite.get('failures')) == (3 if outcome == 'failed' else 0) + assert int(suite.get('errors')) == 0 + assert int(suite.get('skipped')) == (4 if outcome == 'skipped' else 2) From 39dc7b5ab3d1ed357cffe59adc07dfee2dae56a5 Mon Sep 17 00:00:00 2001 From: scarreras Date: Sat, 21 Nov 2026 08:02:12 -0300 Subject: [PATCH 2/6] ci: keep draft pull requests free of live B2 operations --- .github/workflows/ci.yml | 25 ++-- .../+draft-ci-no-live.infrastructure.md | 1 + doc/source/contributing.rst | 8 ++ pyproject.toml | 3 + test/static/test_ci_draft_safety.py | 135 ++++++++++++++++++ uv.lock | 8 +- 6 files changed, 169 insertions(+), 11 deletions(-) create mode 100644 changelog.d/+draft-ci-no-live.infrastructure.md create mode 100644 test/static/test_ci_draft_safety.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4acc6e4b..d93b55b1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,6 +5,7 @@ on: branches: [master] pull_request: branches: [master] + types: [opened, synchronize, reopened, ready_for_review] env: PYTHON_DEFAULT_VERSION: "3.14" @@ -74,11 +75,13 @@ jobs: - name: Build the distribution run: uv run nox -vs build cleanup_buckets: + # Draft validation must never touch B2 resources. + if: ${{ github.event_name != 'pull_request' || !github.event.pull_request.draft }} timeout-minutes: 30 needs: lint env: - B2_TEST_APPLICATION_KEY: ${{ secrets.B2_TEST_APPLICATION_KEY }} - B2_TEST_APPLICATION_KEY_ID: ${{ secrets.B2_TEST_APPLICATION_KEY_ID }} + B2_TEST_APPLICATION_KEY: ${{ (github.event_name != 'pull_request' || !github.event.pull_request.draft) && secrets.B2_TEST_APPLICATION_KEY || '' }} + B2_TEST_APPLICATION_KEY_ID: ${{ (github.event_name != 'pull_request' || !github.event.pull_request.draft) && secrets.B2_TEST_APPLICATION_KEY_ID || '' }} runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -103,10 +106,11 @@ jobs: run: uv run nox -vs cleanup_old_buckets test: timeout-minutes: 90 - needs: cleanup_buckets + needs: [lint, cleanup_buckets] + if: ${{ !cancelled() && needs.lint.result == 'success' && (needs.cleanup_buckets.result == 'success' || needs.cleanup_buckets.result == 'skipped') }} env: - B2_TEST_APPLICATION_KEY: ${{ secrets.B2_TEST_APPLICATION_KEY }} - B2_TEST_APPLICATION_KEY_ID: ${{ secrets.B2_TEST_APPLICATION_KEY_ID }} + B2_TEST_APPLICATION_KEY: ${{ (github.event_name != 'pull_request' || !github.event.pull_request.draft) && secrets.B2_TEST_APPLICATION_KEY || '' }} + B2_TEST_APPLICATION_KEY_ID: ${{ (github.event_name != 'pull_request' || !github.event.pull_request.draft) && secrets.B2_TEST_APPLICATION_KEY_ID || '' }} NOX_EXTRAS: ${{ matrix.extras }} runs-on: ${{ matrix.os }} strategy: @@ -141,14 +145,15 @@ jobs: - name: Run unit tests run: uv run nox -vs unit -- -v - name: Run integration tests - if: ${{ env.B2_TEST_APPLICATION_KEY != '' && env.B2_TEST_APPLICATION_KEY_ID != '' }} + if: ${{ (github.event_name != 'pull_request' || !github.event.pull_request.draft) && env.B2_TEST_APPLICATION_KEY != '' && env.B2_TEST_APPLICATION_KEY_ID != '' }} run: uv run nox -vs integration -- --dont-cleanup-old-buckets -v cli-compat: timeout-minutes: 90 - needs: cleanup_buckets + needs: [lint, cleanup_buckets] + if: ${{ !cancelled() && needs.lint.result == 'success' && (needs.cleanup_buckets.result == 'success' || needs.cleanup_buckets.result == 'skipped') }} env: - B2_TEST_APPLICATION_KEY: ${{ secrets.B2_TEST_APPLICATION_KEY }} - B2_TEST_APPLICATION_KEY_ID: ${{ secrets.B2_TEST_APPLICATION_KEY_ID }} + B2_TEST_APPLICATION_KEY: ${{ (github.event_name != 'pull_request' || !github.event.pull_request.draft) && secrets.B2_TEST_APPLICATION_KEY || '' }} + B2_TEST_APPLICATION_KEY_ID: ${{ (github.event_name != 'pull_request' || !github.event.pull_request.draft) && secrets.B2_TEST_APPLICATION_KEY_ID || '' }} INSTALL_SDK_FROM: ../b2-sdk-python runs-on: ${{ matrix.os }} strategy: @@ -196,7 +201,7 @@ jobs: - name: Run CLI integration tests (without secrets) run: uv run --directory ./B2_Command_Line_Tool nox -vs integration -p ${{ matrix.python-version }} -- -m "not require_secrets" - name: Run CLI integration tests (with secrets) - if: ${{ env.B2_TEST_APPLICATION_KEY != '' && env.B2_TEST_APPLICATION_KEY_ID != '' && matrix.os == 'ubuntu-latest' && matrix.python-version == '3.14' }} + if: ${{ (github.event_name != 'pull_request' || !github.event.pull_request.draft) && env.B2_TEST_APPLICATION_KEY != '' && env.B2_TEST_APPLICATION_KEY_ID != '' && matrix.os == 'ubuntu-latest' && matrix.python-version == '3.14' }} working-directory: B2_Command_Line_Tool run: | export VIRTUAL_ENV="$PWD/.nox/integration-3-14" diff --git a/changelog.d/+draft-ci-no-live.infrastructure.md b/changelog.d/+draft-ci-no-live.infrastructure.md new file mode 100644 index 00000000..55dcc175 --- /dev/null +++ b/changelog.d/+draft-ci-no-live.infrastructure.md @@ -0,0 +1 @@ +Keep draft pull request CI credential-free: skip bucket cleanup and SDK/CLI live tests while retaining unit and build checks. diff --git a/doc/source/contributing.rst b/doc/source/contributing.rst index ecae4ac9..89177c96 100644 --- a/doc/source/contributing.rst +++ b/doc/source/contributing.rst @@ -155,6 +155,14 @@ To run tests matching a keyword expression:: $ nox -s unit-3.10 -- -k keyword +Draft pull requests run credential-free checks. CI skips the bucket-cleanup job, +withholds B2 test credentials, and skips SDK and CLI live integration steps. +Unit and CLI compatibility jobs still run when cleanup is deliberately skipped; +failed lint or cleanup jobs continue to block them. Marking a pull request ready +for review starts the normal CI lane, which can use configured test-account +credentials for live tests and cleanup. Keep a pull request draft while live +validation is pending. + JUnit reports include each subtest as its own case, as well as its parent test. The repository's ``test.junit_subtests`` compatibility plugin supplies those cases because pytest 9's reporter counts subtest outcomes without emitting separate diff --git a/pyproject.toml b/pyproject.toml index 9e80c3e3..fd8a09a7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,6 +45,8 @@ format = [ "ruff~=0.8.4", ] lint = [ + "pytest-xdist>=3.8.0,<4", + "pyyaml>=6,<7", "pip>=26.2.2", "ruff~=0.8.4", "pytest>=9.0.3,<10", @@ -53,6 +55,7 @@ lint = [ "setuptools<81", # liccheck 0.9.2 imports pkg_resources, removed in setuptools 81 ] test = [ + "pyyaml>=6,<7", "pip>=26.2.2", "coverage>=7.13.5,<8", "pytest>=9.0.3,<10", diff --git a/test/static/test_ci_draft_safety.py b/test/static/test_ci_draft_safety.py new file mode 100644 index 00000000..39a84b02 --- /dev/null +++ b/test/static/test_ci_draft_safety.py @@ -0,0 +1,135 @@ +###################################################################### +# +# File: test/static/test_ci_draft_safety.py +# +# Copyright 2026 Backblaze Inc. All Rights Reserved. +# +# License https://www.backblaze.com/using_b2_code.html +# +###################################################################### +"""Exercise CI's draft, credential and dependency gates without running jobs.""" + +from __future__ import annotations + +import ast +import re +from pathlib import Path + +import pytest +import yaml + +JOBS = yaml.safe_load((Path(__file__).parents[2] / '.github/workflows/ci.yml').read_text())['jobs'] + + +def expression(value, context): + """Evaluate only the boolean/string expression subset used by these gates. + + actionlint validates the complete workflow separately. Unknown contexts or + expression syntax fail this check rather than being assumed safe. + """ + if not isinstance(value, str) or not value.startswith('${{'): + return value + code = value[3:-2].strip().replace('cancelled()', repr(context['cancelled'])) + code = re.sub( + r'[a-zA-Z_][\w-]*(?:\.[a-zA-Z_][\w-]*)+', lambda match: repr(context[match.group()]), code + ) + code = code.replace('&&', ' and ').replace('||', ' or ') + code = re.sub(r'!(?!=)', 'not ', code) + code = re.sub(r'\btrue\b', 'True', code) + code = re.sub(r'\bfalse\b', 'False', code) + tree = ast.parse(code.strip(), mode='eval') + allowed = ( + ast.Expression, + ast.Constant, + ast.BoolOp, + ast.And, + ast.Or, + ast.UnaryOp, + ast.Not, + ast.Compare, + ast.Eq, + ast.NotEq, + ) + assert all(isinstance(node, allowed) for node in ast.walk(tree)), code + return eval(compile(tree, '', 'eval'), {'__builtins__': {}}, {}) + + +def context(draft=True, credentials=True, event='pull_request', lint='success', cleanup='skipped'): + return { + 'github.event_name': event, + 'github.event.pull_request.draft': draft, + 'needs.lint.result': lint, + 'needs.cleanup_buckets.result': cleanup, + 'cancelled': False, + 'secrets.B2_TEST_APPLICATION_KEY': 'test-key' if credentials else '', + 'secrets.B2_TEST_APPLICATION_KEY_ID': 'test-id' if credentials else '', + } + + +@pytest.mark.parametrize('credentials', [True, False]) +def test_draft_never_starts_bucket_cleanup(credentials): + assert not expression( + JOBS['cleanup_buckets'].get('if', '${{ true }}'), context(credentials=credentials) + ) + + +@pytest.mark.parametrize('job_id', ['test', 'cli-compat']) +@pytest.mark.parametrize( + 'draft,event', [(True, 'pull_request'), (False, 'pull_request'), (False, 'push')] +) +def test_b2_credentials_are_empty_only_for_drafts(job_id, draft, event): + values = context(draft=draft, event=event) + for name in ('B2_TEST_APPLICATION_KEY', 'B2_TEST_APPLICATION_KEY_ID'): + actual = expression(JOBS[job_id]['env'][name], values) + assert actual == ('' if draft else values[f'secrets.{name}']) + + +@pytest.mark.parametrize('job_id', ['test', 'cli-compat']) +@pytest.mark.parametrize( + 'draft,credentials,allowed', + [(True, True, False), (True, False, False), (False, True, True), (False, False, False)], +) +def test_live_steps_require_non_draft_and_credentials(job_id, draft, credentials, allowed): + values = context(draft=draft, credentials=credentials) + values.update( + { + f'env.{name}': expression(value, values) + for name, value in JOBS[job_id]['env'].items() + if name.startswith('B2_TEST_') + } + ) + values.update({'matrix.os': 'ubuntu-latest', 'matrix.python-version': '3.14'}) + steps = [ + step + for step in JOBS[job_id]['steps'] + if step.get('name') in ('Run integration tests', 'Run CLI integration tests (with secrets)') + ] + assert len(steps) == 1 + assert bool(expression(steps[0]['if'], values)) == allowed + + +@pytest.mark.parametrize('job_id', ['test', 'cli-compat']) +@pytest.mark.parametrize( + 'lint,cleanup,allowed', + [ + ('success', 'skipped', True), + ('success', 'success', True), + ('failure', 'skipped', False), + ('success', 'failure', False), + ('success', 'cancelled', False), + ], +) +def test_unit_jobs_survive_skipped_cleanup_but_not_failed_dependencies( + job_id, lint, cleanup, allowed +): + assert 'lint' in JOBS[job_id]['needs'] + assert 'cleanup_buckets' in JOBS[job_id]['needs'] + values = context(lint=lint, cleanup=cleanup) + assert bool(expression(JOBS[job_id]['if'], values)) == allowed + values['cancelled'] = True + assert not expression(JOBS[job_id]['if'], values) + + +@pytest.mark.parametrize('event', ['pull_request', 'push']) +def test_non_draft_cleanup_is_retained(event): + assert expression(JOBS['cleanup_buckets']['if'], context(draft=False, event=event)) diff --git a/uv.lock b/uv.lock index 46673c7f..322bda99 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 3 +revision = 5 requires-python = ">=3.10" resolution-markers = [ "python_full_version >= '3.11'", @@ -94,6 +94,8 @@ lint = [ { name = "liccheck" }, { name = "pip" }, { name = "pytest" }, + { name = "pytest-xdist" }, + { name = "pyyaml" }, { name = "ruff" }, { name = "setuptools" }, { name = "tenacity" }, @@ -118,6 +120,7 @@ test = [ { name = "pytest-watcher" }, { name = "pytest-xdist" }, { name = "pywin32", marker = "platform_python_implementation != 'PyPy' and sys_platform == 'win32'" }, + { name = "pyyaml" }, { name = "responses" }, { name = "tenacity" }, { name = "tqdm" }, @@ -151,6 +154,8 @@ lint = [ { name = "liccheck", specifier = "==0.9.2" }, { name = "pip", specifier = ">=26.2.2" }, { name = "pytest", specifier = ">=9.0.3,<10" }, + { name = "pytest-xdist", specifier = ">=3.8.0,<4" }, + { name = "pyyaml", specifier = ">=6,<7" }, { name = "ruff", specifier = "~=0.8.4" }, { name = "setuptools", specifier = "<81" }, { name = "tenacity", specifier = ">=9.1.4,<10" }, @@ -173,6 +178,7 @@ test = [ { name = "pytest-watcher", specifier = ">=0.6.3,<1" }, { name = "pytest-xdist", specifier = ">=3.8.0,<4" }, { name = "pywin32", marker = "platform_python_implementation != 'PyPy' and sys_platform == 'win32'", specifier = ">=311,<312" }, + { name = "pyyaml", specifier = ">=6,<7" }, { name = "responses", specifier = ">=0.26.0,<1" }, { name = "tenacity", specifier = ">=9.1.4,<10" }, { name = "tqdm", specifier = ">=4.5.0,<5.0.0" }, From 190ef775edb3336459f225f8f33dddb372ffc643 Mon Sep 17 00:00:00 2001 From: scarreras Date: Sat, 21 Nov 2026 08:11:21 -0300 Subject: [PATCH 3/6] test: interpret CI gate expressions without executing code --- test/static/test_ci_draft_safety.py | 41 +++++++++++++++++++---------- 1 file changed, 27 insertions(+), 14 deletions(-) diff --git a/test/static/test_ci_draft_safety.py b/test/static/test_ci_draft_safety.py index 39a84b02..5a89e48d 100644 --- a/test/static/test_ci_draft_safety.py +++ b/test/static/test_ci_draft_safety.py @@ -38,20 +38,33 @@ def expression(value, context): code = re.sub(r'\btrue\b', 'True', code) code = re.sub(r'\bfalse\b', 'False', code) tree = ast.parse(code.strip(), mode='eval') - allowed = ( - ast.Expression, - ast.Constant, - ast.BoolOp, - ast.And, - ast.Or, - ast.UnaryOp, - ast.Not, - ast.Compare, - ast.Eq, - ast.NotEq, - ) - assert all(isinstance(node, allowed) for node in ast.walk(tree)), code - return eval(compile(tree, '', 'eval'), {'__builtins__': {}}, {}) + + def resolve(node): + if isinstance(node, ast.Constant) and isinstance(node.value, (str, bool)): + return node.value + if isinstance(node, ast.UnaryOp) and isinstance(node.op, ast.Not): + return not resolve(node.operand) + if isinstance(node, ast.BoolOp): + result = resolve(node.values[0]) + for child in node.values[1:]: + if isinstance(node.op, ast.And) and not result: + return result + if isinstance(node.op, ast.Or) and result: + return result + result = resolve(child) + return result + if isinstance(node, ast.Compare): + left = resolve(node.left) + for operator, child in zip(node.ops, node.comparators): + right = resolve(child) + assert isinstance(operator, (ast.Eq, ast.NotEq)), code + if (left == right) != isinstance(operator, ast.Eq): + return False + left = right + return True + raise AssertionError(f'Unsupported workflow expression: {code}') + + return resolve(tree.body) def context(draft=True, credentials=True, event='pull_request', lint='success', cleanup='skipped'): From 95e3a173ab7bb620be5c9b972c89c55743b8f11c Mon Sep 17 00:00:00 2001 From: scarreras Date: Sat, 21 Nov 2026 09:55:56 -0300 Subject: [PATCH 4/6] fix: preserve JUnit teardown totals and subtest diagnostics --- .../+junit-review-errors.infrastructure.md | 1 + doc/source/contributing.rst | 7 +- test/junit_subtests.py | 15 +++ test/static/test_junit_subtests.py | 106 ++++++++++++++++++ 4 files changed, 127 insertions(+), 2 deletions(-) create mode 100644 changelog.d/+junit-review-errors.infrastructure.md diff --git a/changelog.d/+junit-review-errors.infrastructure.md b/changelog.d/+junit-review-errors.infrastructure.md new file mode 100644 index 00000000..3d14a089 --- /dev/null +++ b/changelog.d/+junit-review-errors.infrastructure.md @@ -0,0 +1 @@ +Keep JUnit case totals consistent when a failed subtest parent also has a teardown error. Preserve requested captured output on subtest cases while respecting passing-test logging settings. diff --git a/doc/source/contributing.rst b/doc/source/contributing.rst index 89177c96..c6042682 100644 --- a/doc/source/contributing.rst +++ b/doc/source/contributing.rst @@ -166,8 +166,11 @@ validation is pending. JUnit reports include each subtest as its own case, as well as its parent test. The repository's ``test.junit_subtests`` compatibility plugin supplies those cases because pytest 9's reporter counts subtest outcomes without emitting separate -case elements. It copies reports only for JUnit serialization; assertions, test -selection, terminal outcomes and exit codes keep their normal pytest behavior. +case elements. Totals count emitted cases, including separate parent-call and +teardown-error cases. Requested captured output is retained on subtest cases, +respecting ``junit_logging`` and ``junit_log_passing_tests``. It copies reports +only for JUnit serialization; assertions, test selection, terminal outcomes and +exit codes keep their normal pytest behavior. The plugin uses pytest's internal reporter API, so dependency updates should run ``pytest test/static/test_junit_subtests.py`` (serial and xdist regression cases). Remove the compatibility layer only after upstream reporting passes those checks. diff --git a/test/junit_subtests.py b/test/junit_subtests.py index 7c084561..c666be10 100644 --- a/test/junit_subtests.py +++ b/test/junit_subtests.py @@ -37,10 +37,25 @@ def pytest_runtest_logreport(self, report): report = copy(report) report.nodeid += f'::subtest-{next(self._subtest_ids)}' super().pytest_runtest_logreport(report) + # The normal reporter writes output at teardown, which subtests lack. + # Failure-only logging and regular skips have already written theirs. + if ( + report.passed + or (report.failed and self.log_passing_tests) + or (report.skipped and hasattr(report, 'wasxfail')) + ): + self.node_reporter(report).write_captured_output(report) # Subtests have a call report, but no later teardown to close the case. self.finalize(report) self.open_reports = [entry for entry in self.open_reports if entry is not report] + def pytest_sessionfinish(self): + # Upstream subtracts a logical test when call and teardown both fail, + # although it emits two XML cases. Compensate from the actual case list + # so every outcome remains present and the declared count matches it. + self.cnt_double_fail_tests = sum(self.stats.values()) - len(self.node_reporters_ordered) + super().pytest_sessionfinish() + @pytest.hookimpl(trylast=True) def pytest_configure(config): diff --git a/test/static/test_junit_subtests.py b/test/static/test_junit_subtests.py index ffa452f4..de039b2c 100644 --- a/test/static/test_junit_subtests.py +++ b/test/static/test_junit_subtests.py @@ -96,3 +96,109 @@ def test_xfail(): assert int(suite.get('failures')) == (3 if outcome == 'failed' else 0) assert int(suite.get('errors')) == 0 assert int(suite.get('skipped')) == (4 if outcome == 'skipped' else 2) + + +@pytest.mark.parametrize('workers', [0, 2]) +def test_junit_totals_include_a_failed_parent_and_its_teardown_error(tmp_path, workers): + (tmp_path / 'pytest.ini').write_text('[pytest]\n') + (tmp_path / 'conftest.py').write_text("pytest_plugins = ['test.junit_subtests']\n") + (tmp_path / 'test_sample.py').write_text( + textwrap.dedent(""" + import pytest + + @pytest.fixture + def broken_teardown(): + yield + raise RuntimeError('deliberate teardown error') + + def test_subtest(subtests, broken_teardown): + with subtests.test('deliberate failure'): + assert False + """) + ) + env = { + key: value + for key, value in os.environ.items() + if not key.startswith(('B2_', 'PYTEST_', 'COV_CORE_', 'COVERAGE_')) + } + env['PYTHONPATH'] = str(ROOT) + completed = subprocess.run( + [sys.executable, '-m', 'pytest', '-n', str(workers), '--junitxml=result.xml', '-q'], + cwd=tmp_path, + env=env, + capture_output=True, + text=True, + timeout=90, + ) + assert completed.returncode == 1, completed.stdout + completed.stderr + suite = ET.parse(tmp_path / 'result.xml').getroot().find('testsuite') + cases = suite.findall('testcase') + assert int(suite.get('tests')) == len(cases) + assert len(cases) == 3 # failed subtest, failed parent call, errored teardown + assert int(suite.get('failures')) == 2 + assert int(suite.get('errors')) == 1 + assert sum(case.find('failure') is not None for case in cases) == 2 + assert sum(case.find('error') is not None for case in cases) == 1 + + +@pytest.mark.parametrize('workers', [0, 2]) +@pytest.mark.parametrize('outcome', ['passed', 'failed', 'skipped', 'xfail']) +@pytest.mark.parametrize('log_passing', [True, False]) +def test_junit_subtests_retain_requested_captured_output(tmp_path, workers, outcome, log_passing): + (tmp_path / 'pytest.ini').write_text('[pytest]\n') + (tmp_path / 'conftest.py').write_text("pytest_plugins = ['test.junit_subtests']\n") + (tmp_path / 'test_sample.py').write_text( + textwrap.dedent(f""" + import sys + import pytest + + def test_subtest(subtests): + with subtests.test('captured output'): + print('SUBTEST_STDOUT') + print('SUBTEST_STDERR', file=sys.stderr) + if {outcome!r} == 'failed': + assert False + if {outcome!r} == 'skipped': + pytest.skip('deliberate skip') + if {outcome!r} == 'xfail': + pytest.xfail('deliberate expected failure') + """) + ) + env = { + key: value + for key, value in os.environ.items() + if not key.startswith(('B2_', 'PYTEST_', 'COV_CORE_', 'COVERAGE_')) + } + env['PYTHONPATH'] = str(ROOT) + completed = subprocess.run( + [ + sys.executable, + '-m', + 'pytest', + '-n', + str(workers), + '--junitxml=result.xml', + '-q', + '-o', + 'junit_logging=all', + '-o', + f'junit_log_passing_tests={log_passing}', + ], + cwd=tmp_path, + env=env, + capture_output=True, + text=True, + timeout=90, + ) + assert completed.returncode == (1 if outcome == 'failed' else 0), ( + completed.stdout + completed.stderr + ) + cases = ET.parse(tmp_path / 'result.xml').getroot().findall('.//testcase') + subcase = next(case for case in cases if case.get('name').startswith('subtest-')) + for tag, token in [('system-out', 'SUBTEST_STDOUT'), ('system-err', 'SUBTEST_STDERR')]: + captured = subcase.findall(tag) + if log_passing or outcome != 'passed': + assert len(captured) == 1 + assert token in captured[0].text + else: + assert not captured From 099b642a0b890e2975101b5cdb608413a924d606 Mon Sep 17 00:00:00 2001 From: scarreras Date: Sat, 21 Nov 2026 10:03:15 -0300 Subject: [PATCH 5/6] fix: keep interrupted JUnit reports inconclusive --- .../+junit-review-errors.infrastructure.md | 2 +- doc/source/contributing.rst | 3 +- test/junit_subtests.py | 5 +++- test/static/test_junit_subtests.py | 28 +++++++++++++++++++ 4 files changed, 35 insertions(+), 3 deletions(-) diff --git a/changelog.d/+junit-review-errors.infrastructure.md b/changelog.d/+junit-review-errors.infrastructure.md index 3d14a089..6f074992 100644 --- a/changelog.d/+junit-review-errors.infrastructure.md +++ b/changelog.d/+junit-review-errors.infrastructure.md @@ -1 +1 @@ -Keep JUnit case totals consistent when a failed subtest parent also has a teardown error. Preserve requested captured output on subtest cases while respecting passing-test logging settings. +Keep JUnit case totals consistent when a failed subtest parent also has a teardown error. Preserve requested captured output on subtest cases while respecting passing-test logging settings. Leave interrupted reports unreconciled rather than manufacturing passing outcomes. diff --git a/doc/source/contributing.rst b/doc/source/contributing.rst index c6042682..6b326f39 100644 --- a/doc/source/contributing.rst +++ b/doc/source/contributing.rst @@ -167,7 +167,8 @@ JUnit reports include each subtest as its own case, as well as its parent test. The repository's ``test.junit_subtests`` compatibility plugin supplies those cases because pytest 9's reporter counts subtest outcomes without emitting separate case elements. Totals count emitted cases, including separate parent-call and -teardown-error cases. Requested captured output is retained on subtest cases, +teardown-error cases. Incomplete reports stay unreconciled rather than counting +unfinished cases as passes. Requested captured output is retained on subtest cases, respecting ``junit_logging`` and ``junit_log_passing_tests``. It copies reports only for JUnit serialization; assertions, test selection, terminal outcomes and exit codes keep their normal pytest behavior. diff --git a/test/junit_subtests.py b/test/junit_subtests.py index c666be10..46622c1e 100644 --- a/test/junit_subtests.py +++ b/test/junit_subtests.py @@ -53,7 +53,10 @@ def pytest_sessionfinish(self): # Upstream subtracts a logical test when call and teardown both fail, # although it emits two XML cases. Compensate from the actual case list # so every outcome remains present and the declared count matches it. - self.cnt_double_fail_tests = sum(self.stats.values()) - len(self.node_reporters_ordered) + # An interrupted call can leave a bare, unfinished XML case. Do not + # reconcile such a report: that would manufacture a passing outcome. + if not self.node_reporters: + self.cnt_double_fail_tests = sum(self.stats.values()) - len(self.node_reporters_ordered) super().pytest_sessionfinish() diff --git a/test/static/test_junit_subtests.py b/test/static/test_junit_subtests.py index de039b2c..f24c3733 100644 --- a/test/static/test_junit_subtests.py +++ b/test/static/test_junit_subtests.py @@ -202,3 +202,31 @@ def test_subtest(subtests): assert token in captured[0].text else: assert not captured + + +def test_junit_does_not_count_an_interrupted_test_as_passed(tmp_path): + (tmp_path / 'pytest.ini').write_text('[pytest]\n') + (tmp_path / 'conftest.py').write_text("pytest_plugins = ['test.junit_subtests']\n") + (tmp_path / 'test_sample.py').write_text( + 'def test_interrupted():\n raise KeyboardInterrupt\n' + ) + env = { + key: value + for key, value in os.environ.items() + if not key.startswith(('B2_', 'PYTEST_', 'COV_CORE_', 'COVERAGE_')) + } + env['PYTHONPATH'] = str(ROOT) + completed = subprocess.run( + [sys.executable, '-m', 'pytest', '--junitxml=result.xml', '-q'], + cwd=tmp_path, + env=env, + capture_output=True, + text=True, + timeout=30, + ) + assert completed.returncode == 2, completed.stdout + completed.stderr + suite = ET.parse(tmp_path / 'result.xml').getroot().find('testsuite') + declared_passes = int(suite.get('tests')) - sum( + int(suite.get(key)) for key in ('failures', 'errors', 'skipped') + ) + assert declared_passes == 0, 'an interrupted call has no passing test outcome' From 53522acffcf9b4d9a8dbcd3b7b76f6df64518eb8 Mon Sep 17 00:00:00 2001 From: scarreras Date: Sat, 21 Nov 2026 12:39:57 -0300 Subject: [PATCH 6/6] ci: require explicit opt-in for live B2 operations --- .github/workflows/ci.yml | 20 +++++------ changelog.d/+live-ci-opt-in.infrastructure.md | 1 + doc/source/contributing.rst | 10 +++--- test/static/test_ci_draft_safety.py | 33 ++++++++++++++++++- 4 files changed, 49 insertions(+), 15 deletions(-) create mode 100644 changelog.d/+live-ci-opt-in.infrastructure.md diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d93b55b1..bf27eeda 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -75,13 +75,13 @@ jobs: - name: Build the distribution run: uv run nox -vs build cleanup_buckets: - # Draft validation must never touch B2 resources. - if: ${{ github.event_name != 'pull_request' || !github.event.pull_request.draft }} + # Live B2 access requires explicit opt-in and is never allowed for drafts. + if: ${{ vars.B2_LIVE_TESTS_ENABLED == 'true' && (github.event_name != 'pull_request' || !github.event.pull_request.draft) }} timeout-minutes: 30 needs: lint env: - B2_TEST_APPLICATION_KEY: ${{ (github.event_name != 'pull_request' || !github.event.pull_request.draft) && secrets.B2_TEST_APPLICATION_KEY || '' }} - B2_TEST_APPLICATION_KEY_ID: ${{ (github.event_name != 'pull_request' || !github.event.pull_request.draft) && secrets.B2_TEST_APPLICATION_KEY_ID || '' }} + B2_TEST_APPLICATION_KEY: ${{ (vars.B2_LIVE_TESTS_ENABLED == 'true' && (github.event_name != 'pull_request' || !github.event.pull_request.draft)) && secrets.B2_TEST_APPLICATION_KEY || '' }} + B2_TEST_APPLICATION_KEY_ID: ${{ (vars.B2_LIVE_TESTS_ENABLED == 'true' && (github.event_name != 'pull_request' || !github.event.pull_request.draft)) && secrets.B2_TEST_APPLICATION_KEY_ID || '' }} runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -109,8 +109,8 @@ jobs: needs: [lint, cleanup_buckets] if: ${{ !cancelled() && needs.lint.result == 'success' && (needs.cleanup_buckets.result == 'success' || needs.cleanup_buckets.result == 'skipped') }} env: - B2_TEST_APPLICATION_KEY: ${{ (github.event_name != 'pull_request' || !github.event.pull_request.draft) && secrets.B2_TEST_APPLICATION_KEY || '' }} - B2_TEST_APPLICATION_KEY_ID: ${{ (github.event_name != 'pull_request' || !github.event.pull_request.draft) && secrets.B2_TEST_APPLICATION_KEY_ID || '' }} + B2_TEST_APPLICATION_KEY: ${{ (vars.B2_LIVE_TESTS_ENABLED == 'true' && (github.event_name != 'pull_request' || !github.event.pull_request.draft)) && secrets.B2_TEST_APPLICATION_KEY || '' }} + B2_TEST_APPLICATION_KEY_ID: ${{ (vars.B2_LIVE_TESTS_ENABLED == 'true' && (github.event_name != 'pull_request' || !github.event.pull_request.draft)) && secrets.B2_TEST_APPLICATION_KEY_ID || '' }} NOX_EXTRAS: ${{ matrix.extras }} runs-on: ${{ matrix.os }} strategy: @@ -145,15 +145,15 @@ jobs: - name: Run unit tests run: uv run nox -vs unit -- -v - name: Run integration tests - if: ${{ (github.event_name != 'pull_request' || !github.event.pull_request.draft) && env.B2_TEST_APPLICATION_KEY != '' && env.B2_TEST_APPLICATION_KEY_ID != '' }} + if: ${{ (vars.B2_LIVE_TESTS_ENABLED == 'true' && (github.event_name != 'pull_request' || !github.event.pull_request.draft)) && env.B2_TEST_APPLICATION_KEY != '' && env.B2_TEST_APPLICATION_KEY_ID != '' }} run: uv run nox -vs integration -- --dont-cleanup-old-buckets -v cli-compat: timeout-minutes: 90 needs: [lint, cleanup_buckets] if: ${{ !cancelled() && needs.lint.result == 'success' && (needs.cleanup_buckets.result == 'success' || needs.cleanup_buckets.result == 'skipped') }} env: - B2_TEST_APPLICATION_KEY: ${{ (github.event_name != 'pull_request' || !github.event.pull_request.draft) && secrets.B2_TEST_APPLICATION_KEY || '' }} - B2_TEST_APPLICATION_KEY_ID: ${{ (github.event_name != 'pull_request' || !github.event.pull_request.draft) && secrets.B2_TEST_APPLICATION_KEY_ID || '' }} + B2_TEST_APPLICATION_KEY: ${{ (vars.B2_LIVE_TESTS_ENABLED == 'true' && (github.event_name != 'pull_request' || !github.event.pull_request.draft)) && secrets.B2_TEST_APPLICATION_KEY || '' }} + B2_TEST_APPLICATION_KEY_ID: ${{ (vars.B2_LIVE_TESTS_ENABLED == 'true' && (github.event_name != 'pull_request' || !github.event.pull_request.draft)) && secrets.B2_TEST_APPLICATION_KEY_ID || '' }} INSTALL_SDK_FROM: ../b2-sdk-python runs-on: ${{ matrix.os }} strategy: @@ -201,7 +201,7 @@ jobs: - name: Run CLI integration tests (without secrets) run: uv run --directory ./B2_Command_Line_Tool nox -vs integration -p ${{ matrix.python-version }} -- -m "not require_secrets" - name: Run CLI integration tests (with secrets) - if: ${{ (github.event_name != 'pull_request' || !github.event.pull_request.draft) && env.B2_TEST_APPLICATION_KEY != '' && env.B2_TEST_APPLICATION_KEY_ID != '' && matrix.os == 'ubuntu-latest' && matrix.python-version == '3.14' }} + if: ${{ (vars.B2_LIVE_TESTS_ENABLED == 'true' && (github.event_name != 'pull_request' || !github.event.pull_request.draft)) && env.B2_TEST_APPLICATION_KEY != '' && env.B2_TEST_APPLICATION_KEY_ID != '' && matrix.os == 'ubuntu-latest' && matrix.python-version == '3.14' }} working-directory: B2_Command_Line_Tool run: | export VIRTUAL_ENV="$PWD/.nox/integration-3-14" diff --git a/changelog.d/+live-ci-opt-in.infrastructure.md b/changelog.d/+live-ci-opt-in.infrastructure.md new file mode 100644 index 00000000..ac7ec132 --- /dev/null +++ b/changelog.d/+live-ci-opt-in.infrastructure.md @@ -0,0 +1 @@ +Require the B2_LIVE_TESTS_ENABLED repository variable to opt into live tests and bucket cleanup, including ready pull requests and master pushes. diff --git a/doc/source/contributing.rst b/doc/source/contributing.rst index 6b326f39..65259a82 100644 --- a/doc/source/contributing.rst +++ b/doc/source/contributing.rst @@ -158,10 +158,12 @@ To run tests matching a keyword expression:: Draft pull requests run credential-free checks. CI skips the bucket-cleanup job, withholds B2 test credentials, and skips SDK and CLI live integration steps. Unit and CLI compatibility jobs still run when cleanup is deliberately skipped; -failed lint or cleanup jobs continue to block them. Marking a pull request ready -for review starts the normal CI lane, which can use configured test-account -credentials for live tests and cleanup. Keep a pull request draft while live -validation is pending. +failed lint or cleanup jobs continue to block them. Live tests, B2 credential +access and bucket cleanup also require the repository Actions variable +``B2_LIVE_TESTS_ENABLED`` to be explicitly set to ``true``. The default is off, +including ready pull requests and pushes to master. Enable that variable only +when live validation with an approved test account is intended; marking a pull +request ready or merging it does not itself enable live operations. JUnit reports include each subtest as its own case, as well as its parent test. The repository's ``test.junit_subtests`` compatibility plugin supplies those cases diff --git a/test/static/test_ci_draft_safety.py b/test/static/test_ci_draft_safety.py index 5a89e48d..8a497394 100644 --- a/test/static/test_ci_draft_safety.py +++ b/test/static/test_ci_draft_safety.py @@ -67,13 +67,21 @@ def resolve(node): return resolve(tree.body) -def context(draft=True, credentials=True, event='pull_request', lint='success', cleanup='skipped'): +def context( + draft=True, + credentials=True, + event='pull_request', + lint='success', + cleanup='skipped', + live_enabled='true', +): return { 'github.event_name': event, 'github.event.pull_request.draft': draft, 'needs.lint.result': lint, 'needs.cleanup_buckets.result': cleanup, 'cancelled': False, + 'vars.B2_LIVE_TESTS_ENABLED': live_enabled, 'secrets.B2_TEST_APPLICATION_KEY': 'test-key' if credentials else '', 'secrets.B2_TEST_APPLICATION_KEY_ID': 'test-id' if credentials else '', } @@ -146,3 +154,26 @@ def test_unit_jobs_survive_skipped_cleanup_but_not_failed_dependencies( @pytest.mark.parametrize('event', ['pull_request', 'push']) def test_non_draft_cleanup_is_retained(event): assert expression(JOBS['cleanup_buckets']['if'], context(draft=False, event=event)) + + +@pytest.mark.parametrize('event', ['pull_request', 'push']) +@pytest.mark.parametrize('live_enabled', ['', 'false', 'true']) +def test_non_draft_live_operations_require_explicit_repository_opt_in(event, live_enabled): + values = context(draft=False, event=event, live_enabled=live_enabled) + allowed = live_enabled == 'true' + assert bool(expression(JOBS['cleanup_buckets']['if'], values)) == allowed + for job_id in ('test', 'cli-compat'): + job_values = dict(values) + for name in ('B2_TEST_APPLICATION_KEY', 'B2_TEST_APPLICATION_KEY_ID'): + actual = expression(JOBS[job_id]['env'][name], values) + assert actual == (values[f'secrets.{name}'] if allowed else '') + job_values[f'env.{name}'] = actual + job_values.update({'matrix.os': 'ubuntu-latest', 'matrix.python-version': '3.14'}) + steps = [ + step + for step in JOBS[job_id]['steps'] + if step.get('name') + in ('Run integration tests', 'Run CLI integration tests (with secrets)') + ] + assert len(steps) == 1 + assert bool(expression(steps[0]['if'], job_values)) == allowed