Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Add regression tests for known defects; tests of defects that are still present are strict expected failures marked `known_defect` (run them with `pytest --runxfail -m known_defect`).
8 changes: 8 additions & 0 deletions test/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,14 @@
pytest_plugins = ['b2sdk.v3.testing']


def pytest_configure(config):
config.addinivalue_line(
'markers',
'known_defect: test of a defect that is still present, marked as a strict expected failure '
'(see test/known_defect.py)',
)


@pytest.fixture
def bg_executor():
with concurrent.futures.ThreadPoolExecutor() as executor:
Expand Down
89 changes: 89 additions & 0 deletions test/known_defect.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
######################################################################
#
# File: test/known_defect.py
#
# Copyright 2026 Backblaze Inc. All Rights Reserved.
#
# License https://www.backblaze.com/using_b2_code.html
#
######################################################################
from __future__ import annotations

import contextlib
import re

import pytest


class DefectStillPresent(AssertionError):
"""
Raised by :func:`expect` when the behaviour that a ``@known_defect`` test describes is wrong.

A test that has ordinary prerequisite assertions (setup that must hold whether or not the defect
is fixed) marks itself ``raises=DefectStillPresent`` and checks the defect with :func:`expect`.
A failed prerequisite is then a plain ``AssertionError`` and shows up as a real failure instead
of being swallowed as the expected one.
"""


def expect(condition: object, message: str = '') -> None:
"""Assert the correct behaviour of a known defect; fails with :class:`DefectStillPresent`."""
if not condition:
raise DefectStillPresent(message)


@contextlib.contextmanager
def defect_raises(exc_type: type[BaseException], match: str | None = None):
"""
Turn the one specific exception that a known defect produces into :class:`DefectStillPresent`.

An exception of another type, or whose message does not match ``match`` (a regular expression
searched in ``str(exception)``), propagates unchanged and is a real failure of the test.
"""
try:
yield
except exc_type as exc:
if match is not None and not re.search(match, str(exc)):
raise
raise DefectStillPresent(f'{type(exc).__name__}: {exc}') from exc


def known_defect(
defect_id: str,
summary: str,
*,
raises: type[BaseException] | tuple[type[BaseException], ...],
condition: bool = True,
):
"""
Mark a test that asserts the correct behaviour of a defect that is still present.

The test is a strict expected failure (``xfail(strict=True)``), so the default test run stays
green, and the run fails as soon as the defect is fixed ("XPASS(strict)"). When that happens,
delete the ``@known_defect(...)`` line: the test then becomes a normal regression test.

The tests carry the ``known_defect`` marker, so they can be run (and are reported as real
failures) on demand with one command, for example::

pytest --runxfail -m known_defect --api=v3 test/unit test/static

:param defect_id: identifier of the defect, for example ``'C16'``
:param summary: one-line description of the defect
:param raises: the exception (or tuple of exceptions) the test raises while the defect is present.
Any other failure, in the test or in its setup, is reported as a real failure.
Use ``AssertionError`` for a test whose only assertion is the defect check,
:class:`DefectStillPresent` when the test also has prerequisite assertions, or
``pytest.fail.Exception`` when the defect is "DID NOT RAISE" from ``pytest.raises``
:param condition: when false the defect does not apply (for example to another apiver) and the
test is a normal test
"""

def decorator(func):
if not condition:
return func
func = pytest.mark.known_defect(func)
return pytest.mark.xfail(
strict=True, raises=raises, reason=f'Known defect {defect_id}: {summary}'
)(func)

return decorator
118 changes: 118 additions & 0 deletions test/static/test_regression_repo_checks.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,118 @@
######################################################################
#
# File: test/static/test_regression_repo_checks.py
#
# Copyright 2026 Backblaze Inc. All Rights Reserved.
#
# License https://www.backblaze.com/using_b2_code.html
#
######################################################################
"""Static regression checks of repository files (docs, changelog, packaging, sources).

They read repository files only and are skipped when the repository files are not available.

Tests marked with ``@known_defect`` describe a defect that is still present: they assert the correct
behaviour and are strict expected failures, so the default run stays green and fails with
"XPASS(strict)" once the defect is fixed (then remove the decorator). To see them fail, run
``pytest --runxfail -m known_defect`` (see ``test/known_defect.py``). The ``C<n>`` identifiers are
short labels that also appear in the test names and decorators.
"""

from __future__ import annotations

import re
import subprocess
import sys
import textwrap
from pathlib import Path

import pytest

from test.known_defect import DefectStillPresent, expect, known_defect

ROOT = Path(__file__).resolve().parents[2]


def _read(rel: str) -> str:
path = ROOT / rel
if not path.is_file():
pytest.skip(f'{rel} is not available (not a full repository checkout)')
return path.read_text(encoding='utf-8')


@known_defect('C30', 'typo in .github/SUPPORT.md', raises=AssertionError)
def test_c30_support_file_has_no_known_typo():
"""C30: typo 'Issuew' in .github/SUPPORT.md (codespell did not catch it)."""
assert 'Issuew' not in _read('.github/SUPPORT.md'), 'typo is still present'


@known_defect('C59', 'the README does not mention the "full" install extra', raises=AssertionError)
def test_c59_readme_mentions_full_extra():
"""C59: README must mention the `full` install extra (tqdm/pydantic support)."""
assert re.search(r'b2sdk\[full\]', _read('README.md')), 'README does not mention b2sdk[full]'


# an EncryptionKey(...) call whose arguments contain `id=`; quoted strings are skipped as a whole,
# so a ")" inside the secret does not end the argument list early
ENCRYPTION_KEY_WITH_ID = re.compile(r"""EncryptionKey\((?:[^()'"]|'[^']*'|"[^"]*")*?\bid=""")
# installs the local checkout (`pip install .` or `pip install -e .`), not a package index or a URL
PIP_INSTALL_CHECKOUT = re.compile(r'pip install (?:-e\s+)?\.(?:\[[^\]]*\])?(?=\s|$)')


@known_defect('C4', 'the Quick Start uses a class that does not exist', raises=AssertionError)
def test_c4_quick_start_does_not_use_encryption_settings_class():
"""C4: Quick Start must not use the non-existent class EncryptionSettings."""
assert 'EncryptionSettings(' not in _read(
'doc/source/quick_start.rst'
), 'EncryptionSettings does not exist'


@known_defect(
'C4',
'the Quick Start uses an EncryptionKey argument that does not exist',
raises=AssertionError,
)
def test_c4_quick_start_uses_key_id_argument_of_encryption_key():
"""C4: Quick Start must call EncryptionKey with `key_id=`, which is the argument it has."""
assert not ENCRYPTION_KEY_WITH_ID.search(
_read('doc/source/quick_start.rst')
), 'EncryptionKey has no id= argument'


@known_defect(
'C4',
'the development-version install instructions install from PyPI, not the checkout',
raises=AssertionError,
)
def test_c4_install_doc_development_version_installs_from_checkout():
"""C4: 'Installing a development version' must install the checkout, not PyPI."""
section = _read('doc/source/install.rst').split('Installing a development version', 1)[1]
assert PIP_INSTALL_CHECKOUT.search(section), 'development-version section installs from PyPI'


@known_defect(
'C36',
'importing b2sdk fails when the package metadata is not installed',
raises=DefectStillPresent,
)
def test_c36_version_import_works_when_package_metadata_is_missing(tmp_path):
"""C36: importing a vendored/uninstalled copy of b2sdk must not fail for missing dist metadata."""
script = textwrap.dedent(
f"""
import importlib.metadata as m, sys
orig = m.version
def fake(name):
if name == 'b2sdk':
raise m.PackageNotFoundError(name)
return orig(name)
m.version = fake
sys.path.insert(0, {str(ROOT)!r})
import b2sdk.version
"""
)
result = subprocess.run(
[sys.executable, '-c', script], capture_output=True, text=True, cwd=tmp_path
)
# only the missing-metadata error is the defect; any other failure is a real failure
expect('PackageNotFoundError' not in result.stderr, result.stderr[-300:])
assert result.returncode == 0, result.stderr[-300:]
89 changes: 89 additions & 0 deletions test/unit/account_info/test_regression_sqlite.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
######################################################################
#
# File: test/unit/account_info/test_regression_sqlite.py
#
# Copyright 2026 Backblaze Inc. All Rights Reserved.
#
# License https://www.backblaze.com/using_b2_code.html
#
######################################################################
"""Regression tests for SqliteAccountInfo.

Tests marked with ``@known_defect`` describe a defect that is still present: they assert the correct
behaviour and are strict expected failures, so the default run stays green and fails with
"XPASS(strict)" once the defect is fixed (then remove the decorator). To see them fail, run
``pytest --runxfail -m known_defect`` (see ``test/known_defect.py``). The ``C<n>`` identifiers are
short labels that also appear in the test names and decorators.
"""

from __future__ import annotations

import os
import sqlite3
import threading

import pytest
from apiver_deps import (
B2_ACCOUNT_INFO_ENV_VAR,
XDG_CONFIG_HOME_ENV_VAR,
SqliteAccountInfo,
)

from test.known_defect import DefectStillPresent, expect, known_defect

pytestmark = [pytest.mark.apiver(from_ver=2)]


@pytest.fixture
def home(monkeypatch, tmp_path):
monkeypatch.setenv('HOME', str(tmp_path))
monkeypatch.setenv('USERPROFILE', str(tmp_path))
monkeypatch.delenv(B2_ACCOUNT_INFO_ENV_VAR, raising=False)
monkeypatch.delenv(XDG_CONFIG_HOME_ENV_VAR, raising=False)
return tmp_path


class TestProfilePaths:
@known_defect(
'C47',
'an existing ~/.b2db-<profile>.sqlite is ignored because "~" is not expanded before the existence check',
raises=AssertionError,
)
def test_c47_existing_legacy_profile_database_is_used_with_xdg(self, home, monkeypatch):
"""C47: an existing ~/.b2db-<profile>.sqlite must be found even when XDG_CONFIG_HOME is set."""
monkeypatch.setenv(XDG_CONFIG_HOME_ENV_VAR, str(home / 'xdg'))
legacy = home / '.b2db-foo.sqlite'
legacy.write_bytes(b'')
path = SqliteAccountInfo.get_user_account_info_path(profile='foo')
assert os.path.realpath(path) == os.path.realpath(str(legacy)), f'got {path}'


class TestThreadConnections:
@known_defect(
'C37',
"close() closes only the calling thread's database connection",
raises=DefectStillPresent,
)
def test_c37_close_closes_connections_of_other_threads(self, tmp_path):
"""C37: close() must close the database connections opened by other threads, too."""
info = SqliteAccountInfo(file_name=str(tmp_path / 'acct.sqlite'))
opened, closed_by_main, result = threading.Event(), threading.Event(), {}

def worker():
conn = info._get_connection() # connection of this worker thread
opened.set()
assert closed_by_main.wait(timeout=10), 'main thread did not call close()'
try:
conn.execute('select 1')
result['closed'] = False
except sqlite3.ProgrammingError: # Cannot operate on a closed database
result['closed'] = True

thread = threading.Thread(target=worker)
thread.start()
assert opened.wait(timeout=10), 'worker thread did not open a connection'
info.close()
closed_by_main.set()
thread.join(timeout=10)
assert 'closed' in result, 'the worker thread did not finish its check'
expect(result['closed'], 'connection of another thread is still open after close()')
Loading
Loading