Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions .github/dependabot.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
version: 2
updates:
# The release the optional evidence replay checker runs on.
- package-ecosystem: pip
directory: /docs/examples/evidence-replay
target-branch: dev
schedule:
interval: weekly
18 changes: 18 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -155,3 +155,21 @@ jobs:

- name: cargo check (MSRV)
run: cargo check --workspace

evidence-checker:
name: Evidence replay checker
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4

- uses: actions/setup-python@v5
with:
python-version: "3.12"

# The optional checker's release is pinned, with hashes, in
# requirements.txt; Dependabot proposes each new release.
- name: Install the pinned checker release
run: python -m pip install --require-hashes -r docs/examples/evidence-replay/requirements.txt

- name: Run the checker controls
run: python -m unittest discover -s docs/examples/evidence-replay -p test_checker.py -v
11 changes: 11 additions & 0 deletions atomic-repository/src/triage/model.rs
Original file line number Diff line number Diff line change
Expand Up @@ -170,6 +170,17 @@ pub struct EvidenceReplayDecision {
pub claim_type: String,
pub decision: String,
pub reason: String,
/// The checker release the response names in `details.checker`, so a
/// reviewer can hold the decision to the checker they allowlisted. `None`
/// when the response names none, or names it in any other shape.
pub checker: Option<CheckerRelease>,
}

/// The package and version a replay checker reports for itself.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct CheckerRelease {
pub package: String,
pub version: String,
}

/// A single acceptance criterion under an intent (a "testcase").
Expand Down
65 changes: 64 additions & 1 deletion atomic-repository/src/triage/report.rs
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,30 @@ fn failing_view_verification(node: &CanonicalNode) -> Option<String> {
None
}

/// Longest package name or version accepted from a checker's self-report.
const CHECKER_FIELD_MAX: usize = 128;

/// The `{package, version}` a checker names in `details.checker`. Anything
/// else -- a missing member, an extra member, a non-string, an empty or
/// overlong value -- names no release, so a summary never shows a guess.
fn checker_release(details: &serde_json::Value) -> Option<CheckerRelease> {
let checker = details.get("checker")?.as_object()?;
if checker.len() != 2 {
return None;
}
let field = |name: &str| {
checker
.get(name)?
.as_str()
.filter(|value| !value.is_empty() && value.len() <= CHECKER_FIELD_MAX)
.map(str::to_owned)
};
Some(CheckerRelease {
package: field("package")?,
version: field("version")?,
})
}

/// Map a replay/report failure into the repository error surface. Repository
/// failures stay typed; report parsing/hash failures are invalid operations.
fn evidence_error(error: intent_evidence::EvidenceError) -> RepositoryError {
Expand Down Expand Up @@ -341,6 +365,7 @@ pub fn build_report_with_evidence(
claim_type: decision.claim_type.clone(),
decision: decision.decision.clone(),
reason: decision.reason.clone(),
checker: checker_release(&decision.details),
})
.collect(),
});
Expand Down Expand Up @@ -1532,6 +1557,34 @@ mod tests {
use atomic_core::change::ChangeHeader;
use tempfile::TempDir;

#[test]
fn checker_release_reads_only_the_exact_self_report() {
use serde_json::json;
let named = |details: serde_json::Value| checker_release(&details);
assert_eq!(
named(
json!({"checker": {"package": "probity-verify", "version": "0.2.0"}, "scope": {}})
),
Some(CheckerRelease {
package: "probity-verify".into(),
version: "0.2.0".into(),
})
);
let long = "v".repeat(CHECKER_FIELD_MAX + 1);
for details in [
serde_json::Value::Null,
json!({}),
json!({"checker": "probity-verify 0.2.0"}),
json!({"checker": {"package": "probity-verify"}}),
json!({"checker": {"package": "probity-verify", "version": 2}}),
json!({"checker": {"package": "", "version": "0.2.0"}}),
json!({"checker": {"package": "probity-verify", "version": long}}),
json!({"checker": {"package": "p", "version": "1", "extra": "x"}}),
] {
assert_eq!(named(details.clone()), None, "{details}");
}
}

fn record_all(repo: &Repository, message: &str) {
let header = ChangeHeader::new(message);
let options = RecordOptions::new()
Expand Down Expand Up @@ -1753,7 +1806,9 @@ Edit a.txt.
claim_type: "source_text_coverage/v1".into(),
decision: "supported".into(),
reason: "all_selected_spans_present".into(),
details: serde_json::Value::Null,
details: serde_json::json!({
"checker": {"package": "probity-verify", "version": "0.2.0"}
}),
}],
},
};
Expand All @@ -1778,6 +1833,14 @@ Edit a.txt.
.expect("reached intent");
assert_eq!(intent_report.evidence_replays.len(), 1);
assert!(intent_report.evidence_replays[0].fresh);
assert_eq!(
intent_report.evidence_replays[0].decisions[0].checker,
Some(CheckerRelease {
package: "probity-verify".into(),
version: "0.2.0".into(),
}),
"the summary carries the checker release the response names"
);
assert!(with
.findings
.iter()
Expand Down
14 changes: 10 additions & 4 deletions docs/examples/evidence-replay/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -22,15 +22,21 @@ On Windows, select a local executable implementing the protocol below. A checker
that cannot start, fails, or returns an invalid response remains
`not_established`; it cannot make validation pass.

Use a dedicated Python environment for the optional checker. The pinned Verify
revision below implements the four claim adapters listed later in this document:
Use a dedicated Python 3.11 or newer environment for the optional checker.
`requirements.txt` pins the probity-verify release, with hashes for it and every
dependency; that release implements the four claim adapters listed later in this
document. CI installs the same file and runs the checker controls below, and each
response names the installed release in `details.checker`:

```sh
python3 -m venv /tmp/atomic-evidence-python
/tmp/atomic-evidence-python/bin/pip install \
'git+https://github.com/probityai/probity-verify.git@e835ce2bd6a960e7a1cc2fa6522f16d55dce728a'
/tmp/atomic-evidence-python/bin/pip install --require-hashes \
-r /path/to/atomic/docs/examples/evidence-replay/requirements.txt
```

To move to a newer release, change `requirements.in` and regenerate the lock with
`uv pip compile requirements.in --generate-hashes --python-version 3.11 --universal -o requirements.txt`.

Create a **draft** with one criterion in an Atomic repository:

```sh
Expand Down
14 changes: 10 additions & 4 deletions docs/examples/evidence-replay/probity-checker.py
Original file line number Diff line number Diff line change
@@ -1,17 +1,22 @@
#!/usr/bin/env python3
"""Optional local Atomic checker using Probity Verify's bounded adapters.

Install probity-verify in the Python environment selected by this executable.
This wrapper authenticates neither witnesses nor Atomic signatures. Its result
Install the pinned probity-verify from requirements.txt in the Python
environment selected by this executable. Every response names the installed
probity-verify version in `details.checker`, so a retained report records which
release produced it. This wrapper authenticates neither witnesses nor Atomic signatures. Its result
only applies to the selected claim, retained bytes, and supplied consumer policy.
"""
import json
import sys
from importlib.metadata import version
from pathlib import Path

from probity_verify.common import CaseError
from probity_verify.core import adjudicate

CHECKER = {"package": "probity-verify", "version": version("probity-verify")}


def check(envelope):
if not isinstance(envelope, dict) or set(envelope) != {"request", "request_digest"}:
Expand Down Expand Up @@ -42,9 +47,10 @@ def check(envelope):
if assessment["claim_type"] != claim["claim_type"]:
raise CaseError("claim type differs from consumer policy")
result = adjudicate(case, policy, Path(claim["artifact_root"]))
response.update(decision=result["decision"], reason=result["reason"], details=result)
response.update(decision=result["decision"], reason=result["reason"],
details={**result, "checker": CHECKER})
except (CaseError, KeyError, TypeError, ValueError, OSError) as error:
response["details"] = {"error": str(error)}
response["details"] = {"error": str(error), "checker": CHECKER}
return response


Expand Down
1 change: 1 addition & 0 deletions docs/examples/evidence-replay/requirements.in
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
probity-verify==0.2.0
Loading