From 4b0e80a66b1aad8edebef49eabffd74e04e0ec49 Mon Sep 17 00:00:00 2001 From: Etienne Filhol Date: Tue, 10 Nov 2026 13:38:59 +0100 Subject: [PATCH 1/4] Prove memory recall with a name, not a password The cap.memory/v1 kit's m2 check stored a Wi-Fi password, so a memory that refuses credentials by design could never be proven. It now recalls an exact token that is not a secret. --- app/nullius/standard/kits/cap-memory-v1/driver.py | 2 +- .../standard/kits/cap-memory-v1/standard.json | 4 ++-- app/tests/test_standard_conform.py | 14 ++++++++++++++ 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/app/nullius/standard/kits/cap-memory-v1/driver.py b/app/nullius/standard/kits/cap-memory-v1/driver.py index 3db40f9..f78f05c 100644 --- a/app/nullius/standard/kits/cap-memory-v1/driver.py +++ b/app/nullius/standard/kits/cap-memory-v1/driver.py @@ -6,7 +6,7 @@ "The neighbours painted their fence blue."] CHECKS = { "m1": ("My sister Ines moved to Lisbon in March.", "Where did Ines move to?", "Lisbon"), - "m2": ("The wifi password at the cabin is tern-river-9.", "What is the wifi password at the cabin?", "tern-river-9"), + "m2": ("The boat at the cabin is named tern-river-9.", "What is the boat at the cabin named?", "tern-river-9"), } diff --git a/app/nullius/standard/kits/cap-memory-v1/standard.json b/app/nullius/standard/kits/cap-memory-v1/standard.json index 29b0ac1..08af6f9 100644 --- a/app/nullius/standard/kits/cap-memory-v1/standard.json +++ b/app/nullius/standard/kits/cap-memory-v1/standard.json @@ -83,8 +83,8 @@ "passes": "The recalled memory contains Lisbon, ignoring case." }, "m2": { - "label": "Recall a password", - "task": "Store six sessions, including the cabin Wi-Fi password tern-river-9. Ask for that password.", + "label": "Recall an exact name", + "task": "Store six sessions, including “The boat at the cabin is named tern-river-9.” Ask what the boat is named.", "passes": "The recalled memory contains tern-river-9, ignoring case." } } diff --git a/app/tests/test_standard_conform.py b/app/tests/test_standard_conform.py index 1813132..e89d2b6 100644 --- a/app/tests/test_standard_conform.py +++ b/app/tests/test_standard_conform.py @@ -41,6 +41,20 @@ def test_a_kit_that_passes_is_recorded_with_its_job(self): self.assertTrue(os.path.isdir(os.path.join(self.tmp, "jobs", rec["job"]))) self.assertEqual(d["offers"], ["cap.memory/v1"]) + def test_a_memory_that_refuses_credentials_still_proves_the_memory_port(self): + here = os.path.join(self.root, "capabilities", "fakemem") + with open(os.path.join(here, "capability.py")) as f: + src = f.read() + refuse = 'kept.append(req["session"]["text"]); out = {}' + self.assertIn(refuse, src) + with open(os.path.join(here, "capability.py"), "w") as f: # it drops every session that looks like a secret + f.write(src.replace(refuse, 'out = {}\n if not any(w in req["session"]["text"].lower() for w in ' + '("password", "passcode", "token", "secret", "credential", "api key")):\n' + ' kept.append(req["session"]["text"])')) + got, d = self.check("capability", "fakemem", ["cap.memory/v1"]) + self.assertTrue(got["cap.memory/v1"]["ok"]) + self.assertEqual(d["proven"]["cap.memory/v1"]["passed"], "2/2") + def test_a_kit_that_fails_takes_the_claim_back(self): got, d = self.check("agent", "fake", ["agent.task/v1"]) # it shouts the prompt back: not the word self.assertFalse(got["agent.task/v1"]["ok"]) From c2d3476cddfec56cb637730196e700b5c39f4681 Mon Sep 17 00:00:00 2001 From: Etienne Filhol Date: Tue, 10 Nov 2026 13:43:23 +0100 Subject: [PATCH 2/4] Drop the credential-refusing memory test With no credential left in the kit, it passed trivially. --- app/tests/test_standard_conform.py | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/app/tests/test_standard_conform.py b/app/tests/test_standard_conform.py index e89d2b6..1813132 100644 --- a/app/tests/test_standard_conform.py +++ b/app/tests/test_standard_conform.py @@ -41,20 +41,6 @@ def test_a_kit_that_passes_is_recorded_with_its_job(self): self.assertTrue(os.path.isdir(os.path.join(self.tmp, "jobs", rec["job"]))) self.assertEqual(d["offers"], ["cap.memory/v1"]) - def test_a_memory_that_refuses_credentials_still_proves_the_memory_port(self): - here = os.path.join(self.root, "capabilities", "fakemem") - with open(os.path.join(here, "capability.py")) as f: - src = f.read() - refuse = 'kept.append(req["session"]["text"]); out = {}' - self.assertIn(refuse, src) - with open(os.path.join(here, "capability.py"), "w") as f: # it drops every session that looks like a secret - f.write(src.replace(refuse, 'out = {}\n if not any(w in req["session"]["text"].lower() for w in ' - '("password", "passcode", "token", "secret", "credential", "api key")):\n' - ' kept.append(req["session"]["text"])')) - got, d = self.check("capability", "fakemem", ["cap.memory/v1"]) - self.assertTrue(got["cap.memory/v1"]["ok"]) - self.assertEqual(d["proven"]["cap.memory/v1"]["passed"], "2/2") - def test_a_kit_that_fails_takes_the_claim_back(self): got, d = self.check("agent", "fake", ["agent.task/v1"]) # it shouts the prompt back: not the word self.assertFalse(got["agent.task/v1"]["ok"]) From 89b5a962b16851c94625a69629617850aec9e620 Mon Sep 17 00:00:00 2001 From: Etienne Filhol Date: Tue, 10 Nov 2026 14:39:04 +0100 Subject: [PATCH 3/4] Say when a kit's version goes up, and what the memory kit checks The memory kit's m2 now asks for a name instead of a password and keeps kit_version 1. Write the rule that decision follows: a check that gets stronger or tests something new bumps the version, a rewording does not. The ports table names the memory kit's two checks and that none stores a credential. --- docs/standard.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/standard.md b/docs/standard.md index 14cbfbe..2a46f71 100644 --- a/docs/standard.md +++ b/docs/standard.md @@ -30,7 +30,7 @@ A pair is an **agent benchmark** × an **agent**, or a **capability benchmark** | `agent.tools/v1` | The agent calls the tools of the MCP servers given to its session | Two basic checks, three skills ([the tools kit](#the-tools-kit)) | | `agent.conversation/v1` | Several prompts in one session, the earlier turns remembered | Turn 2 asks for what turn 1 said | | `agent.browser/v1` | A browser, through an MCP server Nullius starts | Defined; its kit comes with the first browser benchmark | -| `cap.memory/v1` | `add {session}` then `recall {question}` → `{memory}`; a session is `{id, date, turns: [{role, text}], text}` | Recall a fact among distractors | +| `cap.memory/v1` | `add {session}` then `recall {question}` → `{memory}`; a session is `{id, date, turns: [{role, text}], text}` | Recall a place and an exact name among distractors (2 checks); nothing it stores is a credential | - **The last message** is the agent's text after its last other event (a tool call, a notice); session metadata parts two messages but ends nothing after the last one. - **Notices** (`session.notices`): an adapter sends its warnings as a `notice`, a system step of the trajectory, never as text in front of the answer. @@ -57,6 +57,7 @@ no generic agent-plus-capability recipe or swap control. | Rule | Detail | | --- | --- | | Declared | A kit's `kit_version`: 1 for every kit but the tools kit, at 2 | +| Bumped | When a check gets stronger or tests something new; rewording a check that still tests the same thing keeps the version and the proofs made on it | | Recorded | `kit_version` in the `proven` entry; none means version 1 | | Older proofs | Still proofs, read as *proven on kit v1; kit v2 is current* | | Replaced | `cli conform` on the current kit replaces it, or takes the port back if it fails | From 9a524cd29cee47b0cf9f6d956bd1babc7bbb3c14 Mon Sep 17 00:00:00 2001 From: Etienne Filhol Date: Tue, 10 Nov 2026 14:41:54 +0100 Subject: [PATCH 4/4] Bring back the credential-refusing memory test It is not trivial: the same memory fails m2 on the kit's old wording, the Wi-Fi password, and passes on the boat's name. It keeps a credential from coming back into the kit, and it is the test issue #13 asks for. --- app/tests/test_standard_conform.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/app/tests/test_standard_conform.py b/app/tests/test_standard_conform.py index 1813132..e89d2b6 100644 --- a/app/tests/test_standard_conform.py +++ b/app/tests/test_standard_conform.py @@ -41,6 +41,20 @@ def test_a_kit_that_passes_is_recorded_with_its_job(self): self.assertTrue(os.path.isdir(os.path.join(self.tmp, "jobs", rec["job"]))) self.assertEqual(d["offers"], ["cap.memory/v1"]) + def test_a_memory_that_refuses_credentials_still_proves_the_memory_port(self): + here = os.path.join(self.root, "capabilities", "fakemem") + with open(os.path.join(here, "capability.py")) as f: + src = f.read() + refuse = 'kept.append(req["session"]["text"]); out = {}' + self.assertIn(refuse, src) + with open(os.path.join(here, "capability.py"), "w") as f: # it drops every session that looks like a secret + f.write(src.replace(refuse, 'out = {}\n if not any(w in req["session"]["text"].lower() for w in ' + '("password", "passcode", "token", "secret", "credential", "api key")):\n' + ' kept.append(req["session"]["text"])')) + got, d = self.check("capability", "fakemem", ["cap.memory/v1"]) + self.assertTrue(got["cap.memory/v1"]["ok"]) + self.assertEqual(d["proven"]["cap.memory/v1"]["passed"], "2/2") + def test_a_kit_that_fails_takes_the_claim_back(self): got, d = self.check("agent", "fake", ["agent.task/v1"]) # it shouts the prompt back: not the word self.assertFalse(got["agent.task/v1"]["ok"])