diff --git a/app/nullius/standard/kits/cap-memory-v1/driver.py b/app/nullius/standard/kits/cap-memory-v1/driver.py index 3db40f9..f78f05c 100644 --- a/app/nullius/standard/kits/cap-memory-v1/driver.py +++ b/app/nullius/standard/kits/cap-memory-v1/driver.py @@ -6,7 +6,7 @@ "The neighbours painted their fence blue."] CHECKS = { "m1": ("My sister Ines moved to Lisbon in March.", "Where did Ines move to?", "Lisbon"), - "m2": ("The wifi password at the cabin is tern-river-9.", "What is the wifi password at the cabin?", "tern-river-9"), + "m2": ("The boat at the cabin is named tern-river-9.", "What is the boat at the cabin named?", "tern-river-9"), } diff --git a/app/nullius/standard/kits/cap-memory-v1/standard.json b/app/nullius/standard/kits/cap-memory-v1/standard.json index 29b0ac1..08af6f9 100644 --- a/app/nullius/standard/kits/cap-memory-v1/standard.json +++ b/app/nullius/standard/kits/cap-memory-v1/standard.json @@ -83,8 +83,8 @@ "passes": "The recalled memory contains Lisbon, ignoring case." }, "m2": { - "label": "Recall a password", - "task": "Store six sessions, including the cabin Wi-Fi password tern-river-9. Ask for that password.", + "label": "Recall an exact name", + "task": "Store six sessions, including “The boat at the cabin is named tern-river-9.” Ask what the boat is named.", "passes": "The recalled memory contains tern-river-9, ignoring case." } } diff --git a/app/tests/test_standard_conform.py b/app/tests/test_standard_conform.py index 1813132..e89d2b6 100644 --- a/app/tests/test_standard_conform.py +++ b/app/tests/test_standard_conform.py @@ -41,6 +41,20 @@ def test_a_kit_that_passes_is_recorded_with_its_job(self): self.assertTrue(os.path.isdir(os.path.join(self.tmp, "jobs", rec["job"]))) self.assertEqual(d["offers"], ["cap.memory/v1"]) + def test_a_memory_that_refuses_credentials_still_proves_the_memory_port(self): + here = os.path.join(self.root, "capabilities", "fakemem") + with open(os.path.join(here, "capability.py")) as f: + src = f.read() + refuse = 'kept.append(req["session"]["text"]); out = {}' + self.assertIn(refuse, src) + with open(os.path.join(here, "capability.py"), "w") as f: # it drops every session that looks like a secret + f.write(src.replace(refuse, 'out = {}\n if not any(w in req["session"]["text"].lower() for w in ' + '("password", "passcode", "token", "secret", "credential", "api key")):\n' + ' kept.append(req["session"]["text"])')) + got, d = self.check("capability", "fakemem", ["cap.memory/v1"]) + self.assertTrue(got["cap.memory/v1"]["ok"]) + self.assertEqual(d["proven"]["cap.memory/v1"]["passed"], "2/2") + def test_a_kit_that_fails_takes_the_claim_back(self): got, d = self.check("agent", "fake", ["agent.task/v1"]) # it shouts the prompt back: not the word self.assertFalse(got["agent.task/v1"]["ok"]) diff --git a/docs/standard.md b/docs/standard.md index 14cbfbe..2a46f71 100644 --- a/docs/standard.md +++ b/docs/standard.md @@ -30,7 +30,7 @@ A pair is an **agent benchmark** × an **agent**, or a **capability benchmark** | `agent.tools/v1` | The agent calls the tools of the MCP servers given to its session | Two basic checks, three skills ([the tools kit](#the-tools-kit)) | | `agent.conversation/v1` | Several prompts in one session, the earlier turns remembered | Turn 2 asks for what turn 1 said | | `agent.browser/v1` | A browser, through an MCP server Nullius starts | Defined; its kit comes with the first browser benchmark | -| `cap.memory/v1` | `add {session}` then `recall {question}` → `{memory}`; a session is `{id, date, turns: [{role, text}], text}` | Recall a fact among distractors | +| `cap.memory/v1` | `add {session}` then `recall {question}` → `{memory}`; a session is `{id, date, turns: [{role, text}], text}` | Recall a place and an exact name among distractors (2 checks); nothing it stores is a credential | - **The last message** is the agent's text after its last other event (a tool call, a notice); session metadata parts two messages but ends nothing after the last one. - **Notices** (`session.notices`): an adapter sends its warnings as a `notice`, a system step of the trajectory, never as text in front of the answer. @@ -57,6 +57,7 @@ no generic agent-plus-capability recipe or swap control. | Rule | Detail | | --- | --- | | Declared | A kit's `kit_version`: 1 for every kit but the tools kit, at 2 | +| Bumped | When a check gets stronger or tests something new; rewording a check that still tests the same thing keeps the version and the proofs made on it | | Recorded | `kit_version` in the `proven` entry; none means version 1 | | Older proofs | Still proofs, read as *proven on kit v1; kit v2 is current* | | Replaced | `cli conform` on the current kit replaces it, or takes the port back if it fails |