Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion app/nullius/standard/kits/cap-memory-v1/driver.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
"The neighbours painted their fence blue."]
CHECKS = {
"m1": ("My sister Ines moved to Lisbon in March.", "Where did Ines move to?", "Lisbon"),
"m2": ("The wifi password at the cabin is tern-river-9.", "What is the wifi password at the cabin?", "tern-river-9"),
"m2": ("The boat at the cabin is named tern-river-9.", "What is the boat at the cabin named?", "tern-river-9"),
}


Expand Down
4 changes: 2 additions & 2 deletions app/nullius/standard/kits/cap-memory-v1/standard.json
Original file line number Diff line number Diff line change
Expand Up @@ -83,8 +83,8 @@
"passes": "The recalled memory contains Lisbon, ignoring case."
},
"m2": {
"label": "Recall a password",
"task": "Store six sessions, including the cabin Wi-Fi password tern-river-9. Ask for that password.",
"label": "Recall an exact name",
"task": "Store six sessions, including “The boat at the cabin is named tern-river-9.” Ask what the boat is named.",
"passes": "The recalled memory contains tern-river-9, ignoring case."
}
}
Expand Down
14 changes: 14 additions & 0 deletions app/tests/test_standard_conform.py
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,20 @@ def test_a_kit_that_passes_is_recorded_with_its_job(self):
self.assertTrue(os.path.isdir(os.path.join(self.tmp, "jobs", rec["job"])))
self.assertEqual(d["offers"], ["cap.memory/v1"])

def test_a_memory_that_refuses_credentials_still_proves_the_memory_port(self):
here = os.path.join(self.root, "capabilities", "fakemem")
with open(os.path.join(here, "capability.py")) as f:
src = f.read()
refuse = 'kept.append(req["session"]["text"]); out = {}'
self.assertIn(refuse, src)
with open(os.path.join(here, "capability.py"), "w") as f: # it drops every session that looks like a secret
f.write(src.replace(refuse, 'out = {}\n if not any(w in req["session"]["text"].lower() for w in '
'("password", "passcode", "token", "secret", "credential", "api key")):\n'
' kept.append(req["session"]["text"])'))
got, d = self.check("capability", "fakemem", ["cap.memory/v1"])
self.assertTrue(got["cap.memory/v1"]["ok"])
self.assertEqual(d["proven"]["cap.memory/v1"]["passed"], "2/2")

def test_a_kit_that_fails_takes_the_claim_back(self):
got, d = self.check("agent", "fake", ["agent.task/v1"]) # it shouts the prompt back: not the word
self.assertFalse(got["agent.task/v1"]["ok"])
Expand Down
3 changes: 2 additions & 1 deletion docs/standard.md
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ A pair is an **agent benchmark** × an **agent**, or a **capability benchmark**
| `agent.tools/v1` | The agent calls the tools of the MCP servers given to its session | Two basic checks, three skills ([the tools kit](#the-tools-kit)) |
| `agent.conversation/v1` | Several prompts in one session, the earlier turns remembered | Turn 2 asks for what turn 1 said |
| `agent.browser/v1` | A browser, through an MCP server Nullius starts | Defined; its kit comes with the first browser benchmark |
| `cap.memory/v1` | `add {session}` then `recall {question}` → `{memory}`; a session is `{id, date, turns: [{role, text}], text}` | Recall a fact among distractors |
| `cap.memory/v1` | `add {session}` then `recall {question}` → `{memory}`; a session is `{id, date, turns: [{role, text}], text}` | Recall a place and an exact name among distractors (2 checks); nothing it stores is a credential |

- **The last message** is the agent's text after its last other event (a tool call, a notice); session metadata parts two messages but ends nothing after the last one.
- **Notices** (`session.notices`): an adapter sends its warnings as a `notice`, a system step of the trajectory, never as text in front of the answer.
Expand All @@ -57,6 +57,7 @@ no generic agent-plus-capability recipe or swap control.
| Rule | Detail |
| --- | --- |
| Declared | A kit's `kit_version`: 1 for every kit but the tools kit, at 2 |
| Bumped | When a check gets stronger or tests something new; rewording a check that still tests the same thing keeps the version and the proofs made on it |
| Recorded | `kit_version` in the `proven` entry; none means version 1 |
| Older proofs | Still proofs, read as *proven on kit v1; kit v2 is current* |
| Replaced | `cli conform` on the current kit replaces it, or takes the port back if it fails |
Expand Down
Loading