feat(catalogue): run LongMemEval on one question type

app/nullius/catalogue/benchmarks/longmemeval/driver.py

@@ -16,7 +16,8 @@
 The oracle control supplies the dataset's answer, including its explanation
 for abstention questions.
 
-Questions retain dataset order and question_id. Large datasets are streamed
+Questions retain dataset order and question_id; the question_type option
+keeps one type's questions, abstention ones included. Large datasets are streamed
 once into a line-per-question index so workers need not load the whole file.
 """
 import ast
@@ -72,6 +73,7 @@ def gen_length():
 READER = reader_template()
 GEN_LENGTH = gen_length()
 INDEX = {}
+TYPES = {}                                          # question_id -> question_type, read with the index
 
 
 def entries(path, size=1 << 20):
@@ -122,7 +124,9 @@ def index():
         with open(lines_path(), "rb") as f:
             at = 0
             for line in f:
-                INDEX[json.loads(line)["question_id"]] = at
+                q = json.loads(line)
+                INDEX[q["question_id"]] = at
+                TYPES[q["question_id"]] = q.get("question_type")
                 at += len(line)
     return INDEX
 
@@ -134,7 +138,10 @@ def entry(qid):
 
 
 def units(dataset, setup=None):
-    return {"units": list(index())}                 # a setup's unit is a question too, as run_generation reads them
+    # a setup's unit is a question too, as run_generation reads them; one type keeps its abstention
+    # questions too (their question_type is the type's), in dataset order, before any --limit
+    kind = os.environ.get("NULLIUS_OPTION_QUESTION_TYPE", "all")
+    return {"units": [q for q in index() if kind == "all" or TYPES[q] == kind]}
 
 
 def setup(unit, dir):

app/nullius/catalogue/benchmarks/longmemeval/standard.json

@@ -118,6 +118,25 @@
     "turn_seconds": 900
   },
   "options": {
+    "question_type": {
+      "label": "Question type",
+      "summary": "Which memory ability the run measures: every question, or only the questions of one type.",
+      "values": [
+        "all",
+        "knowledge-update",
+        "temporal-reasoning",
+        "multi-session",
+        "single-session-user",
+        "single-session-assistant",
+        "single-session-preference"
+      ],
+      "default": "all",
+      "setups": [
+        "original",
+        "model",
+        "yours"
+      ]
+    },
     "reading": {
       "summary": "Whether the model answers from recalled memory directly or first identifies relevant facts and reasons through them.",
       "values": [

app/nullius/catalogue/benchmarks/longmemeval/upstream-settings.json

@@ -4,6 +4,7 @@
   {"entry": "data (Hugging Face)", "setting": "longmemeval-cleaned/longmemeval_oracle.json", "values": "500 questions, evidence sessions only (the same file in both releases)", "default": "", "nullius": "dataset:longmemeval_oracle"},
   {"entry": "data (Hugging Face)", "setting": "longmemeval/longmemeval_s (first release, uncleaned)", "values": "500 questions", "default": "", "nullius": "dataset:longmemeval_s_original"},
   {"entry": "data (Hugging Face)", "setting": "longmemeval/longmemeval_m (first release, uncleaned)", "values": "500 questions", "default": "", "nullius": "dataset:longmemeval_m_original"},
+  {"entry": "data (Hugging Face)", "setting": "question_type (a field of every question)", "values": "knowledge-update, temporal-reasoning, multi-session, single-session-user, single-session-assistant, single-session-preference; an abstention question (its question_id ends in _abs) keeps its type", "default": "", "nullius": "option:question_type"},
   {"entry": "data/custom_history/sample_haystack_and_timestamp.py", "setting": "task n_questions min_n_haystack_filler max_n_haystack_filler enforce_json_length", "values": "a task name, counts, a length (80 / 500 sessions, 115000 for S)", "default": "", "nullius": "not offered - it compiles new histories from a corpus released on Google Drive without a digest; the released files are the paper's histories, offered as datasets"},
   {"entry": "src/generation/run_generation.sh", "setting": "$1 in_file", "values": "a released file, or a retrieval log", "default": "required", "nullius": "fixed:the unit's question, written as a one-question file of the run's dataset (or run_retrieval.py's log of it) - one question a unit"},
   {"entry": "src/generation/run_generation.sh", "setting": "$2 model_alias gpt-4o, gpt-4o-mini", "values": "gpt-4o (gpt-4o-2024-08-06), gpt-4o-mini (gpt-4o-mini-2024-07-18)", "default": "required", "nullius": "setup:original"},

app/tests/test_longmemeval_setups.py

@@ -156,19 +156,20 @@ def chat_completions_with_backoff(client, **kw): pass
 
 
 class Driver(unittest.TestCase):
-    def run_driver(self, reqs, options, keep=None):
+    def run_driver(self, reqs, options, keep=None, questions=None):
         with tempfile.TemporaryDirectory() as tmp:
             t = keep or tmp
             up, ws = os.path.join(t, "up"), os.path.join(t, "ws")
             os.makedirs(ws)
             _fake_upstream(up)
             data = os.path.join(t, "data.json")
             with open(data, "w") as f:
-                json.dump([{"question_id": "q1", "question_type": "multi-session", "question": "Q?", "answer": "A",
-                            "question_date": "", "haystack_session_ids": [], "haystack_dates": [], "haystack_sessions": []}], f)
+                json.dump([{"question_id": q, "question_type": t, "question": "Q?", "answer": "A", "question_date": "",
+                            "haystack_session_ids": [], "haystack_dates": [], "haystack_sessions": []}
+                           for q, t in (questions or [("q1", "multi-session")])], f)
             env = dict(os.environ, NULLIUS_UPSTREAM=up, NULLIUS_DATASET=data,
                        **{"NULLIUS_OPTION_" + k.upper(): v for k, v in options.items()})
-            lines = "".join(json.dumps(dict(r, dir=ws)) + "\n" for r in reqs)
+            lines = "".join(json.dumps(r if r["op"] == "units" else dict(r, dir=ws)) + "\n" for r in reqs)
             out = subprocess.run([sys.executable, DRIVER], input=lines, capture_output=True, text=True, env=env, timeout=60)
             return [json.loads(ln) for ln in out.stdout.splitlines()]
 
@@ -211,6 +212,28 @@ def test_the_options_reach_the_scripts_arguments(self):
                          ("orig-turn", "50", "nl", "true", "replace", "gpt-4o-2024-08-06"))
         self.assertEqual(argv[-4:], ["--cot", "true", "--con", "true"])
 
+    QUESTIONS = [("a1", "temporal-reasoning"), ("k1", "knowledge-update"), ("m1", "multi-session"),
+                 ("k2_abs", "knowledge-update"), ("k3", "knowledge-update")]
+
+    def test_a_question_type_keeps_that_types_questions_abstention_included_in_dataset_order(self):
+        got = self.run_driver([{"op": "units", "dataset": "longmemeval_oracle"}], {"question_type": "knowledge-update"},
+                              questions=self.QUESTIONS)
+        self.assertEqual(got[0]["units"], ["k1", "k2_abs", "k3"])
+
+    def test_without_a_question_type_every_question_runs(self):
+        for options in ({}, {"question_type": "all"}):
+            got = self.run_driver([{"op": "units", "dataset": "longmemeval_oracle"}], options, questions=self.QUESTIONS)
+            self.assertEqual(got[0]["units"], [q for q, _ in self.QUESTIONS])
+
+    def test_question_type_offers_every_type_and_applies_to_every_setup(self):
+        o = BENCH["options"]["question_type"]
+        self.assertEqual(o["default"], "all")
+        self.assertEqual(set(o["values"]), {"all", "knowledge-update", "temporal-reasoning", "multi-session",
+                                            "single-session-user", "single-session-assistant", "single-session-preference"})
+        for setup in ("original", "model", "yours"):
+            vals, bad = definitions.bench_options_of(BENCH, "longmemeval_oracle", {"question_type": "knowledge-update"}, setup)
+            self.assertEqual((bad, vals["question_type"]), ([], "knowledge-update"))
+
     def test_a_dense_retriever_runs_run_retrieval_with_the_repositorys_shared_model_cache(self):
         with tempfile.TemporaryDirectory() as t:
             got = self.run_driver([{"op": "answer", "unit": "q1", "setup": "original", "model": "gpt-4o", "base_url": "u",

app/tests/test_standard_bench_options.py

@@ -145,7 +145,7 @@ class OtherBenchmarks(unittest.TestCase):
     def test_longmemeval_keeps_all_options_in_one_entry(self):
         b = definitions.get("benchmark", "longmemeval")
         self.assertNotIn("@", b["_ref"])
-        self.assertEqual(definitions.bench_options_of(b, "longmemeval_s")[0], {"reading": "con"})
+        self.assertEqual(definitions.bench_options_of(b, "longmemeval_s")[0], {"question_type": "all", "reading": "con"})
         self.assertEqual(set(b["options"]["retriever"]["values"]),
                          {"full-history-session", "full-history-turn", "no-retrieval",
                           "flat-bm25-session", "flat-bm25-turn", "flat-contriever-session",

docs/benchmarks/supported/longmemeval.md

@@ -34,14 +34,15 @@ Long-term chat memory: questions over timestamped chat histories.
 
 | Version | Setups as published, whose model | Options |
 |---|---|---|
-| `longmemeval` | Original · Original memory · Like the paper or Any of your choice | `history_format`, `merge_key_expansion`, `reading`, `retriever`, `separate_notes`, `topk`, `useronly` |
+| `longmemeval` | Original · Original memory · Like the paper or Any of your choice | `history_format`, `merge_key_expansion`, `question_type`, `reading`, `retriever`, `separate_notes`, `topk`, `useronly` |
 
 ## Options
 
 | Option | Values, default in bold | What it sets | Setups |
 |---|---|---|---|
 | `history_format` | **`json`**, `nl` | Whether conversation history is shown to the model as JSON or as lines of text labelled by speaker. | `original`, `model` |
 | `merge_key_expansion` | **`none`**, `merge`, `replace` | Whether retrieved conversations are shown in full, replaced by their user text, or combined with that text. | `original`, `model` |
+| `question_type` | **`all`**, `knowledge-update`, `temporal-reasoning`, `multi-session`, `single-session-user`, `single-session-assistant`, `single-session-preference` | Which memory ability the run measures: every question, or only the questions of one type. | `original`, `model`, `yours` |
 | `reading` | **`con`**, `direct` | Whether the model answers from recalled memory directly or first identifies relevant facts and reasons through them. | `original`, `model`, `yours` |
 | `retriever` | **`full-history-session`**, `full-history-turn`, `no-retrieval`, `flat-bm25-session`, `flat-bm25-turn`, `flat-contriever-session`, `flat-contriever-turn`, `flat-stella-session`, `flat-stella-turn` | Whether the model receives the full conversation history, no history, or sessions or turns selected by a retrieval method. | `original`, `model` |
 | `separate_notes` | **`false`**, `true` | Write a separate note for each retrieved session or turn before reasoning. Original memory only. | `original`, `model` |