From 448e3e7eb252905764feb4819c054d22bee306ad Mon Sep 17 00:00:00 2001 From: Etienne Filhol Date: Tue, 10 Nov 2026 13:42:40 +0100 Subject: [PATCH] Run LongMemEval on one question type A question_type option keeps one type's questions, abstention ones included, in dataset order and before --limit. The default, all, runs every question as before. --- .../benchmarks/longmemeval/driver.py | 13 ++++++-- .../benchmarks/longmemeval/standard.json | 19 ++++++++++++ .../longmemeval/upstream-settings.json | 1 + app/tests/test_longmemeval_setups.py | 31 ++++++++++++++++--- app/tests/test_standard_bench_options.py | 2 +- docs/benchmarks/supported/longmemeval.md | 3 +- 6 files changed, 60 insertions(+), 9 deletions(-) diff --git a/app/nullius/catalogue/benchmarks/longmemeval/driver.py b/app/nullius/catalogue/benchmarks/longmemeval/driver.py index fb395a7..66573ef 100644 --- a/app/nullius/catalogue/benchmarks/longmemeval/driver.py +++ b/app/nullius/catalogue/benchmarks/longmemeval/driver.py @@ -16,7 +16,8 @@ The oracle control supplies the dataset's answer, including its explanation for abstention questions. -Questions retain dataset order and question_id. Large datasets are streamed +Questions retain dataset order and question_id; the question_type option +keeps one type's questions, abstention ones included. Large datasets are streamed once into a line-per-question index so workers need not load the whole file. """ import ast @@ -72,6 +73,7 @@ def gen_length(): READER = reader_template() GEN_LENGTH = gen_length() INDEX = {} +TYPES = {} # question_id -> question_type, read with the index def entries(path, size=1 << 20): @@ -122,7 +124,9 @@ def index(): with open(lines_path(), "rb") as f: at = 0 for line in f: - INDEX[json.loads(line)["question_id"]] = at + q = json.loads(line) + INDEX[q["question_id"]] = at + TYPES[q["question_id"]] = q.get("question_type") at += len(line) return INDEX @@ -134,7 +138,10 @@ def entry(qid): def units(dataset, setup=None): - return {"units": list(index())} # a setup's unit is a question too, as run_generation reads them + # a setup's unit is a question too, as run_generation reads them; one type keeps its abstention + # questions too (their question_type is the type's), in dataset order, before any --limit + kind = os.environ.get("NULLIUS_OPTION_QUESTION_TYPE", "all") + return {"units": [q for q in index() if kind == "all" or TYPES[q] == kind]} def setup(unit, dir): diff --git a/app/nullius/catalogue/benchmarks/longmemeval/standard.json b/app/nullius/catalogue/benchmarks/longmemeval/standard.json index 1fa54f2..59ef207 100644 --- a/app/nullius/catalogue/benchmarks/longmemeval/standard.json +++ b/app/nullius/catalogue/benchmarks/longmemeval/standard.json @@ -118,6 +118,25 @@ "turn_seconds": 900 }, "options": { + "question_type": { + "label": "Question type", + "summary": "Which memory ability the run measures: every question, or only the questions of one type.", + "values": [ + "all", + "knowledge-update", + "temporal-reasoning", + "multi-session", + "single-session-user", + "single-session-assistant", + "single-session-preference" + ], + "default": "all", + "setups": [ + "original", + "model", + "yours" + ] + }, "reading": { "summary": "Whether the model answers from recalled memory directly or first identifies relevant facts and reasons through them.", "values": [ diff --git a/app/nullius/catalogue/benchmarks/longmemeval/upstream-settings.json b/app/nullius/catalogue/benchmarks/longmemeval/upstream-settings.json index dba2d99..3971359 100644 --- a/app/nullius/catalogue/benchmarks/longmemeval/upstream-settings.json +++ b/app/nullius/catalogue/benchmarks/longmemeval/upstream-settings.json @@ -4,6 +4,7 @@ {"entry": "data (Hugging Face)", "setting": "longmemeval-cleaned/longmemeval_oracle.json", "values": "500 questions, evidence sessions only (the same file in both releases)", "default": "", "nullius": "dataset:longmemeval_oracle"}, {"entry": "data (Hugging Face)", "setting": "longmemeval/longmemeval_s (first release, uncleaned)", "values": "500 questions", "default": "", "nullius": "dataset:longmemeval_s_original"}, {"entry": "data (Hugging Face)", "setting": "longmemeval/longmemeval_m (first release, uncleaned)", "values": "500 questions", "default": "", "nullius": "dataset:longmemeval_m_original"}, + {"entry": "data (Hugging Face)", "setting": "question_type (a field of every question)", "values": "knowledge-update, temporal-reasoning, multi-session, single-session-user, single-session-assistant, single-session-preference; an abstention question (its question_id ends in _abs) keeps its type", "default": "", "nullius": "option:question_type"}, {"entry": "data/custom_history/sample_haystack_and_timestamp.py", "setting": "task n_questions min_n_haystack_filler max_n_haystack_filler enforce_json_length", "values": "a task name, counts, a length (80 / 500 sessions, 115000 for S)", "default": "", "nullius": "not offered - it compiles new histories from a corpus released on Google Drive without a digest; the released files are the paper's histories, offered as datasets"}, {"entry": "src/generation/run_generation.sh", "setting": "$1 in_file", "values": "a released file, or a retrieval log", "default": "required", "nullius": "fixed:the unit's question, written as a one-question file of the run's dataset (or run_retrieval.py's log of it) - one question a unit"}, {"entry": "src/generation/run_generation.sh", "setting": "$2 model_alias gpt-4o, gpt-4o-mini", "values": "gpt-4o (gpt-4o-2024-08-06), gpt-4o-mini (gpt-4o-mini-2024-07-18)", "default": "required", "nullius": "setup:original"}, diff --git a/app/tests/test_longmemeval_setups.py b/app/tests/test_longmemeval_setups.py index e9c5688..939ed93 100644 --- a/app/tests/test_longmemeval_setups.py +++ b/app/tests/test_longmemeval_setups.py @@ -156,7 +156,7 @@ def chat_completions_with_backoff(client, **kw): pass class Driver(unittest.TestCase): - def run_driver(self, reqs, options, keep=None): + def run_driver(self, reqs, options, keep=None, questions=None): with tempfile.TemporaryDirectory() as tmp: t = keep or tmp up, ws = os.path.join(t, "up"), os.path.join(t, "ws") @@ -164,11 +164,12 @@ def run_driver(self, reqs, options, keep=None): _fake_upstream(up) data = os.path.join(t, "data.json") with open(data, "w") as f: - json.dump([{"question_id": "q1", "question_type": "multi-session", "question": "Q?", "answer": "A", - "question_date": "", "haystack_session_ids": [], "haystack_dates": [], "haystack_sessions": []}], f) + json.dump([{"question_id": q, "question_type": t, "question": "Q?", "answer": "A", "question_date": "", + "haystack_session_ids": [], "haystack_dates": [], "haystack_sessions": []} + for q, t in (questions or [("q1", "multi-session")])], f) env = dict(os.environ, NULLIUS_UPSTREAM=up, NULLIUS_DATASET=data, **{"NULLIUS_OPTION_" + k.upper(): v for k, v in options.items()}) - lines = "".join(json.dumps(dict(r, dir=ws)) + "\n" for r in reqs) + lines = "".join(json.dumps(r if r["op"] == "units" else dict(r, dir=ws)) + "\n" for r in reqs) out = subprocess.run([sys.executable, DRIVER], input=lines, capture_output=True, text=True, env=env, timeout=60) return [json.loads(ln) for ln in out.stdout.splitlines()] @@ -211,6 +212,28 @@ def test_the_options_reach_the_scripts_arguments(self): ("orig-turn", "50", "nl", "true", "replace", "gpt-4o-2024-08-06")) self.assertEqual(argv[-4:], ["--cot", "true", "--con", "true"]) + QUESTIONS = [("a1", "temporal-reasoning"), ("k1", "knowledge-update"), ("m1", "multi-session"), + ("k2_abs", "knowledge-update"), ("k3", "knowledge-update")] + + def test_a_question_type_keeps_that_types_questions_abstention_included_in_dataset_order(self): + got = self.run_driver([{"op": "units", "dataset": "longmemeval_oracle"}], {"question_type": "knowledge-update"}, + questions=self.QUESTIONS) + self.assertEqual(got[0]["units"], ["k1", "k2_abs", "k3"]) + + def test_without_a_question_type_every_question_runs(self): + for options in ({}, {"question_type": "all"}): + got = self.run_driver([{"op": "units", "dataset": "longmemeval_oracle"}], options, questions=self.QUESTIONS) + self.assertEqual(got[0]["units"], [q for q, _ in self.QUESTIONS]) + + def test_question_type_offers_every_type_and_applies_to_every_setup(self): + o = BENCH["options"]["question_type"] + self.assertEqual(o["default"], "all") + self.assertEqual(set(o["values"]), {"all", "knowledge-update", "temporal-reasoning", "multi-session", + "single-session-user", "single-session-assistant", "single-session-preference"}) + for setup in ("original", "model", "yours"): + vals, bad = definitions.bench_options_of(BENCH, "longmemeval_oracle", {"question_type": "knowledge-update"}, setup) + self.assertEqual((bad, vals["question_type"]), ([], "knowledge-update")) + def test_a_dense_retriever_runs_run_retrieval_with_the_repositorys_shared_model_cache(self): with tempfile.TemporaryDirectory() as t: got = self.run_driver([{"op": "answer", "unit": "q1", "setup": "original", "model": "gpt-4o", "base_url": "u", diff --git a/app/tests/test_standard_bench_options.py b/app/tests/test_standard_bench_options.py index 25cfc96..e275d6a 100644 --- a/app/tests/test_standard_bench_options.py +++ b/app/tests/test_standard_bench_options.py @@ -145,7 +145,7 @@ class OtherBenchmarks(unittest.TestCase): def test_longmemeval_keeps_all_options_in_one_entry(self): b = definitions.get("benchmark", "longmemeval") self.assertNotIn("@", b["_ref"]) - self.assertEqual(definitions.bench_options_of(b, "longmemeval_s")[0], {"reading": "con"}) + self.assertEqual(definitions.bench_options_of(b, "longmemeval_s")[0], {"question_type": "all", "reading": "con"}) self.assertEqual(set(b["options"]["retriever"]["values"]), {"full-history-session", "full-history-turn", "no-retrieval", "flat-bm25-session", "flat-bm25-turn", "flat-contriever-session", diff --git a/docs/benchmarks/supported/longmemeval.md b/docs/benchmarks/supported/longmemeval.md index 6952dc4..2d97456 100644 --- a/docs/benchmarks/supported/longmemeval.md +++ b/docs/benchmarks/supported/longmemeval.md @@ -34,7 +34,7 @@ Long-term chat memory: questions over timestamped chat histories. | Version | Setups as published, whose model | Options | |---|---|---| -| `longmemeval` | Original · Original memory · Like the paper or Any of your choice | `history_format`, `merge_key_expansion`, `reading`, `retriever`, `separate_notes`, `topk`, `useronly` | +| `longmemeval` | Original · Original memory · Like the paper or Any of your choice | `history_format`, `merge_key_expansion`, `question_type`, `reading`, `retriever`, `separate_notes`, `topk`, `useronly` | ## Options @@ -42,6 +42,7 @@ Long-term chat memory: questions over timestamped chat histories. |---|---|---|---| | `history_format` | **`json`**, `nl` | Whether conversation history is shown to the model as JSON or as lines of text labelled by speaker. | `original`, `model` | | `merge_key_expansion` | **`none`**, `merge`, `replace` | Whether retrieved conversations are shown in full, replaced by their user text, or combined with that text. | `original`, `model` | +| `question_type` | **`all`**, `knowledge-update`, `temporal-reasoning`, `multi-session`, `single-session-user`, `single-session-assistant`, `single-session-preference` | Which memory ability the run measures: every question, or only the questions of one type. | `original`, `model`, `yours` | | `reading` | **`con`**, `direct` | Whether the model answers from recalled memory directly or first identifies relevant facts and reasons through them. | `original`, `model`, `yours` | | `retriever` | **`full-history-session`**, `full-history-turn`, `no-retrieval`, `flat-bm25-session`, `flat-bm25-turn`, `flat-contriever-session`, `flat-contriever-turn`, `flat-stella-session`, `flat-stella-turn` | Whether the model receives the full conversation history, no history, or sessions or turns selected by a retrieval method. | `original`, `model` | | `separate_notes` | **`false`**, `true` | Write a separate note for each retrieved session or turn before reasoning. Original memory only. | `original`, `model` |