From 8c2fa3b5859fc513d6bfb9a8ec553f50597ead13 Mon Sep 17 00:00:00 2001 From: Etienne Filhol Date: Tue, 10 Nov 2026 13:46:59 +0100 Subject: [PATCH 1/2] Give a driver only the nullius package on PYTHONPATH From the PyPI install the folder holding the package is the tool's whole site-packages, which came before the driver's own environment. --- app/nullius/standard/prove.py | 2 +- app/nullius/standard/run.py | 22 +++++++++++++-- app/tests/test_driver_path.py | 52 +++++++++++++++++++++++++++++++++++ 3 files changed, 72 insertions(+), 4 deletions(-) create mode 100644 app/tests/test_driver_path.py diff --git a/app/nullius/standard/prove.py b/app/nullius/standard/prove.py index 93318ee..cab6bb8 100644 --- a/app/nullius/standard/prove.py +++ b/app/nullius/standard/prove.py @@ -75,7 +75,7 @@ def start(self, n: int) -> jsonl.Proc: os.makedirs(home, exist_ok=True) cmd, extra = install.driver_argv(self.bench) env = run.base_env(home, {**extra, "NULLIUS_UPSTREAM": self.up or "", "NULLIUS_DATASET": self.data or "", - "NULLIUS_DATASET_ID": self.ds["id"], "PYTHONPATH": run.APP, "PYTHONDONTWRITEBYTECODE": "1", + "NULLIUS_DATASET_ID": self.ds["id"], "PYTHONPATH": run.driver_path(), "PYTHONDONTWRITEBYTECODE": "1", "NULLIUS_VERSION": str(self.bench.get("_version") or ""), **run.option_env(definitions.bench_options_of(self.bench, self.ds["id"])[0])}) # the options at their defaults return jsonl.Proc(cmd, self.bench["_here"], env, os.path.join(self.work, f"driver-{n}.log")).start() diff --git a/app/nullius/standard/run.py b/app/nullius/standard/run.py index b9a8403..94fd6ec 100644 --- a/app/nullius/standard/run.py +++ b/app/nullius/standard/run.py @@ -47,6 +47,7 @@ """ from __future__ import annotations +import hashlib import json import os import re @@ -61,7 +62,7 @@ from concurrent.futures import CancelledError, ThreadPoolExecutor, as_completed from typing import Callable, Dict, List, Optional -from nullius import progress, providers +from nullius import paths, progress, providers from nullius.redact import redact from nullius.harbor import feed as live_feed from nullius.standard import acp, container, definitions, hostreq, install, job, jsonl, match, mcp_machine, relay, shard as shards @@ -72,7 +73,22 @@ # The variables of this machine's environment a run's processes keep: where # programs are, the locale, the temporary folder. Not a key among them. KEEP = ("PATH", "LANG", "LC_ALL", "TMPDIR", "TERM", "NVM_DIR", "NODE_PATH", "SSL_CERT_FILE", "SSL_CERT_DIR") -APP = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) # app/: what a kit's driver imports +APP = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) # the folder holding the nullius package + + +def driver_path() -> str: + """A driver's ``PYTHONPATH``: a folder whose one entry links to the ``nullius`` package, for the + kits' drivers that import it. Not ``APP``: installed from a wheel, that is the tool's whole + site-packages, and every package there would come before the driver's own environment's.""" + pkg = os.path.join(APP, "nullius") + root = os.path.join(paths.cache_dir(), "driver-path", hashlib.sha256(pkg.encode()).hexdigest()[:16]) + link = os.path.join(root, "nullius") + if os.path.realpath(link) != os.path.realpath(pkg): + os.makedirs(root, exist_ok=True) + part = f"{link}.{os.getpid()}.{threading.get_ident()}.part" # written whole, then renamed: drivers start in parallel + os.symlink(pkg, part) + os.replace(part, link) + return root PROVIDERS = { "openai": {"base": "https://api.openai.com/v1", "key": "OPENAI_API_KEY"}, @@ -447,7 +463,7 @@ def driver(self) -> jsonl.Proc: raise Refused(f"could not install the driver's environment: {e}") env = base_env(os.path.join(self.job_dir, ".work", "driver-home"), {**extra, "NULLIUS_UPSTREAM": self.up or "", "NULLIUS_DATASET": self.data or "", - "NULLIUS_DATASET_ID": self.ds["id"], "PYTHONPATH": APP, "PYTHONDONTWRITEBYTECODE": "1", + "NULLIUS_DATASET_ID": self.ds["id"], "PYTHONPATH": driver_path(), "PYTHONDONTWRITEBYTECODE": "1", "NULLIUS_VERSION": str(self.bench.get("_version") or ""), **({container.CHECK_IMAGE: self.image, **{k: os.environ[k] for k in container.CLIENT_KEEP if k.startswith("DOCKER") and k in os.environ}} diff --git a/app/tests/test_driver_path.py b/app/tests/test_driver_path.py new file mode 100644 index 0000000..4795e63 --- /dev/null +++ b/app/tests/test_driver_path.py @@ -0,0 +1,52 @@ +"""A driver's PYTHONPATH gives it the nullius package and nothing else of the +environment Nullius runs in, from a checkout and from a wheel's site-packages.""" +from __future__ import annotations + +import json +import os +import subprocess +import sys +import tempfile +import unittest + +from nullius.standard import run + + +def imports(path: str, *names: str) -> dict: + """Which of ``names`` a fresh interpreter finds with ``path`` as its PYTHONPATH.""" + code = ("import importlib.util, json, sys\n" + "print(json.dumps({n: importlib.util.find_spec(n) is not None for n in sys.argv[1:]}))") + with tempfile.TemporaryDirectory() as cwd: + out = subprocess.run([sys.executable, "-c", code, *names], cwd=cwd, capture_output=True, text=True, timeout=60, + env={"PATH": os.environ.get("PATH", ""), "PYTHONPATH": path, "PYTHONNOUSERSITE": "1"}) + return json.loads(out.stdout) + + +class DriverPath(unittest.TestCase): + def test_from_a_checkout_the_driver_sees_nullius_and_not_the_rest_of_app(self): + path = run.driver_path() + self.assertEqual(os.listdir(path), ["nullius"]) + self.assertEqual(os.path.realpath(os.path.join(path, "nullius")), os.path.realpath(os.path.join(run.APP, "nullius"))) + self.assertEqual(imports(path, "nullius.standard.jsonl", "web"), {"nullius.standard.jsonl": True, "web": False}) + + def test_from_a_wheel_the_driver_does_not_see_the_tools_site_packages(self): + with tempfile.TemporaryDirectory() as site: + os.symlink(os.path.join(run.APP, "nullius"), os.path.join(site, "nullius")) + os.makedirs(os.path.join(site, "nullius_tool_only_dep")) # a package of the tool's own environment + open(os.path.join(site, "nullius_tool_only_dep", "__init__.py"), "w").close() + app, run.APP = run.APP, site + try: + path = run.driver_path() + finally: + run.APP = app + self.assertNotEqual(path, site) + self.assertEqual(os.listdir(path), ["nullius"]) + self.assertEqual(imports(path, "nullius.standard.jsonl", "nullius_tool_only_dep"), + {"nullius.standard.jsonl": True, "nullius_tool_only_dep": False}) + + def test_it_is_made_once_and_kept(self): + self.assertEqual(run.driver_path(), run.driver_path()) + + +if __name__ == "__main__": + unittest.main() From 382b07673d17424568ffd21aa38153cfd64cf196 Mon Sep 17 00:00:00 2001 From: Etienne Filhol Date: Tue, 10 Nov 2026 14:26:31 +0100 Subject: [PATCH 2/2] Make a driver's path link beside its folder, not in it The link was written as nullius...part inside the folder on a driver's PYTHONPATH, then renamed. A driver or a test listing that folder in the meantime saw the .part entry too. Write it beside the folder and rename it in, so the folder only ever holds nullius. --- app/nullius/standard/run.py | 4 +++- app/tests/test_driver_path.py | 20 ++++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/app/nullius/standard/run.py b/app/nullius/standard/run.py index 94fd6ec..101e05d 100644 --- a/app/nullius/standard/run.py +++ b/app/nullius/standard/run.py @@ -85,7 +85,9 @@ def driver_path() -> str: link = os.path.join(root, "nullius") if os.path.realpath(link) != os.path.realpath(pkg): os.makedirs(root, exist_ok=True) - part = f"{link}.{os.getpid()}.{threading.get_ident()}.part" # written whole, then renamed: drivers start in parallel + # written whole beside the folder, then renamed in: drivers start in parallel, and the folder + # on their PYTHONPATH never holds anything but the link + part = f"{root}.{os.getpid()}.{threading.get_ident()}.part" os.symlink(pkg, part) os.replace(part, link) return root diff --git a/app/tests/test_driver_path.py b/app/tests/test_driver_path.py index 4795e63..fea8226 100644 --- a/app/tests/test_driver_path.py +++ b/app/tests/test_driver_path.py @@ -8,6 +8,7 @@ import sys import tempfile import unittest +from unittest import mock from nullius.standard import run @@ -47,6 +48,25 @@ def test_from_a_wheel_the_driver_does_not_see_the_tools_site_packages(self): def test_it_is_made_once_and_kept(self): self.assertEqual(run.driver_path(), run.driver_path()) + def test_while_it_is_made_the_folder_holds_only_the_link(self): + """A driver starting while another's link is being made sees only ``nullius``.""" + with tempfile.TemporaryDirectory() as site: + os.symlink(os.path.join(run.APP, "nullius"), os.path.join(site, "nullius")) + seen, replace = [], os.replace + + def watch(src, dst): + seen.append(sorted(os.listdir(os.path.dirname(dst)))) + return replace(src, dst) + + app, run.APP = run.APP, site + try: + with mock.patch.object(run.os, "replace", watch): + path = run.driver_path() + finally: + run.APP = app + self.assertEqual(seen, [[]]) + self.assertEqual(os.listdir(path), ["nullius"]) + if __name__ == "__main__": unittest.main()