fix(standard): give drivers only the nullius package on PYTHONPATH

app/nullius/standard/prove.py

@@ -75,7 +75,7 @@ def start(self, n: int) -> jsonl.Proc:
         os.makedirs(home, exist_ok=True)
         cmd, extra = install.driver_argv(self.bench)
         env = run.base_env(home, {**extra, "NULLIUS_UPSTREAM": self.up or "", "NULLIUS_DATASET": self.data or "",
-                                  "NULLIUS_DATASET_ID": self.ds["id"], "PYTHONPATH": run.APP, "PYTHONDONTWRITEBYTECODE": "1",
+                                  "NULLIUS_DATASET_ID": self.ds["id"], "PYTHONPATH": run.driver_path(), "PYTHONDONTWRITEBYTECODE": "1",
                                   "NULLIUS_VERSION": str(self.bench.get("_version") or ""),
                                   **run.option_env(definitions.bench_options_of(self.bench, self.ds["id"])[0])})   # the options at their defaults
         return jsonl.Proc(cmd, self.bench["_here"], env, os.path.join(self.work, f"driver-{n}.log")).start()

app/nullius/standard/run.py

@@ -47,6 +47,7 @@
 """
 from __future__ import annotations
 
+import hashlib
 import json
 import os
 import re
@@ -61,7 +62,7 @@
 from concurrent.futures import CancelledError, ThreadPoolExecutor, as_completed
 from typing import Callable, Dict, List, Optional
 
-from nullius import progress, providers
+from nullius import paths, progress, providers
 from nullius.redact import redact
 from nullius.harbor import feed as live_feed
 from nullius.standard import acp, container, definitions, hostreq, install, job, jsonl, match, mcp_machine, relay, shard as shards
@@ -72,7 +73,24 @@
 # The variables of this machine's environment a run's processes keep: where
 # programs are, the locale, the temporary folder. Not a key among them.
 KEEP = ("PATH", "LANG", "LC_ALL", "TMPDIR", "TERM", "NVM_DIR", "NODE_PATH", "SSL_CERT_FILE", "SSL_CERT_DIR")
-APP = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))      # app/: what a kit's driver imports
+APP = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))      # the folder holding the nullius package
+
+
+def driver_path() -> str:
+    """A driver's ``PYTHONPATH``: a folder whose one entry links to the ``nullius`` package, for the
+    kits' drivers that import it. Not ``APP``: installed from a wheel, that is the tool's whole
+    site-packages, and every package there would come before the driver's own environment's."""
+    pkg = os.path.join(APP, "nullius")
+    root = os.path.join(paths.cache_dir(), "driver-path", hashlib.sha256(pkg.encode()).hexdigest()[:16])
+    link = os.path.join(root, "nullius")
+    if os.path.realpath(link) != os.path.realpath(pkg):
+        os.makedirs(root, exist_ok=True)
+        # written whole beside the folder, then renamed in: drivers start in parallel, and the folder
+        # on their PYTHONPATH never holds anything but the link
+        part = f"{root}.{os.getpid()}.{threading.get_ident()}.part"
+        os.symlink(pkg, part)
+        os.replace(part, link)
+    return root
 
 PROVIDERS = {
     "openai": {"base": "https://api.openai.com/v1", "key": "OPENAI_API_KEY"},
@@ -447,7 +465,7 @@ def driver(self) -> jsonl.Proc:
                 raise Refused(f"could not install the driver's environment: {e}")
             env = base_env(os.path.join(self.job_dir, ".work", "driver-home"),
                            {**extra, "NULLIUS_UPSTREAM": self.up or "", "NULLIUS_DATASET": self.data or "",
-                            "NULLIUS_DATASET_ID": self.ds["id"], "PYTHONPATH": APP, "PYTHONDONTWRITEBYTECODE": "1",
+                            "NULLIUS_DATASET_ID": self.ds["id"], "PYTHONPATH": driver_path(), "PYTHONDONTWRITEBYTECODE": "1",
                             "NULLIUS_VERSION": str(self.bench.get("_version") or ""),
                             **({container.CHECK_IMAGE: self.image,
                                 **{k: os.environ[k] for k in container.CLIENT_KEEP if k.startswith("DOCKER") and k in os.environ}}

app/tests/test_driver_path.py

@@ -0,0 +1,72 @@
+"""A driver's PYTHONPATH gives it the nullius package and nothing else of the
+environment Nullius runs in, from a checkout and from a wheel's site-packages."""
+from __future__ import annotations
+
+import json
+import os
+import subprocess
+import sys
+import tempfile
+import unittest
+from unittest import mock
+
+from nullius.standard import run
+
+
+def imports(path: str, *names: str) -> dict:
+    """Which of ``names`` a fresh interpreter finds with ``path`` as its PYTHONPATH."""
+    code = ("import importlib.util, json, sys\n"
+            "print(json.dumps({n: importlib.util.find_spec(n) is not None for n in sys.argv[1:]}))")
+    with tempfile.TemporaryDirectory() as cwd:
+        out = subprocess.run([sys.executable, "-c", code, *names], cwd=cwd, capture_output=True, text=True, timeout=60,
+                             env={"PATH": os.environ.get("PATH", ""), "PYTHONPATH": path, "PYTHONNOUSERSITE": "1"})
+    return json.loads(out.stdout)
+
+
+class DriverPath(unittest.TestCase):
+    def test_from_a_checkout_the_driver_sees_nullius_and_not_the_rest_of_app(self):
+        path = run.driver_path()
+        self.assertEqual(os.listdir(path), ["nullius"])
+        self.assertEqual(os.path.realpath(os.path.join(path, "nullius")), os.path.realpath(os.path.join(run.APP, "nullius")))
+        self.assertEqual(imports(path, "nullius.standard.jsonl", "web"), {"nullius.standard.jsonl": True, "web": False})
+
+    def test_from_a_wheel_the_driver_does_not_see_the_tools_site_packages(self):
+        with tempfile.TemporaryDirectory() as site:
+            os.symlink(os.path.join(run.APP, "nullius"), os.path.join(site, "nullius"))
+            os.makedirs(os.path.join(site, "nullius_tool_only_dep"))    # a package of the tool's own environment
+            open(os.path.join(site, "nullius_tool_only_dep", "__init__.py"), "w").close()
+            app, run.APP = run.APP, site
+            try:
+                path = run.driver_path()
+            finally:
+                run.APP = app
+            self.assertNotEqual(path, site)
+            self.assertEqual(os.listdir(path), ["nullius"])
+            self.assertEqual(imports(path, "nullius.standard.jsonl", "nullius_tool_only_dep"),
+                             {"nullius.standard.jsonl": True, "nullius_tool_only_dep": False})
+
+    def test_it_is_made_once_and_kept(self):
+        self.assertEqual(run.driver_path(), run.driver_path())
+
+    def test_while_it_is_made_the_folder_holds_only_the_link(self):
+        """A driver starting while another's link is being made sees only ``nullius``."""
+        with tempfile.TemporaryDirectory() as site:
+            os.symlink(os.path.join(run.APP, "nullius"), os.path.join(site, "nullius"))
+            seen, replace = [], os.replace
+
+            def watch(src, dst):
+                seen.append(sorted(os.listdir(os.path.dirname(dst))))
+                return replace(src, dst)
+
+            app, run.APP = run.APP, site
+            try:
+                with mock.patch.object(run.os, "replace", watch):
+                    path = run.driver_path()
+            finally:
+                run.APP = app
+            self.assertEqual(seen, [[]])
+            self.assertEqual(os.listdir(path), ["nullius"])
+
+
+if __name__ == "__main__":
+    unittest.main()