diff --git a/app/nullius/standard/prove.py b/app/nullius/standard/prove.py index 93318ee..cab6bb8 100644 --- a/app/nullius/standard/prove.py +++ b/app/nullius/standard/prove.py @@ -75,7 +75,7 @@ def start(self, n: int) -> jsonl.Proc: os.makedirs(home, exist_ok=True) cmd, extra = install.driver_argv(self.bench) env = run.base_env(home, {**extra, "NULLIUS_UPSTREAM": self.up or "", "NULLIUS_DATASET": self.data or "", - "NULLIUS_DATASET_ID": self.ds["id"], "PYTHONPATH": run.APP, "PYTHONDONTWRITEBYTECODE": "1", + "NULLIUS_DATASET_ID": self.ds["id"], "PYTHONPATH": run.driver_path(), "PYTHONDONTWRITEBYTECODE": "1", "NULLIUS_VERSION": str(self.bench.get("_version") or ""), **run.option_env(definitions.bench_options_of(self.bench, self.ds["id"])[0])}) # the options at their defaults return jsonl.Proc(cmd, self.bench["_here"], env, os.path.join(self.work, f"driver-{n}.log")).start() diff --git a/app/nullius/standard/run.py b/app/nullius/standard/run.py index b9a8403..101e05d 100644 --- a/app/nullius/standard/run.py +++ b/app/nullius/standard/run.py @@ -47,6 +47,7 @@ """ from __future__ import annotations +import hashlib import json import os import re @@ -61,7 +62,7 @@ from concurrent.futures import CancelledError, ThreadPoolExecutor, as_completed from typing import Callable, Dict, List, Optional -from nullius import progress, providers +from nullius import paths, progress, providers from nullius.redact import redact from nullius.harbor import feed as live_feed from nullius.standard import acp, container, definitions, hostreq, install, job, jsonl, match, mcp_machine, relay, shard as shards @@ -72,7 +73,24 @@ # The variables of this machine's environment a run's processes keep: where # programs are, the locale, the temporary folder. Not a key among them. KEEP = ("PATH", "LANG", "LC_ALL", "TMPDIR", "TERM", "NVM_DIR", "NODE_PATH", "SSL_CERT_FILE", "SSL_CERT_DIR") -APP = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) # app/: what a kit's driver imports +APP = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) # the folder holding the nullius package + + +def driver_path() -> str: + """A driver's ``PYTHONPATH``: a folder whose one entry links to the ``nullius`` package, for the + kits' drivers that import it. Not ``APP``: installed from a wheel, that is the tool's whole + site-packages, and every package there would come before the driver's own environment's.""" + pkg = os.path.join(APP, "nullius") + root = os.path.join(paths.cache_dir(), "driver-path", hashlib.sha256(pkg.encode()).hexdigest()[:16]) + link = os.path.join(root, "nullius") + if os.path.realpath(link) != os.path.realpath(pkg): + os.makedirs(root, exist_ok=True) + # written whole beside the folder, then renamed in: drivers start in parallel, and the folder + # on their PYTHONPATH never holds anything but the link + part = f"{root}.{os.getpid()}.{threading.get_ident()}.part" + os.symlink(pkg, part) + os.replace(part, link) + return root PROVIDERS = { "openai": {"base": "https://api.openai.com/v1", "key": "OPENAI_API_KEY"}, @@ -447,7 +465,7 @@ def driver(self) -> jsonl.Proc: raise Refused(f"could not install the driver's environment: {e}") env = base_env(os.path.join(self.job_dir, ".work", "driver-home"), {**extra, "NULLIUS_UPSTREAM": self.up or "", "NULLIUS_DATASET": self.data or "", - "NULLIUS_DATASET_ID": self.ds["id"], "PYTHONPATH": APP, "PYTHONDONTWRITEBYTECODE": "1", + "NULLIUS_DATASET_ID": self.ds["id"], "PYTHONPATH": driver_path(), "PYTHONDONTWRITEBYTECODE": "1", "NULLIUS_VERSION": str(self.bench.get("_version") or ""), **({container.CHECK_IMAGE: self.image, **{k: os.environ[k] for k in container.CLIENT_KEEP if k.startswith("DOCKER") and k in os.environ}} diff --git a/app/tests/test_driver_path.py b/app/tests/test_driver_path.py new file mode 100644 index 0000000..fea8226 --- /dev/null +++ b/app/tests/test_driver_path.py @@ -0,0 +1,72 @@ +"""A driver's PYTHONPATH gives it the nullius package and nothing else of the +environment Nullius runs in, from a checkout and from a wheel's site-packages.""" +from __future__ import annotations + +import json +import os +import subprocess +import sys +import tempfile +import unittest +from unittest import mock + +from nullius.standard import run + + +def imports(path: str, *names: str) -> dict: + """Which of ``names`` a fresh interpreter finds with ``path`` as its PYTHONPATH.""" + code = ("import importlib.util, json, sys\n" + "print(json.dumps({n: importlib.util.find_spec(n) is not None for n in sys.argv[1:]}))") + with tempfile.TemporaryDirectory() as cwd: + out = subprocess.run([sys.executable, "-c", code, *names], cwd=cwd, capture_output=True, text=True, timeout=60, + env={"PATH": os.environ.get("PATH", ""), "PYTHONPATH": path, "PYTHONNOUSERSITE": "1"}) + return json.loads(out.stdout) + + +class DriverPath(unittest.TestCase): + def test_from_a_checkout_the_driver_sees_nullius_and_not_the_rest_of_app(self): + path = run.driver_path() + self.assertEqual(os.listdir(path), ["nullius"]) + self.assertEqual(os.path.realpath(os.path.join(path, "nullius")), os.path.realpath(os.path.join(run.APP, "nullius"))) + self.assertEqual(imports(path, "nullius.standard.jsonl", "web"), {"nullius.standard.jsonl": True, "web": False}) + + def test_from_a_wheel_the_driver_does_not_see_the_tools_site_packages(self): + with tempfile.TemporaryDirectory() as site: + os.symlink(os.path.join(run.APP, "nullius"), os.path.join(site, "nullius")) + os.makedirs(os.path.join(site, "nullius_tool_only_dep")) # a package of the tool's own environment + open(os.path.join(site, "nullius_tool_only_dep", "__init__.py"), "w").close() + app, run.APP = run.APP, site + try: + path = run.driver_path() + finally: + run.APP = app + self.assertNotEqual(path, site) + self.assertEqual(os.listdir(path), ["nullius"]) + self.assertEqual(imports(path, "nullius.standard.jsonl", "nullius_tool_only_dep"), + {"nullius.standard.jsonl": True, "nullius_tool_only_dep": False}) + + def test_it_is_made_once_and_kept(self): + self.assertEqual(run.driver_path(), run.driver_path()) + + def test_while_it_is_made_the_folder_holds_only_the_link(self): + """A driver starting while another's link is being made sees only ``nullius``.""" + with tempfile.TemporaryDirectory() as site: + os.symlink(os.path.join(run.APP, "nullius"), os.path.join(site, "nullius")) + seen, replace = [], os.replace + + def watch(src, dst): + seen.append(sorted(os.listdir(os.path.dirname(dst)))) + return replace(src, dst) + + app, run.APP = run.APP, site + try: + with mock.patch.object(run.os, "replace", watch): + path = run.driver_path() + finally: + run.APP = app + self.assertEqual(seen, [[]]) + self.assertEqual(os.listdir(path), ["nullius"]) + + +if __name__ == "__main__": + unittest.main()