diff --git a/CHANGELOG.md b/CHANGELOG.md index c14c32b..86c5045 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,18 @@ # Changelog +## 0.10.1 (unreleased) + +A fix for `gs://` with a stale Google login. No placement change, no unit table change, +and no default changes. + +- A `gs://` open falls back to the bucket's public https URL when Google's default + credentials fail to refresh. An expired gcloud login (a Workspace reauth policy) is + refused with HTTP 400 `invalid_grant`, which tensorstore does not report as a refusal, + so the open raised instead of reading a public bucket. Any error from getting the + access token now counts as a refusal, and so does a token endpoint out of reach. +- When every way in fails after a refusal, the error raised carries the first refusal + as its cause, so a stale login stays in the traceback of a private bucket's 403. + ## 0.10.0 (2026-11-03) Axes in another order, on request. No placement change, no unit table change, and no diff --git a/docs/spec.md b/docs/spec.md index ddc9005..191d027 100644 --- a/docs/spec.md +++ b/docs/spec.md @@ -539,11 +539,18 @@ this document and that one differ, fix one of them. changes it between opens reads the right host. An N5 `units` written once as a string is read as that unit on every axis; tensorstore is given only the array members of `attributes.json`, so attributes it would reject do not stop the open. The next way is tried only when the error - starts `PERMISSION_DENIED` or `UNAUTHENTICATED`; the way that worked is remembered per + starts `PERMISSION_DENIED` or `UNAUTHENTICATED`, or came from getting an access token, + which tensorstore marks with its `internal/oauth2/` code among the error's source + locations (0.10.1): Google refuses an expired gcloud login with HTTP 400 `invalid_grant`, + which starts neither (cellmap-flow, on David's Mac), and a token endpoint out of reach + leaves no token either, so the public URL is worth trying. When the last way fails too + after a refusal, its error carries the first refusal as its cause, so a stale login is + still in the traceback of a private bucket's 403. The way that worked is remembered per (driver, bucket) for the process. Format probing runs all probe keys under one way before moving to the next: a bucket with listing off answers 403 for a missing key, so a 403 on one probe key counts as "missing" when another probe key was read under the same way, - and only when every probe key is refused is the next way tried (see Open question 4). + and only when every probe key is refused is the next way tried (see Open question 4); + a failed login answers no key, so it moves on at once. Metadata reads retry for about five seconds in total (`max_retries 5, initial_delay 0.2s, max_delay 2s`); data reads keep tensorstore's defaults plus `Retry`, and each document read and store open is also timed out under `Retry` (0.7, Behaviour 6). @@ -1049,6 +1056,9 @@ chunkmirage derives from raw attributes carries a `layered` note beside the pack any other error raises at once, the way that worked is tried first next time for that bucket, and a 403 on one probe key with another probe key readable does not trigger the fallback; detection ignores the directory suffix and finds a precomputed scale key. + 0.10.1: a failed login counts as a refusal and is the cause of a later failure; end to + end, a `gs://` open whose token refresh is answered 400 `invalid_grant` (an https proxy + playing www.googleapis.com under a test CA) reads through the public URL. Reads: every statement in Behaviour 1, 2, 3 and 9; a document-less group probed through `s0`, integer and `"first"` levels, `detect(missing_ok=True)`, `index_box`; funlib keys on diff --git a/pyproject.toml b/pyproject.toml index 68f21b1..7667ccb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "worldarray" -version = "0.10.0" +version = "0.10.1" description = "Read zarr, N5 and Neuroglancer precomputed volumes in world coordinates through tensorstore: correct voxel placement, pyramid levels, padded region reads, resampling. numpy + tensorstore only." readme = "README.md" license = { file = "LICENSE" } diff --git a/src/worldarray/__init__.py b/src/worldarray/__init__.py index a0ffb3b..ae45100 100644 --- a/src/worldarray/__init__.py +++ b/src/worldarray/__init__.py @@ -28,7 +28,7 @@ parse_chunk_key, ) -__version__ = "0.10.0" +__version__ = "0.10.1" __all__ = [ "ArrayMeta", diff --git a/src/worldarray/store.py b/src/worldarray/store.py index d0bde2f..7cd5262 100644 --- a/src/worldarray/store.py +++ b/src/worldarray/store.py @@ -158,9 +158,19 @@ class Refused(Exception): """A read was refused for want of credentials (not a missing key, not a network error).""" +def login_failed(error: BaseException) -> bool: + """``error`` came from getting an access token, before any request reached the store: a + stale Google login (HTTP 400 ``invalid_grant``, which starts neither refusal code), a + rejected client, or a token endpoint out of reach. tensorstore names its oauth2 code + among the error's source locations.""" + return "tensorstore/internal/oauth2/" in str(error) + + def is_refused(error: BaseException) -> bool: - return isinstance(error, Refused) or str(error).startswith( - ("PERMISSION_DENIED", "UNAUTHENTICATED") + return ( + isinstance(error, Refused) + or str(error).startswith(("PERMISSION_DENIED", "UNAUTHENTICATED")) + or login_failed(error) ) @@ -171,19 +181,23 @@ def is_refused(error: BaseException) -> bool: def with_access(specs: Sequence[dict], attempt: Callable[[dict], T]) -> T: """``attempt(spec)`` with each spec in turn, moving on only while access is refused; the way that worked is tried first next time, for its bucket. Any other error, and the last - refusal, is raised.""" + refusal, is raised; after a refusal, with the first refusal as its cause, since that + can be the reason the read failed (a stale login passed over for a public URL that + refuses too).""" if len(specs) == 1: return attempt(specs[0]) key = (specs[0]["driver"], specs[0].get("bucket", "")) with _admitted_lock: first = _admitted.get(key, 0) order = [first] + [i for i in range(len(specs)) if i != first] + refusal: BaseException | None = None for i in order[:-1]: try: result = attempt(specs[i]) except Exception as e: if not is_refused(e): raise + refusal = refusal or e logger.info( "%s refused a read (%s); trying the next way in", key, str(e).splitlines()[0][:200] ) @@ -191,7 +205,12 @@ def with_access(specs: Sequence[dict], attempt: Callable[[dict], T]) -> T: with _admitted_lock: _admitted[key] = i return result - result = attempt(specs[order[-1]]) + try: + result = attempt(specs[order[-1]]) + except Exception as e: + if refusal is None or e.__cause__ is not None: + raise + raise e from refusal with _admitted_lock: _admitted[key] = order[-1] return result @@ -264,7 +283,8 @@ def _read_keys(spec: dict, keys: Sequence[str]) -> dict[str, bytes | None | Base """``keys`` under one way in, their reads begun at once: one round trip, not one per key (an http open probes five). Each value is the document, None when it is missing, or the error its read raised; after an error other than a refusal the rest are cancelled and - left out, so a server that never answers costs one key's attempts, not five.""" + left out, so a server that never answers costs one key's attempts, not five, and so + does a failed login, which every key would meet.""" kv = _metadata_kv(spec) started = {key: kv.read(key) for key in keys} out: dict[str, bytes | None | BaseException] = {} @@ -278,7 +298,7 @@ def start(key=key, first=first): result = _wait(start, spec, _where(spec, key), "read") except Exception as e: # noqa: BLE001 the caller decides what a failed key means out[key] = e - if not is_refused(e): + if not is_refused(e) or login_failed(e): for future in started.values(): future.cancel() return out @@ -325,14 +345,14 @@ def read_many(self, keys: Sequence[str]) -> dict[str, bytes | None]: """All of ``keys`` under one way in, read at once (``_read_keys``). A bucket with listing off answers a refusal for a missing key, so a refusal on one key counts as missing when another key was read under the same way; only when every key is refused - is the next way tried.""" + is the next way tried, and at once after a failed login, which answers no key.""" def attempt(spec: dict) -> dict[str, bytes | None]: found: dict[str, bytes | None] = {} refusals: dict[str, BaseException] = {} for key, value in _read_keys(spec, keys).items(): if isinstance(value, BaseException): - if not is_refused(value): + if not is_refused(value) or login_failed(value): raise value refusals[key] = value else: diff --git a/tests/test_access.py b/tests/test_access.py index 190ed06..7918118 100644 --- a/tests/test_access.py +++ b/tests/test_access.py @@ -5,9 +5,13 @@ import functools import gc import http.server +import json import logging import os +import shutil import socket +import socketserver +import ssl import subprocess import sys import textwrap @@ -104,6 +108,40 @@ def fake_read_key(spec, key): assert found[".zarray"] == b"{}" and found["zarr.json"] is None +_STALE_LOGIN = ( # tensorstore's error for gcloud's login after a reauth policy expired it + 'Unknown [HTTP code 400] [http_response_body=\'{"error": "invalid_grant"}\'] ' + "[source locations='tensorstore/internal/oauth2/oauth2_auth_provider.cc:83\n" + "tensorstore/internal/oauth2/auth_provider.cc:30']" +) + + +def test_a_failed_login_is_a_refusal_and_the_cause_of_a_later_one(): + unreachable = ( + "UNAVAILABLE: CURL error Could not resolve host [source locations='" + "tensorstore/internal/curl/curl_transport.cc:475\n" + "tensorstore/internal/oauth2/oauth2_auth_provider.cc:81']" + ) + assert store.is_refused(ValueError(_STALE_LOGIN)) and store.is_refused(ValueError(unreachable)) + assert not store.is_refused(ValueError("Unknown [HTTP code 400] [source locations='x.cc:1']")) + specs = [ + {"driver": "gcs", "bucket": "b"}, + {"driver": "http", "base_url": "https://storage.googleapis.com", "path": "/b/"}, + ] + + def attempt(spec, public=True): + if spec["driver"] == "gcs": + raise ValueError(_STALE_LOGIN) + if public: + return "ok" + raise ValueError("PERMISSION_DENIED: Forbidden [http_response_code='403']") + + assert store.with_access(specs, attempt) == "ok" + store._admitted.clear() + with pytest.raises(ValueError, match="PERMISSION_DENIED") as raised: + store.with_access(specs, functools.partial(attempt, public=False)) + assert "invalid_grant" in str(raised.value.__cause__) # the stale login is in the traceback + + def test_detection_ignores_the_suffix_and_finds_precomputed_scale_keys(tmp_path): misnamed = tmp_path / "actually_zarr.n5" write_zarr2_group(misnamed) @@ -481,6 +519,121 @@ def log_message(self, *args): server.shutdown() +def _google_tls(tmp_path): + """A test CA's certificate, and a TLS context serving a certificate it signed for + Google's token hosts, with the extensions macOS requires of a TLS server's.""" + (tmp_path / "test.cnf").write_text( + textwrap.dedent(""" + [req] + distinguished_name = dn + [dn] + [ca] + basicConstraints = critical, CA:TRUE + keyUsage = critical, keyCertSign, cRLSign + [leaf] + basicConstraints = critical, CA:FALSE + keyUsage = critical, digitalSignature, keyEncipherment + extendedKeyUsage = serverAuth + subjectAltName = DNS:www.googleapis.com, DNS:oauth2.googleapis.com + """) + ) + for args in ( + "req -x509 -config test.cnf -extensions ca -newkey rsa:2048 -nodes -keyout ca.key" + " -out ca.pem -subj /CN=ca -days 1 -sha256", + "req -new -config test.cnf -newkey rsa:2048 -nodes -keyout leaf.key -out leaf.csr" + " -subj /CN=www.googleapis.com -sha256", + "x509 -req -in leaf.csr -CA ca.pem -CAkey ca.key -CAcreateserial -out leaf.pem -days 1" + " -sha256 -extfile test.cnf -extensions leaf", + ): + subprocess.run(["openssl", *args.split()], cwd=tmp_path, check=True, capture_output=True) + tls = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + tls.load_cert_chain(tmp_path / "leaf.pem", tmp_path / "leaf.key") + return tmp_path / "ca.pem", tls + + +@pytest.mark.skipif(shutil.which("openssl") is None, reason="the test CA is made with openssl") +def test_a_stale_google_login_falls_back_to_the_public_url(tmp_path): + """Google answers the token refresh of an expired gcloud login 400 invalid_grant, which + starts neither refusal code (cellmap-flow, on David's Mac). An https proxy plays + www.googleapis.com, under a CA tensorstore is told to trust, and a local http server is + the bucket's public URL.""" + public = tmp_path / "public" + write_zarr2(public / "bucket" / "vol.zarr" / "s0", pattern((4, 4))) + ca, tls = _google_tls(tmp_path) + refreshes = [] + + class Google(socketserver.StreamRequestHandler): + def handle(self): + target = self.rfile.readline().split()[1] + while self.rfile.readline() not in (b"\r\n", b""): + pass + if target != b"www.googleapis.com:443": + self.wfile.write(b"HTTP/1.1 502 Bad Gateway\r\n\r\n") + return + self.wfile.write(b"HTTP/1.1 200 Connection established\r\n\r\n") + with tls.wrap_socket(self.connection, server_side=True) as conn: + request = conn.makefile("rb") + headers = [] + while (line := request.readline()) not in (b"\r\n", b""): + headers.append(line.decode()) + sizes = [h.split(":")[1] for h in headers if h.lower().startswith("content-length")] + request.read(int(sizes[0]) if sizes else 0) + refreshes.append(headers[0].split()[1]) + body = b'{"error": "invalid_grant", "error_description": "reauth related error"}' + conn.sendall( + b"HTTP/1.1 400 Bad Request\r\nContent-Type: application/json\r\n" + b"Connection: close\r\nContent-Length: %d\r\n\r\n%s" % (len(body), body) + ) + + class Files(http.server.SimpleHTTPRequestHandler): + def log_message(self, *args): + pass + + proxy = socketserver.ThreadingTCPServer(("127.0.0.1", 0), Google) + proxy.daemon_threads = True + files = http.server.ThreadingHTTPServer( + ("127.0.0.1", 0), functools.partial(Files, directory=str(public)) + ) + for server in (proxy, files): + threading.Thread(target=server.serve_forever, daemon=True).start() + login = tmp_path / "adc.json" # gcloud's application default credentials, for a user + login.write_text( + json.dumps( + { + "type": "authorized_user", + "client_id": "c", + "client_secret": "s", + "refresh_token": "r", + } + ) + ) + env = {k: v for k, v in os.environ.items() if "proxy" not in k.lower()} + env.update( + HTTPS_PROXY=f"http://127.0.0.1:{proxy.server_address[1]}", + NO_PROXY="127.0.0.1", + TENSORSTORE_CA_BUNDLE=str(ca), + GOOGLE_APPLICATION_CREDENTIALS=str(login), + TENSORSTORE_GCS_HTTP_URL=f"http://127.0.0.1:{files.server_address[1]}", + TENSORSTORE_CURL_VERBOSE="1", # why a handshake with the proxy failed, if one does + ) + read = ( + "import json, worldarray as wa; " + "vol = wa.open('gs://bucket/vol.zarr/s0', units=None); " + "print(json.dumps(vol.ts.read().result().tolist()))" + ) + try: + done = subprocess.run( + [sys.executable, "-c", read], env=env, capture_output=True, text=True, timeout=120 + ) + finally: + proxy.shutdown() + files.shutdown() + assert done.returncode == 0, done.stderr + assert json.loads(done.stdout) == pattern((4, 4)).tolist() + # the login was tried, and refused + assert refreshes and set(refreshes) == {"/oauth2/v3/token"}, done.stderr[-4000:] + + def test_capped_contexts_size_file_io_apart_and_write_without_sync_or_locks(): spec = wa.capped_context_spec(concurrency=1, network=False) assert spec["data_copy_concurrency"] == spec["file_io_concurrency"] == {"limit": 1} diff --git a/uv.lock b/uv.lock index 293a063..53b9a46 100644 --- a/uv.lock +++ b/uv.lock @@ -1247,7 +1247,7 @@ wheels = [ [[package]] name = "worldarray" -version = "0.10.0" +version = "0.10.1" source = { editable = "." } dependencies = [ { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" },