From 7cf4df08a646af5fc7d6b42d8fb8c2aecc61488c Mon Sep 17 00:00:00 2001 From: Chase Martin Date: Thu, 24 Sep 2026 11:57:22 -0700 Subject: [PATCH 1/7] Make the pygeoapi image build and serve DIE products locally MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Dockerfile: use orchestration/ as the build context so products.yaml can be copied, and run generate_config.py with /venv/bin/python3 (the Noble base image has no `python` and blocks system pip). - cloudbuild.yaml: match the new build context and derive PYGEOAPI_SERVER_URL from $PROJECT_NUMBER/$_REGION instead of the _SUFFIX placeholder. - config.yml.j2: fix fields current pygeoapi requires — layer name `latest` (GDAL names GeoJSON layers after the file), `extents`, metadata.contact.name, server.map, and server.limits in place of the deprecated server.limit (default 500, max 10000 for NewWeaver). Co-Authored-By: Claude Opus 5.5 (1M context) --- orchestration/pygeoapi/Dockerfile | 11 ++++++----- orchestration/pygeoapi/cloudbuild.yaml | 5 ++--- orchestration/pygeoapi/config.yml.j2 | 12 +++++++++--- 3 files changed, 17 insertions(+), 11 deletions(-) diff --git a/orchestration/pygeoapi/Dockerfile b/orchestration/pygeoapi/Dockerfile index b87d49b2..c6d83cfd 100644 --- a/orchestration/pygeoapi/Dockerfile +++ b/orchestration/pygeoapi/Dockerfile @@ -6,14 +6,15 @@ FROM geopython/pygeoapi:latest WORKDIR /pygeoapi # Copy generation inputs -COPY config.yml.j2 /tmp/config.yml.j2 -COPY generate_config.py /tmp/generate_config.py -COPY ../config/products.yaml /tmp/products.yaml +# Build context is orchestration/ (see cloudbuild.yaml) +COPY pygeoapi/config.yml.j2 /tmp/config.yml.j2 +COPY pygeoapi/generate_config.py /tmp/generate_config.py +COPY config/products.yaml /tmp/products.yaml # Bake config into image at build time. # §V: config generated from products.yaml, not hand-edited. -RUN pip install jinja2 pyyaml --quiet && \ - python /tmp/generate_config.py \ +# jinja2 and PyYAML are pygeoapi dependencies, already in /venv. +RUN /venv/bin/python3 /tmp/generate_config.py \ --products /tmp/products.yaml \ --template /tmp/config.yml.j2 \ --output /pygeoapi/local.config.yml diff --git a/orchestration/pygeoapi/cloudbuild.yaml b/orchestration/pygeoapi/cloudbuild.yaml index dabf0800..ad0e1b72 100644 --- a/orchestration/pygeoapi/cloudbuild.yaml +++ b/orchestration/pygeoapi/cloudbuild.yaml @@ -8,7 +8,7 @@ steps: - '$_REGION-docker.pkg.dev/$PROJECT_ID/die/die-pygeoapi:latest' - '-f' - 'orchestration/pygeoapi/Dockerfile' - - 'orchestration/pygeoapi' # build context — allows COPY ../config/products.yaml + - 'orchestration' # build context holds pygeoapi/ and config/products.yaml - name: 'gcr.io/cloud-builders/docker' args: @@ -28,11 +28,10 @@ steps: - '--allow-unauthenticated' - '--port=80' - '--memory=512Mi' - - '--set-env-vars=PYGEOAPI_SERVER_URL=https://die-pygeoapi-$_SUFFIX.run.app' + - '--set-env-vars=PYGEOAPI_SERVER_URL=https://die-pygeoapi-$PROJECT_NUMBER.$_REGION.run.app' substitutions: _REGION: us-central1 - _SUFFIX: placeholder # replace with Cloud Run URL suffix after first deploy images: - '$_REGION-docker.pkg.dev/$PROJECT_ID/die/die-pygeoapi:$COMMIT_SHA' diff --git a/orchestration/pygeoapi/config.yml.j2 b/orchestration/pygeoapi/config.yml.j2 index 6eb32baf..6619161c 100644 --- a/orchestration/pygeoapi/config.yml.j2 +++ b/orchestration/pygeoapi/config.yml.j2 @@ -8,7 +8,12 @@ server: language: en-US cors: true pretty_print: false - limit: 500 + limits: + default_items: 500 + max_items: 10000 # NewWeaver requests limit=10000 + map: + url: https://tile.openstreetmap.org/{z}/{x}/{y}.png + attribution: '© OpenStreetMap contributors' logging: level: ERROR @@ -32,6 +37,7 @@ metadata: name: NM Bureau of Geology & Mineral Resources url: https://geoinfo.nmt.edu contact: + name: NM Bureau of Geology & Mineral Resources address: 801 Leroy Place city: Socorro stateorprovince: New Mexico @@ -48,7 +54,7 @@ resources: - water - groundwater - New Mexico - extent: + extents: spatial: bbox: - -109.05 @@ -74,7 +80,7 @@ resources: EMPTY_AS_NULL: "NO" GDAL_CACHEMAX: "64" id_field: id - layer: OGRGeoJSON + layer: latest # GDAL names a GeoJSON layer after the file basename {% if product.output_type == 'ogc_timeseries' %} time_field: datetime {% endif %} From 249145766f93686efee7cf7d8d2fa92dcce39554 Mon Sep 17 00:00:00 2001 From: Chase Martin Date: Thu, 24 Sep 2026 12:38:55 -0700 Subject: [PATCH 2/7] Fix pygeoapi temporal extent format pygeoapi's config expects extents.temporal.begin/end, not the interval array used in API responses. With interval, any datetime= request crashed with KeyError: 'begin'. begin is left unquoted so YAML parses it as a tz-aware datetime, which pygeoapi requires. Co-Authored-By: Claude Opus 5.5 (1M context) --- orchestration/pygeoapi/config.yml.j2 | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/orchestration/pygeoapi/config.yml.j2 b/orchestration/pygeoapi/config.yml.j2 index 6619161c..614ed4d4 100644 --- a/orchestration/pygeoapi/config.yml.j2 +++ b/orchestration/pygeoapi/config.yml.j2 @@ -64,9 +64,8 @@ resources: crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 {% if product.output_type == 'ogc_timeseries' %} temporal: - interval: - - - "1900-01-01T00:00:00Z" - - null + begin: 1900-01-01T00:00:00Z # unquoted: pygeoapi needs a tz-aware datetime, not a string + end: null {% endif %} providers: - type: feature From fab53356fcdb01eba3b6c4db5643a9df061fb134 Mon Sep 17 00:00:00 2001 From: Chase Martin Date: Thu, 24 Sep 2026 12:59:40 -0700 Subject: [PATCH 3/7] Serve the timeseries product from GeoParquet in pygeoapi The OGR provider ignores datetime filters and parses the whole 733 MB GeoJSON on every request (~9-10s locally). pygeoapi's Parquet provider applies datetime and serves the same layer in well under a second. - Dockerfile: install pyarrow and geopandas (pinned to the tested versions), which the Parquet provider needs and the base image lacks. - config.yml.j2: ogc_timeseries products use the Parquet provider on gs://.../latest.parquet with time_field: datetime; all other products stay on OGR + /vsigs/ GeoJSON. - generate_config.py: the GCS path check expects the parquet path for timeseries products. - cloudbuild.yaml: 2Gi memory; the Parquet provider peaked at ~1.8 GB across 4 workers locally. latest.parquet is uploaded manually for now; the pipeline doesn't write it yet. Known gaps: date-only datetime queries 500 against a tz-aware column, and numberMatched is wrong. Co-Authored-By: Claude Opus 5.5 (1M context) --- orchestration/pygeoapi/Dockerfile | 6 ++++++ orchestration/pygeoapi/cloudbuild.yaml | 2 +- orchestration/pygeoapi/config.yml.j2 | 13 +++++++++++-- orchestration/pygeoapi/generate_config.py | 12 ++++++++---- 4 files changed, 26 insertions(+), 7 deletions(-) diff --git a/orchestration/pygeoapi/Dockerfile b/orchestration/pygeoapi/Dockerfile index c6d83cfd..5c6a0886 100644 --- a/orchestration/pygeoapi/Dockerfile +++ b/orchestration/pygeoapi/Dockerfile @@ -5,6 +5,12 @@ FROM geopython/pygeoapi:latest WORKDIR /pygeoapi +# pygeoapi's Parquet provider needs pyarrow and geopandas, which the base +# image doesn't ship (s3fs, shapely, pandas are already in /venv). +# Pinned to the versions tested locally against pygeoapi 0.25. +RUN /venv/bin/python3 -m pip install --no-cache-dir --quiet \ + pyarrow==25.0.1 geopandas==1.1.4 + # Copy generation inputs # Build context is orchestration/ (see cloudbuild.yaml) COPY pygeoapi/config.yml.j2 /tmp/config.yml.j2 diff --git a/orchestration/pygeoapi/cloudbuild.yaml b/orchestration/pygeoapi/cloudbuild.yaml index ad0e1b72..e747f9e6 100644 --- a/orchestration/pygeoapi/cloudbuild.yaml +++ b/orchestration/pygeoapi/cloudbuild.yaml @@ -27,7 +27,7 @@ steps: - '--platform=managed' - '--allow-unauthenticated' - '--port=80' - - '--memory=512Mi' + - '--memory=2Gi' # Parquet provider peaked at 1.2-2.0 GB across 4 workers locally - '--set-env-vars=PYGEOAPI_SERVER_URL=https://die-pygeoapi-$PROJECT_NUMBER.$_REGION.run.app' substitutions: diff --git a/orchestration/pygeoapi/config.yml.j2 b/orchestration/pygeoapi/config.yml.j2 index 614ed4d4..a2f6a503 100644 --- a/orchestration/pygeoapi/config.yml.j2 +++ b/orchestration/pygeoapi/config.yml.j2 @@ -68,6 +68,17 @@ resources: end: null {% endif %} providers: +{% if product.output_type == 'ogc_timeseries' %} + # GeoParquet via the Parquet provider: the OGR provider ignores datetime + # filters and parses the whole GeoJSON on every request. The file must be + # GeoParquet with a bbox covering column and a tz-aware UTC datetime. + - type: feature + name: Parquet + data: + source: gs://{{ gcs_bucket }}/products/{{ product.id }}/latest.parquet + id_field: id + time_field: datetime +{% else %} - type: feature name: OGR data: @@ -80,8 +91,6 @@ resources: GDAL_CACHEMAX: "64" id_field: id layer: latest # GDAL names a GeoJSON layer after the file basename -{% if product.output_type == 'ogc_timeseries' %} - time_field: datetime {% endif %} {% endfor %} diff --git a/orchestration/pygeoapi/generate_config.py b/orchestration/pygeoapi/generate_config.py index a9e4a906..46102002 100644 --- a/orchestration/pygeoapi/generate_config.py +++ b/orchestration/pygeoapi/generate_config.py @@ -2,7 +2,8 @@ Generate pygeoapi config.yml from products.yaml + Jinja2 template. §V: pygeoapi config MUST be generated from products.yaml — never hand-edited. -§V: pygeoapi OGR provider MUST use /vsigs/ path (GCS). +§V: pygeoapi OGR provider MUST use /vsigs/ path (GCS). Timeseries products use + the Parquet provider and read gs://.../latest.parquet instead. Usage: python generate_config.py \ @@ -29,13 +30,16 @@ def generate(products_path: Path, template_path: Path, output_path: Path) -> Non gcs_bucket=products_config["gcs_bucket"], ) - # Sanity check: every product must produce an OGR /vsigs/ entry + # Sanity check: every product must produce a GCS source entry for product in products_config["products"]: pid = product["id"] bucket = products_config["gcs_bucket"] - expected = f"/vsigs/{bucket}/products/{pid}/latest.geojson" + if product.get("output_type") == "ogc_timeseries": + expected = f"gs://{bucket}/products/{pid}/latest.parquet" + else: + expected = f"/vsigs/{bucket}/products/{pid}/latest.geojson" assert expected in rendered, ( - f"§V violated: OGR provider for '{pid}' missing /vsigs/ path in generated config" + f"§V violated: provider for '{pid}' missing {expected} in generated config" ) output_path.parent.mkdir(parents=True, exist_ok=True) From 5c8e617ec4ad97c08b342a433030f654ad3d9bbb Mon Sep 17 00:00:00 2001 From: Chase Martin Date: Thu, 24 Sep 2026 13:01:53 -0700 Subject: [PATCH 4/7] =?UTF-8?q?Update=20SPEC=20=C2=A76=20for=20the=20pygeo?= =?UTF-8?q?api=20Parquet=20provider?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Timeseries products are now served by the Parquet provider from gs://.../latest.parquet; all other products stay on OGR + /vsigs/ GeoJSON. §3.3 and §6 now describe both paths, the GeoParquet file requirements, the manual upload, and known provider limitations. §6.2, §6.3 and §6.5 now mirror the current template, generator and Dockerfile, and the invariants gain a gs:// rule for the Parquet provider. Co-Authored-By: Claude Opus 5.5 (1M context) --- SPEC.md | 203 ++++++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 154 insertions(+), 49 deletions(-) diff --git a/SPEC.md b/SPEC.md index c4f5a3ea..603ea884 100644 --- a/SPEC.md +++ b/SPEC.md @@ -94,10 +94,11 @@ Cloud Scheduler (cron) GCS bucket (public read) │ - GDAL /vsigs/ virtual FS + GDAL /vsigs/ (GeoJSON) · PyArrow gs:// (GeoParquet) │ Cloud Run Service ← pygeoapi (always-on) - (OGR provider reads GeoJSON from GCS) + (OGR provider: GeoJSON products; + Parquet provider: timeseries products) │ HTTP clients (OGC API - Features) @@ -105,10 +106,11 @@ Cloud Scheduler (cron) **Two Cloud Run deployments:** - **Cloud Run Job** (Dagster, stateless): triggered by Cloud Scheduler, runs pipeline per product -- **Cloud Run Service** (pygeoapi, always-on): serves OGC API - Features via GDAL OGR + GCS +- **Cloud Run Service** (pygeoapi, always-on): serves OGC API - Features from GCS -No PostgreSQL required. GCS is authoritative store. pygeoapi reads GeoJSON directly from -GCS via GDAL's `/vsigs/` virtual filesystem — no proxy, no DB, no sync step. +No PostgreSQL required. GCS is authoritative store. pygeoapi reads product files directly +from GCS — GeoJSON via GDAL's `/vsigs/` virtual filesystem, timeseries GeoParquet via +PyArrow's `gs://` filesystem — no proxy, no DB, no sync step (§6). > GeoServer/PostGIS persister in `backend/persisters/geoserver.py` is unchanged and > still usable via CLI `--output-format geoserver`. Not part of this pipeline. @@ -261,16 +263,40 @@ Structure supports later migration to persistent Dagster daemon (change Cloud Ru ### §6.1 Role -pygeoapi serves the GCS-stored GeoJSON files as OGC API - Features collections. -No DB. pygeoapi uses the **OGR provider** backed by GDAL's `/vsigs/` virtual filesystem, -which reads GeoJSON directly from GCS using Application Default Credentials on Cloud Run. +pygeoapi serves the GCS-stored product files as OGC API - Features collections. No DB. +The provider depends on the product's `output_type`: + +| Products | Provider | Source | +|---|---|---| +| `ogc_timeseries` | **Parquet** (`time_field: datetime`) | `gs://{bucket}/products/{id}/latest.parquet` | +| all others | **OGR** (`source_type: GeoJSON`) | `/vsigs/{bucket}/products/{id}/latest.geojson` | + +Timeseries products use Parquet because the OGR provider ignores `datetime` filters and +parses the whole GeoJSON on every request (~9–10 s per request for the 733 MB +`nm_waterlevels_timeseries` file in local testing, vs. under 1 s from GeoParquet). + +The timeseries GeoParquet file must have: +- GeoParquet `geo` metadata with WKB geometry and a **bbox covering column** (required for + `bbox` queries; geopandas `to_parquet(..., write_covering_bbox=True)`) +- `datetime` as a **tz-aware UTC** timestamp +- `string` rather than `large_string` columns (the provider can't describe `large_string` + fields) + +The pipeline does not write `latest.parquet` yet; it is uploaded manually. It must exist +in the bucket before pygeoapi starts — the entrypoint fails if any collection can't open. + +Known provider limitations: +- Parquet: date-only `datetime` queries (`2020-01-01/2020-12-31`) return 500 against a + tz-aware column — full RFC 3339 values (`2020-01-01T00:00:00Z/..`) work. + `numberMatched` is wrong. +- OGR: no `numberReturned`; DateTime properties are returned as `YYYY/MM/DD HH:MM:SS+00`. ``` GET /collections GET /collections/{id}/items GET /collections/{id}/items/{feature_id} GET /collections/{id}/items?bbox=-107,32,-103,37 -GET /collections/{id}/items?datetime=2020-01-01/2024-12-31 ← timeseries only +GET /collections/{id}/items?datetime=2020-01-01T00:00:00Z/2024-12-31T23:59:59Z ← timeseries only ``` ### §6.2 pygeoapi Config Template (`orchestration/pygeoapi/config.yml.j2`) @@ -286,7 +312,12 @@ server: language: en-US cors: true pretty_print: false - limit: 500 + limits: + default_items: 500 + max_items: 10000 # NewWeaver requests limit=10000 + map: + url: https://tile.openstreetmap.org/{z}/{x}/{y}.png + attribution: '© OpenStreetMap contributors' logging: level: ERROR @@ -295,7 +326,11 @@ metadata: identification: title: NM Unified Water Data description: OGC API - Features for New Mexico water data - keywords: [water, groundwater, "New Mexico", NMBGMR] + keywords: + - water + - groundwater + - New Mexico + - NMBGMR keywords_type: theme terms_of_service: https://creativecommons.org/licenses/by/4.0/ url: https://waterdata.nmt.edu @@ -305,92 +340,161 @@ metadata: provider: name: NM Bureau of Geology & Mineral Resources url: https://geoinfo.nmt.edu + contact: + name: NM Bureau of Geology & Mineral Resources + address: 801 Leroy Place + city: Socorro + stateorprovince: New Mexico + postalcode: "87801" + country: USA resources: {% for product in products %} {{ product.id }}: type: collection - title: {{ product.title }} - description: {{ product.description }} - keywords: [water, groundwater, "New Mexico"] - extent: + title: {{ product.title | tojson }} + description: {{ product.description | tojson }} + keywords: + - water + - groundwater + - New Mexico + extents: spatial: - bbox: [-109.05, 31.33, -103.00, 37.00] + bbox: + - -109.05 + - 31.33 + - -103.00 + - 37.00 crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 {% if product.output_type == 'ogc_timeseries' %} temporal: - interval: [["1900-01-01T00:00:00Z", null]] + begin: 1900-01-01T00:00:00Z # unquoted: pygeoapi needs a tz-aware datetime, not a string + end: null {% endif %} providers: +{% if product.output_type == 'ogc_timeseries' %} + # GeoParquet via the Parquet provider: the OGR provider ignores datetime + # filters and parses the whole GeoJSON on every request. The file must be + # GeoParquet with a bbox covering column and a tz-aware UTC datetime. + - type: feature + name: Parquet + data: + source: gs://{{ gcs_bucket }}/products/{{ product.id }}/latest.parquet + id_field: id + time_field: datetime +{% else %} - type: feature name: OGR data: source_type: GeoJSON - source: /vsigs/{{ gcs_bucket }}/{{ product.id }}/latest.geojson + source: /vsigs/{{ gcs_bucket }}/products/{{ product.id }}/latest.geojson source_options: - GDAL_HTTP_UNSAFESSL: NO + GDAL_HTTP_UNSAFESSL: "NO" gdal_ogr_options: - EMPTY_AS_NULL: NO - GDAL_CACHEMAX: 64 + EMPTY_AS_NULL: "NO" + GDAL_CACHEMAX: "64" id_field: id - layer: OGRGeoJSON -{% if product.output_type == 'ogc_timeseries' %} - time_field: datetime + layer: latest # GDAL names a GeoJSON layer after the file basename {% endif %} {% endfor %} ``` +Notes on fields current pygeoapi requires (each fails at boot or first request if wrong): +`metadata.contact.name`, `server.map`, `extents` (plural), `server.limits` (the old +`server.limit` is deprecated and leaves the HTML items page broken), `temporal.begin`/`end` +(not the `interval` array used in API responses; `begin` unquoted so YAML parses a +tz-aware datetime), and `layer: latest` for GeoJSON (GDAL names the layer after the file +basename). + ### §6.3 Config Generation (`orchestration/pygeoapi/generate_config.py`) ```python -import yaml -from jinja2 import Environment, FileSystemLoader -from pathlib import Path - -def generate(products_path: Path, template_path: Path, output_path: Path): - products = yaml.safe_load(products_path.read_text()) - env = Environment(loader=FileSystemLoader(str(template_path.parent))) +def generate(products_path: Path, template_path: Path, output_path: Path) -> None: + products_config = yaml.safe_load(products_path.read_text()) + env = Environment( + loader=FileSystemLoader(str(template_path.parent)), + keep_trailing_newline=True, + ) tmpl = env.get_template(template_path.name) - output_path.write_text(tmpl.render( - products=products["products"], - gcs_bucket=products["gcs_bucket"], - )) + rendered = tmpl.render( + products=products_config["products"], + gcs_bucket=products_config["gcs_bucket"], + ) + + # Sanity check: every product must produce a GCS source entry + for product in products_config["products"]: + pid = product["id"] + bucket = products_config["gcs_bucket"] + if product.get("output_type") == "ogc_timeseries": + expected = f"gs://{bucket}/products/{pid}/latest.parquet" + else: + expected = f"/vsigs/{bucket}/products/{pid}/latest.geojson" + assert expected in rendered, ( + f"§V violated: provider for '{pid}' missing {expected} in generated config" + ) + + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text(rendered) + print(f"Generated {output_path} ({len(products_config['products'])} collections)") ``` -Run at Docker build time in `cloudbuild.yaml` — baked into image, not runtime. +Run at Docker build time — baked into the image, not runtime. -### §6.4 GDAL + GCS Auth +### §6.4 GCS Auth -On Cloud Run, GDAL `/vsigs/` uses the service account's ADC automatically. -No credentials file needed. Require the pygeoapi Cloud Run Service account to have -`roles/storage.objectViewer` on the `die-products` bucket. +On Cloud Run, both GDAL `/vsigs/` and PyArrow's `gs://` filesystem use the service +account's ADC automatically. No credentials file needed. Require the pygeoapi Cloud Run +Service account to have `roles/storage.objectViewer` on the products bucket +(`gcs_bucket` in `products.yaml`). -For local dev: +For local dev, render the config and rewrite GCS paths to a mounted folder instead: ```bash -export GOOGLE_APPLICATION_CREDENTIALS=/path/to/sa-key.json +sed -i '' -E 's#(/vsigs/|gs://)dataservices-die-products#/data#' local.config.yml ``` ### §6.5 Dockerfile (`orchestration/pygeoapi/Dockerfile`) +Build context is `orchestration/` so `config/products.yaml` can be copied. + ```dockerfile FROM geopython/pygeoapi:latest -# Generate config from products.yaml at build time -COPY ../config/products.yaml /tmp/products.yaml -COPY config.yml.j2 /tmp/config.yml.j2 -COPY generate_config.py /tmp/generate_config.py -RUN python /tmp/generate_config.py \ +# pygeoapi base image includes GDAL with /vsigs/ GCS support. +# Auth uses Application Default Credentials — no key file needed on Cloud Run. + +WORKDIR /pygeoapi + +# pygeoapi's Parquet provider needs pyarrow and geopandas, which the base +# image doesn't ship (s3fs, shapely, pandas are already in /venv). +# Pinned to the versions tested locally against pygeoapi 0.25. +RUN /venv/bin/python3 -m pip install --no-cache-dir --quiet \ + pyarrow==25.0.1 geopandas==1.1.4 + +# Copy generation inputs +# Build context is orchestration/ (see cloudbuild.yaml) +COPY pygeoapi/config.yml.j2 /tmp/config.yml.j2 +COPY pygeoapi/generate_config.py /tmp/generate_config.py +COPY config/products.yaml /tmp/products.yaml + +# Bake config into image at build time. +# §V: config generated from products.yaml, not hand-edited. +# jinja2 and PyYAML are pygeoapi dependencies, already in /venv. +RUN /venv/bin/python3 /tmp/generate_config.py \ --products /tmp/products.yaml \ --template /tmp/config.yml.j2 \ --output /pygeoapi/local.config.yml EXPOSE 80 + +# pygeoapi reads PYGEOAPI_CONFIG env var; default is local.config.yml +ENV PYGEOAPI_CONFIG=/pygeoapi/local.config.yml ``` -Cloud Run Service env vars: -- `PYGEOAPI_SERVER_URL` — public Cloud Run URL +Cloud Run Service (`orchestration/pygeoapi/cloudbuild.yaml`): +- `PYGEOAPI_SERVER_URL` — `https://die-pygeoapi-$PROJECT_NUMBER.$_REGION.run.app` - Port: 80 +- Memory: 2Gi — the Parquet provider peaked at ~1.8 GB across 4 gunicorn workers locally --- @@ -940,6 +1044,7 @@ feature/composition-refactor ← branch off main after §T.9 merged - All existing tests pass under `uv run pytest` - pygeoapi config MUST be generated from `products.yaml` — never hand-edited - pygeoapi OGR provider MUST use `/vsigs/` path (GCS), never local filesystem path +- pygeoapi Parquet provider (`ogc_timeseries` products) MUST use `gs://` path (GCS), never local filesystem path - No database introduced in orchestration pipeline — GCS is sole store - `latest.geojson` MUST be overwritten atomically (upload to tmp key, then copy/rename) From f31b24befea02d666fa5b307dd76287a8839d2c2 Mon Sep 17 00:00:00 2001 From: Chase Martin Date: Thu, 24 Sep 2026 13:52:16 -0700 Subject: [PATCH 5/7] Run die-pygeoapi as a dedicated service account MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Deploy the Cloud Run service as die-pygeoapi@$PROJECT_ID instead of the Compute Engine default account. It needs storage.objectViewer on the products bucket, and Cloud Build's account needs iam.serviceAccountUser on it. SPEC §6.5 lists the same requirements. Co-Authored-By: Claude Opus 5.5 (1M context) --- SPEC.md | 3 +++ orchestration/pygeoapi/cloudbuild.yaml | 1 + 2 files changed, 4 insertions(+) diff --git a/SPEC.md b/SPEC.md index 603ea884..a3dce37a 100644 --- a/SPEC.md +++ b/SPEC.md @@ -494,6 +494,9 @@ ENV PYGEOAPI_CONFIG=/pygeoapi/local.config.yml Cloud Run Service (`orchestration/pygeoapi/cloudbuild.yaml`): - `PYGEOAPI_SERVER_URL` — `https://die-pygeoapi-$PROJECT_NUMBER.$_REGION.run.app` - Port: 80 +- Service account: `die-pygeoapi@$PROJECT_ID.iam.gserviceaccount.com` — needs + `roles/storage.objectViewer` on the products bucket; Cloud Build's account needs + `roles/iam.serviceAccountUser` on it - Memory: 2Gi — the Parquet provider peaked at ~1.8 GB across 4 gunicorn workers locally --- diff --git a/orchestration/pygeoapi/cloudbuild.yaml b/orchestration/pygeoapi/cloudbuild.yaml index e747f9e6..e92feb0b 100644 --- a/orchestration/pygeoapi/cloudbuild.yaml +++ b/orchestration/pygeoapi/cloudbuild.yaml @@ -27,6 +27,7 @@ steps: - '--platform=managed' - '--allow-unauthenticated' - '--port=80' + - '--service-account=die-pygeoapi@$PROJECT_ID.iam.gserviceaccount.com' # needs storage.objectViewer on the products bucket - '--memory=2Gi' # Parquet provider peaked at 1.2-2.0 GB across 4 workers locally - '--set-env-vars=PYGEOAPI_SERVER_URL=https://die-pygeoapi-$PROJECT_NUMBER.$_REGION.run.app' From bdcc8c84ce89659c1176cb6d9ecc2ed57365322f Mon Sep 17 00:00:00 2001 From: Chase Martin Date: Thu, 24 Sep 2026 14:27:46 -0700 Subject: [PATCH 6/7] Set CPL_MACHINE_IS_GCE so GDAL can read GCS on Cloud Run MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On the live service all 30 GeoJSON collections returned 500 with "No valid GCS credentials found": GDAL doesn't detect Cloud Run as GCE, so /vsigs/ never asks the metadata server for the service account's credentials. The Parquet collection was unaffected (PyArrow finds ADC on its own). Set the variable in the deploy step, since --set-env-vars replaces the service's env on every deploy. SPEC §6.4/§6.5 updated. Co-Authored-By: Claude Opus 5.5 (1M context) --- SPEC.md | 5 ++++- orchestration/pygeoapi/cloudbuild.yaml | 4 +++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/SPEC.md b/SPEC.md index a3dce37a..b808d66f 100644 --- a/SPEC.md +++ b/SPEC.md @@ -444,7 +444,9 @@ Run at Docker build time — baked into the image, not runtime. ### §6.4 GCS Auth On Cloud Run, both GDAL `/vsigs/` and PyArrow's `gs://` filesystem use the service -account's ADC automatically. No credentials file needed. Require the pygeoapi Cloud Run +account's ADC. No credentials file needed. GDAL only uses the metadata server when it +detects GCE, which it can't on Cloud Run — `CPL_MACHINE_IS_GCE=YES` must be set or every +OGR collection fails with `No valid GCS credentials found`. PyArrow needs no setting. Require the pygeoapi Cloud Run Service account to have `roles/storage.objectViewer` on the products bucket (`gcs_bucket` in `products.yaml`). @@ -493,6 +495,7 @@ ENV PYGEOAPI_CONFIG=/pygeoapi/local.config.yml Cloud Run Service (`orchestration/pygeoapi/cloudbuild.yaml`): - `PYGEOAPI_SERVER_URL` — `https://die-pygeoapi-$PROJECT_NUMBER.$_REGION.run.app` +- `CPL_MACHINE_IS_GCE=YES` — lets GDAL `/vsigs/` use Cloud Run credentials (§6.4) - Port: 80 - Service account: `die-pygeoapi@$PROJECT_ID.iam.gserviceaccount.com` — needs `roles/storage.objectViewer` on the products bucket; Cloud Build's account needs diff --git a/orchestration/pygeoapi/cloudbuild.yaml b/orchestration/pygeoapi/cloudbuild.yaml index e92feb0b..b7bcb32f 100644 --- a/orchestration/pygeoapi/cloudbuild.yaml +++ b/orchestration/pygeoapi/cloudbuild.yaml @@ -29,7 +29,9 @@ steps: - '--port=80' - '--service-account=die-pygeoapi@$PROJECT_ID.iam.gserviceaccount.com' # needs storage.objectViewer on the products bucket - '--memory=2Gi' # Parquet provider peaked at 1.2-2.0 GB across 4 workers locally - - '--set-env-vars=PYGEOAPI_SERVER_URL=https://die-pygeoapi-$PROJECT_NUMBER.$_REGION.run.app' + # CPL_MACHINE_IS_GCE: GDAL can't detect Cloud Run as GCE, so /vsigs/ skips the + # metadata-server credentials and every GeoJSON collection fails without it. + - '--set-env-vars=PYGEOAPI_SERVER_URL=https://die-pygeoapi-$PROJECT_NUMBER.$_REGION.run.app,CPL_MACHINE_IS_GCE=YES' substitutions: _REGION: us-central1 From 507ea9d4fbc07a1d2020b1af0b46f7fef2a09359 Mon Sep 17 00:00:00 2001 From: Chase Martin Date: Thu, 24 Sep 2026 14:46:24 -0700 Subject: [PATCH 7/7] Update pygeoapi landing-page metadata Use a descriptive summary, point identification.url at the deployed service via PYGEOAPI_SERVER_URL, and add the data services contact email. Co-Authored-By: Claude Opus 5.5 (1M context) --- orchestration/pygeoapi/config.yml.j2 | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/orchestration/pygeoapi/config.yml.j2 b/orchestration/pygeoapi/config.yml.j2 index a2f6a503..ea00c1a9 100644 --- a/orchestration/pygeoapi/config.yml.j2 +++ b/orchestration/pygeoapi/config.yml.j2 @@ -20,8 +20,12 @@ logging: metadata: identification: - title: NM Unified Water Data - description: OGC API - Features for New Mexico water data + title: New Mexico Unified Water Data + description: >- + This tool integrates groundwater-level and water-quality data for New Mexico + from state and federal sources, including NMBGMR, USGS, and the Water Quality Portal. + Its collections provide per-well summaries, trends, time series, and derived + water-quality indicators through the OGC API - Features standard. keywords: - water - groundwater @@ -29,7 +33,7 @@ metadata: - NMBGMR keywords_type: theme terms_of_service: https://creativecommons.org/licenses/by/4.0/ - url: https://waterdata.nmt.edu + url: ${PYGEOAPI_SERVER_URL} # the live Cloud Run URL, set in cloudbuild.yaml license: name: CC-BY 4.0 url: https://creativecommons.org/licenses/by/4.0/ @@ -43,6 +47,7 @@ metadata: stateorprovince: New Mexico postalcode: "87801" country: USA + email: data-services-nmbg@nmt.edu resources: {% for product in products %}