From b63b0b26d035d6b136c5127fd0ccd3b5f1344125 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Thu, 16 Jul 2026 23:38:59 -0700 Subject: [PATCH 1/5] feat(types): add gpu_attest_proxy_url to sys-config Plumb an optional NVIDIA GPU attestation collateral proxy URL from the VMM config through sys-config, following the existing pccs_url pattern. Empty (the default) keeps guests talking to NVIDIA's endpoints directly. --- dstack/dstack-types/src/lib.rs | 6 ++++++ dstack/vmm/src/app.rs | 1 + dstack/vmm/src/config.rs | 5 +++++ dstack/vmm/vmm.toml | 5 +++++ 4 files changed, 17 insertions(+) diff --git a/dstack/dstack-types/src/lib.rs b/dstack/dstack-types/src/lib.rs index 87893679a..3ac49af31 100644 --- a/dstack/dstack-types/src/lib.rs +++ b/dstack/dstack-types/src/lib.rs @@ -710,6 +710,12 @@ pub struct SysConfig { #[serde(default, alias = "tproxy_urls")] pub gateway_urls: Vec, pub pccs_url: Option, + /// Base URL of an NVIDIA GPU attestation collateral proxy (OCSP + RIM + /// cache, e.g. gpu-attest-proxy). When set, the guest routes nvattest's + /// OCSP and RIM requests through it instead of NVIDIA's endpoints and + /// relaxes the OCSP nonce check via a packaged relying-party policy. + #[serde(default)] + pub gpu_attest_proxy_url: Option, pub docker_registry: Option, pub host_api_url: Option, /// MrConfigV3 document string for platform app/config binding. diff --git a/dstack/vmm/src/app.rs b/dstack/vmm/src/app.rs index f4dcd0d12..b6960c13f 100644 --- a/dstack/vmm/src/app.rs +++ b/dstack/vmm/src/app.rs @@ -1368,6 +1368,7 @@ pub(crate) fn make_sys_config( "kms_urls": kms_urls, "gateway_urls": gateway_urls, "pccs_url": cfg.cvm.pccs_url, + "gpu_attest_proxy_url": cfg.cvm.gpu_attest_proxy_url, "docker_registry": cfg.cvm.docker_registry, "host_api_url": format!("vsock://2:{}/api", cfg.host_api.port), "vm_config": serde_json::to_string(&vm_config)?, diff --git a/dstack/vmm/src/config.rs b/dstack/vmm/src/config.rs index 330a2e440..8c58ffe32 100644 --- a/dstack/vmm/src/config.rs +++ b/dstack/vmm/src/config.rs @@ -260,6 +260,11 @@ pub struct CvmConfig { /// The URL of the PCCS server #[serde(default)] pub pccs_url: String, + /// Base URL of an NVIDIA GPU attestation collateral proxy (OCSP + RIM + /// cache). Passed to guests via sys-config; empty disables the proxy and + /// guests talk to NVIDIA's endpoints directly. + #[serde(default)] + pub gpu_attest_proxy_url: Option, /// The URL of the Docker registry pub docker_registry: String, /// The start of the CID pool that allocates CIDs to VMs diff --git a/dstack/vmm/vmm.toml b/dstack/vmm/vmm.toml index 42ac81079..7c9a62a17 100644 --- a/dstack/vmm/vmm.toml +++ b/dstack/vmm/vmm.toml @@ -28,6 +28,11 @@ kms_urls = ["http://127.0.0.1:8081"] gateway_urls = ["http://127.0.0.1:8082"] # PCCS URL used by guest to verify the quote from local key provider pccs_url = "" +# Base URL of an NVIDIA GPU attestation collateral proxy (OCSP + RIM cache, +# e.g. gpu-attest-proxy). Guests route nvattest's OCSP/RIM requests through it +# and relax the OCSP nonce check via a packaged relying-party policy. Empty +# means guests talk to NVIDIA's endpoints directly. +#gpu_attest_proxy_url = "http://10.0.2.1:8090" docker_registry = "" cid_start = 1000 cid_pool_size = 1000 From 74241ee6acbc72aeb516c6c8bd215e5456a49026 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Thu, 16 Jul 2026 23:38:59 -0700 Subject: [PATCH 2/5] feat(nvattest): package allow_trust_outpost_ocsp policy in the image NVIDIA's sample relying-party policy keeps every built-in appraisal check except the OCSP nonce match. dstack-util passes it to nvattest when a caching collateral proxy is configured, because a cached OCSP response can never echo the per-request nonce. --- .../recipes-graphics/nvattest/nvattest_2026.06.09.bb | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/os/yocto/layers/meta-nvidia/recipes-graphics/nvattest/nvattest_2026.06.09.bb b/os/yocto/layers/meta-nvidia/recipes-graphics/nvattest/nvattest_2026.06.09.bb index 3494a8809..fa6967138 100644 --- a/os/yocto/layers/meta-nvidia/recipes-graphics/nvattest/nvattest_2026.06.09.bb +++ b/os/yocto/layers/meta-nvidia/recipes-graphics/nvattest/nvattest_2026.06.09.bb @@ -100,10 +100,19 @@ do_install() { install -d ${D}${systemd_system_unitdir}/dstack-prepare.service.d install -m 0644 ${UNPACKDIR}/10-nvidia-gpu-ordering.conf \ ${D}${systemd_system_unitdir}/dstack-prepare.service.d/10-nvidia-gpu-ordering.conf + + # NVIDIA's sample relying-party policy that keeps every default check + # except the OCSP nonce match. dstack-util passes it to nvattest when a + # caching OCSP/RIM proxy (gpu_attest_proxy_url in sys-config) is in use, + # because a cached response can never echo the request nonce. + install -d ${D}${datadir}/nvattest/policies + install -m 0644 ${S}/relying_party_policy_examples/allow_trust_outpost_ocsp.rego \ + ${D}${datadir}/nvattest/policies/allow_trust_outpost_ocsp.rego } FILES:${PN} += " \ ${systemd_system_unitdir}/dstack-prepare.service.d/10-nvidia-gpu-ordering.conf \ + ${datadir}/nvattest/policies/allow_trust_outpost_ocsp.rego \ ${libdir}/lib*.so \ ${libdir}/lib*.so.* \ " From e5952750c901c05d3598df6e824ac751b7749b7e Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Thu, 16 Jul 2026 23:38:59 -0700 Subject: [PATCH 3/5] feat(guest): route GPU attestation collateral through optional proxy When sys-config carries gpu_attest_proxy_url, run nvattest with --ocsp-url/--rim-url pointing at the proxy and --relying-party-policy allow_trust_outpost_ocsp.rego so cached OCSP responses (which cannot match the request nonce) are still appraised for signature, validity window and measurements. Fail closed if the policy file is missing. --- dstack/dstack-util/src/system_setup.rs | 118 ++++++++++++++++++++----- 1 file changed, 96 insertions(+), 22 deletions(-) diff --git a/dstack/dstack-util/src/system_setup.rs b/dstack/dstack-util/src/system_setup.rs index ae02dc5db..8d94140dd 100644 --- a/dstack/dstack-util/src/system_setup.rs +++ b/dstack/dstack-util/src/system_setup.rs @@ -1083,6 +1083,10 @@ mod gpu { const POLICY_ENTRYPOINT: &str = "data.policy.nv_match"; /// Bound Rego evaluation so a runaway application policy cannot hang boot. const POLICY_TIMEOUT: Duration = Duration::from_secs(10); + /// NVIDIA's sample relying-party policy packaged by the nvattest recipe. + /// It keeps every built-in appraisal check except the OCSP nonce match, + /// which a caching collateral proxy can never satisfy. + const OUTPOST_POLICY: &str = "/usr/share/nvattest/policies/allow_trust_outpost_ocsp.rego"; #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(super) struct GpuInventory { @@ -1368,11 +1372,70 @@ mod gpu { serde_json::to_vec(&event).context("failed to serialize GPU attestation event") } - /// Run local GPU attestation via nvattest with a fresh nonce and no custom - /// relying-party policy. nvattest's built-in appraisal still applies. The - /// complete JSON output is preserved under /run and retained with its - /// claims for the application policy and versioned RTMR3 summary. - pub(super) async fn attest_gpu(expected_devices: u32) -> Result { + /// Build the nvattest command line. When `proxy_url` is set, OCSP and RIM + /// requests are routed through the caching collateral proxy and the + /// packaged outpost policy replaces nvattest's built-in appraisal (a + /// cached OCSP response can never echo the per-request nonce). The policy + /// file must exist — it is packaged by the nvattest recipe; fail closed + /// with a clear error rather than silently skipping appraisal. + fn nvattest_args(nonce: &str, proxy_url: Option<&str>) -> Result> { + let proxy_base = match proxy_url { + Some(url) => { + let base = url.trim_end_matches('/'); + if base.is_empty() { + bail!("sys-config gpu_attest_proxy_url is empty"); + } + Some(base) + } + None => None, + }; + if proxy_base.is_some() && !Path::new(OUTPOST_POLICY).exists() { + bail!( + "sys-config gpu_attest_proxy_url is set but {} is not packaged in this image", + OUTPOST_POLICY + ); + } + Ok(nvattest_cmdline(nonce, proxy_base)) + } + + fn nvattest_cmdline(nonce: &str, proxy_base: Option<&str>) -> Vec { + let mut args = vec![ + "attest".to_string(), + "--device".to_string(), + "gpu".to_string(), + "--verifier".to_string(), + "local".to_string(), + "--nonce".to_string(), + nonce.to_string(), + "--format".to_string(), + "json".to_string(), + ]; + if let Some(base) = proxy_base.map(|url| url.trim_end_matches('/')) { + args.extend([ + "--ocsp-url".to_string(), + format!("{base}/ocsp"), + "--rim-url".to_string(), + base.to_string(), + "--relying-party-policy".to_string(), + OUTPOST_POLICY.to_string(), + ]); + } + args + } + + /// Run local GPU attestation via nvattest with a fresh nonce. Without a + /// collateral proxy no custom relying-party policy is given, so nvattest's + /// built-in appraisal applies. With `proxy_url` (sys-config + /// `gpu_attest_proxy_url`), OCSP and RIM requests go through the caching + /// proxy and the packaged outpost policy replaces the built-in one: a + /// cached OCSP response can never echo the request nonce, while signature, + /// validity-window and measurement checks are unchanged. The complete JSON + /// output is preserved under /run and retained with its claims for the + /// application policy and versioned RTMR3 summary. + pub(super) async fn attest_gpu( + expected_devices: u32, + proxy_url: Option<&str>, + ) -> Result { if !Path::new(NVATTEST).exists() { bail!("nvattest is not available in this image"); } @@ -1382,22 +1445,15 @@ mod gpu { warn!("failed to step system clock: {err:?}"); } let nonce = hex::encode(rand::thread_rng().gen::<[u8; 32]>()); - let output = run_command( - NVATTEST, - &[ - "attest", - "--device", - "gpu", - "--verifier", - "local", - "--nonce", - &nonce, - "--format", - "json", - ], - ATTESTATION_TIMEOUT, - ) - .await?; + let args = nvattest_args(&nonce, proxy_url)?; + if let Some(base) = proxy_url { + info!( + "routing GPU attestation collateral through proxy: {}", + base.trim_end_matches('/') + ); + } + let arg_refs = args.iter().map(String::as_str).collect::>(); + let output = run_command(NVATTEST, &arg_refs, ATTESTATION_TIMEOUT).await?; if !output.stderr.is_empty() { info!("nvattest: {}", truncated_lossy(&output.stderr, 2048)); } @@ -1556,6 +1612,20 @@ mod gpu { assert_eq!(nvidia_gpu_count(nvidia).unwrap(), 2); } + #[test] + fn nvattest_cmdline_routes_collateral_through_proxy() { + let args = nvattest_cmdline("aa", Some("http://10.0.2.1:8090/")); + let joined = args.join(" "); + assert!(joined.contains("--ocsp-url http://10.0.2.1:8090/ocsp")); + assert!(joined.contains("--rim-url http://10.0.2.1:8090")); + assert!(joined.contains(&format!("--relying-party-policy {OUTPOST_POLICY}"))); + + let direct = nvattest_cmdline("aa", None).join(" "); + assert!(!direct.contains("--ocsp-url")); + assert!(!direct.contains("--rim-url")); + assert!(!direct.contains("--relying-party-policy")); + } + #[test] fn basic_policy_requires_cc_and_rejects_devtools_by_default() { let nonce = "44".repeat(32); @@ -1874,7 +1944,11 @@ impl Stage0<'_> { } self.vmm.notify_q("boot.progress", "attesting GPU").await; info!("verifying GPU TEE attestation"); - let attestation = gpu::attest_gpu(expected_devices).await?; + let attestation = gpu::attest_gpu( + expected_devices, + self.shared.sys_config.gpu_attest_proxy_url.as_deref(), + ) + .await?; let gpu_state = gpu::query_gpu_state(expected_devices)?; attestation From 9d3551db95465a7787f0d00d3204d19235dc8617 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Thu, 16 Jul 2026 23:38:59 -0700 Subject: [PATCH 4/5] feat(gpu-attest-proxy): add caching OCSP/RIM collateral proxy A PCCS-style cache for NVIDIA GPU attestation collateral, run on the host or at site level: - POST /ocsp relays requests byte-for-byte and caches successful responses keyed by CertID until their nextUpdate, so reboots and fleets stop depending on the responder at every boot. - GET /v1/rim/{id} caches RIM documents with a TTL and serves stale within a bounded window when the upstream is unreachable. - A background refresher renews entries before expiry, letting a warm cache ride through upstream outages with close to a full validity window. - The proxy holds no secrets: guests still verify OCSP signatures, validity windows and RIM signatures themselves. The minimal DER walker extracts CertIDs from requests (cache key) and thisUpdate/nextUpdate from responses (expiry). Includes unit tests and a documented default configuration. --- dstack/Cargo.lock | 25 ++ dstack/Cargo.toml | 1 + dstack/gpu-attest-proxy/Cargo.toml | 33 ++ dstack/gpu-attest-proxy/gpu-attest-proxy.toml | 25 ++ dstack/gpu-attest-proxy/src/cache.rs | 221 +++++++++++ dstack/gpu-attest-proxy/src/der.rs | 364 ++++++++++++++++++ dstack/gpu-attest-proxy/src/lib.rs | 7 + dstack/gpu-attest-proxy/src/main.rs | 94 +++++ dstack/gpu-attest-proxy/src/proxy.rs | 351 +++++++++++++++++ 9 files changed, 1121 insertions(+) create mode 100644 dstack/gpu-attest-proxy/Cargo.toml create mode 100644 dstack/gpu-attest-proxy/gpu-attest-proxy.toml create mode 100644 dstack/gpu-attest-proxy/src/cache.rs create mode 100644 dstack/gpu-attest-proxy/src/der.rs create mode 100644 dstack/gpu-attest-proxy/src/lib.rs create mode 100644 dstack/gpu-attest-proxy/src/main.rs create mode 100644 dstack/gpu-attest-proxy/src/proxy.rs diff --git a/dstack/Cargo.lock b/dstack/Cargo.lock index 4cb0b55f1..967860d76 100644 --- a/dstack/Cargo.lock +++ b/dstack/Cargo.lock @@ -2831,6 +2831,31 @@ dependencies = [ "regex-syntax", ] +[[package]] +name = "gpu-attest-proxy" +version = "0.6.0" +dependencies = [ + "anyhow", + "base64 0.22.1", + "clap", + "dashmap", + "fs-err", + "git-version", + "hex", + "load_config", + "or-panic", + "reqwest", + "rocket", + "serde", + "serde-duration", + "serde_json", + "sha2 0.10.9", + "tempfile", + "tokio", + "tracing", + "tracing-subscriber", +] + [[package]] name = "group" version = "0.13.0" diff --git a/dstack/Cargo.toml b/dstack/Cargo.toml index 265eae995..100516140 100644 --- a/dstack/Cargo.toml +++ b/dstack/Cargo.toml @@ -59,6 +59,7 @@ members = [ "dstack-mr", "dstack-mr/cli", "verifier", + "gpu-attest-proxy", "size-parser", "port-forward", "crates/dstack-cli-core", diff --git a/dstack/gpu-attest-proxy/Cargo.toml b/dstack/gpu-attest-proxy/Cargo.toml new file mode 100644 index 000000000..2a0880cc0 --- /dev/null +++ b/dstack/gpu-attest-proxy/Cargo.toml @@ -0,0 +1,33 @@ +# SPDX-FileCopyrightText: © 2026 Phala Network +# +# SPDX-License-Identifier: Apache-2.0 + +[package] +name = "gpu-attest-proxy" +version.workspace = true +authors.workspace = true +edition.workspace = true +license.workspace = true + +[dependencies] +anyhow.workspace = true +base64.workspace = true +clap = { workspace = true, features = ["derive", "string"] } +dashmap.workspace = true +fs-err.workspace = true +git-version.workspace = true +hex = { workspace = true, features = ["alloc"] } +load_config.workspace = true +or-panic.workspace = true +reqwest.workspace = true +rocket = { workspace = true, features = ["json"] } +serde = { workspace = true, features = ["derive"] } +serde-duration.workspace = true +serde_json.workspace = true +sha2.workspace = true +tokio = { workspace = true, features = ["macros", "rt-multi-thread", "time"] } +tracing.workspace = true +tracing-subscriber.workspace = true + +[dev-dependencies] +tempfile.workspace = true diff --git a/dstack/gpu-attest-proxy/gpu-attest-proxy.toml b/dstack/gpu-attest-proxy/gpu-attest-proxy.toml new file mode 100644 index 000000000..d5c1f46f3 --- /dev/null +++ b/dstack/gpu-attest-proxy/gpu-attest-proxy.toml @@ -0,0 +1,25 @@ +# SPDX-FileCopyrightText: © 2026 Phala Network +# +# SPDX-License-Identifier: Apache-2.0 + +address = "0.0.0.0" +port = 8090 +log_level = "info" + +# NVIDIA attestation collateral upstreams. +upstream_ocsp_url = "https://ocsp.ndis.nvidia.com" +upstream_rim_url = "https://rim.attestation.nvidia.com" + +# Persistent cache location: one JSON file per entry under ocsp/ and rim/. +cache_dir = "/var/lib/dstack/gpu-attest-proxy" + +# RIM documents are re-fetched after rim_ttl; when the upstream is +# unreachable a stale document keeps being served for up to rim_max_stale. +rim_ttl = "24h" +rim_max_stale = "7d" + +# OCSP responses are cached until their nextUpdate, bounded by ocsp_max_ttl; +# entries within refresh_margin of expiry are refreshed in the background so +# a warm cache rides through upstream outages. +ocsp_max_ttl = "7d" +refresh_margin = "24h" diff --git a/dstack/gpu-attest-proxy/src/cache.rs b/dstack/gpu-attest-proxy/src/cache.rs new file mode 100644 index 000000000..f0c246c48 --- /dev/null +++ b/dstack/gpu-attest-proxy/src/cache.rs @@ -0,0 +1,221 @@ +// SPDX-FileCopyrightText: © 2026 Phala Network +// +// SPDX-License-Identifier: Apache-2.0 + +//! File-backed collateral cache. Every entry is one JSON file under +//! `//`, so a proxy restart keeps the cache warm and +//! operators can inspect or evict entries with plain file tools. + +use std::path::{Path, PathBuf}; +use std::time::{SystemTime, UNIX_EPOCH}; + +use anyhow::{Context, Result}; +use dashmap::DashMap; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct CacheEntry { + /// "ocsp" or "rim" — also the subdirectory the entry persists under. + pub kind: String, + /// Cache key: hex(CertID DER) for OCSP, the RIM id for RIM. + pub key: String, + pub content_type: String, + #[serde(with = "base64serde")] + pub body: Vec, + /// Original upstream request body for OCSP entries, letting the + /// background refresher re-run the exact query. RIM refreshes by key. + #[serde(default, with = "base64serde_opt")] + pub refresh_body: Option>, + pub fetched_at: i64, + pub expires_at: i64, +} + +impl CacheEntry { + pub fn is_fresh(&self, now: i64) -> bool { + now < self.expires_at + } + + pub fn is_usable_stale(&self, now: i64, max_stale_secs: u64) -> bool { + now < self.expires_at.saturating_add(max_stale_secs as i64) + } +} + +mod base64serde { + use base64::{engine::general_purpose::STANDARD, Engine as _}; + use serde::{Deserialize, Deserializer, Serializer}; + + pub fn serialize(bytes: &Vec, s: S) -> Result { + s.serialize_str(&STANDARD.encode(bytes)) + } + + pub fn deserialize<'de, D: Deserializer<'de>>(d: D) -> Result, D::Error> { + let text = String::deserialize(d)?; + STANDARD.decode(text).map_err(serde::de::Error::custom) + } +} + +mod base64serde_opt { + use base64::{engine::general_purpose::STANDARD, Engine as _}; + use serde::{Deserialize, Deserializer, Serializer}; + + pub fn serialize(bytes: &Option>, s: S) -> Result { + match bytes { + Some(bytes) => s.serialize_str(&STANDARD.encode(bytes)), + None => s.serialize_none(), + } + } + + pub fn deserialize<'de, D: Deserializer<'de>>(d: D) -> Result>, D::Error> { + let text = Option::::deserialize(d)?; + text.map(|text| STANDARD.decode(text).map_err(serde::de::Error::custom)) + .transpose() + } +} + +pub struct Cache { + dir: PathBuf, + entries: DashMap, +} + +fn now() -> i64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_secs() as i64) + .unwrap_or(0) +} + +impl Cache { + /// Load every well-formed entry from disk; entries past `now + + /// keep_stale_secs` are dropped instead. + pub fn load(dir: impl Into, keep_stale_secs: u64) -> Result { + let dir = dir.into(); + let cache = Self { + dir, + entries: DashMap::new(), + }; + let now = now(); + for kind in ["ocsp", "rim"] { + let kind_dir = cache.dir.join(kind); + let Ok(read_dir) = fs_err::read_dir(&kind_dir) else { + continue; + }; + for file in read_dir.flatten() { + let path = file.path(); + if path.extension().and_then(|e| e.to_str()) != Some("json") { + continue; + } + match Self::read_entry(&path) { + Ok(entry) if entry.is_usable_stale(now, keep_stale_secs) => { + cache.entries.insert(map_key(kind, &entry.key), entry); + } + Ok(_) => { + fs_err::remove_file(&path).ok(); + } + Err(err) => { + tracing::warn!( + "dropping unreadable cache entry {}: {err:#}", + path.display() + ); + } + } + } + } + Ok(cache) + } + + fn read_entry(path: &Path) -> Result { + let json = fs_err::read(path)?; + serde_json::from_slice(&json).context("invalid cache entry JSON") + } + + fn entry_path(&self, kind: &str, key: &str) -> PathBuf { + let name = hex::encode(Sha256::digest(key.as_bytes())); + self.dir.join(kind).join(format!("{name}.json")) + } + + pub fn get(&self, kind: &str, key: &str) -> Option { + self.entries.get(&map_key(kind, key)).map(|e| e.clone()) + } + + /// Insert or replace an entry, persisting it. The entry stays in memory + /// even if the write fails — availability of the proxy matters more than + /// durability of one entry. + pub fn put(&self, entry: CacheEntry) { + let path = self.entry_path(&entry.kind, &entry.key); + if let Some(parent) = path.parent() { + fs_err::create_dir_all(parent).ok(); + } + match serde_json::to_vec(&entry) { + Ok(json) => { + if let Err(err) = fs_err::write(&path, json) { + tracing::warn!("failed to persist cache entry {}: {err:#}", path.display()); + } + } + Err(err) => tracing::warn!("failed to serialize cache entry: {err:#}"), + } + self.entries.insert(map_key(&entry.kind, &entry.key), entry); + } + + /// Entries whose freshness ends within `margin_secs`, for the background + /// refresher. + pub fn expiring_soon(&self, margin_secs: u64) -> Vec { + let deadline = now().saturating_add(margin_secs as i64); + self.entries + .iter() + .filter(|entry| entry.expires_at < deadline) + .map(|entry| entry.clone()) + .collect() + } + + pub fn stats(&self) -> (usize, usize) { + let now = now(); + let fresh = self + .entries + .iter() + .filter(|entry| entry.is_fresh(now)) + .count(); + (fresh, self.entries.len().saturating_sub(fresh)) + } +} + +fn map_key(kind: &str, key: &str) -> String { + format!("{kind}:{key}") +} + +#[cfg(test)] +mod tests { + use super::*; + + fn entry(kind: &str, key: &str, expires_at: i64) -> CacheEntry { + CacheEntry { + kind: kind.into(), + key: key.into(), + content_type: "application/ocsp-response".into(), + body: b"body".to_vec(), + refresh_body: Some(b"request".to_vec()), + fetched_at: now(), + expires_at, + } + } + + #[test] + fn entries_survive_a_reload() { + let dir = tempfile::tempdir().unwrap(); + let cache = Cache::load(dir.path(), 0).unwrap(); + cache.put(entry("ocsp", "aabb", now() + 3600)); + let reloaded = Cache::load(dir.path(), 0).unwrap(); + let entry = reloaded.get("ocsp", "aabb").unwrap(); + assert_eq!(entry.body, b"body"); + assert_eq!(entry.refresh_body, Some(b"request".to_vec())); + } + + #[test] + fn load_drops_entries_past_the_stale_window() { + let dir = tempfile::tempdir().unwrap(); + let cache = Cache::load(dir.path(), 0).unwrap(); + cache.put(entry("rim", "RIM_OLD", now() - 7200)); + let reloaded = Cache::load(dir.path(), 3600).unwrap(); + assert!(reloaded.get("rim", "RIM_OLD").is_none()); + } +} diff --git a/dstack/gpu-attest-proxy/src/der.rs b/dstack/gpu-attest-proxy/src/der.rs new file mode 100644 index 000000000..9f8d46226 --- /dev/null +++ b/dstack/gpu-attest-proxy/src/der.rs @@ -0,0 +1,364 @@ +// SPDX-FileCopyrightText: © 2026 Phala Network +// +// SPDX-License-Identifier: Apache-2.0 + +//! Minimal DER walking for the two OCSP structures the proxy needs: +//! extracting CertIDs from a request (cache key) and the +//! thisUpdate/nextUpdate window from a response (cache expiry). Only +//! low-tag-number, definite-length DER is accepted, which covers everything +//! an OCSP responder is allowed to send. + +use anyhow::{bail, Context, Result}; + +pub const TAG_ENUMERATED: u8 = 0x0a; +pub const TAG_GENERALIZED_TIME: u8 = 0x18; +pub const TAG_SEQUENCE: u8 = 0x30; +pub const TAG_OCTET_STRING: u8 = 0x04; + +/// One parsed TLV element. `raw` covers the full TLV, `content` only the V. +#[derive(Debug, Clone, Copy)] +pub struct Tlv<'a> { + pub tag: u8, + pub content: &'a [u8], + pub raw: &'a [u8], +} + +impl<'a> Tlv<'a> { + pub fn children(&self) -> Result>> { + read_all(self.content) + } +} + +/// Read one TLV element, returning it and the remaining bytes. +pub fn read_tlv(buf: &[u8]) -> Result<(Tlv<'_>, &[u8])> { + let (&tag, rest) = buf.split_first().context("truncated DER: missing tag")?; + if tag & 0x1f == 0x1f { + bail!("multi-byte DER tags are not supported"); + } + let (&len0, mut rest) = rest + .split_first() + .context("truncated DER: missing length")?; + let len = if len0 & 0x80 == 0 { + len0 as usize + } else { + let n = (len0 & 0x7f) as usize; + if n == 0 || n > 4 || rest.len() < n { + bail!("invalid DER long-form length"); + } + let mut len = 0usize; + for &b in &rest[..n] { + len = len + .checked_mul(256) + .and_then(|l| l.checked_add(b as usize)) + .context("DER length overflow")?; + } + rest = &rest[n..]; + len + }; + if rest.len() < len { + bail!("truncated DER: content shorter than declared length"); + } + let header_len = buf.len() - rest.len(); + let tlv = Tlv { + tag, + content: &rest[..len], + raw: &buf[..header_len + len], + }; + Ok((tlv, &rest[len..])) +} + +fn read_all(mut buf: &[u8]) -> Result>> { + let mut out = Vec::new(); + while !buf.is_empty() { + let (tlv, rest) = read_tlv(buf)?; + out.push(tlv); + buf = rest; + } + Ok(out) +} + +fn expect_tag(tlv: &Tlv, tag: u8, what: &str) -> Result<()> { + if tlv.tag != tag { + bail!( + "expected {what} (tag 0x{tag:02x}), got tag 0x{:02x}", + tlv.tag + ); + } + Ok(()) +} + +/// Extract the raw CertID TLVs from a DER OCSP request. They fully determine +/// the revocation query (issuer name/key hashes + serial), so they are the +/// cache key; the per-request nonce in requestExtensions is ignored. +pub fn ocsp_request_cert_ids(request_der: &[u8]) -> Result>> { + let (outer, rest) = read_tlv(request_der).context("invalid OCSP request")?; + if !rest.is_empty() { + bail!("trailing bytes after OCSP request"); + } + expect_tag(&outer, TAG_SEQUENCE, "OCSPRequest")?; + let tbs = outer + .children()? + .into_iter() + .next() + .context("OCSPRequest has no tbsRequest")?; + expect_tag(&tbs, TAG_SEQUENCE, "tbsRequest")?; + // Skip optional [0] version and [1] requestorName; requestList is the + // first SEQUENCE child. + let request_list = tbs + .children()? + .into_iter() + .find(|child| child.tag == TAG_SEQUENCE) + .context("tbsRequest has no requestList")?; + let mut cert_ids = Vec::new(); + for request in request_list.children()? { + expect_tag(&request, TAG_SEQUENCE, "Request")?; + let cert_id = request + .children()? + .into_iter() + .next() + .context("Request has no reqCert")?; + expect_tag(&cert_id, TAG_SEQUENCE, "reqCert")?; + cert_ids.push(cert_id.raw.to_vec()); + } + if cert_ids.is_empty() { + bail!("OCSP request carries no CertID"); + } + Ok(cert_ids) +} + +/// The validity window (thisUpdate, nextUpdate) the response asserts for +/// `cert_id`. Returns Ok(None) when the responder signalled a non-successful +/// status or the response cannot be parsed — callers must then skip caching +/// but still relay the body. +pub fn ocsp_response_validity( + response_der: &[u8], + cert_id: &[u8], +) -> Result)>> { + match response_validity_inner(response_der, cert_id) { + Ok(window) => Ok(Some(window)), + Err(err) => { + tracing::debug!("not caching unparseable OCSP response: {err:#}"); + Ok(None) + } + } +} + +fn response_validity_inner(response_der: &[u8], cert_id: &[u8]) -> Result<(i64, Option)> { + let (outer, rest) = read_tlv(response_der).context("invalid OCSP response")?; + if !rest.is_empty() { + bail!("trailing bytes after OCSP response"); + } + expect_tag(&outer, TAG_SEQUENCE, "OCSPResponse")?; + let mut parts = outer.children()?.into_iter(); + let status = parts.next().context("OCSPResponse has no status")?; + expect_tag(&status, TAG_ENUMERATED, "responseStatus")?; + if status.content != [0] { + bail!("OCSP responder status is not successful"); + } + let response_bytes = parts.next().context("OCSPResponse has no responseBytes")?; + expect_tag(&response_bytes, 0xa0, "responseBytes")?; + let rb = response_bytes + .children()? + .into_iter() + .next() + .context("empty responseBytes")?; + expect_tag(&rb, TAG_SEQUENCE, "ResponseBytes")?; + let mut rb_parts = rb.children()?.into_iter(); + rb_parts.next(); // responseType OID + let basic = rb_parts.next().context("ResponseBytes has no response")?; + expect_tag(&basic, TAG_OCTET_STRING, "BasicOCSPResponse wrapper")?; + + let (basic, _) = read_tlv(basic.content).context("invalid BasicOCSPResponse")?; + expect_tag(&basic, TAG_SEQUENCE, "BasicOCSPResponse")?; + let tbs = basic + .children()? + .into_iter() + .next() + .context("BasicOCSPResponse has no tbsResponseData")?; + expect_tag(&tbs, TAG_SEQUENCE, "tbsResponseData")?; + // Skip optional [0] version, responderID ([1]/[2]) and producedAt; + // responses is the first SEQUENCE child. + let responses = tbs + .children()? + .into_iter() + .find(|child| child.tag == TAG_SEQUENCE) + .context("tbsResponseData has no responses")?; + for single in responses.children()? { + expect_tag(&single, TAG_SEQUENCE, "SingleResponse")?; + let fields = single.children()?; + let Some(id) = fields.first() else { continue }; + expect_tag(id, TAG_SEQUENCE, "certID")?; + if id.raw != cert_id { + continue; + } + let this_update = fields + .iter() + .find(|field| field.tag == TAG_GENERALIZED_TIME) + .context("SingleResponse has no thisUpdate")?; + let next_update = fields + .iter() + .find(|field| field.tag == 0xa0) + .map(|ext| -> Result { + let inner = ext + .children()? + .into_iter() + .next() + .context("empty nextUpdate")?; + expect_tag(&inner, TAG_GENERALIZED_TIME, "nextUpdate")?; + parse_generalized_time(inner.content) + }) + .transpose()?; + return Ok((parse_generalized_time(this_update.content)?, next_update)); + } + bail!("response contains no SingleResponse for the requested CertID") +} + +/// Parse `YYYYMMDDHHMMSSZ` (GeneralizedTime as OCSP responders emit it). +pub fn parse_generalized_time(bytes: &[u8]) -> Result { + let text = std::str::from_utf8(bytes).context("GeneralizedTime is not ASCII")?; + let digits = text.strip_suffix('Z').unwrap_or(text); + if digits.len() != 14 || !digits.bytes().all(|b| b.is_ascii_digit()) { + bail!("unsupported GeneralizedTime format: {text}"); + } + let num = |range: std::ops::Range| digits[range].parse::(); + let (Ok(year), Ok(month), Ok(day), Ok(hour), Ok(min), Ok(sec)) = ( + num(0..4), + num(4..6), + num(6..8), + num(8..10), + num(10..12), + num(12..14), + ) else { + bail!("unparseable GeneralizedTime: {text}"); + }; + if !(1..=12).contains(&month) || !(1..=31).contains(&day) { + bail!("out-of-range GeneralizedTime: {text}"); + } + // days-from-civil (Howard Hinnant's algorithm), no leap seconds. + let y = if month <= 2 { year - 1 } else { year }; + let era = if y >= 0 { y } else { y - 399 } / 400; + let yoe = y - era * 400; + let mp = (month + 9) % 12; + let doy = (153 * mp + 2) / 5 + day - 1; + let doe = yoe * 365 + yoe / 4 - yoe / 100 + doy; + let days = era * 146097 + doe - 719468; + Ok(days * 86400 + hour * 3600 + min * 60 + sec) +} + +#[cfg(test)] +mod tests { + use super::*; + + // Build a tiny synthetic OCSP request/response pair with a hand-rolled + // DER writer; the structures mirror RFC 6960. + fn tlv(tag: u8, content: &[u8]) -> Vec { + let mut out = vec![tag]; + if content.len() < 0x80 { + out.push(content.len() as u8); + } else { + out.push(0x81); + out.push(content.len() as u8); + } + out.extend_from_slice(content); + out + } + + fn cert_id(serial: u8) -> Vec { + let mut content = tlv(0x30, &[]); // hashAlgorithm + content.extend(tlv(0x04, b"issuer-name-hash")); + content.extend(tlv(0x04, b"issuer-key-hash")); + content.extend(tlv(0x02, &[serial])); + tlv(0x30, &content) + } + + fn ocsp_request(cert_ids: &[Vec]) -> Vec { + let requests = cert_ids + .iter() + .map(|id| tlv(0x30, id)) + .flatten() + .collect::>(); + let mut tbs = tlv(0xa0, &tlv(0x02, &[1])); // version [0] + tbs.extend(tlv(0x30, &requests)); + tbs.extend(tlv(0xa2, &[])); // requestExtensions [2] (nonce would live here) + tlv(0x30, &tlv(0x30, &tbs)) + } + + fn single_response(id: &[u8], this: &[u8], next: Option<&[u8]>) -> Vec { + let mut fields = id.to_vec(); + fields.extend(tlv(0x80, &[])); // certStatus good + fields.extend(tlv(TAG_GENERALIZED_TIME, this)); + if let Some(next) = next { + fields.extend(tlv(0xa0, &tlv(TAG_GENERALIZED_TIME, next))); + } + tlv(0x30, &fields) + } + + fn ocsp_response(singles: &[Vec]) -> Vec { + let responses = singles.iter().flatten().copied().collect::>(); + let mut tbs_data = tlv(0xa1, &tlv(0x04, b"responder")); // responderID byName + tbs_data.extend(tlv(TAG_GENERALIZED_TIME, b"20260701000000Z")); + tbs_data.extend(tlv(0x30, &responses)); + let mut basic = tlv(0x30, &tbs_data); + basic.extend(tlv(0x30, &[])); // signatureAlgorithm + basic.extend(tlv(0x03, &[0])); // signature BIT STRING + let mut response_bytes = tlv( + 0x06, + &[0x2b, 0x06, 0x01, 0x05, 0x05, 0x07, 0x30, 0x01, 0x01], + ); + response_bytes.extend(tlv(0x04, &tlv(0x30, &basic))); + let mut body = tlv(TAG_ENUMERATED, &[0]); + body.extend(tlv(0xa0, &tlv(0x30, &response_bytes))); + tlv(0x30, &body) + } + + #[test] + fn extracts_cert_ids_from_request() { + let ids = vec![cert_id(7), cert_id(8)]; + let parsed = ocsp_request_cert_ids(&ocsp_request(&ids)).unwrap(); + assert_eq!(parsed, ids); + } + + #[test] + fn extracts_validity_window_for_matching_cert_id() { + let (wanted, other) = (cert_id(1), cert_id(2)); + let response = ocsp_response(&[ + single_response(&other, b"20260701000000Z", None), + single_response(&wanted, b"20260701120000Z", Some(b"20260708120000Z")), + ]); + let (this, next) = ocsp_response_validity(&response, &wanted).unwrap().unwrap(); + assert_eq!(this, 1782907200); // 2026-07-01 12:00:00Z + assert_eq!(next, Some(1783512000)); // 2026-07-08 12:00:00Z + } + + #[test] + fn absent_next_update_is_none_not_an_error() { + let id = cert_id(1); + let response = ocsp_response(&[single_response(&id, b"20260701120000Z", None)]); + let (_, next) = ocsp_response_validity(&response, &id).unwrap().unwrap(); + assert_eq!(next, None); + } + + #[test] + fn non_successful_responses_are_not_cached() { + let mut body = tlv(TAG_ENUMERATED, &[3]); // tryLater + let response = tlv(0x30, &body); + body.clear(); + assert_eq!( + ocsp_response_validity(&response, &cert_id(1)).unwrap(), + None + ); + } + + #[test] + fn generalized_time_conversion_matches_known_epoch() { + assert_eq!(parse_generalized_time(b"19700101000000Z").unwrap(), 0); + assert_eq!( + parse_generalized_time(b"20260701120000Z").unwrap(), + 1782907200 + ); + // Leap year boundary: 2024-03-01 is one day after 2024-02-29. + let feb29 = parse_generalized_time(b"20240229000000Z").unwrap(); + let mar01 = parse_generalized_time(b"20240301000000Z").unwrap(); + assert_eq!(mar01 - feb29, 86400); + } +} diff --git a/dstack/gpu-attest-proxy/src/lib.rs b/dstack/gpu-attest-proxy/src/lib.rs new file mode 100644 index 000000000..4a417bbeb --- /dev/null +++ b/dstack/gpu-attest-proxy/src/lib.rs @@ -0,0 +1,7 @@ +// SPDX-FileCopyrightText: © 2026 Phala Network +// +// SPDX-License-Identifier: Apache-2.0 + +pub mod cache; +pub mod der; +pub mod proxy; diff --git a/dstack/gpu-attest-proxy/src/main.rs b/dstack/gpu-attest-proxy/src/main.rs new file mode 100644 index 000000000..4a0dafd33 --- /dev/null +++ b/dstack/gpu-attest-proxy/src/main.rs @@ -0,0 +1,94 @@ +// SPDX-FileCopyrightText: © 2026 Phala Network +// +// SPDX-License-Identifier: Apache-2.0 + +use std::sync::Arc; +use std::time::Duration; + +use anyhow::{anyhow, Context, Result}; +use clap::Parser; +use gpu_attest_proxy::cache::Cache; +use gpu_attest_proxy::proxy::{refresh_loop, rocket, ProxyConfig, ProxyState}; +use load_config::load_config; +use rocket::figment::Figment; +use tracing::error; +use tracing_subscriber::EnvFilter; + +pub const DEFAULT_CONFIG: &str = include_str!("../gpu-attest-proxy.toml"); + +pub fn load_config_figment(config_file: Option<&str>) -> Figment { + load_config("gpu-attest-proxy", DEFAULT_CONFIG, config_file, false) +} + +fn app_version() -> String { + const CARGO_PKG_VERSION: &str = env!("CARGO_PKG_VERSION"); + const VERSION: &str = git_version::git_version!( + args = ["--abbrev=20", "--always", "--dirty=-modified"], + prefix = "git:", + fallback = "unknown" + ); + format!("v{CARGO_PKG_VERSION} ({VERSION})") +} + +#[derive(Parser)] +#[command(author, version, about, long_version = app_version())] +struct Args { + /// Path to the configuration file + #[arg(short, long)] + config: Option, + /// bind address + #[arg(short, long)] + address: Option, + /// bind port + #[arg(short, long)] + port: Option, +} + +fn main() -> Result<()> { + let args = Args::parse(); + tracing_subscriber::fmt() + .with_env_filter(EnvFilter::from_default_env()) + .init(); + if let Err(err) = async_main(args) { + error!("{err:?}"); + return Err(err); + } + Ok(()) +} + +#[tokio::main] +async fn async_main(args: Args) -> Result<()> { + let mut figment = load_config_figment(args.config.as_deref()); + if let Some(address) = args.address { + figment = figment.join(("address", address)); + } + if let Some(port) = args.port { + figment = figment.join(("port", port)); + } + let config: ProxyConfig = figment.extract().context("failed to parse config")?; + let http = reqwest::Client::builder() + .connect_timeout(Duration::from_secs(10)) + .timeout(Duration::from_secs(30)) + .build() + .context("failed to build HTTP client")?; + let cache = Cache::load(&config.cache_dir, config.rim_max_stale.as_secs()) + .context("failed to load collateral cache")?; + let state = Arc::new(ProxyState { + config, + cache, + http, + }); + tokio::spawn(refresh_loop(state.clone())); + let rocket = rocket(figment, state); + let ignite = rocket + .ignite() + .await + .map_err(|err| anyhow!("{err:?}")) + .context("failed to ignite rocket")?; + ignite + .launch() + .await + .map_err(|err| anyhow!("{err:?}")) + .context("failed to launch rocket")?; + Ok(()) +} diff --git a/dstack/gpu-attest-proxy/src/proxy.rs b/dstack/gpu-attest-proxy/src/proxy.rs new file mode 100644 index 000000000..10e92fe56 --- /dev/null +++ b/dstack/gpu-attest-proxy/src/proxy.rs @@ -0,0 +1,351 @@ +// SPDX-FileCopyrightText: © 2026 Phala Network +// +// SPDX-License-Identifier: Apache-2.0 + +//! HTTP surface: the two endpoints NVIDIA's attestation SDK talks to, plus +//! health/info endpoints for operators. + +use std::io::Cursor; +use std::sync::Arc; +use std::time::Duration; + +use anyhow::{Context, Result}; +use rocket::figment::Figment; +use rocket::http::{ContentType, Status}; +use rocket::response::{self, Responder, Response}; +use rocket::{get, post, routes, Build, Rocket, State}; +use serde::Deserialize; +use tracing::{debug, warn}; + +use crate::cache::{Cache, CacheEntry}; +use crate::der; + +/// How long before nominal expiry an entry stops being served and starts +/// being refreshed. Covers clock skew between proxy and guests plus the +/// guest-side validity check. +const SERVING_SKEW_SECS: i64 = 300; +/// Matches the SDK's fallback (NvHttpOcspClient::DEFAULT_NEXT_UPDATE_TTL_SECONDS) +/// when a response carries no nextUpdate. +const DEFAULT_OCSP_TTL_SECS: i64 = 3600; +const OCSP_REQUEST_TYPE: &str = "application/ocsp-request"; +const OCSP_RESPONSE_TYPE: &str = "application/ocsp-response"; + +#[derive(Debug, Clone, Deserialize)] +#[serde(default)] +pub struct ProxyConfig { + pub upstream_ocsp_url: String, + pub upstream_rim_url: String, + pub cache_dir: String, + /// RIM documents are re-fetched after this long. + #[serde(with = "serde_duration")] + pub rim_ttl: Duration, + /// A stale RIM document is served (when the upstream cannot provide a + /// fresh copy) for this long after `rim_ttl` elapsed. + #[serde(with = "serde_duration")] + pub rim_max_stale: Duration, + /// Upper bound on OCSP caching, applied on top of the response's own + /// nextUpdate. + #[serde(with = "serde_duration")] + pub ocsp_max_ttl: Duration, + /// Entries expiring within this window are refreshed in the background. + #[serde(with = "serde_duration")] + pub refresh_margin: Duration, +} + +impl Default for ProxyConfig { + fn default() -> Self { + Self { + upstream_ocsp_url: "https://ocsp.ndis.nvidia.com".into(), + upstream_rim_url: "https://rim.attestation.nvidia.com".into(), + cache_dir: "/var/lib/dstack/gpu-attest-proxy".into(), + rim_ttl: Duration::from_secs(24 * 3600), + rim_max_stale: Duration::from_secs(7 * 24 * 3600), + ocsp_max_ttl: Duration::from_secs(7 * 24 * 3600), + refresh_margin: Duration::from_secs(24 * 3600), + } + } +} + +pub struct ProxyState { + pub config: ProxyConfig, + pub cache: Cache, + pub http: reqwest::Client, +} + +struct RawResponse { + status: Status, + content_type: ContentType, + body: Vec, +} + +impl<'r> Responder<'r, 'static> for RawResponse { + fn respond_to(self, _request: &'r rocket::Request<'_>) -> response::Result<'static> { + Response::build() + .status(self.status) + .header(self.content_type) + .sized_body(self.body.len(), Cursor::new(self.body)) + .ok() + } +} + +fn plain(status: Status, text: &'static str) -> RawResponse { + RawResponse { + status, + content_type: ContentType::Plain, + body: text.as_bytes().to_vec(), + } +} + +fn unix_now() -> i64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_secs() as i64) + .unwrap_or(0) +} + +fn content_type_or(saved: &str, default: &str) -> ContentType { + saved + .parse::() + .unwrap_or_else(|_| default.parse().expect("default content types parse")) +} + +/// POST endpoint for OCSP requests. Successful responses are cached keyed by +/// CertID until their nextUpdate; everything else is relayed untouched. +#[post("/ocsp", data = "")] +async fn ocsp(state: &State>, body: Vec) -> RawResponse { + let cert_ids = match der::ocsp_request_cert_ids(&body) { + Ok(ids) => ids, + Err(err) => { + warn!("rejecting malformed OCSP request: {err:#}"); + return plain(Status::BadRequest, "malformed OCSP request"); + } + }; + // NVIDIA's SDK queries one certificate per request. + let key = hex::encode(&cert_ids[0]); + let now = unix_now(); + if let Some(entry) = state.cache.get("ocsp", &key) { + if entry.expires_at > now + SERVING_SKEW_SECS { + debug!("OCSP cache hit for {key}"); + return RawResponse { + status: Status::Ok, + content_type: content_type_or(&entry.content_type, OCSP_RESPONSE_TYPE), + body: entry.body, + }; + } + } + match fetch_and_cache_ocsp(state, &body, &key, &cert_ids[0]).await { + Ok((status, content_type, upstream_body)) => RawResponse { + status, + content_type, + body: upstream_body, + }, + Err(err) => { + warn!("OCSP upstream request failed: {err:#}"); + plain(Status::BadGateway, "OCSP upstream unavailable") + } + } +} + +/// Relay one OCSP request upstream, caching successful, parseable responses. +/// Errors mean the upstream could not be reached at all; HTTP error statuses +/// are relayed (and deliberately not cached). +async fn fetch_and_cache_ocsp( + state: &ProxyState, + request_body: &[u8], + key: &str, + cert_id: &[u8], +) -> Result<(Status, ContentType, Vec)> { + let resp = state + .http + .post(&state.config.upstream_ocsp_url) + .header(reqwest::header::CONTENT_TYPE, OCSP_REQUEST_TYPE) + .header(reqwest::header::ACCEPT, OCSP_RESPONSE_TYPE) + .body(request_body.to_vec()) + .send() + .await + .context("OCSP upstream POST failed")?; + let status = Status::from_code(resp.status().as_u16()).unwrap_or(Status::BadGateway); + let content_type = resp + .headers() + .get(reqwest::header::CONTENT_TYPE) + .and_then(|value| value.to_str().ok()) + .and_then(|value| value.parse::().ok()) + .unwrap_or(ContentType::Binary); + let body = resp + .bytes() + .await + .context("failed to read OCSP upstream body")?; + if status == Status::Ok { + let validity = der::ocsp_response_validity(&body, cert_id); + if let Ok(Some((this_update, next_update))) = validity { + let now = unix_now(); + let nominal = next_update.unwrap_or(this_update + DEFAULT_OCSP_TTL_SECS); + let expires_at = nominal.min(now + state.config.ocsp_max_ttl.as_secs() as i64); + state.cache.put(CacheEntry { + kind: "ocsp".into(), + key: key.to_string(), + content_type: OCSP_RESPONSE_TYPE.into(), + body: body.to_vec(), + refresh_body: Some(request_body.to_vec()), + fetched_at: now, + expires_at, + }); + } + } + Ok((status, content_type, body.to_vec())) +} + +/// GET endpoint mirroring NVIDIA's RIM service path layout, so the proxy base +/// URL can be handed to the SDK as-is (`{base}/v1/rim/{rim_id}`). RIM +/// documents are signed and version-pinned, so staleness is far less +/// sensitive than OCSP: fresh entries are served directly, and on upstream +/// failure a stale entry is served within `rim_max_stale`. +#[get("/v1/rim/")] +async fn rim(state: &State>, rim_id: &str) -> RawResponse { + if !valid_rim_id(rim_id) { + return plain(Status::BadRequest, "invalid RIM id"); + } + let now = unix_now(); + let cached = state.cache.get("rim", rim_id); + if let Some(entry) = &cached { + if entry.is_fresh(now) { + debug!("RIM cache hit for {rim_id}"); + return RawResponse { + status: Status::Ok, + content_type: content_type_or(&entry.content_type, "application/xml"), + body: entry.body.clone(), + }; + } + } + match fetch_and_cache_rim(state, rim_id).await { + Ok((status, content_type, body)) => RawResponse { + status, + content_type, + body, + }, + Err(err) => { + warn!("RIM upstream request failed for {rim_id}: {err:#}"); + if let Some(entry) = cached { + if entry.is_usable_stale(now, state.config.rim_max_stale.as_secs()) { + warn!( + "serving stale RIM {rim_id} (fetched at {})", + entry.fetched_at + ); + return RawResponse { + status: Status::Ok, + content_type: content_type_or(&entry.content_type, "application/xml"), + body: entry.body, + }; + } + } + plain( + Status::BadGateway, + "RIM upstream unavailable and no usable cache", + ) + } + } +} + +fn valid_rim_id(rim_id: &str) -> bool { + !rim_id.is_empty() + && rim_id.len() <= 256 + && rim_id + .bytes() + .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'_' | b'-' | b'.' | b'=')) +} + +async fn fetch_and_cache_rim( + state: &ProxyState, + rim_id: &str, +) -> Result<(Status, ContentType, Vec)> { + let url = format!( + "{}/v1/rim/{rim_id}", + state.config.upstream_rim_url.trim_end_matches('/') + ); + let resp = state + .http + .get(&url) + .send() + .await + .context("RIM upstream GET failed")?; + let status = Status::from_code(resp.status().as_u16()).unwrap_or(Status::BadGateway); + let content_type = resp + .headers() + .get(reqwest::header::CONTENT_TYPE) + .and_then(|value| value.to_str().ok()) + .and_then(|value| value.parse::().ok()) + .unwrap_or(ContentType::XML); + let body = resp + .bytes() + .await + .context("failed to read RIM upstream body")?; + if status == Status::Ok { + let now = unix_now(); + state.cache.put(CacheEntry { + kind: "rim".into(), + key: rim_id.into(), + content_type: content_type.to_string(), + body: body.to_vec(), + refresh_body: None, + fetched_at: now, + expires_at: now + state.config.rim_ttl.as_secs() as i64, + }); + } + Ok((status, content_type, body.to_vec())) +} + +#[get("/health")] +fn health() -> &'static str { + "ok" +} + +#[get("/info")] +fn info(state: &State>) -> String { + let (fresh, stale) = state.cache.stats(); + format!("cache entries: {fresh} fresh, {stale} stale\n") +} + +/// Refresh entries nearing expiry so a warm cache rides through upstream +/// outages with close to a full validity window. Failures are retried on the +/// next tick; the existing entry keeps serving until it expires. +pub async fn refresh_loop(state: Arc) { + let mut ticker = tokio::time::interval(Duration::from_secs(600)); + loop { + ticker.tick().await; + let margin = state.config.refresh_margin.as_secs(); + for entry in state.cache.expiring_soon(margin) { + let result = match entry.kind.as_str() { + "ocsp" => match &entry.refresh_body { + Some(body) => { + let key = entry.key.clone(); + match hex::decode(&key) { + Ok(cert_id) => fetch_and_cache_ocsp(&state, body, &key, &cert_id).await, + Err(_) => continue, + } + } + None => continue, + }, + "rim" => fetch_and_cache_rim(&state, &entry.key).await, + _ => continue, + }; + match result { + Ok((status, _, _)) if status == Status::Ok => { + debug!("refreshed {} {}", entry.kind, entry.key) + } + Ok((status, _, _)) => { + debug!( + "refresh of {} {} got HTTP {}", + entry.kind, entry.key, status.code + ) + } + Err(err) => debug!("refresh of {} {} failed: {err:#}", entry.kind, entry.key), + } + } + } +} + +pub fn rocket(figment: Figment, state: Arc) -> Rocket { + rocket::custom(figment) + .manage(state) + .mount("/", routes![ocsp, rim, health, info]) +} From 0f15ec9eab7d37068cc9eb2e4b88c4709b361d45 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Thu, 16 Jul 2026 23:38:59 -0700 Subject: [PATCH 5/5] docs: document the GPU attestation collateral proxy Covers the motivation (issue #778: OCSP as a boot DoS), the nonce relaxation trade-off, setup via vmm.toml, and operational notes. --- docs/gpu-attestation-proxy.md | 90 +++++++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 docs/gpu-attestation-proxy.md diff --git a/docs/gpu-attestation-proxy.md b/docs/gpu-attestation-proxy.md new file mode 100644 index 000000000..3d229a49a --- /dev/null +++ b/docs/gpu-attestation-proxy.md @@ -0,0 +1,90 @@ +# GPU Attestation Collateral Proxy + +Local GPU attestation (`nvattest --verifier local`) makes live calls to two +NVIDIA services at every CVM boot: the OCSP responder +(`ocsp.ndis.nvidia.com`, one request per certificate in three chains) and the +RIM service (`rim.attestation.nvidia.com`, driver + VBIOS RIM documents). +Because the GPU attestation gate fails closed, anything that breaks those +calls — an egress-restricted or air-gapped network, an upstream outage, or a +middlebox — keeps every GPU CVM from booting. + +`gpu-attest-proxy` is a small host-side (or site-level) caching proxy that +removes the guest's direct dependency on those services, in the same spirit +as Intel's PCCS for TDX collateral: + +- **OCSP** requests are relayed byte-for-byte; successful responses are + cached keyed by CertID until their `nextUpdate`, so repeat boots and + fleets of CVMs sharing GPUs stop hitting the responder. +- **RIM** documents are cached by RIM id and re-fetched on a TTL; when the + upstream is unreachable, a stale document keeps being served within a + bounded window. +- A background refresher renews entries before they expire, so a warm cache + rides through upstream outages with close to a full validity window. + +Everything the proxy serves is signed collateral that the guest verifies +itself (OCSP response signatures and validity windows via +`OCSP_basic_verify`, RIM signatures and certificate chains by the attestation +SDK). The proxy needs no secrets and cannot forge responses; its worst +misbehavior is withholding service, which fails closed — the same outcome as +blocking NVIDIA's endpoints directly. + +## Why the OCSP nonce check is relaxed + +The attestation SDK adds a random nonce to every OCSP request and its +built-in appraisal policy requires the response to echo it +(`x-nvidia-cert-ocsp-nonce-matches`). A response cached for another request +can never echo this request's nonce, so caching is only possible if the +guest relaxes that single check. + +When `gpu_attest_proxy_url` is set, `dstack-util` runs nvattest with +`--relying-party-policy` pointing at NVIDIA's +`allow_trust_outpost_ocsp.rego` (packaged in the image at +`/usr/share/nvattest/policies/`), which keeps every built-in check except +the nonce match. Freshness is then bounded by the response validity window +instead of the nonce: a revoked certificate may keep passing until the +cached response's `nextUpdate`. This is the same trade-off the SDK's own +in-process OCSP cache makes. Guests without the proxy configured keep the +default policy and the nonce check. + +## Setup + +Run the proxy somewhere every CVM can reach (typically on each host, next to +dstack-vmm): + +```console +$ gpu-attest-proxy -c /etc/gpu-attest-proxy/gpu-attest-proxy.toml +``` + +See `dstack/gpu-attest-proxy/gpu-attest-proxy.toml` for the default +configuration (upstreams, cache directory, TTLs). The cache persists as one +JSON file per entry under `cache_dir`, so restarts keep it warm. + +Point guests at it through `vmm.toml`: + +```toml +[cvm] +gpu_attest_proxy_url = "http://:8090" +``` + +The VMM passes the URL to the guest via sys-config. At boot, `dstack-util` +derives the SDK endpoints from it: + +- `--ocsp-url {base}/ocsp` +- `--rim-url {base}` (the SDK appends `/v1/rim/{rim_id}` itself) +- `--relying-party-policy /usr/share/nvattest/policies/allow_trust_outpost_ocsp.rego` + +When the value is empty (default), guests talk to NVIDIA directly and the +built-in appraisal policy applies unchanged. + +## Operational notes + +- `GET /health` and `GET /info` expose liveness and cache statistics. +- A cold cache cannot help: the first boot after an outage begins still + needs a reachable upstream. Keep the proxy running so entries stay warm; + `refresh_margin` controls how far ahead of expiry entries are renewed. +- The guest-side clock check is unchanged: OCSP validity windows are + verified by the guest, so an expired cached response never passes. +- Revocation visibility degrades from "immediate" (nonce) to the OCSP + response validity window (`nextUpdate`). Measure the actual window for + your certificate chains — it determines both how long an outage the cache + can absorb and how long a revocation can go unnoticed.