diff --git a/Cargo.lock b/Cargo.lock index 43f840655..af9453d74 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4084,6 +4084,7 @@ dependencies = [ "prism-artifacts", "prism-pipeline", "prism-store", + "prism-tree", "reqwest 0.12.28", "serde", "serde_json", diff --git a/crates/prism-registry/Cargo.toml b/crates/prism-registry/Cargo.toml index cbd7aa264..2ee1bb47c 100644 --- a/crates/prism-registry/Cargo.toml +++ b/crates/prism-registry/Cargo.toml @@ -14,6 +14,7 @@ hex = "0.4" prism-artifacts = { path = "../prism-artifacts" } prism-pipeline = { path = "../prism-pipeline" } prism-store = { path = "../prism-store" } +prism-tree = { path = "../prism-tree" } reqwest = { version = "0.12", default-features = false, features = ["rustls-tls", "json"] } serde = { version = "1", features = ["derive"] } serde_json = "1" diff --git a/crates/prism-registry/src/hf.rs b/crates/prism-registry/src/hf.rs index 8331897c8..e991adaed 100644 --- a/crates/prism-registry/src/hf.rs +++ b/crates/prism-registry/src/hf.rs @@ -1,22 +1,32 @@ //! HuggingFace Hub top-model publisher. //! -//! When a submission becomes the new global-best bpb, source files -//! (`architecture.py`, `training.py`, `METRICS.json`, `README.md`) are -//! committed to `PRISM_TOPMODEL_HF_REPO` (default -//! `BaseIntelligence/prism-top-model`) via the Hub ndjson commit API. +//! When a submission becomes the new global-best bpb, the master publishes a +//! **reloadable** Hub model card to `PRISM_TOPMODEL_HF_REPO` (default +//! `BaseIntelligence/top-prism-architecture`): //! -//! Token discipline mirrors GitHub: read from -//! `PRISM_TOPMODEL_HF_TOKEN_FILE` (e.g. `deploy/secrets/huggingface/token`), -//! never from env text. Absent/empty → graceful no-op. +//! - custom architecture / AutoModel novelty sources (`architecture.py`, +//! `training.py`, plus touched `.py` / patch files from `tree_blob`) +//! - `config.json` + thin `configuration_prism.py` / `modeling_prism.py` +//! wrappers so `trust_remote_code=True` consumers can locate the code +//! - trained `checkpoint.pt` (LFS when large) when secure-receive parked it +//! +//! Token discipline: read from `PRISM_TOPMODEL_HF_TOKEN_FILE` only (never env +//! text). Absent/empty → graceful no-op. + +use std::collections::BTreeMap; +use std::path::Path; use base64::{engine::general_purpose::STANDARD as B64, Engine as _}; +use sha2::{Digest, Sha256}; use tracing::info; use crate::publish::{PublishError, TopModelRequest}; const DEFAULT_API_BASE: &str = "https://huggingface.co"; -const DEFAULT_REPO: &str = "BaseIntelligence/prism-top-model"; +const DEFAULT_REPO: &str = "BaseIntelligence/top-prism-architecture"; const DEFAULT_REVISION: &str = "main"; +/// Hub regular (non-LFS) file ceiling used by the ndjson commit API. +const REGULAR_FILE_MAX: usize = 5 * 1024 * 1024; /// HuggingFace Hub publisher (token never `Debug`/`Display`'d). pub struct HfTopModelPublisher { @@ -76,11 +86,11 @@ impl HfTopModelPublisher { headers.insert(reqwest::header::AUTHORIZATION, hv); headers.insert( reqwest::header::USER_AGENT, - reqwest::header::HeaderValue::from_static("base-prism-topmodel-hf/0.1"), + reqwest::header::HeaderValue::from_static("base-prism-topmodel-hf/0.2"), ); let http = reqwest::Client::builder() .default_headers(headers) - .timeout(std::time::Duration::from_mins(10)) + .timeout(std::time::Duration::from_mins(30)) .build() .map_err(|e| PublishError::Transport(e.to_string()))?; Ok(Self { @@ -91,54 +101,29 @@ impl HfTopModelPublisher { }) } - /// Ensure the model repo exists, then commit top-model sources. + /// Ensure the model repo exists, then commit a reloadable custom-arch pack. /// /// # Errors /// Transport / Hub API failures. pub async fn publish(&self, req: &TopModelRequest) -> Result { self.ensure_repo().await?; - let arch = req.arch_id.as_deref().unwrap_or("arch-unregistered"); - let readme = format!( - "# PRISM top model (HuggingFace)\n\n\ - Global-best bpb champion published by the Base master.\n\n\ - | field | value |\n|---|---|\n\ - | arch_id | `{arch}` |\n\ - | bpb | `{:.6}` |\n\ - | submission | `{}` |\n\ - | owner_hotkey | `{}…` |\n\n\ - Companion GitHub publish (when configured) lives under\n\ - `BaseIntelligence/prism` `top-model/`.\n", - req.bpb, - req.submission_id, - req.owner_hotkey.chars().take(12).collect::(), - ); - let metrics = serde_json::to_string_pretty(&serde_json::json!({ - "submission_id": req.submission_id, - "arch_id": req.arch_id, - "owner_hotkey": req.owner_hotkey, - "bpb": req.bpb, - "n_params": req.metrics_json.as_ref().and_then(|m| m.get("n_params")), - "tokens_seen": req.metrics_json.as_ref().and_then(|m| m.get("tokens_seen")), - "wall_clock_seconds": req.metrics_json.as_ref().and_then(|m| m.get("wall_clock_seconds")), - "battery": req.metrics_json.as_ref().and_then(|m| m.get("battery")), - "eval_tier": req.metrics_json.as_ref().and_then(|m| m.get("eval_tier")), - "flow": req.metrics_json.as_ref().and_then(|m| m.get("flow")), - })) - .unwrap_or_else(|_| "{}".into()); - let files: [(&str, &[u8]); 4] = [ - ("architecture.py", req.architecture_py.as_bytes()), - ("training.py", req.training_py.as_bytes()), - ("METRICS.json", metrics.as_bytes()), - ("README.md", readme.as_bytes()), - ]; + let files = build_hub_files(req)?; let oid = self - .commit_files(&format!("top-model: {arch} bpb={:.4}", req.bpb), &files) + .commit_payload( + &format!( + "top-model: {} bpb={:.4}", + req.arch_id.as_deref().unwrap_or("arch-unregistered"), + req.bpb + ), + &files, + ) .await?; info!( submission_id = %req.submission_id, repo = %self.repo, commit = %oid, - "top model published to HuggingFace" + files = files.len(), + "top model published to HuggingFace (custom-arch pack)" ); Ok(oid) } @@ -160,12 +145,10 @@ impl HfTopModelPublisher { .await .map_err(|e| PublishError::Transport(e.to_string()))?; let status = resp.status(); - // 409 / already exists → ok; 200/201 → created. if status.is_success() || status.as_u16() == 409 { return Ok(()); } let text = resp.text().await.unwrap_or_default(); - // Hub sometimes returns 400 "You already created this repository". if text.to_ascii_lowercase().contains("already") { return Ok(()); } @@ -174,10 +157,10 @@ impl HfTopModelPublisher { ))) } - async fn commit_files( + async fn commit_payload( &self, summary: &str, - files: &[(&str, &[u8])], + files: &[(String, Vec)], ) -> Result { let url = format!( "{}/api/models/{}/commit/{}", @@ -187,22 +170,38 @@ impl HfTopModelPublisher { ndjson.push_str( &serde_json::json!({ "key": "header", - "value": {"summary": summary, "description": ""} + "value": {"summary": summary, "description": "PRISM custom-arch top model"} }) .to_string(), ); ndjson.push('\n'); for (path, bytes) in files { - let line = serde_json::json!({ - "key": "file", - "value": { - "content": B64.encode(bytes), - "path": path, - "encoding": "base64", - } - }); - ndjson.push_str(&line.to_string()); - ndjson.push('\n'); + if bytes.len() > REGULAR_FILE_MAX { + let oid = hex::encode(Sha256::digest(bytes)); + self.upload_lfs(path, bytes, &oid).await?; + let line = serde_json::json!({ + "key": "lfsFile", + "value": { + "path": path, + "algo": "sha256", + "size": bytes.len(), + "oid": oid, + } + }); + ndjson.push_str(&line.to_string()); + ndjson.push('\n'); + } else { + let line = serde_json::json!({ + "key": "file", + "value": { + "content": B64.encode(bytes), + "path": path, + "encoding": "base64", + } + }); + ndjson.push_str(&line.to_string()); + ndjson.push('\n'); + } } let resp = self .http @@ -224,6 +223,102 @@ impl HfTopModelPublisher { .unwrap_or("ok") .to_owned()) } + + /// LFS batch + PUT for one large file (Hub oid = hex sha256, no prefix). + async fn upload_lfs( + &self, + path: &str, + bytes: &[u8], + oid_hex: &str, + ) -> Result<(), PublishError> { + let batch_url = format!("{}/{}.git/info/lfs/objects/batch", self.api_base, self.repo); + let batch = serde_json::json!({ + "operation": "upload", + "transfers": ["basic"], + "objects": [{ + "oid": oid_hex, + "size": bytes.len(), + }], + "hash_algo": "sha256", + }); + let resp = self + .http + .post(&batch_url) + .header(reqwest::header::ACCEPT, "application/vnd.git-lfs+json") + .header( + reqwest::header::CONTENT_TYPE, + "application/vnd.git-lfs+json", + ) + .json(&batch) + .send() + .await + .map_err(|e| PublishError::Transport(e.to_string()))?; + let status = resp.status(); + let body = resp.text().await.unwrap_or_default(); + if !status.is_success() { + return Err(PublishError::Api(format!( + "hf lfs batch {status} ({path}): {body}" + ))); + } + let v: serde_json::Value = + serde_json::from_str(&body).map_err(|e| PublishError::Api(e.to_string()))?; + let obj = v + .get("objects") + .and_then(|o| o.as_array()) + .and_then(|a| a.first()) + .ok_or_else(|| PublishError::Api("hf lfs batch: empty objects".into()))?; + if obj.get("error").is_some() { + return Err(PublishError::Api(format!( + "hf lfs object error ({path}): {obj}" + ))); + } + // Already present on the server → nothing to upload. + let Some(actions) = obj.get("actions") else { + return Ok(()); + }; + let upload = actions + .get("upload") + .ok_or_else(|| PublishError::Api(format!("hf lfs missing upload action ({path})")))?; + let href = upload + .get("href") + .and_then(|h| h.as_str()) + .ok_or_else(|| PublishError::Api("hf lfs upload href missing".into()))?; + let mut req = self.http.put(href).body(bytes.to_vec()); + if let Some(headers) = upload.get("header").and_then(|h| h.as_object()) { + for (k, val) in headers { + if let Some(s) = val.as_str() { + req = req.header(k.as_str(), s); + } + } + } + let put = req + .send() + .await + .map_err(|e| PublishError::Transport(e.to_string()))?; + if !put.status().is_success() { + let st = put.status(); + let t = put.text().await.unwrap_or_default(); + return Err(PublishError::Api(format!("hf lfs put {st} ({path}): {t}"))); + } + // Optional verify action. + if let Some(verify) = actions.get("verify") { + if let Some(vhref) = verify.get("href").and_then(|h| h.as_str()) { + let mut vreq = self.http.post(vhref).json(&serde_json::json!({ + "oid": oid_hex, + "size": bytes.len(), + })); + if let Some(headers) = verify.get("header").and_then(|h| h.as_object()) { + for (k, val) in headers { + if let Some(s) = val.as_str() { + vreq = vreq.header(k.as_str(), s); + } + } + } + let _ = vreq.send().await; + } + } + Ok(()) + } } fn split_repo(repo: &str) -> Result<(&str, &str), PublishError> { @@ -239,6 +334,301 @@ fn split_repo(repo: &str) -> Result<(&str, &str), PublishError> { Ok((org, name)) } +/// Build the Hub file set for a top-model publish (sources + wrappers + weights). +fn build_hub_files(req: &TopModelRequest) -> Result)>, PublishError> { + let arch = req.arch_id.as_deref().unwrap_or("arch-unregistered"); + let mut files: BTreeMap> = BTreeMap::new(); + + files.insert( + "architecture.py".into(), + req.architecture_py.as_bytes().to_vec(), + ); + files.insert("training.py".into(), req.training_py.as_bytes().to_vec()); + files.insert( + "configuration_prism.py".into(), + CONFIGURATION_PRISM_PY.as_bytes().to_vec(), + ); + files.insert( + "modeling_prism.py".into(), + MODELING_PRISM_PY.as_bytes().to_vec(), + ); + files.insert( + "config.json".into(), + serde_json::to_vec_pretty(&serde_json::json!({ + "model_type": "prism_custom", + "architectures": ["PrismCustomModel"], + "auto_map": { + "AutoConfig": "configuration_prism.PrismConfig", + "AutoModel": "modeling_prism.PrismCustomModel", + "AutoModelForCausalLM": "modeling_prism.PrismCustomModel", + }, + "prism_arch_id": arch, + "prism_submission_id": req.submission_id, + "prism_bpb": req.bpb, + "torch_dtype": "float32", + "checkpoint_file": "checkpoint.pt", + })) + .map_err(|e| PublishError::Transport(e.to_string()))?, + ); + + let metrics = serde_json::to_vec_pretty(&serde_json::json!({ + "submission_id": req.submission_id, + "arch_id": req.arch_id, + "owner_hotkey": req.owner_hotkey, + "bpb": req.bpb, + "n_params": req.metrics_json.as_ref().and_then(|m| m.get("n_params")), + "tokens_seen": req.metrics_json.as_ref().and_then(|m| m.get("tokens_seen")), + "wall_clock_seconds": req.metrics_json.as_ref().and_then(|m| m.get("wall_clock_seconds")), + "battery": req.metrics_json.as_ref().and_then(|m| m.get("battery")), + "eval_tier": req.metrics_json.as_ref().and_then(|m| m.get("eval_tier")), + "flow": req.metrics_json.as_ref().and_then(|m| m.get("flow")), + })) + .unwrap_or_else(|_| b"{}".to_vec()); + files.insert("METRICS.json".into(), metrics); + + for (rel, bytes) in &req.extra_files { + let clean = sanitize_hub_path(rel); + if clean.is_empty() || files.contains_key(&clean) { + continue; + } + files.insert(clean, bytes.clone()); + } + + if let Some(path) = &req.checkpoint_path { + let bytes = std::fs::read(path).map_err(|e| PublishError::Transport(e.to_string()))?; + if bytes.is_empty() { + return Err(PublishError::Transport("empty checkpoint".into())); + } + let name = path + .file_name() + .and_then(|s| s.to_str()) + .unwrap_or("checkpoint.pt"); + files.insert(name.to_owned(), bytes); + } + + let has_ckpt = files.keys().any(|k| k.ends_with("checkpoint.pt")); + files.insert( + "README.md".into(), + hub_readme(req, arch, has_ckpt).into_bytes(), + ); + + Ok(files.into_iter().collect()) +} + +fn sanitize_hub_path(rel: &str) -> String { + let p = Path::new(rel); + if p.is_absolute() + || p.components() + .any(|c| matches!(c, std::path::Component::ParentDir)) + { + return String::new(); + } + let s = rel.trim_start_matches("./").replace('\\', "/"); + if s.is_empty() || s.contains('\0') { + return String::new(); + } + // Keep novel modules under sources/ so they never collide with wrappers. + if s == "architecture.py" + || s == "training.py" + || s == "config.json" + || s == "README.md" + || s == "METRICS.json" + || s == "modeling_prism.py" + || s == "configuration_prism.py" + || s == "checkpoint.pt" + { + return s; + } + if s.starts_with("sources/") { + s + } else { + format!("sources/{s}") + } +} + +fn hub_readme(req: &TopModelRequest, arch: &str, has_ckpt: bool) -> String { + let ckpt_note = if has_ckpt { + "Weights: `checkpoint.pt` (LFS when large). Load via `PrismCustomModel.from_pretrained` with `trust_remote_code=True`, or apply `sources/.prism/automodel.patch` onto the live AutoModel pin and `torch.load` the checkpoint." + } else { + "Weights were not parked on the master for this champion — sources/config only." + }; + format!( + "---\n\ + library_name: transformers\n\ + tags:\n\ + - prism\n\ + - custom-architecture\n\ + - trust-remote-code\n\ + ---\n\n\ + # PRISM top model — custom architecture\n\n\ + Global-best champion published by the Base master. Novel Prism / AutoModel\n\ + modules are included under `sources/` (and seam files at repo root) so the\n\ + Hub card is reloadable even when the arch is **not** a stock transformers\n\ + GPT-2-like config.\n\n\ + | field | value |\n|---|---|\n\ + | arch_id | `{arch}` |\n\ + | bpb | `{:.6}` |\n\ + | submission | `{}` |\n\ + | owner_hotkey | `{}…` |\n\ + | hub repo | `{DEFAULT_REPO}` (override via `PRISM_TOPMODEL_HF_REPO`) |\n\n\ + ## Load (trust_remote_code)\n\n\ + ```python\n\ + from transformers import AutoModel, AutoConfig\n\ + cfg = AutoConfig.from_pretrained(\"{DEFAULT_REPO}\", trust_remote_code=True)\n\ + model = AutoModel.from_pretrained(\"{DEFAULT_REPO}\", trust_remote_code=True)\n\ + ```\n\n\ + {ckpt_note}\n\n\ + Companion GitHub publish (when configured) lives under\n\ + `BaseIntelligence/prism` `top-model/`.\n", + req.bpb, + req.submission_id, + req.owner_hotkey.chars().take(12).collect::(), + ) +} + +const CONFIGURATION_PRISM_PY: &str = r#" +"""Prism custom-arch Hub config (trust_remote_code).""" + +from transformers import PretrainedConfig + + +class PrismConfig(PretrainedConfig): + model_type = "prism_custom" + + def __init__(self, prism_arch_id=None, checkpoint_file="checkpoint.pt", **kwargs): + super().__init__(**kwargs) + self.prism_arch_id = prism_arch_id + self.checkpoint_file = checkpoint_file +"#; + +const MODELING_PRISM_PY: &str = r#" +"""Prism custom-arch loader (trust_remote_code). + +Supports: + 1. Legacy seam: ``architecture.build_model(ctx)`` + ``checkpoint.pt`` + 2. AutoModel novelty: sources under ``sources/`` (apply patch offline) + +This is intentionally permissive — novel arches are not limited to stock +GPT-2 ``transformers`` configs. +""" + +from __future__ import annotations + +import importlib.util +from pathlib import Path +from typing import Any, Optional + +import torch +from torch import nn +from transformers import PreTrainedModel + +try: + from configuration_prism import PrismConfig +except ImportError: # package-style local import + from .configuration_prism import PrismConfig + + +def _load_architecture_module(root: Path): + path = root / "architecture.py" + if not path.is_file(): + raise FileNotFoundError(f"architecture.py missing under {root}") + spec = importlib.util.spec_from_file_location("prism_architecture", path) + if spec is None or spec.loader is None: + raise ImportError("cannot load architecture.py") + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +class PrismCustomModel(PreTrainedModel): + config_class = PrismConfig + _no_split_modules = [] + + def __init__(self, config: PrismConfig, inner: Optional[nn.Module] = None): + super().__init__(config) + self.inner = inner if inner is not None else nn.Identity() + self.post_init() + + def forward(self, *args: Any, **kwargs: Any): + return self.inner(*args, **kwargs) + + @classmethod + def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs): + trust = kwargs.pop("trust_remote_code", True) + config = kwargs.pop("config", None) + if config is None: + config = PrismConfig.from_pretrained( + pretrained_model_name_or_path, trust_remote_code=trust, **kwargs + ) + root = Path(pretrained_model_name_or_path) + # Hub download may leave us with a cache dir; prefer local folder layout. + if not (root / "architecture.py").is_file(): + # Fall back to empty shell when only config is present. + return cls(config) + + mod = _load_architecture_module(root) + ctx = { + "device": "cpu", + "dtype": torch.float32, + "seed": 0, + "vocab_size": getattr(config, "vocab_size", 50257), + } + if hasattr(mod, "build_model"): + inner = mod.build_model(ctx) + elif hasattr(mod, "Model"): + inner = mod.Model(**{k: v for k, v in ctx.items() if k in ("vocab_size",)}) + else: + raise AttributeError( + "architecture.py must define build_model(ctx) or Model for Hub reload" + ) + model = cls(config, inner=inner) + ckpt_name = getattr(config, "checkpoint_file", "checkpoint.pt") or "checkpoint.pt" + ckpt = root / ckpt_name + if ckpt.is_file(): + blob = torch.load(ckpt, map_location="cpu", weights_only=False) + state = blob.get("state_dict", blob) if isinstance(blob, dict) else blob + if isinstance(state, dict): + try: + model.inner.load_state_dict(state, strict=False) + except Exception: + # Novel arches / tied keys — best-effort; sources remain authoritative. + pass + return model +"#; + +/// Collect novel source files from a packed `tree_blob` for Hub `sources/`. +#[must_use] +pub fn extra_files_from_tree_blob(tree_blob: Option<&[u8]>) -> Vec<(String, Vec)> { + let Some(blob) = tree_blob else { + return Vec::new(); + }; + let Ok(tree) = prism_tree::StagedTree::unpack(blob) else { + return Vec::new(); + }; + let mut out = Vec::new(); + for (path, bytes) in tree.files() { + if !is_publishable_source(path) { + continue; + } + if bytes.len() > 8 * 1024 * 1024 { + continue; + } + out.push((path.clone(), bytes.clone())); + } + out +} + +fn is_publishable_source(path: &str) -> bool { + let Some(ext) = Path::new(path).extension().and_then(|e| e.to_str()) else { + // Allow extensionless patch paths under .prism/ + return path.ends_with("automodel.patch"); + }; + matches!( + ext.to_ascii_lowercase().as_str(), + "py" | "toml" | "json" | "md" | "patch" | "yaml" | "yml" + ) +} + #[cfg(test)] mod tests { #![allow(clippy::unwrap_used)] @@ -252,7 +642,7 @@ mod tests { arch_id: Some("arch_hf".into()), owner_hotkey: "cd".repeat(32), bpb: 1.1, - architecture_py: "def build_model(ctx):\n pass\n".into(), + architecture_py: "def build_model(ctx):\n return None\n".into(), training_py: "def train(model, ctx):\n return {}\n".into(), metrics_json: Some(serde_json::json!({ "n_params": 1, @@ -261,11 +651,15 @@ mod tests { "eval_tier": "public", })), checkpoint_path: None, + extra_files: vec![( + "nemo_automodel/components/models/toy/layers.py".into(), + b"class ToyLayer: pass\n".to_vec(), + )], } } #[tokio::test] - async fn commits_sources_via_ndjson() { + async fn commits_custom_arch_pack() { let server = MockServer::start().await; Mock::given(method("POST")) .and(path("/api/repos/create")) @@ -273,22 +667,34 @@ mod tests { .mount(&server) .await; Mock::given(method("POST")) - .and(path("/api/models/BaseIntelligence/prism-top-model/commit/main")) + .and(path( + "/api/models/BaseIntelligence/top-prism-architecture/commit/main", + )) .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ "commitOid": "hfoid123", - "commitUrl": "https://huggingface.co/BaseIntelligence/prism-top-model/commit/hfoid123" + "commitUrl": "https://huggingface.co/BaseIntelligence/top-prism-architecture/commit/hfoid123" }))) .mount(&server) .await; let p = HfTopModelPublisher::with_config( "hf_tok_test", server.uri(), - "BaseIntelligence/prism-top-model", + "BaseIntelligence/top-prism-architecture", "main", ) .unwrap(); let oid = p.publish(&req()).await.unwrap(); assert_eq!(oid, "hfoid123"); + let built = build_hub_files(&req()).unwrap(); + let keys: Vec<_> = built.iter().map(|(k, _)| k.as_str()).collect(); + assert!(keys.contains(&"config.json")); + assert!(keys.contains(&"modeling_prism.py")); + assert!(keys.contains(&"sources/nemo_automodel/components/models/toy/layers.py")); + } + + #[test] + fn default_repo_is_top_prism_architecture() { + assert_eq!(DEFAULT_REPO, "BaseIntelligence/top-prism-architecture"); } #[test] @@ -299,8 +705,14 @@ mod tests { #[test] fn split_repo_ok() { assert_eq!( - split_repo("BaseIntelligence/prism-top-model").unwrap(), - ("BaseIntelligence", "prism-top-model") + split_repo("BaseIntelligence/top-prism-architecture").unwrap(), + ("BaseIntelligence", "top-prism-architecture") ); } + + #[test] + fn sanitize_rejects_traversal() { + assert!(sanitize_hub_path("../evil.py").is_empty()); + assert_eq!(sanitize_hub_path("foo/bar.py"), "sources/foo/bar.py"); + } } diff --git a/crates/prism-registry/src/hooks.rs b/crates/prism-registry/src/hooks.rs index 11689ec9e..5e32cf13e 100644 --- a/crates/prism-registry/src/hooks.rs +++ b/crates/prism-registry/src/hooks.rs @@ -122,6 +122,9 @@ pub async fn post_score_hooks( training_py: row.training_py.clone(), metrics_json: row.metrics_json.clone(), checkpoint_path: ckpt, + // Novel AutoModel / tree modules so HF cards reload without a stock + // transformers arch already on the Hub. + extra_files: crate::hf::extra_files_from_tree_blob(row.tree_blob.as_deref()), }; let mut journaled = false; if let Some(publisher) = publisher { diff --git a/crates/prism-registry/src/lib.rs b/crates/prism-registry/src/lib.rs index 71302e03a..11e072875 100644 --- a/crates/prism-registry/src/lib.rs +++ b/crates/prism-registry/src/lib.rs @@ -9,9 +9,9 @@ //! public `BaseIntelligence/prism` GitHub repo under `top-model/`, via a //! token read from a deploy secret file (`PRISM_TOPMODEL_GITHUB_TOKEN_FILE`; //! graceful no-op when absent). -//! - [`HfTopModelPublisher`] — same trigger, commits sources to HuggingFace -//! (`PRISM_TOPMODEL_HF_TOKEN_FILE`; default repo -//! `BaseIntelligence/prism-top-model`). +//! - [`HfTopModelPublisher`] — same trigger, commits a reloadable custom-arch +//! pack to HuggingFace (`PRISM_TOPMODEL_HF_TOKEN_FILE`; default repo +//! `BaseIntelligence/top-prism-architecture`). #![forbid(unsafe_code)] #![allow(clippy::cast_precision_loss)] diff --git a/crates/prism-registry/src/publish.rs b/crates/prism-registry/src/publish.rs index 2a4a0d646..c5f15fa08 100644 --- a/crates/prism-registry/src/publish.rs +++ b/crates/prism-registry/src/publish.rs @@ -48,6 +48,9 @@ pub struct TopModelRequest { /// Master-parked checkpoint path (harvested from the Lium pod). /// When absent and weights are required, publish is refused. pub checkpoint_path: Option, + /// Extra source files for Hub publish (AutoModel novelty / tree_blob + /// modules). Paths are relative; HF publisher nests them under `sources/`. + pub extra_files: Vec<(String, Vec)>, } /// GitHub contents-API publisher. Token is never `Debug`/`Display`'d. @@ -273,6 +276,7 @@ mod tests { "telemetry": {"finish_reason": "finish_evaluation", "loss_series": [{"step": 1, "loss": 2.0}]}, })), checkpoint_path: None, + extra_files: Vec::new(), } } diff --git a/deploy/compose/env-prod.yml b/deploy/compose/env-prod.yml index 254426998..fdb15b82f 100644 --- a/deploy/compose/env-prod.yml +++ b/deploy/compose/env-prod.yml @@ -75,7 +75,7 @@ services: BASE_CHAIN_ENDPOINTS: "wss://bittensor-finney.api.onfinality.io/public-ws,wss://entrypoint-finney.opentensor.ai:443" # Top-model HuggingFace publish (optional; no-op without token file). PRISM_TOPMODEL_HF_TOKEN_FILE: "/run/base/huggingface/token" - PRISM_TOPMODEL_HF_REPO: "BaseIntelligence/prism-top-model" + PRISM_TOPMODEL_HF_REPO: "BaseIntelligence/top-prism-architecture" volumes: - /var/lib/prism/automodel-pin:/var/lib/prism/automodel-pin:ro - /var/lib/prism/payer-vault:/var/lib/prism/payer-vault diff --git a/deploy/compose/env-staging.yml b/deploy/compose/env-staging.yml index 2b13dc261..0c5655171 100644 --- a/deploy/compose/env-staging.yml +++ b/deploy/compose/env-staging.yml @@ -70,6 +70,7 @@ services: PRISM_PAYER_VAULT_DIR: "/var/lib/prism/payer-vault" PRISM_PAYER_VAULT_KEY_FILE: "/run/secrets/prism_payer_vault_key" PRISM_TOPMODEL_HF_TOKEN_FILE: "/run/base/huggingface/token" + PRISM_TOPMODEL_HF_REPO: "BaseIntelligence/top-prism-architecture" volumes: - /var/lib/prism/automodel-pin:/var/lib/prism/automodel-pin:ro - /var/lib/prism/payer-vault:/var/lib/prism/payer-vault diff --git a/deploy/env/prism-challenge.env.example b/deploy/env/prism-challenge.env.example index b02e54180..47bfdc35d 100644 --- a/deploy/env/prism-challenge.env.example +++ b/deploy/env/prism-challenge.env.example @@ -39,4 +39,4 @@ BASE_NETUID=541 # Top-model HuggingFace publish (optional). Token file only — never paste # the token into this env file. Missing/empty → HF publish no-ops. # PRISM_TOPMODEL_HF_TOKEN_FILE=/run/base/huggingface/token -# PRISM_TOPMODEL_HF_REPO=BaseIntelligence/prism-top-model +# PRISM_TOPMODEL_HF_REPO=BaseIntelligence/top-prism-architecture diff --git a/deploy/secrets/README.md b/deploy/secrets/README.md index 897232522..65fe385de 100644 --- a/deploy/secrets/README.md +++ b/deploy/secrets/README.md @@ -54,7 +54,7 @@ chmod 0400 deploy/secrets/design/annotator_tokens deploy/secrets/openrouter/api_ empty file = top-model publish silently disabled. Mode **0400**, uid **65532** — never commit it. - `huggingface/token` — prism-challenge HuggingFace top-model publisher: Hub - **write** token for `BaseIntelligence/prism-top-model` (override + **write** token for `BaseIntelligence/top-prism-architecture` (override `PRISM_TOPMODEL_HF_REPO`). Read via `PRISM_TOPMODEL_HF_TOKEN_FILE` (`/run/base/huggingface/token`); missing or empty = HF publish no-ops. Mode **0400**, uid **65532** — never commit it. diff --git a/docker-compose.yml b/docker-compose.yml index e5edf3f02..25c8570ac 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -213,10 +213,10 @@ services: # Top-model GitHub publish (BaseIntelligence/prism top-model/): no-op # when the token file is absent/empty. PRISM_TOPMODEL_GITHUB_TOKEN_FILE: /run/base/github/token - # Top-model HuggingFace publish (BaseIntelligence/prism-top-model): no-op - # when the token file is absent/empty. + # Top-model HuggingFace publish (BaseIntelligence/top-prism-architecture): + # no-op when the token file is absent/empty. PRISM_TOPMODEL_HF_TOKEN_FILE: /run/base/huggingface/token - PRISM_TOPMODEL_HF_REPO: "${PRISM_TOPMODEL_HF_REPO:-BaseIntelligence/prism-top-model}" + PRISM_TOPMODEL_HF_REPO: "${PRISM_TOPMODEL_HF_REPO:-BaseIntelligence/top-prism-architecture}" # Require harvested checkpoint for top-model journal (set 0 for source-only). PRISM_TOPMODEL_REQUIRE_WEIGHTS: "${PRISM_TOPMODEL_REQUIRE_WEIGHTS:-1}" # Parked checkpoints harvested from Lium pods (master-local). diff --git a/docs/PRISM.md b/docs/PRISM.md index 6d4352095..e7c33e145 100644 --- a/docs/PRISM.md +++ b/docs/PRISM.md @@ -229,10 +229,13 @@ publishes `architecture.py` + `training.py` + `METRICS.json` + `ARTIFACT.json` + a `README.md` block to the public [`BaseIntelligence/prism`](https://github.com/BaseIntelligence/prism) repo under `top-model/` via the GitHub contents API; large checkpoints upload as -a mutable Release tag `prism-top-model`. The same trigger also commits -sources to HuggingFace (`PRISM_TOPMODEL_HF_TOKEN_FILE`, default repo -`BaseIntelligence/prism-top-model`) when that token file is present. The -publication is journaled (`prism_topmodel_publication`). GitHub token: +a mutable Release tag `prism-top-model`. The same trigger also commits a +**reloadable custom-arch pack** to HuggingFace +(`PRISM_TOPMODEL_HF_TOKEN_FILE`, default repo +`BaseIntelligence/top-prism-architecture`): seam sources, AutoModel novelty +under `sources/`, `config.json` + `trust_remote_code` wrappers, and +`checkpoint.pt` (LFS when large). The publication is journaled +(`prism_topmodel_publication`). GitHub token: `PRISM_TOPMODEL_GITHUB_TOKEN_FILE` (`deploy/secrets/github/token`); absent/empty → publish no-op. With `PRISM_TOPMODEL_REQUIRE_WEIGHTS=1` (default), a missing/invalid receipt fails the publish (no journal). diff --git a/docs/external-miner/prism.md b/docs/external-miner/prism.md index f510f200d..8bf9c337d 100644 --- a/docs/external-miner/prism.md +++ b/docs/external-miner/prism.md @@ -209,7 +209,8 @@ The global-best model (sources + `ARTIFACT.json` / checkpoint release) is published to [`BaseIntelligence/prism`](https://github.com/BaseIntelligence/prism) `top-model/` and (when configured) a HuggingFace model repo -`BaseIntelligence/prism-top-model`. See [`PRISM.md`](../PRISM.md). +`BaseIntelligence/top-prism-architecture` (custom-arch / AutoModel novelty + +weights, `trust_remote_code`). See [`PRISM.md`](../PRISM.md). ## v3 scoring (shadow-by-default)