From a56192c617affad677edecb52a5e6d07de6bc098 Mon Sep 17 00:00:00 2001 From: John Myers <9696606+johntmyers@users.noreply.github.com> Date: Tue, 29 Sep 2026 16:08:11 -0700 Subject: [PATCH] refactor(providers): remove profile telemetry buckets Remove the static inference provider table and stop emitting profile identity in provider lifecycle events. Keep operation and outcome for aggregate reporting. Refs #3906 Signed-off-by: John Myers <9696606+johntmyers@users.noreply.github.com> --- crates/openshell-core/src/telemetry.rs | 100 +++--------- crates/openshell-server/src/grpc/provider.rs | 157 ++----------------- docs/how-it-works/providers/overview.mdx | 10 +- docs/observability/telemetry.mdx | 2 +- 4 files changed, 37 insertions(+), 232 deletions(-) diff --git a/crates/openshell-core/src/telemetry.rs b/crates/openshell-core/src/telemetry.rs index 9541598fea..fab8409735 100644 --- a/crates/openshell-core/src/telemetry.rs +++ b/crates/openshell-core/src/telemetry.rs @@ -193,60 +193,6 @@ impl Default for TelemetryComputeDriver { } } -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum ProviderProfile { - Anthropic, - Claude, - Codex, - Copilot, - Deepinfra, - Github, - Gitlab, - Nvidia, - Openai, - Opencode, - Outlook, - Custom, -} - -impl ProviderProfile { - #[must_use] - pub const fn as_str(self) -> &'static str { - match self { - Self::Anthropic => "anthropic", - Self::Claude => "claude", - Self::Codex => "codex", - Self::Copilot => "copilot", - Self::Deepinfra => "deepinfra", - Self::Github => "github", - Self::Gitlab => "gitlab", - Self::Nvidia => "nvidia", - Self::Openai => "openai", - Self::Opencode => "opencode", - Self::Outlook => "outlook", - Self::Custom => "custom", - } - } - - #[must_use] - pub fn from_raw(raw: &str) -> Self { - match raw.trim().to_ascii_lowercase().as_str() { - "anthropic" => Self::Anthropic, - "claude" | "claude-code" => Self::Claude, - "codex" => Self::Codex, - "copilot" => Self::Copilot, - "deepinfra" => Self::Deepinfra, - "github" | "gh" => Self::Github, - "gitlab" | "glab" => Self::Gitlab, - "nvidia" => Self::Nvidia, - "openai" => Self::Openai, - "opencode" => Self::Opencode, - "outlook" => Self::Outlook, - _ => Self::Custom, - } - } -} - #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] pub enum DenyGroup { Bypass, @@ -487,22 +433,24 @@ pub fn emit_lifecycle( ); } -pub fn emit_provider_lifecycle( - operation: LifecycleOperation, - outcome: TelemetryOutcome, - provider_profile: ProviderProfile, -) { +pub fn emit_provider_lifecycle(operation: LifecycleOperation, outcome: TelemetryOutcome) { emit_event( "openshell_provider_lifecycle_event", - json!({ - "nvidiaSource": SOURCE.as_str(), - "operation": operation.as_str(), - "outcome": outcome.as_str(), - "providerProfile": provider_profile.as_str(), - }), + provider_lifecycle_parameters(operation, outcome), ); } +fn provider_lifecycle_parameters( + operation: LifecycleOperation, + outcome: TelemetryOutcome, +) -> Value { + json!({ + "nvidiaSource": SOURCE.as_str(), + "operation": operation.as_str(), + "outcome": outcome.as_str(), + }) +} + pub fn emit_sandbox_create( outcome: TelemetryOutcome, requested_gpu: bool, @@ -704,16 +652,24 @@ mod tests { #[test] fn telemetry_validation_maps_privacy_sensitive_strings_to_safe_buckets() { - assert_eq!( - ProviderProfile::from_raw("corp-llm-prod"), - ProviderProfile::Custom - ); assert_eq!( DenyGroup::from_raw("host=private.example"), DenyGroup::Unknown ); } + #[test] + fn provider_lifecycle_parameters_exclude_profile_identity() { + assert_eq!( + provider_lifecycle_parameters(LifecycleOperation::Create, TelemetryOutcome::Success), + json!({ + "nvidiaSource": "openshell", + "operation": "create", + "outcome": "success", + }) + ); + } + #[test] fn telemetry_enums_serialize_to_expected_strings() { assert_eq!(LifecycleResource::SandboxPolicy.as_str(), "sandbox_policy"); @@ -757,11 +713,7 @@ mod disabled_tests { LifecycleOperation::Create, TelemetryOutcome::Success, ); - emit_provider_lifecycle( - LifecycleOperation::Create, - TelemetryOutcome::Success, - ProviderProfile::Custom, - ); + emit_provider_lifecycle(LifecycleOperation::Create, TelemetryOutcome::Success); emit_sandbox_create( TelemetryOutcome::Success, false, diff --git a/crates/openshell-server/src/grpc/provider.rs b/crates/openshell-server/src/grpc/provider.rs index 2141ecb4c2..e840b9a15b 100644 --- a/crates/openshell-server/src/grpc/provider.rs +++ b/crates/openshell-server/src/grpc/provider.rs @@ -27,9 +27,7 @@ use openshell_core::proto::{ ProviderProfile, ProviderProfileCredential, Sandbox, StaticCredentialBinding, StaticCredentialEndpointBinding, }; -use openshell_core::telemetry::{ - LifecycleOperation, ProviderProfile as TelemetryProviderProfile, TelemetryOutcome, -}; +use openshell_core::telemetry::{LifecycleOperation, TelemetryOutcome, emit_provider_lifecycle}; use openshell_policy::ProviderPolicyLayer; use prost::Message; use sha2::{Digest, Sha256}; @@ -2532,11 +2530,7 @@ pub(super) async fn handle_create_provider( .await? .ensure_active()?; let Some(mut provider) = req.provider else { - emit_provider_lifecycle( - "custom", - LifecycleOperation::Create, - TelemetryOutcome::Failure, - ); + emit_provider_lifecycle(LifecycleOperation::Create, TelemetryOutcome::Failure); return Err(Status::invalid_argument("provider is required")); }; if let Some(metadata) = provider.metadata.as_mut() { @@ -2547,7 +2541,6 @@ pub(super) async fn handle_create_provider( "provider.credential_handles is internal gateway state and cannot be supplied", )); } - let provider_type = provider.r#type.clone(); let _sandbox_sync_guard = state.compute.sandbox_sync_guard().await.map_err(|error| { super::persistence_error_to_status(error, "acquire provider mutation lock") })?; @@ -2570,22 +2563,14 @@ pub(super) async fn handle_create_provider( .await; match result { Ok(provider) => { - emit_provider_lifecycle( - &provider.r#type, - LifecycleOperation::Create, - TelemetryOutcome::Success, - ); + emit_provider_lifecycle(LifecycleOperation::Create, TelemetryOutcome::Success); Ok(Response::new(ProviderResponse { provider: Some(provider), ..Default::default() })) } Err(err) => { - emit_provider_lifecycle( - &provider_type, - LifecycleOperation::Create, - TelemetryOutcome::Failure, - ); + emit_provider_lifecycle(LifecycleOperation::Create, TelemetryOutcome::Failure); Err(err) } } @@ -3846,14 +3831,9 @@ pub(super) async fn handle_update_provider( super::persistence_error_to_status(error, "acquire provider mutation lock") })?; let Some(mut provider) = req.provider else { - emit_provider_lifecycle( - "custom", - LifecycleOperation::Update, - TelemetryOutcome::Failure, - ); + emit_provider_lifecycle(LifecycleOperation::Update, TelemetryOutcome::Failure); return Err(Status::invalid_argument("provider is required")); }; - let provider_type = provider.r#type.clone(); provider .credential_expiration_times .extend(req.credential_expiration_times); @@ -3900,11 +3880,7 @@ pub(super) async fn handle_update_provider( .await?, ); } - emit_provider_lifecycle( - &provider.r#type, - LifecycleOperation::Update, - TelemetryOutcome::Success, - ); + emit_provider_lifecycle(LifecycleOperation::Update, TelemetryOutcome::Success); Ok(Response::new(ProviderResponse { provider: Some(provider), target_receipts, @@ -3912,11 +3888,7 @@ pub(super) async fn handle_update_provider( })) } Err(err) => { - emit_provider_lifecycle( - &provider_type, - LifecycleOperation::Update, - TelemetryOutcome::Failure, - ); + emit_provider_lifecycle(LifecycleOperation::Update, TelemetryOutcome::Failure); Err(err) } } @@ -5048,7 +5020,6 @@ pub(super) async fn handle_delete_provider( .await? .name; let name = req.name; - let provider_profile = provider_profile_for_name(state.store.as_ref(), &workspace, &name).await; let result = delete_provider_record_with_credentials( state.store.as_ref(), &workspace, @@ -5059,74 +5030,18 @@ pub(super) async fn handle_delete_provider( match result { Ok(deleted) => { let outcome = TelemetryOutcome::from_success(deleted || req.allow_missing); - emit_provider_profile_lifecycle( - provider_profile.unwrap_or(TelemetryProviderProfile::Custom), - LifecycleOperation::Delete, - outcome, - ); + emit_provider_lifecycle(LifecycleOperation::Delete, outcome); Ok(Response::new(DeleteProviderResponse { outcome: super::deletion_outcome(deleted, req.allow_missing, "provider")?, })) } Err(err) => { - emit_provider_profile_lifecycle( - provider_profile.unwrap_or(TelemetryProviderProfile::Custom), - LifecycleOperation::Delete, - TelemetryOutcome::Failure, - ); + emit_provider_lifecycle(LifecycleOperation::Delete, TelemetryOutcome::Failure); Err(err) } } } -fn emit_provider_lifecycle( - provider_type: &str, - operation: LifecycleOperation, - outcome: TelemetryOutcome, -) { - let provider_profile = telemetry_provider_profile(provider_type); - emit_provider_profile_lifecycle(provider_profile, operation, outcome); -} - -fn emit_provider_profile_lifecycle( - provider_profile: TelemetryProviderProfile, - operation: LifecycleOperation, - outcome: TelemetryOutcome, -) { - openshell_core::telemetry::emit_provider_lifecycle(operation, outcome, provider_profile); -} - -async fn provider_profile_for_name( - store: &Store, - workspace: &str, - name: &str, -) -> Option { - store - .get_message_by_name::(workspace, name) - .await - .ok() - .flatten() - .map(|provider| telemetry_provider_profile(&provider.r#type)) -} - -/// Bucket a provider type for telemetry. -/// -/// Matches the profile ID exactly. Any ID without a bucket, including every -/// operator-authored profile, reports as `Custom`. -fn telemetry_provider_profile(provider_type: &str) -> TelemetryProviderProfile { - match normalize_profile_id(provider_type).as_deref() { - Some("anthropic") => TelemetryProviderProfile::Anthropic, - Some("claude-code") => TelemetryProviderProfile::Claude, - Some("codex") => TelemetryProviderProfile::Codex, - Some("copilot") => TelemetryProviderProfile::Copilot, - Some("deepinfra") => TelemetryProviderProfile::Deepinfra, - Some("github") => TelemetryProviderProfile::Github, - Some("nvidia") => TelemetryProviderProfile::Nvidia, - Some("openai") => TelemetryProviderProfile::Openai, - _ => TelemetryProviderProfile::Custom, - } -} - // --------------------------------------------------------------------------- // Tests // --------------------------------------------------------------------------- @@ -5243,60 +5158,6 @@ mod tests { validate_provider_create_credentials(&profile, &provider).unwrap(); } - #[test] - fn telemetry_provider_profile_maps_unknown_to_custom() { - assert_eq!( - telemetry_provider_profile("CLAUDE-CODE"), - TelemetryProviderProfile::Claude - ); - // A legacy alias is not a profile ID, so it buckets as custom. - assert_eq!( - telemetry_provider_profile("claude"), - TelemetryProviderProfile::Custom - ); - assert_eq!( - telemetry_provider_profile("github"), - TelemetryProviderProfile::Github - ); - // Legacy aliases are not profile IDs. - assert_eq!( - telemetry_provider_profile("gh"), - TelemetryProviderProfile::Custom - ); - assert_eq!( - telemetry_provider_profile("glab"), - TelemetryProviderProfile::Custom - ); - assert_eq!( - telemetry_provider_profile("gitlab"), - TelemetryProviderProfile::Custom - ); - assert_eq!( - telemetry_provider_profile("opencode"), - TelemetryProviderProfile::Custom - ); - assert_eq!( - telemetry_provider_profile("outlook"), - TelemetryProviderProfile::Custom - ); - assert_eq!( - telemetry_provider_profile("generic"), - TelemetryProviderProfile::Custom - ); - assert_eq!( - telemetry_provider_profile("unknown-private"), - TelemetryProviderProfile::Custom - ); - assert_eq!( - telemetry_provider_profile("acme-internal"), - TelemetryProviderProfile::Custom - ); - assert_eq!( - telemetry_provider_profile("corp-llm-prod"), - TelemetryProviderProfile::Custom - ); - } - #[test] fn dynamic_credentials_expand_endpoint_audience_overrides() { let service_audiences = [ diff --git a/docs/how-it-works/providers/overview.mdx b/docs/how-it-works/providers/overview.mdx index 908adf64d8..cb28e8576b 100644 --- a/docs/how-it-works/providers/overview.mdx +++ b/docs/how-it-works/providers/overview.mdx @@ -444,7 +444,7 @@ The provider types available to `--type` are the IDs of the profiles you imported. List them: ```shell -openshell provider list-profiles +openshell profile list ``` The repository's @@ -464,14 +464,6 @@ Attach an inference provider to each sandbox that needs it and configure the workload to call the provider's native endpoint. The workload, not OpenShell, selects the model and request timeout. -| Provider | Example profile | Native base URL | Credential variable | -|---|---|---|---| -| OpenAI | `openai` | `https://api.openai.com/v1` | `OPENAI_API_KEY` | -| Anthropic | `anthropic` | `https://api.anthropic.com` | `ANTHROPIC_API_KEY` | -| NVIDIA API Catalog | `nvidia` | `https://integrate.api.nvidia.com/v1` | `NVIDIA_API_KEY` | -| DeepInfra | `deepinfra` | `https://api.deepinfra.com/v1/openai` | `DEEPINFRA_API_KEY` | -| Google Vertex AI | `google-vertex-ai` | Region and model dependent | `GOOGLE_VERTEX_AI_TOKEN` or `GOOGLE_VERTEX_AI_SERVICE_ACCOUNT_TOKEN` | - An OpenAI-compatible protocol does not make the `openai` profile safe for an arbitrary host. Baseten, Bitdeer, Groq, Ollama, LM Studio, self-hosted NIM, and other alternate endpoints need their own profile declaring the actual host, diff --git a/docs/observability/telemetry.mdx b/docs/observability/telemetry.mdx index e78e0f7d00..4130d68a8d 100644 --- a/docs/observability/telemetry.mdx +++ b/docs/observability/telemetry.mdx @@ -11,7 +11,7 @@ OpenShell collects anonymous telemetry to help improve the project. The data sho ## What Is Collected -Telemetry events are limited to anonymous operational categories and counts, such as sandbox lifecycle outcomes, provider profile buckets, policy decision counts, and aggregate network activity denial categories. +Telemetry events are limited to anonymous operational categories and counts, such as sandbox lifecycle outcomes, provider lifecycle operations and outcomes, policy decision counts, and aggregate network activity denial categories. Provider lifecycle events do not report profile IDs or names. OpenShell telemetry does not collect sandbox names or IDs, hostnames, file paths, binary paths, prompts, credentials, provider names, model names, or user content.