From 90285eeb856e846ac21caa116ba0ab4579b9323a Mon Sep 17 00:00:00 2001 From: Mikola Lysenko Date: Tue, 3 Nov 2026 11:44:21 -0600 Subject: [PATCH 1/3] Rebuild gem native extensions in agent-mode apply (#1996) Agent-mode apply rewrote a gem's ext/**/*.c in place and reported it patched, while Ruby kept loading the extension compiled at install time. When a gem record changes a native-extension build input, apply now writes the patched sources and rebuilds the extension with RubyGems' own builder (the step `gem pristine --extensions` runs), using the Ruby that owns the install. The old build is moved aside first (ExtConfBuilder never overwrites files already in the extension dir) and backed up once inside the gem dir; a marker records the source hashes it was built from and every artifact's hash, including the lib/ copies Ruby may load. - A failed build puts the sources and the previous build back: native_extension_build_failed. - A copy that cannot be rebuilt (precompiled platform gem, no specifications entry, no matching Ruby) is refused with native_extensions_unsupported and nothing is written. - Sources an earlier release patched without a rebuild are rebuilt (native_extension_rebuilt warning). - vex / apply --check attest only when the marker matches (native_extension_not_rebuilt otherwise); hosted copies are unaffected. - rollback restores the backed-up original build, or rebuilds from the restored sources without one (native_extension_restore_failed). Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 8 +- crates/socket-patch-cli/CLI_CONTRACT.md | 33 + crates/socket-patch-cli/src/commands/apply.rs | 49 +- .../socket-patch-cli/src/commands/rollback.rs | 21 +- crates/socket-patch-cli/src/commands/vex.rs | 4 + .../tests/e2e_gem_native_ext_build.rs | 460 +++++ .../socket-patch-core/src/patch/copy_tree.rs | 2 +- crates/socket-patch-core/src/patch/gem_ext.rs | 1605 +++++++++++++++++ crates/socket-patch-core/src/patch/mod.rs | 1 + crates/socket-patch-core/src/vex/verify.rs | 17 + docs/ecosystems.md | 25 + 11 files changed, 2210 insertions(+), 15 deletions(-) create mode 100644 crates/socket-patch-cli/tests/e2e_gem_native_ext_build.rs create mode 100644 crates/socket-patch-core/src/patch/gem_ext.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7040d2ede..a6b071c44 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1271,8 +1271,10 @@ jobs: # Ruby <= 3.3 and 1.17-2.1 need Ruby <= 3.1 (`untaint`). 2.7.2 # (no boundary of its own) is in e2e-full. # Hosted and vendored share each era's leg (one Ruby + bundler - # setup; the run step loops over `suite`). - - {os: ubuntu-latest, suite: e2e_redirect_gem_build e2e_vendor_gem_build, ruby: '3.4', bundler: '4.0.21'} + # setup; the run step loops over `suite`), as does the agent-mode + # C-extension rebuild capstone (#1996), which needs the same Ruby + # plus the runner's C compiler. + - {os: ubuntu-latest, suite: e2e_redirect_gem_build e2e_vendor_gem_build e2e_gem_native_ext_build, ruby: '3.4', bundler: '4.0.21'} # The live-API smoke suites (e2e_npm, e2e_pypi, e2e_gem, # e2e_scan) are intentionally NOT in the PR matrix — their # `#[ignore]`-gated tests hit the real public proxy at @@ -2008,7 +2010,7 @@ jobs: # composer 1.10 keeps the PHP 8.2 it has always run on here # (composer-compatibility.yml runs 1.10.28 on 8.1). - {os: ubuntu-latest, suite: e2e_vendor_composer_build e2e_redirect_composer_build, composer: '1', php: '8.2'} - - {os: ubuntu-latest, suite: e2e_redirect_gem_build e2e_vendor_gem_build, ruby: '3.1', bundler: '1.17.3'} + - {os: ubuntu-latest, suite: e2e_redirect_gem_build e2e_vendor_gem_build e2e_gem_native_ext_build, ruby: '3.1', bundler: '1.17.3'} - {os: ubuntu-latest, suite: e2e_redirect_gem_build e2e_vendor_gem_build, ruby: '3.1', bundler: '2.1.4'} - {os: ubuntu-latest, suite: e2e_redirect_gem_build e2e_vendor_gem_build, ruby: '3.1', bundler: '2.2.33'} - {os: ubuntu-latest, suite: e2e_redirect_gem_build e2e_vendor_gem_build, ruby: '3.3', bundler: '2.5.23'} diff --git a/crates/socket-patch-cli/CLI_CONTRACT.md b/crates/socket-patch-cli/CLI_CONTRACT.md index 0c5ec2d95..10080e117 100644 --- a/crates/socket-patch-cli/CLI_CONTRACT.md +++ b/crates/socket-patch-cli/CLI_CONTRACT.md @@ -705,6 +705,39 @@ are patched by `socket-patch apply` like every other ecosystem: or a bare `--global-prefix` cache with no known `go install` directory) the same warning code says so, and VEX is not withheld (#422). +### Ruby gems with native extensions in agent mode + +Ruby loads the extension RubyGems compiled at install time +(`extensions///-/`, and its copy under the gem's `lib/`), never +the `ext/` sources beside it. When a gem record changes a native-extension build input, `apply` +writes the patched sources and then rebuilds the extension from them with RubyGems' own builder +(the step `gem pristine --extensions` runs), using the Ruby that owns the install: the one in +`/bin` of a `/lib/ruby/gems/` home, else the `ruby` on `PATH` run from +`--cwd`. A build input is a changed `.c`, `.h`, `.cc`, `.cpp`, `.cxx`, `.hh`, `.hpp`, `.hxx`, +`.m`, `.y` or `.rl` file, an `extconf.rb`, a `depend` under `ext/`, or any file under the +directory of a build script the installed gemspec's `extensions` names (#1996). A grammar file +(`.y`, `.rl`) in a gem that declares no extensions is not one: nothing compiles it at install +time. + +The first rebuild of a copy backs up the install-time build (extension directory and `lib/` +copies) to `.socket-patch-native-orig/` in the gem directory, and every rebuild records +`.socket-patch-native-build.json` there: the source hashes it was built from and the hash of every +artifact it produced. A record whose sources are patched and whose marker matches is +`already_patched`; sources an earlier release patched without a rebuild are rebuilt, with a +`native_extension_rebuilt` warning. `rollback` (and `remove`) puts the backed-up build back with +the sources; without a backup it rebuilds from the restored sources. + +| Code | Run result | Meaning | +|---|---|---| +| `native_extensions_unsupported` | refused, exit 1 | The record changes build inputs that cannot be rebuilt in place: the copy is not a RubyGems install with a `specifications/` entry (a Bundler git checkout), the gem declares no extensions (a precompiled platform gem), or no Ruby whose extension directory for the gem is the installed one was found. Nothing is written; use `scan --mode hosted`, where RubyGems compiles the patched gem. | +| `native_extension_build_failed` | refused, exit 1 | The extension did not build from the patched sources (no compiler or Ruby headers, a build error). The previous build and the original sources are put back; the error carries the build output's tail. | +| `native_extension_rebuilt` | warning | The sources were already patched (by a release before #1996) and the extension was rebuilt from them. | +| `native_extension_restore_failed` | rollback: that copy fails | The sources were restored, but the original compiled extension could not be put back. Run `gem pristine --extensions`. | + +`vex` and `apply --check` attest such a copy only while its marker matches the record's patched +sources and every artifact it lists; otherwise the purl fails with `native_extension_not_rebuilt`. +Hosted copies are judged on their files alone, since RubyGems compiled them from the patched gem. + ### sbt / Mill / scala-cli caches in agent mode A JVM project root (any `JVM_PROJECT_MARKERS` file, `build.sbt` / `build.mill` / `build.sc` / diff --git a/crates/socket-patch-cli/src/commands/apply.rs b/crates/socket-patch-cli/src/commands/apply.rs index d7b8f728e..4ade92d4f 100644 --- a/crates/socket-patch-cli/src/commands/apply.rs +++ b/crates/socket-patch-cli/src/commands/apply.rs @@ -12,6 +12,7 @@ use socket_patch_core::patch::apply::{ apply_package_patch, verify_file_patch, ApplyResult, MismatchPolicy, PatchSources, VerifyStatus, }; use socket_patch_core::patch::apply_lock::LockGuard; +use socket_patch_core::patch::gem_ext; use socket_patch_core::patch::redirect::golang_local::{ apply_go_redirect, reconcile_go_redirects, verify_go_redirect_state, }; @@ -717,6 +718,10 @@ fn describe_check_failure(reason: &str) -> &'static str { "hash_mismatch" => "an installed copy matches neither the original nor the patched bytes", "file_not_found" => "a patched file is missing from an installed copy", "no_files" => "the patch record lists no files to verify", + "native_extension_not_rebuilt" => { + "the patched native-extension sources were not compiled into the extension \ + Ruby loads" + } "no_matching_variant" => { "an installed copy matches none of the manifest's release variants \ (no matching variant found)" @@ -2703,15 +2708,41 @@ async fn apply_patches_inner( } attempted = true; - let result = apply_package_patch( - variant_purl, - pkg_path, - &patch.files, - &sources, - args.common.dry_run, - policy, - ) - .await; + // A gem patch to native-extension sources rebuilds the + // extension, or is refused with nothing written (#1996). + let result = if Ecosystem::from_purl(purl) == Some(Ecosystem::Gem) { + let applied = gem_ext::apply_gem_package_patch( + variant_purl, + pkg_path, + &patch.files, + &sources, + args.common.dry_run, + policy, + &gem_ext::RubyEnv { + cwd: args.common.cwd.clone(), + }, + ) + .await; + // A best-effort fallback-home copy's refusal is + // its per-copy skip below, not a run warning. + if let Some((code, detail)) = applied + .warning + .filter(|_| applied.result.success || !best_effort) + { + run_warnings.push(RunWarning { code, detail }); + } + applied.result + } else { + apply_package_patch( + variant_purl, + pkg_path, + &patch.files, + &sources, + args.common.dry_run, + policy, + ) + .await + }; warn_mismatch_overwrites(&result, &args.common); // A variant that reached apply is the installed distribution diff --git a/crates/socket-patch-cli/src/commands/rollback.rs b/crates/socket-patch-cli/src/commands/rollback.rs index 6d9a3c040..7725df72b 100644 --- a/crates/socket-patch-cli/src/commands/rollback.rs +++ b/crates/socket-patch-cli/src/commands/rollback.rs @@ -3276,14 +3276,31 @@ pub(crate) async fn rollback_patches_inner( match try_rollback_local_go(purl, pkg_path, patch, common).await { Some(r) => r, None => { - rollback_package_patch( + let mut result = rollback_package_patch( purl, pkg_path, &patch.files, &blobs_path, common.dry_run, ) - .await + .await; + // A gem whose native extension `apply` rebuilt gets its + // original compiled extension back with its sources + // (#1996). + if result.success + && !common.dry_run + && Ecosystem::from_purl(purl) == Some(Ecosystem::Gem) + { + use socket_patch_core::patch::gem_ext; + let env = gem_ext::RubyEnv { + cwd: common.cwd.clone(), + }; + if let Err(e) = gem_ext::restore_original_build(pkg_path, &env).await { + result.success = false; + result.error = Some(format!("{}: {e}", gem_ext::RESTORE_FAILED_CODE)); + } + } + result } } }; diff --git a/crates/socket-patch-cli/src/commands/vex.rs b/crates/socket-patch-cli/src/commands/vex.rs index 78f8111d5..65acda5d0 100644 --- a/crates/socket-patch-cli/src/commands/vex.rs +++ b/crates/socket-patch-cli/src/commands/vex.rs @@ -2091,6 +2091,10 @@ fn omission_phrase(reason: &str) -> &'static str { against" } "stale_install" => "the installed copy is not patched", + "native_extension_not_rebuilt" => { + "the patched native-extension sources were not compiled into the extension Ruby \ + loads; run socket-patch apply to rebuild it" + } DEPENDENCY_NOT_INSTALLED => { "the patched package.json declares a dependency that is not installed beside \ the package" diff --git a/crates/socket-patch-cli/tests/e2e_gem_native_ext_build.rs b/crates/socket-patch-cli/tests/e2e_gem_native_ext_build.rs new file mode 100644 index 000000000..b465d42ed --- /dev/null +++ b/crates/socket-patch-cli/tests/e2e_gem_native_ext_build.rs @@ -0,0 +1,460 @@ +//! Agent-mode patches to a gem's C extension (#1996). +//! +//! Ruby loads the extension RubyGems compiled at install time, never the +//! `ext/**/*.c` beside it, so `apply` must rebuild the extension from the +//! patched sources (or refuse, writing nothing), `apply --check` / `vex` +//! must not attest patched sources over a stale build, and `rollback` must +//! put the original build back. +//! +//! The capstones build a tiny C-extension gem (`sp_native`, whose +//! `SpNative.value` the patch changes from 1 to 2) with the host's real +//! `gem`, install it into a Bundler-style `vendor/bundle/ruby/` home, +//! and run the real binary over it. They need ruby, its headers and a C +//! compiler: `#[ignore]`-gated, run by the pinned-Ruby CI leg, which sets +//! `SOCKET_PATCH_BUNDLER_E2E_REQUIRED=1` so a missing toolchain fails +//! instead of skipping. The refusal tests lay a native gem down by hand +//! and need no Ruby. + +use std::path::{Path, PathBuf}; +use std::process::Command; + +#[path = "common/mod.rs"] +mod common; +use common::{git_sha256, parse_json_envelope, write_blob, write_minimal_manifest, PatchEntry}; + +const PURL: &str = "pkg:gem/sp_native@1.0.0"; +const UUID: &str = "19961996-1996-4996-8996-199619961996"; +const FULL_NAME: &str = "sp_native-1.0.0"; +const C_KEY: &str = "ext/sp_native/sp_native.c"; + +const EXTCONF: &str = "require \"mkmf\"\ncreate_makefile(\"sp_native/sp_native\")\n"; +const LIB_RB: &str = "require \"sp_native/sp_native\"\n"; +const GEMSPEC: &str = "Gem::Specification.new do |s|\n s.name = \"sp_native\"\n \ + s.version = \"1.0.0\"\n s.summary = \"t\"\n s.authors = [\"t\"]\n \ + s.files = Dir[\"lib/**/*.rb\", \"ext/**/*.{c,rb}\"]\n \ + s.extensions = [\"ext/sp_native/extconf.rb\"]\nend\n"; + +fn c_source(value: u32) -> String { + format!( + "#include \"ruby.h\"\n\ + static VALUE sp_value(VALUE self) {{ return INT2NUM({value}); }}\n\ + void Init_sp_native(void) {{\n \ + VALUE m = rb_define_module(\"SpNative\");\n \ + rb_define_singleton_method(m, \"value\", sp_value, 0);\n}}\n" + ) +} + +/// A source that does not compile. +fn broken_source() -> String { + format!("{}this is not C\n", c_source(2)) +} + +fn required() -> bool { + std::env::var("SOCKET_PATCH_BUNDLER_E2E_REQUIRED").is_ok_and(|v| v == "1") +} + +fn tool_works(cmd: &str, args: &[&str]) -> bool { + Command::new(cmd) + .args(args) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) + .status() + .is_ok_and(|s| s.success()) +} + +/// The ruby ABI dir name (`3.4.0`), or `None` (skip) when ruby, gem or a C +/// compiler is missing. Panics instead under the CI gate. +fn toolchain() -> Option { + let have = tool_works("ruby", &["-v"]) + && tool_works("gem", &["--version"]) + && (tool_works("cc", &["--version"]) || tool_works("gcc", &["--version"])); + let abi = have + .then(|| { + Command::new("ruby") + .args(["-e", "print RbConfig::CONFIG['ruby_version']"]) + .output() + .ok() + .filter(|o| o.status.success()) + .map(|o| String::from_utf8_lossy(&o.stdout).trim().to_string()) + .filter(|s| !s.is_empty()) + }) + .flatten(); + if abi.is_none() { + assert!( + !required(), + "SOCKET_PATCH_BUNDLER_E2E_REQUIRED=1 but ruby, gem or a C compiler is missing" + ); + eprintln!("SKIP: ruby, gem or a C compiler is not available"); + } + abi +} + +/// A project with `sp_native` installed into `vendor/bundle/ruby/` by +/// the real `gem`, and the patch's manifest + blobs (`patched` is the +/// patched C source). +struct Fixture { + _tmp: tempfile::TempDir, + proj: PathBuf, + home: PathBuf, + gem_dir: PathBuf, +} + +impl Fixture { + fn new(abi: &str, patched: &str) -> Fixture { + let tmp = tempfile::tempdir().unwrap(); + let src = tmp.path().join("src"); + std::fs::create_dir_all(src.join("ext/sp_native")).unwrap(); + std::fs::create_dir_all(src.join("lib")).unwrap(); + std::fs::write(src.join("ext/sp_native/extconf.rb"), EXTCONF).unwrap(); + std::fs::write(src.join(C_KEY), c_source(1)).unwrap(); + std::fs::write(src.join("lib/sp_native.rb"), LIB_RB).unwrap(); + std::fs::write(src.join("sp_native.gemspec"), GEMSPEC).unwrap(); + let gem_home_scratch = tmp.path().join("gem-user-home"); + std::fs::create_dir_all(&gem_home_scratch).unwrap(); + let run = |cmd: &mut Command, what: &str| { + let out = cmd + .env("HOME", &gem_home_scratch) + .output() + .unwrap_or_else(|e| panic!("{what}: {e}")); + assert!( + out.status.success(), + "{what} failed:\n{}\n{}", + String::from_utf8_lossy(&out.stdout), + String::from_utf8_lossy(&out.stderr) + ); + }; + run( + Command::new("gem") + .args(["build", "sp_native.gemspec"]) + .current_dir(&src), + "gem build", + ); + let proj = tmp.path().join("proj"); + let home = proj.join("vendor/bundle/ruby").join(abi); + std::fs::create_dir_all(&home).unwrap(); + run( + Command::new("gem") + .args(["install", "--local", "--no-document", "--install-dir"]) + .arg(&home) + .arg(src.join("sp_native-1.0.0.gem")), + "gem install", + ); + std::fs::write( + proj.join("Gemfile"), + "source \"https://rubygems.org\"\ngem \"sp_native\"\n", + ) + .unwrap(); + let socket = proj.join(".socket"); + let before = c_source(1); + write_manifest(&socket, &before, patched); + let gem_dir = home.join("gems").join(FULL_NAME); + Fixture { + _tmp: tmp, + proj, + home, + gem_dir, + } + } + + /// `SpNative.value` as the installed gem's Ruby loads it. + fn value(&self) -> String { + let out = Command::new("ruby") + .args(["-e", "require 'sp_native'; print SpNative.value"]) + .env("GEM_PATH", &self.home) + .env_remove("GEM_HOME") + .env_remove("RUBYOPT") + .output() + .unwrap(); + assert!( + out.status.success(), + "loading sp_native failed: {}", + String::from_utf8_lossy(&out.stderr) + ); + String::from_utf8_lossy(&out.stdout).into_owned() + } + + fn c_file(&self) -> String { + std::fs::read_to_string(self.gem_dir.join(C_KEY)).unwrap() + } + + fn marker(&self) -> PathBuf { + self.gem_dir.join(".socket-patch-native-build.json") + } + + fn backup(&self) -> PathBuf { + self.gem_dir.join(".socket-patch-native-orig") + } + + fn run(&self, args: &[&str]) -> (i32, String, String) { + common::run(&self.proj, args) + } +} + +fn write_manifest(socket: &Path, before: &str, after: &str) { + let before_hash = git_sha256(before.as_bytes()); + let after_hash = git_sha256(after.as_bytes()); + write_minimal_manifest( + socket, + PURL, + UUID, + &[PatchEntry { + file_name: C_KEY, + before_hash: &before_hash, + after_hash: &after_hash, + }], + ); + write_blob(socket, &before_hash, before.as_bytes()); + write_blob(socket, &after_hash, after.as_bytes()); +} + +fn event_action(stdout: &str) -> String { + let env = parse_json_envelope(stdout); + common::envelope::event_for(&env, PURL)["action"] + .as_str() + .unwrap_or_default() + .to_string() +} + +#[test] +#[ignore = "host capstone: compiles a C extension with the real ruby/gem toolchain; \ + the unpinned `test` job skips it, the pinned-Ruby e2e leg runs it via --ignored"] +fn native_patch_is_rebuilt_attested_and_rolled_back() { + let Some(abi) = toolchain() else { + return; + }; + let patched = c_source(2); + let fx = Fixture::new(&abi, &patched); + assert_eq!(fx.value(), "1"); + + let (code, _, _) = fx.run(&["apply", "--check"]); + assert_ne!(code, 0, "an unapplied patch must not check clean"); + + let (code, stdout, stderr) = fx.run(&["apply", "--json"]); + assert_eq!(code, 0, "apply failed:\n{stdout}\n{stderr}"); + assert_eq!(event_action(&stdout), "applied", "{stdout}"); + assert_eq!(fx.c_file(), patched); + assert_eq!(fx.value(), "2", "Ruby must load the rebuilt extension"); + assert!(fx.marker().is_file(), "the build marker is recorded"); + assert!(fx.backup().is_dir(), "the install-time build is backed up"); + + let (code, stdout, stderr) = fx.run(&["apply", "--check"]); + assert_eq!(code, 0, "a rebuilt patch checks clean:\n{stdout}\n{stderr}"); + + // Idempotent: nothing to write, nothing to build. + let marker_before = std::fs::read(fx.marker()).unwrap(); + let (code, stdout, _) = fx.run(&["apply", "--json"]); + assert_eq!(code, 0, "{stdout}"); + assert_eq!(event_action(&stdout), "skipped", "{stdout}"); + assert_eq!(std::fs::read(fx.marker()).unwrap(), marker_before); + + let (code, stdout, stderr) = fx.run(&["rollback", "--json"]); + assert_eq!(code, 0, "rollback failed:\n{stdout}\n{stderr}"); + assert_eq!(fx.c_file(), c_source(1)); + assert_eq!(fx.value(), "1", "the original build is back"); + assert!(!fx.marker().exists()); + assert!(!fx.backup().exists()); + + // A release before #1996 patched the sources and never rebuilt: the + // check refuses to attest it, and apply rebuilds it. (`rollback` drops + // the record, so it is written again.) + write_manifest(&fx.proj.join(".socket"), &c_source(1), &patched); + std::fs::write(fx.gem_dir.join(C_KEY), &patched).unwrap(); + let (code, stdout, stderr) = fx.run(&["apply", "--check", "--json"]); + assert_ne!(code, 0, "stale build must not check clean:\n{stdout}"); + assert!( + stdout.contains("native_extension_not_rebuilt") + || stderr.contains("native_extension_not_rebuilt") + || stdout.contains("not compiled into the extension"), + "{stdout}\n{stderr}" + ); + let (code, stdout, stderr) = fx.run(&["apply", "--json"]); + assert_eq!(code, 0, "{stdout}\n{stderr}"); + assert!(stdout.contains("native_extension_rebuilt"), "{stdout}"); + assert_eq!(fx.value(), "2"); +} + +#[test] +#[ignore = "host capstone: compiles a C extension with the real ruby/gem toolchain; \ + the unpinned `test` job skips it, the pinned-Ruby e2e leg runs it via --ignored"] +fn native_patch_that_does_not_build_is_put_back() { + let Some(abi) = toolchain() else { + return; + }; + let fx = Fixture::new(&abi, &broken_source()); + let ext_dir = fx.home.join("extensions"); + let built_before: Vec<_> = walk_files(&ext_dir); + + let (code, stdout, stderr) = fx.run(&["apply", "--json"]); + assert_eq!(code, 1, "{stdout}\n{stderr}"); + assert!(stdout.contains("native_extension_build_failed"), "{stdout}"); + assert!( + stdout.contains("this is not C"), + "the compiler error is shown: {stdout}" + ); + assert_eq!(fx.c_file(), c_source(1), "the original source is back"); + assert_eq!(fx.value(), "1", "the original build is back"); + assert!(!fx.marker().exists()); + assert!(!fx.backup().exists()); + assert_eq!(walk_files(&ext_dir), built_before); +} + +/// `(relative path, bytes)` of every file under `dir`. +fn walk_files(dir: &Path) -> Vec<(String, Vec)> { + let mut out = Vec::new(); + let mut stack = vec![dir.to_path_buf()]; + while let Some(d) = stack.pop() { + for entry in std::fs::read_dir(&d).unwrap().flatten() { + let p = entry.path(); + if p.is_dir() { + stack.push(p); + } else { + let rel = p.strip_prefix(dir).unwrap().display().to_string(); + out.push((rel, std::fs::read(&p).unwrap())); + } + } + } + out.sort(); + out +} + +// ── hermetic refusals (no Ruby needed) ──────────────────────────────── + +/// A hand-laid native gem: `ext/` source, the install-time build, and an +/// installed spec that does (or doesn't) declare extensions. +fn fake_native_gem(declares_extensions: bool) -> (tempfile::TempDir, PathBuf, PathBuf) { + let tmp = tempfile::tempdir().unwrap(); + let proj = tmp.path().to_path_buf(); + let home = proj.join("vendor/bundle/ruby/3.4.0"); + let gem_dir = home.join("gems").join(FULL_NAME); + std::fs::create_dir_all(gem_dir.join("ext/sp_native")).unwrap(); + std::fs::create_dir_all(gem_dir.join("lib/sp_native")).unwrap(); + std::fs::write(gem_dir.join(C_KEY), c_source(1)).unwrap(); + std::fs::write(gem_dir.join("lib/sp_native.rb"), LIB_RB).unwrap(); + std::fs::write( + gem_dir.join("lib/sp_native/sp_native.so"), + b"\x7fELF original", + ) + .unwrap(); + let ext = home.join("extensions/x86_64-linux/3.4.0").join(FULL_NAME); + std::fs::create_dir_all(ext.join("sp_native")).unwrap(); + std::fs::write(ext.join("sp_native/sp_native.so"), b"\x7fELF original").unwrap(); + std::fs::write(ext.join("gem.build_complete"), b"").unwrap(); + let extensions = if declares_extensions { + " s.extensions = [\"ext/sp_native/extconf.rb\".freeze]\n" + } else { + "" + }; + std::fs::create_dir_all(home.join("specifications")).unwrap(); + std::fs::write( + home.join("specifications") + .join(format!("{FULL_NAME}.gemspec")), + format!( + "Gem::Specification.new do |s|\n s.name = \"sp_native\".freeze\n \ + s.version = \"1.0.0\"\n{extensions}end\n" + ), + ) + .unwrap(); + std::fs::write( + proj.join("Gemfile"), + "source \"https://rubygems.org\"\ngem \"sp_native\"\n", + ) + .unwrap(); + (tmp, proj, gem_dir) +} + +/// Run the CLI with a `PATH` holding no `ruby`. +fn run_without_ruby(proj: &Path, args: &[&str]) -> (i32, String, String) { + let empty = proj.join(".empty-path"); + std::fs::create_dir_all(&empty).unwrap(); + let path = empty.display().to_string(); + common::run_with_env(proj, args, &[("PATH", path.as_str())]) +} + +#[test] +fn native_patch_without_a_ruby_is_refused_and_writes_nothing() { + let (_tmp, proj, gem_dir) = fake_native_gem(true); + write_manifest(&proj.join(".socket"), &c_source(1), &c_source(2)); + + let (code, stdout, stderr) = run_without_ruby(&proj, &["apply", "--json"]); + assert_eq!(code, 1, "{stdout}\n{stderr}"); + let env = parse_json_envelope(&stdout); + let event = common::envelope::event_for(&env, PURL); + assert_eq!(event["action"], "failed", "{stdout}"); + assert!( + event["error"] + .as_str() + .is_some_and(|e| e.starts_with("native_extensions_unsupported:")), + "{stdout}" + ); + assert!(stdout.contains("no `ruby` was found"), "{stdout}"); + assert_eq!( + std::fs::read_to_string(gem_dir.join(C_KEY)).unwrap(), + c_source(1) + ); + assert!(!gem_dir.join(".socket-patch-native-build.json").exists()); + assert!(!gem_dir.join(".socket-patch-native-orig").exists()); + + // Dry run reports the same refusal. + let (code, stdout, _) = run_without_ruby(&proj, &["apply", "--json", "--dry-run"]); + assert_eq!(code, 1, "{stdout}"); + assert!(stdout.contains("native_extensions_unsupported"), "{stdout}"); +} + +#[test] +fn native_patch_to_a_gem_without_extensions_is_refused() { + let (_tmp, proj, gem_dir) = fake_native_gem(false); + write_manifest(&proj.join(".socket"), &c_source(1), &c_source(2)); + + let (code, stdout, stderr) = common::run(&proj, &["apply", "--json"]); + assert_eq!(code, 1, "{stdout}\n{stderr}"); + assert!(stdout.contains("native_extensions_unsupported"), "{stdout}"); + assert!(stdout.contains("builds no extension"), "{stdout}"); + assert_eq!( + std::fs::read_to_string(gem_dir.join(C_KEY)).unwrap(), + c_source(1) + ); +} + +#[test] +fn ruby_only_patch_to_a_native_gem_needs_no_ruby() { + let (_tmp, proj, gem_dir) = fake_native_gem(true); + let before = LIB_RB; + let after = "require \"sp_native/sp_native\"\n# patched\n"; + let (bh, ah) = (git_sha256(before.as_bytes()), git_sha256(after.as_bytes())); + let socket = proj.join(".socket"); + write_minimal_manifest( + &socket, + PURL, + UUID, + &[PatchEntry { + file_name: "lib/sp_native.rb", + before_hash: &bh, + after_hash: &ah, + }], + ); + write_blob(&socket, &ah, after.as_bytes()); + + let (code, stdout, stderr) = run_without_ruby(&proj, &["apply", "--json"]); + assert_eq!(code, 0, "{stdout}\n{stderr}"); + assert_eq!(event_action(&stdout), "applied", "{stdout}"); + assert_eq!( + std::fs::read_to_string(gem_dir.join("lib/sp_native.rb")).unwrap(), + after + ); +} + +#[test] +fn patched_sources_over_the_install_time_build_do_not_check_clean() { + let (_tmp, proj, gem_dir) = fake_native_gem(true); + write_manifest(&proj.join(".socket"), &c_source(1), &c_source(2)); + // What a release before #1996 left: the source patched, the build not. + std::fs::write(gem_dir.join(C_KEY), c_source(2)).unwrap(); + + let (code, stdout, stderr) = run_without_ruby(&proj, &["apply", "--check", "--json"]); + assert_ne!(code, 0, "{stdout}\n{stderr}"); + assert!( + stdout.contains("native_extension_not_rebuilt") + || stdout.contains("not compiled into the extension"), + "{stdout}\n{stderr}" + ); +} diff --git a/crates/socket-patch-core/src/patch/copy_tree.rs b/crates/socket-patch-core/src/patch/copy_tree.rs index cc871ffcd..17afa588e 100644 --- a/crates/socket-patch-core/src/patch/copy_tree.rs +++ b/crates/socket-patch-core/src/patch/copy_tree.rs @@ -148,7 +148,7 @@ fn copy_tree_blocking_reference( /// Recursively remove a tree, retrying once after relaxing *directory* perms /// (a previously patched copy may carry read-only dir modes copied from the /// registry/cache; on unix file modes never gate unlinking). -fn force_remove_dir_all(dir: &Path) -> std::io::Result<()> { +pub(crate) fn force_remove_dir_all(dir: &Path) -> std::io::Result<()> { match std::fs::remove_dir_all(dir) { Ok(()) => Ok(()), Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()), diff --git a/crates/socket-patch-core/src/patch/gem_ext.rs b/crates/socket-patch-core/src/patch/gem_ext.rs new file mode 100644 index 000000000..92899b57b --- /dev/null +++ b/crates/socket-patch-core/src/patch/gem_ext.rs @@ -0,0 +1,1605 @@ +//! Agent-mode patches to a gem's native extension (#1996). +//! +//! Ruby loads the shared object RubyGems compiled when it installed the gem +//! (`extensions///-/…` and, with +//! `Gem.install_extension_in_lib`, its copy under the gem's `lib/`), never +//! the `ext/**/*.c` sources beside it. A patch that rewrites only those +//! sources in place leaves the vulnerable compiled code running. So when a +//! gem patch changes a native-extension build input, `apply` rebuilds the +//! extension from the patched sources with RubyGems' own builder, the step +//! `gem pristine --extensions` runs (`extconf.rb` + `make`, a Rakefile, a +//! CMake or Cargo build), against the Ruby that owns the install: +//! +//! 1. Preflight: the copy must be a RubyGems install (a `specifications/` +//! entry beside `gems/`) that declares extensions, and a Ruby whose +//! extension dir for the gem is the one on disk must be found. Otherwise +//! the purl is refused with [`UNSUPPORTED_CODE`] and nothing is written. +//! 2. The patched sources are written ([`apply_package_patch`]). +//! 3. The current build is moved aside (and, the first time, copied to a +//! backup of the original build inside the gem dir, [`BACKUP_DIR`]), and +//! RubyGems builds the extension into a fresh extension dir. +//! 4. A failed build puts the old build and the original sources back and +//! fails the purl with [`BUILD_FAILED_CODE`]. A successful one records a +//! marker ([`MARKER_FILE`]) naming the source hashes it was built from +//! and the hash of every artifact it produced. +//! +//! `vex` and `apply --check` attest such a copy only when that marker +//! matches the record's patched sources and every recorded artifact still +//! hashes as built ([`verify_native_build`]). `rollback` puts the backed-up +//! original build back ([`restore_original_build`]). + +use std::collections::{BTreeMap, HashMap}; +use std::ffi::OsString; +use std::path::{Path, PathBuf}; +use std::time::Duration; + +use serde::{Deserialize, Serialize}; + +use crate::hash::git_sha256::compute_git_sha256_from_bytes; +use crate::manifest::schema::PatchFileInfo; +use crate::patch::apply::{ + apply_file_patch_at, apply_package_patch, is_safe_relative_subpath, normalize_file_path, + ApplyResult, MismatchPolicy, PatchSources, +}; +use crate::patch::file_hash::compute_file_git_sha256_sync; + +/// The refusal when a native-extension patch cannot be rebuilt in place +/// (the vendored backend's code for the same limitation). +pub const UNSUPPORTED_CODE: &str = "native_extensions_unsupported"; +/// The extension failed to build from the patched sources; the old build +/// and the original sources were put back. +pub const BUILD_FAILED_CODE: &str = "native_extension_build_failed"; +/// Advisory: sources an earlier run patched were rebuilt into the extension +/// (a release before #1996 patched them without a rebuild). +pub const REBUILT_CODE: &str = "native_extension_rebuilt"; +/// `rollback` restored the sources but could not put the original compiled +/// extension back. +pub const RESTORE_FAILED_CODE: &str = "native_extension_restore_failed"; +/// The `vex` / `apply --check` routing tag of a copy whose sources carry +/// the patch while its compiled extension was not built from them. +pub const NOT_REBUILT_REASON: &str = "native_extension_not_rebuilt"; + +/// The build marker, in the gem dir (see [`BuildMarker`]). +pub const MARKER_FILE: &str = ".socket-patch-native-build.json"; +/// The original build's backup, in the gem dir (see [`restore_original_build`]). +pub const BACKUP_DIR: &str = ".socket-patch-native-orig"; +/// The backup's description file, inside [`BACKUP_DIR`]. +const BACKUP_INFO: &str = "backup.json"; + +/// How long one Ruby probe may take. +const PROBE_TIMEOUT: Duration = Duration::from_secs(60); +/// How long one extension build may take (nokogiri's vendored libxml2 build +/// takes minutes). +const BUILD_TIMEOUT: Duration = Duration::from_secs(30 * 60); + +/// Files RubyGems writes into an extension dir that are not build products. +const EXT_DIR_METADATA: &[&str] = &["gem.build_complete", "gem_make.out", "mkmf.log"]; + +/// Source and build-script extensions a native extension compiles. +const NATIVE_SOURCE_EXTENSIONS: &[&str] = &[ + "c", "h", "cc", "cpp", "cxx", "hh", "hpp", "hxx", "m", "y", "rl", +]; + +/// Grammar sources a gem ships already generated (racc / ragel output is a +/// `.rb` or `.c` file the gem packs): in a gem that builds no extension they +/// compile nothing at install time. +const GENERATED_GRAMMAR_EXTENSIONS: &[&str] = &["y", "rl"]; + +/// The probe/build program run by the gem's Ruby. One line: `-e` with an +/// embedded newline is fragile through a Windows command line. +const RUBY_SCRIPT: &str = "mode, path = ARGV; \ +spec = Gem::Specification.load(path) or abort(\"cannot load #{path}\"); \ +if mode == 'probe' then \ +puts \"gem_dir\\t#{spec.full_gem_path}\"; \ +puts \"extension_dir\\t#{spec.extension_dir}\"; \ +puts \"lib_dir\\t#{File.join(spec.full_gem_path, spec.raw_require_paths.first.to_s)}\"; \ +spec.extensions.each { |e| puts \"extension\\t#{e}\" }; \ +else \ +require 'rubygems/ext'; \ +Gem::DefaultUserInteraction.ui = Gem::SilentUI.new; \ +Gem::Ext::Builder.new(spec, spec.build_args).build_extensions; \ +end"; + +/// One installed RubyGems copy: `/gems/` with its +/// `/specifications/.gemspec`. +#[derive(Debug, Clone)] +struct GemCopy { + home: PathBuf, + full_name: String, + gem_dir: PathBuf, + spec_path: PathBuf, +} + +impl GemCopy { + /// The copy at `pkg_path`, when it sits in a RubyGems `gems/` dir. + /// Whether its spec exists is checked by the caller. + fn of(pkg_path: &Path) -> Option { + let full_name = pkg_path.file_name()?.to_str()?.to_string(); + let gems = pkg_path.parent()?; + if gems.file_name()? != "gems" { + return None; + } + let home = gems.parent()?.to_path_buf(); + let spec_path = home + .join("specifications") + .join(format!("{full_name}.gemspec")); + Some(GemCopy { + home, + full_name, + gem_dir: pkg_path.to_path_buf(), + spec_path, + }) + } +} + +/// What the installed gemspec says about extensions. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +struct ExtensionDecl { + /// The spec assigns `extensions`. + declared: bool, + /// The declared build scripts (`ext/foo/extconf.rb`), when the + /// assignment lists them as string literals. + entries: Vec, +} + +/// Parse `s.extensions = ["ext/foo/extconf.rb".freeze, …]` out of an +/// installed gemspec (the `Specification#to_ruby` shape RubyGems writes). +/// Comments are stripped per line; an assignment whose entries are not +/// string literals is `declared` with no `entries`. +fn parse_extensions(spec_text: &str) -> ExtensionDecl { + let mut decl = ExtensionDecl::default(); + for raw in spec_text.lines() { + let line = raw.split('#').next().unwrap_or(""); + let Some(idx) = line.find(".extensions") else { + continue; + }; + let after = &line[idx + ".extensions".len()..]; + if after + .chars() + .next() + .is_some_and(|c| c.is_ascii_alphanumeric() || c == '_') + { + continue; + } + let after = after.trim_start(); + let rhs = if let Some(rest) = after.strip_prefix("+=") { + rest + } else if let Some(rest) = after.strip_prefix("<<") { + rest + } else if let Some(rest) = after.strip_prefix('=') { + if rest.starts_with('=') { + continue; + } + rest + } else { + continue; + }; + let literals = string_literals(rhs); + let empty_list = literals.is_empty() && rhs.trim().trim_end_matches(".freeze") == "[]"; + if !empty_list { + decl.declared = true; + } + decl.entries.extend(literals); + } + decl +} + +/// The double- and single-quoted string literals in `text`, in order. +fn string_literals(text: &str) -> Vec { + let mut out = Vec::new(); + let mut chars = text.chars(); + while let Some(c) = chars.next() { + if c != '"' && c != '\'' { + continue; + } + let mut lit = String::new(); + let mut escaped = false; + for d in chars.by_ref() { + if escaped { + lit.push(d); + escaped = false; + } else if d == '\\' { + escaped = true; + } else if d == c { + break; + } else { + lit.push(d); + } + } + out.push(lit); + } + out +} + +/// Whether a (normalized) gem file path is a native-extension build input by +/// its name alone: a C/C++/Objective-C source or header, a racc/ragel +/// grammar, an `extconf.rb`, or a `depend` file under `ext/`. +fn is_native_by_name(path: &str) -> bool { + let leaf = path.rsplit('/').next().unwrap_or(path); + if leaf == "extconf.rb" || (leaf == "depend" && path.starts_with("ext/")) { + return true; + } + leaf.rsplit_once('.').is_some_and(|(stem, ext)| { + !stem.is_empty() && NATIVE_SOURCE_EXTENSIONS.contains(&ext.to_ascii_lowercase().as_str()) + }) +} + +fn is_generated_grammar(path: &str) -> bool { + path.rsplit_once('.').is_some_and(|(_, ext)| { + GENERATED_GRAMMAR_EXTENSIONS.contains(&ext.to_ascii_lowercase().as_str()) + }) +} + +/// A gem record's native-extension build inputs on one installed copy. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +struct NativeScope { + /// The record's keys that are build inputs, sorted. + inputs: Vec, + /// The installed spec declares extensions (RubyGems builds them at + /// install time). `false` with inputs left: the gem's compiled code, if + /// any, ships precompiled. + builds_extensions: bool, +} + +/// Classify `files` against the extension declaration `decl`: a key is a +/// build input by name ([`is_native_by_name`]) or by living under the dir +/// of a declared build script (a Rust extension's `.rs` sources, a +/// `Rakefile`-built tree). A declaration whose scripts are unknown makes +/// everything under `ext/` an input. In a gem that builds no extension, +/// grammar sources are not inputs. +fn classify(decl: &ExtensionDecl, files: &HashMap) -> NativeScope { + let dirs: Vec = decl + .entries + .iter() + .map(|e| { + let e = e.trim_start_matches("./"); + match e.rsplit_once('/') { + Some((dir, _)) => format!("{dir}/"), + None => String::new(), + } + }) + .collect(); + let unknown_scripts = decl.declared && decl.entries.is_empty(); + let mut inputs: Vec = files + .keys() + .filter(|key| { + let path = normalize_file_path(key); + let native = is_native_by_name(path) + || dirs.iter().any(|d| path.starts_with(d.as_str())) + || (unknown_scripts && path.starts_with("ext/")); + native && (decl.declared || !is_generated_grammar(path)) + }) + .cloned() + .collect(); + inputs.sort(); + NativeScope { + inputs, + builds_extensions: decl.declared, + } +} + +/// The record's native scope on the copy at `pkg_path`, and the copy when it +/// is a RubyGems install whose spec could be read. `None` scope inputs mean +/// the record is plain Ruby for this copy. +async fn scope_of( + pkg_path: &Path, + files: &HashMap, +) -> (NativeScope, Option) { + let copy = GemCopy::of(pkg_path); + let spec_text = match © { + Some(c) => crate::utils::fs::read_regular_to_string(&c.spec_path) + .await + .ok(), + None => None, + }; + let decl = spec_text + .as_deref() + .map(parse_extensions) + .unwrap_or_default(); + let scope = classify(&decl, files); + // Without a readable spec nothing proves which scripts the gem builds; + // name-matched inputs still count, and the copy is unusable for a build. + (scope, copy.filter(|_| spec_text.is_some())) +} + +/// The build marker written into the gem dir after a rebuild. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct BuildMarker { + pub schema: u32, + /// Native input (normalized path) → its git-sha256 when it was built. + pub sources: BTreeMap, + /// The extension dir built into: relative to the gem home when under it, + /// otherwise absolute. + pub extension_dir: String, + /// Artifact path relative to the extension dir → git-sha256. + pub ext_artifacts: BTreeMap, + /// The artifacts' copies RubyGems put under the gem's require dir: + /// path relative to the gem dir → git-sha256. + pub lib_artifacts: BTreeMap, +} + +const MARKER_SCHEMA: u32 = 1; + +impl BuildMarker { + fn extension_dir_in(&self, home: &Path) -> PathBuf { + let p = Path::new(&self.extension_dir); + if p.is_absolute() { + p.to_path_buf() + } else { + home.join(p) + } + } +} + +fn read_marker_sync(gem_dir: &Path) -> Option { + let text = crate::utils::fs::read_regular_to_string_sync(&gem_dir.join(MARKER_FILE)).ok()?; + serde_json::from_str::(&text) + .ok() + .filter(|m| m.schema == MARKER_SCHEMA) +} + +/// Whether `marker` was built from exactly the record's patched native +/// inputs, and every artifact it recorded is still on disk with the bytes it +/// was built with. An artifact path that escapes its dir never verifies. +fn marker_holds( + marker: &BuildMarker, + copy: &GemCopy, + scope: &NativeScope, + files: &HashMap, +) -> bool { + let sources_match = scope.inputs.iter().all(|key| { + marker.sources.get(normalize_file_path(key)) == files.get(key).map(|i| &i.after_hash) + }); + if !sources_match || marker.sources.len() != scope.inputs.len() { + return false; + } + let ext_dir = marker.extension_dir_in(©.home); + let artifact_ok = |base: &Path, rel: &str, hash: &str| { + is_safe_relative_subpath(rel) + && compute_file_git_sha256_sync(&base.join(rel)).is_ok_and(|h| h == hash) + }; + marker + .ext_artifacts + .iter() + .all(|(rel, hash)| artifact_ok(&ext_dir, rel, hash)) + && marker + .lib_artifacts + .iter() + .all(|(rel, hash)| artifact_ok(©.gem_dir, rel, hash)) +} + +/// `vex` / `apply --check`: the installed gem copy at `pkg_path`, whose +/// record files already verify, runs code built from them. A record with no +/// native input on this copy passes; one with native inputs passes only when +/// the copy's build marker holds ([`BuildMarker`]). Errors carry +/// [`NOT_REBUILT_REASON`]. +pub async fn verify_native_build( + pkg_path: &Path, + files: &HashMap, +) -> Result<(), String> { + let (scope, copy) = scope_of(pkg_path, files).await; + if scope.inputs.is_empty() { + return Ok(()); + } + let Some(copy) = copy.filter(|_| scope.builds_extensions) else { + return Err(NOT_REBUILT_REASON.to_string()); + }; + let files = files.clone(); + let holds = crate::utils::fs::run_blocking(move || { + read_marker_sync(©.gem_dir).is_some_and(|m| marker_holds(&m, ©, &scope, &files)) + }) + .await; + if holds { + Ok(()) + } else { + Err(NOT_REBUILT_REASON.to_string()) + } +} + +/// [`apply_gem_package_patch`]'s outcome: the copy's result, and a run +/// warning `(code, detail)` the caller surfaces next to it. +#[derive(Debug, Clone)] +pub struct GemApplied { + pub result: ApplyResult, + pub warning: Option<(String, String)>, +} + +/// Where the Ruby that builds a copy's extension is looked for and run. +#[derive(Debug, Clone)] +pub struct RubyEnv { + /// The project dir the Ruby on `PATH` runs from (a version manager's + /// `.ruby-version` there picks the project's Ruby). + pub cwd: PathBuf, +} + +/// What a Ruby reports about a gem copy. +#[derive(Debug, Clone, PartialEq, Eq)] +struct Probe { + ruby: PathBuf, + gem_dir: PathBuf, + extension_dir: PathBuf, + lib_dir: PathBuf, + extensions: Vec, +} + +fn parse_probe(ruby: &Path, stdout: &str) -> Option { + let mut gem_dir = None; + let mut extension_dir = None; + let mut lib_dir = None; + let mut extensions = Vec::new(); + for line in stdout.lines() { + let (key, value) = line.split_once('\t')?; + match key { + "gem_dir" => gem_dir = Some(PathBuf::from(value)), + "extension_dir" => extension_dir = Some(PathBuf::from(value)), + "lib_dir" => lib_dir = Some(PathBuf::from(value)), + "extension" => extensions.push(value.to_string()), + _ => {} + } + } + Some(Probe { + ruby: ruby.to_path_buf(), + gem_dir: gem_dir?, + extension_dir: extension_dir?, + lib_dir: lib_dir?, + extensions, + }) +} + +/// The Rubies that may own `copy`, best first: the one installed beside a +/// `/lib/ruby/gems/` home (rbenv, asdf, rvm, a system Ruby), +/// then the `ruby` on `PATH`. A prefix inside the project (`project`) is +/// never one: the scanned repository does not choose the program run. +fn ruby_candidates(copy: &GemCopy, project: &Path) -> Vec { + let mut out = Vec::new(); + let exe = if cfg!(windows) { "ruby.exe" } else { "ruby" }; + let mut parts = copy.home.ancestors(); + parts.next(); // + if let (Some(gems), Some(ruby), Some(lib), Some(prefix)) = + (parts.next(), parts.next(), parts.next(), parts.next()) + { + if gems.file_name().is_some_and(|n| n == "gems") + && ruby.file_name().is_some_and(|n| n == "ruby") + && lib.file_name().is_some_and(|n| n == "lib") + { + let candidate = prefix.join("bin").join(exe); + let in_project = match (canonical(prefix), canonical(project)) { + (Some(prefix), Some(project)) => prefix.starts_with(project), + _ => prefix.starts_with(project), + }; + if !in_project && candidate.is_file() { + out.push(candidate); + } + } + } + if let Some(on_path) = crate::utils::process::resolve_tool("ruby") { + if !out.contains(&on_path) { + out.push(on_path); + } + } + out +} + +/// The `GEM_PATH` a build runs with: the copy's home first (an `extconf.rb` +/// requiring `mini_portile2` or `rb_sys` finds the copy installed beside +/// it), then the caller's, then RubyGems' defaults (the trailing separator). +fn gem_path_for(home: &Path) -> Option { + let mut paths = vec![home.to_path_buf()]; + if let Some(existing) = std::env::var_os("GEM_PATH") { + paths.extend(std::env::split_paths(&existing).filter(|p| !p.as_os_str().is_empty())); + } + let mut joined = std::env::join_paths(paths).ok()?; + joined.push(if cfg!(windows) { ";" } else { ":" }); + Some(joined) +} + +/// Run the Ruby program in `mode` (`probe` / `build`) on `copy`. `Ok` +/// carries stdout; `Err` a short reason (stderr's tail on a failed exit). +async fn run_ruby( + ruby: &Path, + mode: &str, + copy: &GemCopy, + env: &RubyEnv, + budget: Duration, +) -> Result { + let mut command = tokio::process::Command::from(crate::utils::process::command_for(ruby)); + command + .arg("-e") + .arg(RUBY_SCRIPT) + .arg(mode) + .arg(©.spec_path) + .current_dir(&env.cwd) + .stdin(std::process::Stdio::null()) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .kill_on_drop(true); + // A Bundler-run parent (`bundle exec`, the Bundler plugin) must not + // load the project's bundle into this Ruby. + for (key, _) in std::env::vars_os() { + let key_str = key.to_string_lossy(); + if key_str == "RUBYOPT" || key_str == "RUBYLIB" || key_str.starts_with("BUNDLE") { + command.env_remove(&key); + } + } + if let Some(gem_path) = gem_path_for(©.home) { + command.env("GEM_PATH", gem_path); + } + let output = match tokio::time::timeout(budget, command.output()).await { + Err(_) => return Err(format!("{} {mode} timed out", ruby.display())), + Ok(Err(e)) => return Err(format!("could not run {}: {e}", ruby.display())), + Ok(Ok(output)) => output, + }; + if output.status.success() { + Ok(String::from_utf8_lossy(&output.stdout).into_owned()) + } else { + Err(stderr_tail(&String::from_utf8_lossy(&output.stderr))) + } +} + +/// The useful end of a failed build's stderr: RubyGems' trailing +/// "Gem files will remain installed" boilerplate dropped, at most 20 lines. +fn stderr_tail(stderr: &str) -> String { + let text = stderr + .split("\nGem files will remain installed") + .next() + .unwrap_or(stderr); + let lines: Vec<&str> = text + .lines() + .map(str::trim_end) + .filter(|l| !l.is_empty()) + .collect(); + let start = lines.len().saturating_sub(20); + let tail = lines[start..].join("\n"); + if tail.is_empty() { + "the build exited with an error and no output".to_string() + } else { + tail + } +} + +/// The extension dirs of `full_name` under `home/extensions/*/*/`. +fn installed_extension_dirs(copy: &GemCopy) -> Vec { + let mut out = Vec::new(); + let Ok(platforms) = std::fs::read_dir(copy.home.join("extensions")) else { + return out; + }; + for platform in platforms.flatten() { + let Ok(abis) = std::fs::read_dir(platform.path()) else { + continue; + }; + for abi in abis.flatten() { + let dir = abi.path().join(©.full_name); + if dir.is_dir() { + out.push(dir); + } + } + } + out.sort(); + out +} + +fn same_path(a: &Path, b: &Path) -> bool { + match (canonical(a), canonical(b)) { + (Some(a), Some(b)) => a == b, + _ => a == b, + } +} + +fn canonical(p: &Path) -> Option { + std::fs::canonicalize(p).ok() +} + +/// Find the Ruby that owns `copy`'s install: one whose spec resolves to +/// this gem dir and whose extension dir for it is the one on disk (any, when +/// the gem has none yet). `Err` names why none qualifies. +async fn find_ruby(copy: &GemCopy, env: &RubyEnv) -> Result { + let candidates = ruby_candidates(copy, &env.cwd); + if candidates.is_empty() { + return Err("no `ruby` was found on PATH to rebuild it with".to_string()); + } + let installed = { + let copy = copy.clone(); + crate::utils::fs::run_blocking(move || installed_extension_dirs(©)).await + }; + let mut reasons = Vec::new(); + for ruby in candidates { + let stdout = match run_ruby(&ruby, "probe", copy, env, PROBE_TIMEOUT).await { + Ok(s) => s, + Err(e) => { + reasons.push(format!("{}: {e}", ruby.display())); + continue; + } + }; + let Some(probe) = parse_probe(&ruby, &stdout) else { + reasons.push(format!("{}: unreadable probe output", ruby.display())); + continue; + }; + if !same_path(&probe.gem_dir, ©.gem_dir) { + reasons.push(format!( + "{} resolves the gem to {}", + ruby.display(), + probe.gem_dir.display() + )); + continue; + } + let matches_disk = installed.is_empty() + || probe.extension_dir.is_dir() + || installed.iter().any(|d| same_path(d, &probe.extension_dir)); + if !matches_disk { + reasons.push(format!( + "{} builds into {}, but the installed extension is {} (another Ruby's)", + ruby.display(), + probe.extension_dir.display(), + installed + .iter() + .map(|d| d.display().to_string()) + .collect::>() + .join(", ") + )); + continue; + } + if probe.extensions.is_empty() { + reasons.push(format!( + "{} reports no extensions for the gem", + ruby.display() + )); + continue; + } + return Ok(probe); + } + Err(format!( + "no Ruby owning this install was found ({})", + reasons.join("; ") + )) +} + +/// Every regular file under `dir`, relative, `/`-separated, sorted; +/// RubyGems' metadata files at the top level and the stage dirs a build +/// leaves are skipped. +fn artifact_rels(dir: &Path) -> Vec { + let mut out = Vec::new(); + for entry in walkdir::WalkDir::new(dir) + .follow_links(false) + .sort_by_file_name() + .into_iter() + .flatten() + { + if !entry.file_type().is_file() { + continue; + } + let Ok(rel) = entry.path().strip_prefix(dir) else { + continue; + }; + let rel = rel + .components() + .map(|c| c.as_os_str().to_string_lossy().into_owned()) + .collect::>() + .join("/"); + if EXT_DIR_METADATA.contains(&rel.as_str()) || rel == MARKER_FILE { + continue; + } + out.push(rel); + } + out +} + +/// An extension-dir artifact's copy in the gem's require dir, relative to +/// the gem dir (`a/a.so` → `lib/a/a.so`). `None` when the require dir is +/// not inside the gem dir. +fn lib_rel(probe: &Probe, rel: &str) -> Option { + let lib = probe.lib_dir.strip_prefix(&probe.gem_dir).ok()?; + let s = lib + .join(rel) + .components() + .map(|c| c.as_os_str().to_string_lossy().into_owned()) + .collect::>() + .join("/"); + is_safe_relative_subpath(&s).then_some(s) +} + +fn copy_tree_sync(from: &Path, to: &Path) -> std::io::Result<()> { + for entry in walkdir::WalkDir::new(from).follow_links(false) { + let entry = entry.map_err(std::io::Error::other)?; + let rel = entry + .path() + .strip_prefix(from) + .map_err(std::io::Error::other)?; + let target = to.join(rel); + if entry.file_type().is_dir() { + std::fs::create_dir_all(&target)?; + } else if entry.file_type().is_file() { + if let Some(parent) = target.parent() { + std::fs::create_dir_all(parent)?; + } + std::fs::copy(entry.path(), &target)?; + } + } + Ok(()) +} + +fn remove_dir_if_exists(dir: &Path) -> std::io::Result<()> { + match std::fs::symlink_metadata(dir) { + Ok(_) => crate::patch::copy_tree::force_remove_dir_all(dir), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(e) => Err(e), + } +} + +/// The backup's description. +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +struct BackupInfo { + /// As [`BuildMarker::extension_dir`]. + extension_dir: String, + /// The extension dir existed (its tree is under `ext/`). + had_extension_dir: bool, + /// The lib copies backed up under `lib/` (paths relative to the gem dir). + lib_files: Vec, +} + +/// [`BuildMarker::extension_dir`] for `ext_dir`: relative to `home` when it +/// is under it, compared canonically too (Ruby and the crawler may spell +/// the same dir differently: a relative crawl root, `/tmp` vs +/// `/private/tmp`), so the record survives the project moving. +fn ext_dir_label(home: &Path, ext_dir: &Path) -> String { + let canonical_rel = || { + let (home, ext) = (canonical(home)?, canonical(ext_dir)?); + ext.strip_prefix(&home).ok().map(Path::to_path_buf) + }; + let rel = ext_dir + .strip_prefix(home) + .ok() + .map(Path::to_path_buf) + .or_else(canonical_rel); + match rel.as_deref().ok_or(()) { + Ok(rel) => rel + .components() + .map(|c| c.as_os_str().to_string_lossy().into_owned()) + .collect::>() + .join("/"), + Err(_) => ext_dir.display().to_string(), + } +} + +/// Back up the original build of `copy` (its extension dir and the lib +/// copies of its artifacts) to [`BACKUP_DIR`], unless a backup exists or the +/// current build is one socket-patch made (a marker exists: its original is +/// gone). Staged and renamed into place, so a backup dir is always whole. +/// `Ok(true)` when this call made the backup. +fn back_up_original(copy: &GemCopy, probe: &Probe) -> std::io::Result { + let backup = copy.gem_dir.join(BACKUP_DIR); + if backup.exists() || copy.gem_dir.join(MARKER_FILE).exists() { + return Ok(false); + } + let stage = copy.gem_dir.join(format!("{BACKUP_DIR}.tmp")); + remove_dir_if_exists(&stage)?; + std::fs::create_dir_all(&stage)?; + let had_extension_dir = probe.extension_dir.is_dir(); + let mut lib_files = Vec::new(); + if had_extension_dir { + copy_tree_sync(&probe.extension_dir, &stage.join("ext"))?; + for rel in artifact_rels(&probe.extension_dir) { + let Some(lib) = lib_rel(probe, &rel) else { + continue; + }; + let src = copy.gem_dir.join(&lib); + if src.is_file() { + let dst = stage.join("lib").join(&lib); + if let Some(parent) = dst.parent() { + std::fs::create_dir_all(parent)?; + } + std::fs::copy(&src, &dst)?; + lib_files.push(lib); + } + } + } + let info = BackupInfo { + extension_dir: ext_dir_label(©.home, &probe.extension_dir), + had_extension_dir, + lib_files, + }; + std::fs::write( + stage.join(BACKUP_INFO), + serde_json::to_vec_pretty(&info).map_err(std::io::Error::other)?, + )?; + std::fs::rename(&stage, &backup)?; + Ok(true) +} + +/// The lib copies of an extension dir's artifacts, read into memory: +/// `(gem-dir-relative path, bytes)`. +fn snapshot_lib_copies(copy: &GemCopy, probe: &Probe, rels: &[String]) -> Vec<(String, Vec)> { + rels.iter() + .filter_map(|rel| lib_rel(probe, rel)) + .filter_map(|lib| { + let bytes = std::fs::read(copy.gem_dir.join(&lib)).ok()?; + Some((lib, bytes)) + }) + .collect() +} + +/// State a build moved aside, to put back if it fails. +struct Aside { + /// The previous extension dir, renamed beside itself. + old_ext: Option, + /// The previous build's lib copies. + lib: Vec<(String, Vec)>, + /// The previous marker's bytes. + marker: Option>, +} + +/// Move the current build out of the way of a fresh one. +fn set_aside(copy: &GemCopy, probe: &Probe) -> std::io::Result