diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7040d2ede..a6b071c44 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1271,8 +1271,10 @@ jobs: # Ruby <= 3.3 and 1.17-2.1 need Ruby <= 3.1 (`untaint`). 2.7.2 # (no boundary of its own) is in e2e-full. # Hosted and vendored share each era's leg (one Ruby + bundler - # setup; the run step loops over `suite`). - - {os: ubuntu-latest, suite: e2e_redirect_gem_build e2e_vendor_gem_build, ruby: '3.4', bundler: '4.0.21'} + # setup; the run step loops over `suite`), as does the agent-mode + # C-extension rebuild capstone (#1996), which needs the same Ruby + # plus the runner's C compiler. + - {os: ubuntu-latest, suite: e2e_redirect_gem_build e2e_vendor_gem_build e2e_gem_native_ext_build, ruby: '3.4', bundler: '4.0.21'} # The live-API smoke suites (e2e_npm, e2e_pypi, e2e_gem, # e2e_scan) are intentionally NOT in the PR matrix — their # `#[ignore]`-gated tests hit the real public proxy at @@ -2008,7 +2010,7 @@ jobs: # composer 1.10 keeps the PHP 8.2 it has always run on here # (composer-compatibility.yml runs 1.10.28 on 8.1). - {os: ubuntu-latest, suite: e2e_vendor_composer_build e2e_redirect_composer_build, composer: '1', php: '8.2'} - - {os: ubuntu-latest, suite: e2e_redirect_gem_build e2e_vendor_gem_build, ruby: '3.1', bundler: '1.17.3'} + - {os: ubuntu-latest, suite: e2e_redirect_gem_build e2e_vendor_gem_build e2e_gem_native_ext_build, ruby: '3.1', bundler: '1.17.3'} - {os: ubuntu-latest, suite: e2e_redirect_gem_build e2e_vendor_gem_build, ruby: '3.1', bundler: '2.1.4'} - {os: ubuntu-latest, suite: e2e_redirect_gem_build e2e_vendor_gem_build, ruby: '3.1', bundler: '2.2.33'} - {os: ubuntu-latest, suite: e2e_redirect_gem_build e2e_vendor_gem_build, ruby: '3.3', bundler: '2.5.23'} diff --git a/crates/socket-patch-cli/CLI_CONTRACT.md b/crates/socket-patch-cli/CLI_CONTRACT.md index 0c5ec2d95..a3ed909d6 100644 --- a/crates/socket-patch-cli/CLI_CONTRACT.md +++ b/crates/socket-patch-cli/CLI_CONTRACT.md @@ -705,6 +705,49 @@ are patched by `socket-patch apply` like every other ecosystem: or a bare `--global-prefix` cache with no known `go install` directory) the same warning code says so, and VEX is not withheld (#422). +### Ruby gems with native extensions in agent mode + +Ruby loads the extension RubyGems compiled at install time +(`extensions///-/`, and its copy under the gem's `lib/`), never +the `ext/` sources beside it. When a gem record changes a native-extension build input, `apply` +writes the patched sources and then rebuilds the extension from them with RubyGems' own builder +(the step `gem pristine --extensions` runs), using the Ruby that owns the install: the one in +`/bin` of a `/lib/ruby/gems/` home, else the `ruby` on `PATH` run from +`--cwd`. A build input is a changed `.c`, `.h`, `.cc`, `.cpp`, `.cxx`, `.hh`, `.hpp`, `.hxx`, +`.m`, `.y` or `.rl` file, an `extconf.rb`, a `depend` under `ext/`, or any file under the +directory of a build script the installed gemspec's `extensions` names (a script at the gem root: +the files beside it) (#1996). In a gem that declares no extensions, only such a source under +`ext/` is one (where a precompiled platform gem keeps the sources of the binary it ships), and +never a grammar file (`.y`, `.rl`), whose generated output the gem packs. The Ruby must build into +the gem's installed `extensions///` dir; a gem without one (installed by a RubyGems +older than 2.2) is refused. The rebuild runs the gem's own build scripts, as `gem install` did. + +The first rebuild of a copy backs up the install-time build (extension directory and `lib/` +copies) to `.socket-patch-native-orig/` in the gem directory, and every rebuild records +`.socket-patch-native-build.json` there: the source hashes it was built from and the hash of every +artifact it produced. A record whose sources are patched and whose marker matches is +`already_patched`; sources an earlier release patched without a rebuild are rebuilt, with a +`native_extension_rebuilt` warning. `rollback` (and `remove`) puts the backed-up build back with +the sources; without a backup it rebuilds from the restored sources. Only the gem's own +extension dir under its home is ever replaced, whatever path the backup or marker names. + +`--no-gem-native-rebuild` (env `SOCKET_NO_GEM_NATIVE_REBUILD`) is for a caller that compiles the +extension itself (the patches evaluation harness): the patched sources are written as before #1996, +with no Ruby run and no backup or marker, and the run succeeds with a `native_extension_not_rebuilt` +warning. `vex` and `apply --check` then omit the patch until a rebuild by `apply` records it. + +| Code | Run result | Meaning | +|---|---|---| +| `native_extensions_unsupported` | refused, exit 1 | The record changes build inputs that cannot be rebuilt in place: the copy is not a RubyGems install with a `specifications/` entry (a Bundler git checkout), the gem declares no extensions (a precompiled platform gem), or no Ruby whose extension directory for the gem is the installed one was found. Nothing is written; use `scan --mode hosted`, where RubyGems compiles the patched gem. | +| `native_extension_build_failed` | refused, exit 1 | The extension did not build from the patched sources (no compiler or Ruby headers, a build error). The previous build and the original sources are put back; the error carries the build output's tail. | +| `native_extension_rebuilt` | warning | The sources were already patched (by a release before #1996) and the extension was rebuilt from them. | +| `native_extension_not_rebuilt` | warning | `--no-gem-native-rebuild`: the sources were written without a rebuild; Ruby loads the old build until the caller compiles it. | +| `native_extension_restore_failed` | rollback: that copy fails | The sources were restored, but the original compiled extension could not be put back. Run `gem pristine --extensions`. | + +`vex` and `apply --check` attest such a copy only while its marker matches the record's patched +sources and every artifact it lists; otherwise the purl fails with `native_extension_not_rebuilt`. +Hosted copies are judged on their files alone, since RubyGems compiled them from the patched gem. + ### sbt / Mill / scala-cli caches in agent mode A JVM project root (any `JVM_PROJECT_MARKERS` file, `build.sbt` / `build.mill` / `build.sc` / diff --git a/crates/socket-patch-cli/src/args.rs b/crates/socket-patch-cli/src/args.rs index 05f4217ee..128e19687 100644 --- a/crates/socket-patch-cli/src/args.rs +++ b/crates/socket-patch-cli/src/args.rs @@ -350,6 +350,21 @@ pub struct GlobalArgs { value_parser = parse_bool_flag, )] pub no_vlt_install_cleanup: bool, + + /// Agent mode (`apply`, `scan`/`get --mode agent`): write a gem patch + /// that changes native-extension sources WITHOUT rebuilding the + /// extension, for a caller that compiles it itself. Ruby keeps loading + /// the old build until then, so `vex` and `apply --check` do not attest + /// those patches (`native_extension_not_rebuilt`). Other subcommands + /// accept it silently. + #[arg( + help_heading = GLOBAL_OPTIONS, + long = "no-gem-native-rebuild", + env = "SOCKET_NO_GEM_NATIVE_REBUILD", + default_value_t = false, + value_parser = parse_bool_flag, + )] + pub no_gem_native_rebuild: bool, } impl GlobalArgs { @@ -641,6 +656,7 @@ pub const GLOBAL_ARG_ENV_VARS: &[&str] = &[ "SOCKET_NO_TRUST_LOCKFILE_CONFIG", "SOCKET_NO_NPM_ALLOW_REMOTE_CONFIG", "SOCKET_NO_VLT_INSTALL_CLEANUP", + "SOCKET_NO_GEM_NATIVE_REBUILD", ]; /// Every env var a subcommand-local flag binds (one per `env = "..."` @@ -709,6 +725,7 @@ impl Default for GlobalArgs { no_trust_lockfile_config: false, no_npm_allow_remote_config: false, no_vlt_install_cleanup: false, + no_gem_native_rebuild: false, } } } diff --git a/crates/socket-patch-cli/src/commands/apply.rs b/crates/socket-patch-cli/src/commands/apply.rs index d7b8f728e..4a3732800 100644 --- a/crates/socket-patch-cli/src/commands/apply.rs +++ b/crates/socket-patch-cli/src/commands/apply.rs @@ -12,6 +12,7 @@ use socket_patch_core::patch::apply::{ apply_package_patch, verify_file_patch, ApplyResult, MismatchPolicy, PatchSources, VerifyStatus, }; use socket_patch_core::patch::apply_lock::LockGuard; +use socket_patch_core::patch::gem_ext; use socket_patch_core::patch::redirect::golang_local::{ apply_go_redirect, reconcile_go_redirects, verify_go_redirect_state, }; @@ -717,6 +718,10 @@ fn describe_check_failure(reason: &str) -> &'static str { "hash_mismatch" => "an installed copy matches neither the original nor the patched bytes", "file_not_found" => "a patched file is missing from an installed copy", "no_files" => "the patch record lists no files to verify", + "native_extension_not_rebuilt" => { + "the patched native-extension sources were not compiled into the extension \ + Ruby loads" + } "no_matching_variant" => { "an installed copy matches none of the manifest's release variants \ (no matching variant found)" @@ -2703,15 +2708,42 @@ async fn apply_patches_inner( } attempted = true; - let result = apply_package_patch( - variant_purl, - pkg_path, - &patch.files, - &sources, - args.common.dry_run, - policy, - ) - .await; + // A gem patch to native-extension sources rebuilds the + // extension, or is refused with nothing written (#1996). + let result = if Ecosystem::from_purl(purl) == Some(Ecosystem::Gem) { + let applied = gem_ext::apply_gem_package_patch( + variant_purl, + pkg_path, + &patch.files, + &sources, + args.common.dry_run, + policy, + &gem_ext::RubyEnv { + cwd: args.common.cwd.clone(), + rebuild: !args.common.no_gem_native_rebuild, + }, + ) + .await; + // A best-effort fallback-home copy's refusal is + // its per-copy skip below, not a run warning. + if let Some((code, detail)) = applied + .warning + .filter(|_| applied.result.success || !best_effort) + { + run_warnings.push(RunWarning { code, detail }); + } + applied.result + } else { + apply_package_patch( + variant_purl, + pkg_path, + &patch.files, + &sources, + args.common.dry_run, + policy, + ) + .await + }; warn_mismatch_overwrites(&result, &args.common); // A variant that reached apply is the installed distribution diff --git a/crates/socket-patch-cli/src/commands/rollback.rs b/crates/socket-patch-cli/src/commands/rollback.rs index 6d9a3c040..78ac53026 100644 --- a/crates/socket-patch-cli/src/commands/rollback.rs +++ b/crates/socket-patch-cli/src/commands/rollback.rs @@ -3276,14 +3276,32 @@ pub(crate) async fn rollback_patches_inner( match try_rollback_local_go(purl, pkg_path, patch, common).await { Some(r) => r, None => { - rollback_package_patch( + let mut result = rollback_package_patch( purl, pkg_path, &patch.files, &blobs_path, common.dry_run, ) - .await + .await; + // A gem whose native extension `apply` rebuilt gets its + // original compiled extension back with its sources + // (#1996). + if result.success + && !common.dry_run + && Ecosystem::from_purl(purl) == Some(Ecosystem::Gem) + { + use socket_patch_core::patch::gem_ext; + let env = gem_ext::RubyEnv { + cwd: common.cwd.clone(), + rebuild: !common.no_gem_native_rebuild, + }; + if let Err(e) = gem_ext::restore_original_build(pkg_path, &env).await { + result.success = false; + result.error = Some(format!("{}: {e}", gem_ext::RESTORE_FAILED_CODE)); + } + } + result } } }; diff --git a/crates/socket-patch-cli/src/commands/vex.rs b/crates/socket-patch-cli/src/commands/vex.rs index 78f8111d5..65acda5d0 100644 --- a/crates/socket-patch-cli/src/commands/vex.rs +++ b/crates/socket-patch-cli/src/commands/vex.rs @@ -2091,6 +2091,10 @@ fn omission_phrase(reason: &str) -> &'static str { against" } "stale_install" => "the installed copy is not patched", + "native_extension_not_rebuilt" => { + "the patched native-extension sources were not compiled into the extension Ruby \ + loads; run socket-patch apply to rebuild it" + } DEPENDENCY_NOT_INSTALLED => { "the patched package.json declares a dependency that is not installed beside \ the package" diff --git a/crates/socket-patch-cli/tests/cli_global_args.rs b/crates/socket-patch-cli/tests/cli_global_args.rs index f7cb2c710..daba0bc31 100644 --- a/crates/socket-patch-cli/tests/cli_global_args.rs +++ b/crates/socket-patch-cli/tests/cli_global_args.rs @@ -126,6 +126,9 @@ fn global_flag_cases() -> Vec<(&'static str, Option<&'static str>, fn(&GlobalArg ("--no-vlt-install-cleanup", None, |c| { assert!(c.no_vlt_install_cleanup) }), + ("--no-gem-native-rebuild", None, |c| { + assert!(c.no_gem_native_rebuild) + }), ("--lock-timeout", Some("30"), |c| { assert_eq!(c.lock_timeout, Some(30)) }), @@ -233,6 +236,7 @@ fn global_flag_cases_cover_every_global_field() { no_trust_lockfile_config: _, no_npm_allow_remote_config: _, no_vlt_install_cleanup: _, + no_gem_native_rebuild: _, strict: _, allow_patch_conflicts: _, vendor_source: _, @@ -241,11 +245,11 @@ fn global_flag_cases_cover_every_global_field() { maven_config: _, } = common; - // 27 fields ↔ 27 long-flag cases. Bump both this count and add a case when + // 28 fields ↔ 28 long-flag cases. Bump both this count and add a case when // the destructure above forces you to add a field. assert_eq!( global_flag_cases().len(), - 27, + 28, "every GlobalArgs field needs a long-flag case in global_flag_cases()", ); } @@ -592,7 +596,7 @@ fn bool_env_vars_reject_zero_and_falsey() { #[serial_test::serial] fn empty_bool_env_var_resolves_to_false_not_crash() { // (env var, accessor) for every boolean global. - let bool_vars: [(&str, fn(&GlobalArgs) -> bool); 13] = [ + let bool_vars: [(&str, fn(&GlobalArgs) -> bool); 14] = [ ("SOCKET_OFFLINE", |c| c.offline), ("SOCKET_STRICT", |c| c.strict), ("SOCKET_GLOBAL", |c| c.global), @@ -612,6 +616,7 @@ fn empty_bool_env_var_resolves_to_false_not_crash() { ("SOCKET_NO_VLT_INSTALL_CLEANUP", |c| { c.no_vlt_install_cleanup }), + ("SOCKET_NO_GEM_NATIVE_REBUILD", |c| c.no_gem_native_rebuild), ]; for (var, accessor) in bool_vars { diff --git a/crates/socket-patch-cli/tests/common/clap_parse.rs b/crates/socket-patch-cli/tests/common/clap_parse.rs index 6c0d42c9e..39976052d 100644 --- a/crates/socket-patch-cli/tests/common/clap_parse.rs +++ b/crates/socket-patch-cli/tests/common/clap_parse.rs @@ -79,6 +79,7 @@ pub fn global_defaults() -> Globals { no_trust_lockfile_config: false, no_npm_allow_remote_config: false, no_vlt_install_cleanup: false, + no_gem_native_rebuild: false, }) } diff --git a/crates/socket-patch-cli/tests/e2e_gem_native_ext_build.rs b/crates/socket-patch-cli/tests/e2e_gem_native_ext_build.rs new file mode 100644 index 000000000..78cecd730 --- /dev/null +++ b/crates/socket-patch-cli/tests/e2e_gem_native_ext_build.rs @@ -0,0 +1,501 @@ +//! Agent-mode patches to a gem's C extension (#1996). +//! +//! Ruby loads the extension RubyGems compiled at install time, never the +//! `ext/**/*.c` beside it, so `apply` must rebuild the extension from the +//! patched sources (or refuse, writing nothing), `apply --check` / `vex` +//! must not attest patched sources over a stale build, and `rollback` must +//! put the original build back. +//! +//! The capstones build a tiny C-extension gem (`sp_native`, whose +//! `SpNative.value` the patch changes from 1 to 2) with the host's real +//! `gem`, install it into a Bundler-style `vendor/bundle/ruby/` home, +//! and run the real binary over it. They need ruby, its headers and a C +//! compiler: `#[ignore]`-gated, run by the pinned-Ruby CI leg, which sets +//! `SOCKET_PATCH_BUNDLER_E2E_REQUIRED=1` so a missing toolchain fails +//! instead of skipping. The refusal tests lay a native gem down by hand +//! and need no Ruby. + +use std::path::{Path, PathBuf}; +use std::process::Command; + +#[path = "common/mod.rs"] +mod common; +use common::{git_sha256, parse_json_envelope, write_blob, write_minimal_manifest, PatchEntry}; + +const PURL: &str = "pkg:gem/sp_native@1.0.0"; +const UUID: &str = "19961996-1996-4996-8996-199619961996"; +const FULL_NAME: &str = "sp_native-1.0.0"; +const C_KEY: &str = "ext/sp_native/sp_native.c"; + +const EXTCONF: &str = "require \"mkmf\"\ncreate_makefile(\"sp_native/sp_native\")\n"; +const LIB_RB: &str = "require \"sp_native/sp_native\"\n"; +const GEMSPEC: &str = "Gem::Specification.new do |s|\n s.name = \"sp_native\"\n \ + s.version = \"1.0.0\"\n s.summary = \"t\"\n s.authors = [\"t\"]\n \ + s.files = Dir[\"lib/**/*.rb\", \"ext/**/*.{c,rb}\"]\n \ + s.extensions = [\"ext/sp_native/extconf.rb\"]\nend\n"; + +fn c_source(value: u32) -> String { + format!( + "#include \"ruby.h\"\n\ + static VALUE sp_value(VALUE self) {{ return INT2NUM({value}); }}\n\ + void Init_sp_native(void) {{\n \ + VALUE m = rb_define_module(\"SpNative\");\n \ + rb_define_singleton_method(m, \"value\", sp_value, 0);\n}}\n" + ) +} + +/// A source that does not compile. +fn broken_source() -> String { + format!("{}this is not C\n", c_source(2)) +} + +fn required() -> bool { + std::env::var("SOCKET_PATCH_BUNDLER_E2E_REQUIRED").is_ok_and(|v| v == "1") +} + +fn tool_works(cmd: &str, args: &[&str]) -> bool { + Command::new(cmd) + .args(args) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) + .status() + .is_ok_and(|s| s.success()) +} + +/// The ruby ABI dir name (`3.4.0`), or `None` (skip) when ruby, gem or a C +/// compiler is missing. Panics instead under the CI gate. +fn toolchain() -> Option { + let have = tool_works("ruby", &["-v"]) + && tool_works("gem", &["--version"]) + && (tool_works("cc", &["--version"]) || tool_works("gcc", &["--version"])); + let abi = have + .then(|| { + Command::new("ruby") + .args(["-e", "print RbConfig::CONFIG['ruby_version']"]) + .output() + .ok() + .filter(|o| o.status.success()) + .map(|o| String::from_utf8_lossy(&o.stdout).trim().to_string()) + .filter(|s| !s.is_empty()) + }) + .flatten(); + if abi.is_none() { + assert!( + !required(), + "SOCKET_PATCH_BUNDLER_E2E_REQUIRED=1 but ruby, gem or a C compiler is missing" + ); + eprintln!("SKIP: ruby, gem or a C compiler is not available"); + } + abi +} + +/// A project with `sp_native` installed into `vendor/bundle/ruby/` by +/// the real `gem`, and the patch's manifest + blobs (`patched` is the +/// patched C source). +struct Fixture { + _tmp: tempfile::TempDir, + proj: PathBuf, + home: PathBuf, + gem_dir: PathBuf, +} + +impl Fixture { + fn new(abi: &str, patched: &str) -> Fixture { + let tmp = tempfile::tempdir().unwrap(); + let src = tmp.path().join("src"); + std::fs::create_dir_all(src.join("ext/sp_native")).unwrap(); + std::fs::create_dir_all(src.join("lib")).unwrap(); + std::fs::write(src.join("ext/sp_native/extconf.rb"), EXTCONF).unwrap(); + std::fs::write(src.join(C_KEY), c_source(1)).unwrap(); + std::fs::write(src.join("lib/sp_native.rb"), LIB_RB).unwrap(); + std::fs::write(src.join("sp_native.gemspec"), GEMSPEC).unwrap(); + let gem_home_scratch = tmp.path().join("gem-user-home"); + std::fs::create_dir_all(&gem_home_scratch).unwrap(); + let run = |cmd: &mut Command, what: &str| { + let out = cmd + .env("HOME", &gem_home_scratch) + .output() + .unwrap_or_else(|e| panic!("{what}: {e}")); + assert!( + out.status.success(), + "{what} failed:\n{}\n{}", + String::from_utf8_lossy(&out.stdout), + String::from_utf8_lossy(&out.stderr) + ); + }; + run( + Command::new("gem") + .args(["build", "sp_native.gemspec"]) + .current_dir(&src), + "gem build", + ); + let proj = tmp.path().join("proj"); + let home = proj.join("vendor/bundle/ruby").join(abi); + std::fs::create_dir_all(&home).unwrap(); + run( + Command::new("gem") + .args(["install", "--local", "--no-document", "--install-dir"]) + .arg(&home) + .arg(src.join("sp_native-1.0.0.gem")), + "gem install", + ); + std::fs::write( + proj.join("Gemfile"), + "source \"https://rubygems.org\"\ngem \"sp_native\"\n", + ) + .unwrap(); + let socket = proj.join(".socket"); + let before = c_source(1); + write_manifest(&socket, &before, patched); + let gem_dir = home.join("gems").join(FULL_NAME); + Fixture { + _tmp: tmp, + proj, + home, + gem_dir, + } + } + + /// `SpNative.value` as the installed gem's Ruby loads it. + fn value(&self) -> String { + let out = Command::new("ruby") + .args(["-e", "require 'sp_native'; print SpNative.value"]) + .env("GEM_PATH", &self.home) + .env_remove("GEM_HOME") + .env_remove("RUBYOPT") + .output() + .unwrap(); + assert!( + out.status.success(), + "loading sp_native failed: {}", + String::from_utf8_lossy(&out.stderr) + ); + String::from_utf8_lossy(&out.stdout).into_owned() + } + + fn c_file(&self) -> String { + std::fs::read_to_string(self.gem_dir.join(C_KEY)).unwrap() + } + + fn marker(&self) -> PathBuf { + self.gem_dir.join(".socket-patch-native-build.json") + } + + fn backup(&self) -> PathBuf { + self.gem_dir.join(".socket-patch-native-orig") + } + + fn run(&self, args: &[&str]) -> (i32, String, String) { + common::run(&self.proj, args) + } +} + +fn write_manifest(socket: &Path, before: &str, after: &str) { + let before_hash = git_sha256(before.as_bytes()); + let after_hash = git_sha256(after.as_bytes()); + write_minimal_manifest( + socket, + PURL, + UUID, + &[PatchEntry { + file_name: C_KEY, + before_hash: &before_hash, + after_hash: &after_hash, + }], + ); + write_blob(socket, &before_hash, before.as_bytes()); + write_blob(socket, &after_hash, after.as_bytes()); +} + +fn event_action(stdout: &str) -> String { + let env = parse_json_envelope(stdout); + common::envelope::event_for(&env, PURL)["action"] + .as_str() + .unwrap_or_default() + .to_string() +} + +#[test] +#[ignore = "host capstone: compiles a C extension with the real ruby/gem toolchain; \ + the unpinned `test` job skips it, the pinned-Ruby e2e leg runs it via --ignored"] +fn native_patch_is_rebuilt_attested_and_rolled_back() { + let Some(abi) = toolchain() else { + return; + }; + let patched = c_source(2); + let fx = Fixture::new(&abi, &patched); + assert_eq!(fx.value(), "1"); + + let (code, _, _) = fx.run(&["apply", "--check"]); + assert_ne!(code, 0, "an unapplied patch must not check clean"); + + let (code, stdout, stderr) = fx.run(&["apply", "--json"]); + assert_eq!(code, 0, "apply failed:\n{stdout}\n{stderr}"); + assert_eq!(event_action(&stdout), "applied", "{stdout}"); + assert_eq!(fx.c_file(), patched); + assert_eq!(fx.value(), "2", "Ruby must load the rebuilt extension"); + assert!(fx.marker().is_file(), "the build marker is recorded"); + assert!(fx.backup().is_dir(), "the install-time build is backed up"); + + let (code, stdout, stderr) = fx.run(&["apply", "--check"]); + assert_eq!(code, 0, "a rebuilt patch checks clean:\n{stdout}\n{stderr}"); + + // Idempotent: nothing to write, nothing to build. + let marker_before = std::fs::read(fx.marker()).unwrap(); + let (code, stdout, _) = fx.run(&["apply", "--json"]); + assert_eq!(code, 0, "{stdout}"); + assert_eq!(event_action(&stdout), "skipped", "{stdout}"); + assert_eq!(std::fs::read(fx.marker()).unwrap(), marker_before); + + let (code, stdout, stderr) = fx.run(&["rollback", "--json"]); + assert_eq!(code, 0, "rollback failed:\n{stdout}\n{stderr}"); + assert_eq!(fx.c_file(), c_source(1)); + assert_eq!(fx.value(), "1", "the original build is back"); + assert!(!fx.marker().exists()); + assert!(!fx.backup().exists()); + + // A release before #1996 patched the sources and never rebuilt: the + // check refuses to attest it, and apply rebuilds it. (`rollback` drops + // the record, so it is written again.) + write_manifest(&fx.proj.join(".socket"), &c_source(1), &patched); + std::fs::write(fx.gem_dir.join(C_KEY), &patched).unwrap(); + let (code, stdout, stderr) = fx.run(&["apply", "--check", "--json"]); + assert_ne!(code, 0, "stale build must not check clean:\n{stdout}"); + assert!( + stdout.contains("native_extension_not_rebuilt") + || stderr.contains("native_extension_not_rebuilt") + || stdout.contains("not compiled into the extension"), + "{stdout}\n{stderr}" + ); + // A dry run says the rebuild is still to do, and does it not. + let (code, stdout, stderr) = fx.run(&["apply", "--json", "--dry-run"]); + assert_eq!(code, 0, "{stdout}\n{stderr}"); + assert!(stdout.contains("apply would rebuild it"), "{stdout}"); + assert_eq!(fx.value(), "1"); + let (code, stdout, stderr) = fx.run(&["apply", "--json"]); + assert_eq!(code, 0, "{stdout}\n{stderr}"); + assert!(stdout.contains("native_extension_rebuilt"), "{stdout}"); + assert_eq!(fx.value(), "2"); +} + +#[test] +#[ignore = "host capstone: compiles a C extension with the real ruby/gem toolchain; \ + the unpinned `test` job skips it, the pinned-Ruby e2e leg runs it via --ignored"] +fn native_patch_that_does_not_build_is_put_back() { + let Some(abi) = toolchain() else { + return; + }; + let fx = Fixture::new(&abi, &broken_source()); + let ext_dir = fx.home.join("extensions"); + let built_before: Vec<_> = walk_files(&ext_dir); + + let (code, stdout, stderr) = fx.run(&["apply", "--json"]); + assert_eq!(code, 1, "{stdout}\n{stderr}"); + assert!(stdout.contains("native_extension_build_failed"), "{stdout}"); + assert!( + stdout.contains("this is not C"), + "the compiler error is shown: {stdout}" + ); + assert_eq!(fx.c_file(), c_source(1), "the original source is back"); + assert_eq!(fx.value(), "1", "the original build is back"); + assert!(!fx.marker().exists()); + assert!(!fx.backup().exists()); + assert_eq!(walk_files(&ext_dir), built_before); +} + +/// `(relative path, bytes)` of every file under `dir`. +fn walk_files(dir: &Path) -> Vec<(String, Vec)> { + let mut out = Vec::new(); + let mut stack = vec![dir.to_path_buf()]; + while let Some(d) = stack.pop() { + for entry in std::fs::read_dir(&d).unwrap().flatten() { + let p = entry.path(); + if p.is_dir() { + stack.push(p); + } else { + let rel = p.strip_prefix(dir).unwrap().display().to_string(); + out.push((rel, std::fs::read(&p).unwrap())); + } + } + } + out.sort(); + out +} + +// ── hermetic refusals (no Ruby needed) ──────────────────────────────── + +/// A hand-laid native gem: `ext/` source, the install-time build, and an +/// installed spec that does (or doesn't) declare extensions. +fn fake_native_gem(declares_extensions: bool) -> (tempfile::TempDir, PathBuf, PathBuf) { + let tmp = tempfile::tempdir().unwrap(); + let proj = tmp.path().to_path_buf(); + let home = proj.join("vendor/bundle/ruby/3.4.0"); + let gem_dir = home.join("gems").join(FULL_NAME); + std::fs::create_dir_all(gem_dir.join("ext/sp_native")).unwrap(); + std::fs::create_dir_all(gem_dir.join("lib/sp_native")).unwrap(); + std::fs::write(gem_dir.join(C_KEY), c_source(1)).unwrap(); + std::fs::write(gem_dir.join("lib/sp_native.rb"), LIB_RB).unwrap(); + std::fs::write( + gem_dir.join("lib/sp_native/sp_native.so"), + b"\x7fELF original", + ) + .unwrap(); + let ext = home.join("extensions/x86_64-linux/3.4.0").join(FULL_NAME); + std::fs::create_dir_all(ext.join("sp_native")).unwrap(); + std::fs::write(ext.join("sp_native/sp_native.so"), b"\x7fELF original").unwrap(); + std::fs::write(ext.join("gem.build_complete"), b"").unwrap(); + let extensions = if declares_extensions { + " s.extensions = [\"ext/sp_native/extconf.rb\".freeze]\n" + } else { + "" + }; + std::fs::create_dir_all(home.join("specifications")).unwrap(); + std::fs::write( + home.join("specifications") + .join(format!("{FULL_NAME}.gemspec")), + format!( + "Gem::Specification.new do |s|\n s.name = \"sp_native\".freeze\n \ + s.version = \"1.0.0\"\n{extensions}end\n" + ), + ) + .unwrap(); + std::fs::write( + proj.join("Gemfile"), + "source \"https://rubygems.org\"\ngem \"sp_native\"\n", + ) + .unwrap(); + (tmp, proj, gem_dir) +} + +/// Run the CLI with a `PATH` holding no `ruby`. +fn run_without_ruby(proj: &Path, args: &[&str]) -> (i32, String, String) { + let empty = proj.join(".empty-path"); + std::fs::create_dir_all(&empty).unwrap(); + let path = empty.display().to_string(); + common::run_with_env(proj, args, &[("PATH", path.as_str())]) +} + +#[test] +fn native_patch_without_a_ruby_is_refused_and_writes_nothing() { + let (_tmp, proj, gem_dir) = fake_native_gem(true); + write_manifest(&proj.join(".socket"), &c_source(1), &c_source(2)); + + let (code, stdout, stderr) = run_without_ruby(&proj, &["apply", "--json"]); + assert_eq!(code, 1, "{stdout}\n{stderr}"); + let env = parse_json_envelope(&stdout); + let event = common::envelope::event_for(&env, PURL); + assert_eq!(event["action"], "failed", "{stdout}"); + assert!( + event["error"] + .as_str() + .is_some_and(|e| e.starts_with("native_extensions_unsupported:")), + "{stdout}" + ); + assert!(stdout.contains("no `ruby` was found"), "{stdout}"); + assert_eq!( + std::fs::read_to_string(gem_dir.join(C_KEY)).unwrap(), + c_source(1) + ); + assert!(!gem_dir.join(".socket-patch-native-build.json").exists()); + assert!(!gem_dir.join(".socket-patch-native-orig").exists()); + + // Dry run reports the same refusal. + let (code, stdout, _) = run_without_ruby(&proj, &["apply", "--json", "--dry-run"]); + assert_eq!(code, 1, "{stdout}"); + assert!(stdout.contains("native_extensions_unsupported"), "{stdout}"); +} + +#[test] +fn native_patch_to_a_gem_without_extensions_is_refused() { + let (_tmp, proj, gem_dir) = fake_native_gem(false); + write_manifest(&proj.join(".socket"), &c_source(1), &c_source(2)); + + let (code, stdout, stderr) = common::run(&proj, &["apply", "--json"]); + assert_eq!(code, 1, "{stdout}\n{stderr}"); + assert!(stdout.contains("native_extensions_unsupported"), "{stdout}"); + assert!(stdout.contains("builds no extension"), "{stdout}"); + assert_eq!( + std::fs::read_to_string(gem_dir.join(C_KEY)).unwrap(), + c_source(1) + ); +} + +#[test] +fn ruby_only_patch_to_a_native_gem_needs_no_ruby() { + let (_tmp, proj, gem_dir) = fake_native_gem(true); + let before = LIB_RB; + let after = "require \"sp_native/sp_native\"\n# patched\n"; + let (bh, ah) = (git_sha256(before.as_bytes()), git_sha256(after.as_bytes())); + let socket = proj.join(".socket"); + write_minimal_manifest( + &socket, + PURL, + UUID, + &[PatchEntry { + file_name: "lib/sp_native.rb", + before_hash: &bh, + after_hash: &ah, + }], + ); + write_blob(&socket, &ah, after.as_bytes()); + + let (code, stdout, stderr) = run_without_ruby(&proj, &["apply", "--json"]); + assert_eq!(code, 0, "{stdout}\n{stderr}"); + assert_eq!(event_action(&stdout), "applied", "{stdout}"); + assert_eq!( + std::fs::read_to_string(gem_dir.join("lib/sp_native.rb")).unwrap(), + after + ); +} + +#[test] +fn patched_sources_over_the_install_time_build_do_not_check_clean() { + let (_tmp, proj, gem_dir) = fake_native_gem(true); + write_manifest(&proj.join(".socket"), &c_source(1), &c_source(2)); + // What a release before #1996 left: the source patched, the build not. + std::fs::write(gem_dir.join(C_KEY), c_source(2)).unwrap(); + + let (code, stdout, stderr) = run_without_ruby(&proj, &["apply", "--check", "--json"]); + assert_ne!(code, 0, "{stdout}\n{stderr}"); + assert!( + stdout.contains("native_extension_not_rebuilt") + || stdout.contains("not compiled into the extension"), + "{stdout}\n{stderr}" + ); +} + +/// `--no-gem-native-rebuild` / `SOCKET_NO_GEM_NATIVE_REBUILD` (a caller +/// that compiles the extension itself, such as the patches evaluation +/// harness): the sources are written with no Ruby involved, the run +/// succeeds with a `native_extension_not_rebuilt` warning, and nothing of +/// socket-patch's build is left behind. +#[test] +fn opted_out_native_patch_writes_sources_without_a_build() { + for opt_out in [ + &["apply", "--json", "--no-gem-native-rebuild"][..], + &["apply", "--json"][..], + ] { + let (_tmp, proj, gem_dir) = fake_native_gem(true); + write_manifest(&proj.join(".socket"), &c_source(1), &c_source(2)); + let empty = proj.join(".empty-path"); + std::fs::create_dir_all(&empty).unwrap(); + let path = empty.display().to_string(); + let mut env = vec![("PATH", path.as_str())]; + if opt_out.len() == 2 { + env.push(("SOCKET_NO_GEM_NATIVE_REBUILD", "1")); + } + let (code, stdout, stderr) = common::run_with_env(&proj, opt_out, &env); + assert_eq!(code, 0, "{stdout}\n{stderr}"); + assert_eq!(event_action(&stdout), "applied", "{stdout}"); + assert!(stdout.contains("native_extension_not_rebuilt"), "{stdout}"); + assert_eq!( + std::fs::read_to_string(gem_dir.join(C_KEY)).unwrap(), + c_source(2) + ); + assert!(!gem_dir.join(".socket-patch-native-build.json").exists()); + assert!(!gem_dir.join(".socket-patch-native-orig").exists()); + // The build Ruby loads is still the old one: no attestation. + let (code, stdout, _) = run_without_ruby(&proj, &["apply", "--check", "--json"]); + assert_ne!(code, 0, "{stdout}"); + } +} diff --git a/crates/socket-patch-core/src/patch/copy_tree.rs b/crates/socket-patch-core/src/patch/copy_tree.rs index cc871ffcd..f49c1df16 100644 --- a/crates/socket-patch-core/src/patch/copy_tree.rs +++ b/crates/socket-patch-core/src/patch/copy_tree.rs @@ -47,7 +47,11 @@ pub(crate) async fn fresh_copy( /// the skip is not pruned, so its contents are still copied and must create /// the skipped directory on demand (it then exists in the copy only when it /// has contents, as it always has). -fn copy_tree_blocking(src: &Path, dst: &Path, skip_file_name: Option<&str>) -> std::io::Result<()> { +pub(crate) fn copy_tree_blocking( + src: &Path, + dst: &Path, + skip_file_name: Option<&str>, +) -> std::io::Result<()> { force_remove_dir_all(dst)?; std::fs::create_dir_all(dst)?; // Depth of the outermost skipped directory the walk is inside. @@ -148,7 +152,7 @@ fn copy_tree_blocking_reference( /// Recursively remove a tree, retrying once after relaxing *directory* perms /// (a previously patched copy may carry read-only dir modes copied from the /// registry/cache; on unix file modes never gate unlinking). -fn force_remove_dir_all(dir: &Path) -> std::io::Result<()> { +pub(crate) fn force_remove_dir_all(dir: &Path) -> std::io::Result<()> { match std::fs::remove_dir_all(dir) { Ok(()) => Ok(()), Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()), diff --git a/crates/socket-patch-core/src/patch/gem_ext.rs b/crates/socket-patch-core/src/patch/gem_ext.rs new file mode 100644 index 000000000..ffcd89126 --- /dev/null +++ b/crates/socket-patch-core/src/patch/gem_ext.rs @@ -0,0 +1,1848 @@ +//! Agent-mode patches to a gem's native extension (#1996). +//! +//! Ruby loads the shared object RubyGems compiled when it installed the gem +//! (`extensions///-/…` and, with +//! `Gem.install_extension_in_lib`, its copy under the gem's `lib/`), never +//! the `ext/**/*.c` sources beside it. A patch that rewrites only those +//! sources in place leaves the vulnerable compiled code running. So when a +//! gem patch changes a native-extension build input, `apply` rebuilds the +//! extension from the patched sources with RubyGems' own builder, the step +//! `gem pristine --extensions` runs (`extconf.rb` + `make`, a Rakefile, a +//! CMake or Cargo build), against the Ruby that owns the install: +//! +//! 1. Preflight: the copy must be a RubyGems install (a `specifications/` +//! entry beside `gems/`) that declares extensions, and a Ruby whose +//! extension dir for the gem is the one on disk must be found. Otherwise +//! the purl is refused with [`UNSUPPORTED_CODE`] and nothing is written. +//! 2. The patched sources are written ([`apply_package_patch`]). +//! 3. The current build is moved aside (and, the first time, copied to a +//! backup of the original build inside the gem dir, [`BACKUP_DIR`]), and +//! RubyGems builds the extension into a fresh extension dir. +//! 4. A failed build puts the old build and the original sources back and +//! fails the purl with [`BUILD_FAILED_CODE`]. A successful one records a +//! marker ([`MARKER_FILE`]) naming the source hashes it was built from +//! and the hash of every artifact it produced. +//! +//! `vex` and `apply --check` attest such a copy only when that marker +//! matches the record's patched sources and every recorded artifact still +//! hashes as built ([`verify_native_build`]). `rollback` puts the backed-up +//! original build back ([`restore_original_build`]). + +use std::collections::{BTreeMap, HashMap}; +use std::ffi::OsString; +use std::path::{Path, PathBuf}; +use std::time::Duration; + +use serde::{Deserialize, Serialize}; + +use crate::hash::git_sha256::compute_git_sha256_from_bytes; +use crate::manifest::schema::PatchFileInfo; +use crate::patch::apply::{ + apply_file_patch_at, apply_package_patch, is_safe_relative_subpath, normalize_file_path, + ApplyResult, MismatchPolicy, PatchSources, +}; +use crate::patch::file_hash::compute_file_git_sha256_sync; +use crate::utils::fs::read_regular_to_bytes_sync; + +/// The refusal when a native-extension patch cannot be rebuilt in place +/// (the vendored backend's code for the same limitation). +pub const UNSUPPORTED_CODE: &str = "native_extensions_unsupported"; +/// The extension failed to build from the patched sources; the old build +/// and the original sources were put back. +pub const BUILD_FAILED_CODE: &str = "native_extension_build_failed"; +/// Advisory: sources an earlier run patched were rebuilt into the extension +/// (a release before #1996 patched them without a rebuild). +pub const REBUILT_CODE: &str = "native_extension_rebuilt"; +/// `rollback` restored the sources but could not put the original compiled +/// extension back. +pub const RESTORE_FAILED_CODE: &str = "native_extension_restore_failed"; +/// The `vex` / `apply --check` routing tag of a copy whose sources carry +/// the patch while its compiled extension was not built from them. +pub const NOT_REBUILT_REASON: &str = "native_extension_not_rebuilt"; + +/// The build marker, in the gem dir (see [`BuildMarker`]). +pub const MARKER_FILE: &str = ".socket-patch-native-build.json"; +/// The original build's backup, in the gem dir (see [`restore_original_build`]). +pub const BACKUP_DIR: &str = ".socket-patch-native-orig"; +/// The backup's description file, inside [`BACKUP_DIR`]. +const BACKUP_INFO: &str = "backup.json"; + +/// How long one Ruby probe may take. +const PROBE_TIMEOUT: Duration = Duration::from_secs(60); +/// How long one extension build may take (nokogiri's vendored libxml2 build +/// takes minutes). +const BUILD_TIMEOUT: Duration = Duration::from_secs(30 * 60); + +/// Files RubyGems writes into an extension dir that are not build products. +const EXT_DIR_METADATA: &[&str] = &["gem.build_complete", "gem_make.out", "mkmf.log"]; + +/// Source and build-script extensions a native extension compiles. +const NATIVE_SOURCE_EXTENSIONS: &[&str] = &[ + "c", "h", "cc", "cpp", "cxx", "hh", "hpp", "hxx", "m", "y", "rl", +]; + +/// Grammar sources a gem ships already generated (racc / ragel output is a +/// `.rb` or `.c` file the gem packs): in a gem that builds no extension they +/// compile nothing at install time. +const GENERATED_GRAMMAR_EXTENSIONS: &[&str] = &["y", "rl"]; + +/// The probe/build program run by the gem's Ruby. One line: `-e` with an +/// embedded newline is fragile through a Windows command line. Ruby 1.8 does +/// not load RubyGems by default (and `RUBYOPT` is cleared), so it is required +/// explicitly; a RubyGems older than 2.2 (no per-gem extension dir, no +/// `raw_require_paths`) is refused by name. Probe lines carry an `sp_` key +/// so stray output (an `operating_system.rb`, a shim) is skipped. +const RUBY_SCRIPT: &str = "require 'rubygems'; mode, path = ARGV; \ +unless Gem::Specification.respond_to?(:load) && \ +Gem::Specification.method_defined?(:extension_dir) && \ +Gem::Specification.method_defined?(:raw_require_paths) then \ +abort(\"RubyGems #{defined?(Gem::VERSION) ? Gem::VERSION : Gem::RubyGemsVersion} is older than 2.2, \ +which has no per-gem extension dir to rebuild into\"); \ +end; \ +spec = Gem::Specification.load(path) or abort(\"cannot load #{path}\"); \ +if mode == 'probe' then \ +puts \"sp_gem_dir\\t#{spec.full_gem_path}\"; \ +puts \"sp_extension_dir\\t#{spec.extension_dir}\"; \ +puts \"sp_lib_dir\\t#{File.join(spec.full_gem_path, spec.raw_require_paths.first.to_s)}\"; \ +spec.extensions.each { |e| puts \"sp_extension\\t#{e}\" }; \ +else \ +require 'rubygems/ext'; \ +Gem::DefaultUserInteraction.ui = Gem::SilentUI.new; \ +Gem::Ext::Builder.new(spec, spec.build_args).build_extensions; \ +end"; + +/// One installed RubyGems copy: `/gems/` with its +/// `/specifications/.gemspec`. +#[derive(Debug, Clone)] +struct GemCopy { + home: PathBuf, + full_name: String, + gem_dir: PathBuf, + spec_path: PathBuf, +} + +impl GemCopy { + /// The copy at `pkg_path`, when it sits in a RubyGems `gems/` dir. + /// Whether its spec exists is checked by the caller. + fn of(pkg_path: &Path) -> Option { + let full_name = pkg_path.file_name()?.to_str()?.to_string(); + let gems = pkg_path.parent()?; + if gems.file_name()? != "gems" { + return None; + } + let home = gems.parent()?.to_path_buf(); + let spec_path = home + .join("specifications") + .join(format!("{full_name}.gemspec")); + Some(GemCopy { + home, + full_name, + gem_dir: pkg_path.to_path_buf(), + spec_path, + }) + } +} + +/// What the installed gemspec says about extensions. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +struct ExtensionDecl { + /// The spec assigns `extensions`. + declared: bool, + /// The declared build scripts (`ext/foo/extconf.rb`), when the + /// assignment lists them as string literals. + entries: Vec, +} + +/// Parse `s.extensions = ["ext/foo/extconf.rb".freeze, …]` out of an +/// installed gemspec (the `Specification#to_ruby` shape RubyGems writes). +/// Whether extensions are declared is the vendored backend's test +/// ([`gemspec_declares_extensions`](crate::vendor::gem::gemspec_declares_extensions)), +/// so both modes agree on which gems are native; the build scripts are the +/// string literals those assignments list. Comments are stripped per line; +/// an assignment whose entries are not string literals is `declared` with +/// no `entries`. +fn parse_extensions(spec_text: &str) -> ExtensionDecl { + let mut entries = Vec::new(); + for raw in spec_text.lines() { + let line = raw.split('#').next().unwrap_or(""); + let Some(idx) = line.find(".extensions") else { + continue; + }; + let after = &line[idx + ".extensions".len()..]; + if after + .chars() + .next() + .is_some_and(|c| c.is_ascii_alphanumeric() || c == '_') + { + continue; + } + let after = after.trim_start(); + let rhs = ["+=", "<<", ".push", ".concat"] + .iter() + .find_map(|op| after.strip_prefix(op)) + .or_else(|| after.strip_prefix('=').filter(|r| !r.starts_with('='))); + if let Some(rhs) = rhs { + entries.extend(string_literals(rhs)); + } + } + ExtensionDecl { + declared: crate::vendor::gem::gemspec_declares_extensions(spec_text), + entries, + } +} + +/// The double- and single-quoted string literals in `text`, in order. +fn string_literals(text: &str) -> Vec { + let mut out = Vec::new(); + let mut chars = text.chars(); + while let Some(c) = chars.next() { + if c != '"' && c != '\'' { + continue; + } + let mut lit = String::new(); + let mut escaped = false; + for d in chars.by_ref() { + if escaped { + lit.push(d); + escaped = false; + } else if d == '\\' { + escaped = true; + } else if d == c { + break; + } else { + lit.push(d); + } + } + out.push(lit); + } + out +} + +/// Whether a (normalized) gem file path is a native-extension build input by +/// its name alone: a C/C++/Objective-C source or header, a racc/ragel +/// grammar, an `extconf.rb`, or a `depend` file under `ext/`. +fn is_native_by_name(path: &str) -> bool { + let leaf = path.rsplit('/').next().unwrap_or(path); + if leaf == "extconf.rb" || (leaf == "depend" && path.starts_with("ext/")) { + return true; + } + leaf.rsplit_once('.').is_some_and(|(stem, ext)| { + !stem.is_empty() && NATIVE_SOURCE_EXTENSIONS.contains(&ext.to_ascii_lowercase().as_str()) + }) +} + +fn is_generated_grammar(path: &str) -> bool { + path.rsplit_once('.').is_some_and(|(_, ext)| { + GENERATED_GRAMMAR_EXTENSIONS.contains(&ext.to_ascii_lowercase().as_str()) + }) +} + +/// A gem record's native-extension build inputs on one installed copy. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +struct NativeScope { + /// The record's keys that are build inputs, sorted. + inputs: Vec, + /// The installed spec declares extensions (RubyGems builds them at + /// install time). `false` with inputs left: the gem's compiled code, if + /// any, ships precompiled. + builds_extensions: bool, +} + +/// Classify `files` against the extension declaration `decl`. +/// +/// In a gem that declares extensions, a key is a build input by name +/// ([`is_native_by_name`]) or by living under the dir of a declared build +/// script (a Rust extension's `.rs` sources, a `Rakefile`-built tree); a +/// script at the gem root (`extensions = ["extconf.rb"]`) makes the files +/// beside it inputs, the ones mkmf compiles, not every file of the gem. A +/// declaration whose scripts are unknown makes everything under `ext/` an +/// input. +/// +/// In a gem that declares none, only a name-matched source under `ext/` +/// counts: that is where a precompiled platform gem keeps the sources of +/// the binary it ships. C or Objective-C files elsewhere (a template, a +/// test fixture) compile nothing, and neither do grammar sources, whose +/// generated output the gem packs. +fn classify(decl: &ExtensionDecl, files: &HashMap) -> NativeScope { + // `None`: a script at the gem root. + let dirs: Vec> = decl + .entries + .iter() + .map(|e| { + e.trim_start_matches("./") + .rsplit_once('/') + .map(|(dir, _)| format!("{dir}/")) + }) + .collect(); + let under_script_dir = |path: &str| { + dirs.iter().any(|d| match d { + Some(d) => path.starts_with(d.as_str()), + None => !path.contains('/'), + }) + }; + let unknown_scripts = decl.declared && decl.entries.is_empty(); + let mut inputs: Vec = files + .keys() + .filter(|key| { + let path = normalize_file_path(key); + if decl.declared { + is_native_by_name(path) + || under_script_dir(path) + || (unknown_scripts && path.starts_with("ext/")) + } else { + path.starts_with("ext/") && is_native_by_name(path) && !is_generated_grammar(path) + } + }) + .cloned() + .collect(); + inputs.sort(); + NativeScope { + inputs, + builds_extensions: decl.declared, + } +} + +/// The record's native scope on the copy at `pkg_path`, and the copy when it +/// is a RubyGems install whose spec could be read. `None` scope inputs mean +/// the record is plain Ruby for this copy. +async fn scope_of( + pkg_path: &Path, + files: &HashMap, +) -> (NativeScope, Option) { + let copy = GemCopy::of(pkg_path); + let spec_text = match © { + Some(c) => crate::utils::fs::read_regular_to_string(&c.spec_path) + .await + .ok(), + None => None, + }; + let decl = spec_text + .as_deref() + .map(parse_extensions) + .unwrap_or_default(); + let scope = classify(&decl, files); + // Without a readable spec nothing proves which scripts the gem builds; + // name-matched inputs still count, and the copy is unusable for a build. + (scope, copy.filter(|_| spec_text.is_some())) +} + +/// The build marker written into the gem dir after a rebuild. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct BuildMarker { + pub schema: u32, + /// Native input (normalized path) → its git-sha256 when it was built. + pub sources: BTreeMap, + /// The extension dir built into, relative to the gem home: + /// `extensions///` (see [`confined_ext_dir`]). + pub extension_dir: String, + /// Artifact path relative to the extension dir → git-sha256. + pub ext_artifacts: BTreeMap, + /// The artifacts' copies RubyGems put under the gem's require dir: + /// path relative to the gem dir → git-sha256. + pub lib_artifacts: BTreeMap, +} + +const MARKER_SCHEMA: u32 = 1; + +/// The extension dir a marker or backup names, only when it is +/// `extensions///` under the copy's home: those +/// files sit in the gem dir, where a checkout could plant them, so the path +/// they hold is never trusted further than the one dir RubyGems builds this +/// gem into. +fn confined_ext_dir(copy: &GemCopy, label: &str) -> Option { + let parts: Vec<&str> = label.split('/').collect(); + let confined = parts.len() == 4 + && parts[0] == "extensions" + && parts[3] == copy.full_name + && is_safe_relative_subpath(label); + confined.then(|| copy.home.join(label)) +} + +fn is_symlink(path: &Path) -> bool { + std::fs::symlink_metadata(path).is_ok_and(|m| m.file_type().is_symlink()) +} + +fn read_marker_sync(gem_dir: &Path) -> Option { + let text = crate::utils::fs::read_regular_to_string_sync(&gem_dir.join(MARKER_FILE)).ok()?; + serde_json::from_str::(&text) + .ok() + .filter(|m| m.schema == MARKER_SCHEMA) +} + +/// Whether `marker` was built from exactly the record's patched native +/// inputs, and every artifact it recorded is still on disk with the bytes it +/// was built with. An artifact path that escapes its dir never verifies. +fn marker_holds( + marker: &BuildMarker, + copy: &GemCopy, + scope: &NativeScope, + files: &HashMap, +) -> bool { + let sources_match = scope.inputs.iter().all(|key| { + marker.sources.get(normalize_file_path(key)) == files.get(key).map(|i| &i.after_hash) + }); + if !sources_match || marker.sources.len() != scope.inputs.len() { + return false; + } + let Some(ext_dir) = confined_ext_dir(copy, &marker.extension_dir) else { + return false; + }; + let artifact_ok = |base: &Path, rel: &str, hash: &str| { + is_safe_relative_subpath(rel) + && compute_file_git_sha256_sync(&base.join(rel)).is_ok_and(|h| h == hash) + }; + marker + .ext_artifacts + .iter() + .all(|(rel, hash)| artifact_ok(&ext_dir, rel, hash)) + && marker + .lib_artifacts + .iter() + .all(|(rel, hash)| artifact_ok(©.gem_dir, rel, hash)) +} + +/// `vex` / `apply --check`: the installed gem copy at `pkg_path`, whose +/// record files already verify, runs code built from them. A record with no +/// native input on this copy passes; one with native inputs passes only when +/// the copy's build marker holds ([`BuildMarker`]). Errors carry +/// [`NOT_REBUILT_REASON`]. +pub async fn verify_native_build( + pkg_path: &Path, + files: &HashMap, +) -> Result<(), String> { + let (scope, copy) = scope_of(pkg_path, files).await; + if scope.inputs.is_empty() { + return Ok(()); + } + let Some(copy) = copy.filter(|_| scope.builds_extensions) else { + return Err(NOT_REBUILT_REASON.to_string()); + }; + let files = files.clone(); + let holds = crate::utils::fs::run_blocking(move || { + read_marker_sync(©.gem_dir).is_some_and(|m| marker_holds(&m, ©, &scope, &files)) + }) + .await; + if holds { + Ok(()) + } else { + Err(NOT_REBUILT_REASON.to_string()) + } +} + +/// [`apply_gem_package_patch`]'s outcome: the copy's result, and a run +/// warning `(code, detail)` the caller surfaces next to it. +#[derive(Debug, Clone)] +pub struct GemApplied { + pub result: ApplyResult, + pub warning: Option<(String, String)>, +} + +/// Where the Ruby that builds a copy's extension is looked for and run. +#[derive(Debug, Clone)] +pub struct RubyEnv { + /// The project dir the Ruby on `PATH` runs from (a version manager's + /// `.ruby-version` there picks the project's Ruby). + pub cwd: PathBuf, + /// Rebuild native extensions. `false` (`--no-gem-native-rebuild`): the + /// caller compiles them itself, so patched sources are written without + /// a build, as before #1996, with a [`NOT_REBUILT_REASON`] warning. + pub rebuild: bool, +} + +/// What a Ruby reports about a gem copy. +#[derive(Debug, Clone, PartialEq, Eq)] +struct Probe { + ruby: PathBuf, + gem_dir: PathBuf, + extension_dir: PathBuf, + lib_dir: PathBuf, + extensions: Vec, +} + +fn parse_probe(ruby: &Path, stdout: &str) -> Option { + let mut gem_dir = None; + let mut extension_dir = None; + let mut lib_dir = None; + let mut extensions = Vec::new(); + for line in stdout.lines() { + let Some((key, value)) = line.split_once('\t') else { + continue; + }; + match key { + "sp_gem_dir" => gem_dir = Some(PathBuf::from(value)), + "sp_extension_dir" => extension_dir = Some(PathBuf::from(value)), + "sp_lib_dir" => lib_dir = Some(PathBuf::from(value)), + "sp_extension" => extensions.push(value.to_string()), + _ => {} + } + } + Some(Probe { + ruby: ruby.to_path_buf(), + gem_dir: gem_dir?, + extension_dir: extension_dir?, + lib_dir: lib_dir?, + extensions, + }) +} + +/// The Rubies that may own `copy`, best first: the one installed beside a +/// `/lib/ruby/gems/` home (rbenv, asdf, rvm, a system Ruby), +/// then the `ruby` on `PATH`. A prefix inside the project (`project`) is +/// never one: the scanned repository does not choose the program run. +fn ruby_candidates(copy: &GemCopy, project: &Path) -> Vec { + let mut out = Vec::new(); + let exe = if cfg!(windows) { "ruby.exe" } else { "ruby" }; + let mut parts = copy.home.ancestors(); + parts.next(); // + if let (Some(gems), Some(ruby), Some(lib), Some(prefix)) = + (parts.next(), parts.next(), parts.next(), parts.next()) + { + if gems.file_name().is_some_and(|n| n == "gems") + && ruby.file_name().is_some_and(|n| n == "ruby") + && lib.file_name().is_some_and(|n| n == "lib") + { + let candidate = prefix.join("bin").join(exe); + let in_project = match (canonical(prefix), canonical(project)) { + (Some(prefix), Some(project)) => prefix.starts_with(project), + _ => prefix.starts_with(project), + }; + if !in_project && candidate.is_file() { + out.push(candidate); + } + } + } + if let Some(on_path) = crate::utils::process::resolve_tool("ruby") { + if !out.contains(&on_path) { + out.push(on_path); + } + } + out +} + +/// The `GEM_PATH` a build runs with: the copy's home first (an `extconf.rb` +/// requiring `mini_portile2` or `rb_sys` finds the copy installed beside +/// it), then the caller's, then RubyGems' defaults (the trailing separator). +fn gem_path_for(home: &Path) -> Option { + let mut paths = vec![home.to_path_buf()]; + if let Some(existing) = std::env::var_os("GEM_PATH") { + paths.extend(std::env::split_paths(&existing).filter(|p| !p.as_os_str().is_empty())); + } + let mut joined = std::env::join_paths(paths).ok()?; + joined.push(if cfg!(windows) { ";" } else { ":" }); + Some(joined) +} + +/// Run the Ruby program in `mode` (`probe` / `build`) on `copy`, bounded +/// by `budget` through [`build_output_within`](crate::utils::process::build_output_within) +/// (a timeout kills the build's whole process group). `Ok` carries stdout; +/// `Err` a short reason (stderr's tail on a failed exit). +async fn run_ruby( + ruby: &Path, + mode: &str, + copy: &GemCopy, + env: &RubyEnv, + budget: Duration, +) -> Result { + use crate::utils::process::{build_output_within, command_for, BoundedError}; + let mut command = command_for(ruby); + command + .arg("-e") + .arg(RUBY_SCRIPT) + .arg(mode) + .arg(©.spec_path) + .current_dir(&env.cwd); + // A Bundler-run parent (`bundle exec`, the Bundler plugin) must not + // load the project's bundle into this Ruby. + for (key, _) in std::env::vars_os() { + let key_str = key.to_string_lossy(); + if key_str == "RUBYOPT" || key_str == "RUBYLIB" || key_str.starts_with("BUNDLE") { + command.env_remove(&key); + } + } + if let Some(gem_path) = gem_path_for(©.home) { + command.env("GEM_PATH", gem_path); + } + let output = crate::utils::fs::run_blocking(move || build_output_within(command, budget)).await; + let output = match output { + Err(BoundedError::TimedOut) => { + return Err(format!( + "{} {mode} did not finish within {} minutes", + ruby.display(), + budget.as_secs() / 60 + )) + } + Err(BoundedError::Spawn(e)) => { + return Err(format!("could not run {}: {e}", ruby.display())) + } + Ok(output) => output, + }; + if output.status.success() { + Ok(String::from_utf8_lossy(&output.stdout).into_owned()) + } else { + Err(stderr_tail(&String::from_utf8_lossy(&output.stderr))) + } +} + +/// The useful end of a failed build's stderr: RubyGems' trailing +/// "Gem files will remain installed" boilerplate dropped, at most 20 lines. +fn stderr_tail(stderr: &str) -> String { + let text = stderr + .split("\nGem files will remain installed") + .next() + .unwrap_or(stderr); + let lines: Vec<&str> = text + .lines() + .map(str::trim_end) + .filter(|l| !l.is_empty()) + .collect(); + let start = lines.len().saturating_sub(20); + let tail = lines[start..].join("\n"); + if tail.is_empty() { + "the build exited with an error and no output".to_string() + } else { + tail + } +} + +/// The extension dirs of `full_name` under `home/extensions/*/*/`. +fn installed_extension_dirs(copy: &GemCopy) -> Vec { + let mut out = Vec::new(); + let Ok(platforms) = std::fs::read_dir(copy.home.join("extensions")) else { + return out; + }; + for platform in platforms.flatten() { + let Ok(abis) = std::fs::read_dir(platform.path()) else { + continue; + }; + for abi in abis.flatten() { + let dir = abi.path().join(©.full_name); + if std::fs::symlink_metadata(&dir).is_ok_and(|m| m.is_dir()) { + out.push(dir); + } + } + } + out.sort(); + out +} + +fn same_path(a: &Path, b: &Path) -> bool { + match (canonical(a), canonical(b)) { + (Some(a), Some(b)) => a == b, + _ => a == b, + } +} + +fn canonical(p: &Path) -> Option { + std::fs::canonicalize(p).ok() +} + +/// Find the Ruby that owns `copy`'s install: the one whose extension dir +/// for this gem is the installed `/extensions///` one +/// (its platform and ABI). A gem with no such dir (installed by a RubyGems +/// older than 2.2, or its build removed) has no build a Ruby can be matched +/// to, so none qualifies. `Err` names why. +async fn find_ruby(copy: &GemCopy, env: &RubyEnv) -> Result { + let installed = { + let copy = copy.clone(); + crate::utils::fs::run_blocking(move || installed_extension_dirs(©)).await + }; + if installed.is_empty() { + return Err(format!( + "the gem has no compiled extension dir under {} (it was installed by a \ + RubyGems older than 2.2, or its build is missing)", + copy.home.join("extensions").display() + )); + } + let candidates = ruby_candidates(copy, &env.cwd); + if candidates.is_empty() { + return Err("no `ruby` was found on PATH to rebuild it with".to_string()); + } + let mut reasons = Vec::new(); + for ruby in candidates { + let stdout = match run_ruby(&ruby, "probe", copy, env, PROBE_TIMEOUT).await { + Ok(s) => s, + Err(e) => { + reasons.push(format!("{}: {e}", ruby.display())); + continue; + } + }; + let Some(probe) = parse_probe(&ruby, &stdout) else { + reasons.push(format!("{}: unreadable probe output", ruby.display())); + continue; + }; + if !same_path(&probe.gem_dir, ©.gem_dir) { + reasons.push(format!( + "{} resolves the gem to {}", + ruby.display(), + probe.gem_dir.display() + )); + continue; + } + if !installed.iter().any(|d| same_path(d, &probe.extension_dir)) { + reasons.push(format!( + "{} builds into {}, but the installed extension is {} (another Ruby's)", + ruby.display(), + probe.extension_dir.display(), + installed + .iter() + .map(|d| d.display().to_string()) + .collect::>() + .join(", ") + )); + continue; + } + if probe.extensions.is_empty() { + reasons.push(format!( + "{} reports no extensions for the gem", + ruby.display() + )); + continue; + } + return Ok(probe); + } + Err(format!( + "no Ruby owning this install was found ({})", + reasons.join("; ") + )) +} + +/// Every regular file under `dir`, relative, `/`-separated, sorted; +/// RubyGems' metadata files at the top level and the stage dirs a build +/// leaves are skipped. +fn artifact_rels(dir: &Path) -> Vec { + let mut out = Vec::new(); + for entry in walkdir::WalkDir::new(dir) + .follow_links(false) + .sort_by_file_name() + .into_iter() + .flatten() + { + if !entry.file_type().is_file() { + continue; + } + let Ok(rel) = entry.path().strip_prefix(dir) else { + continue; + }; + let rel = rel + .components() + .map(|c| c.as_os_str().to_string_lossy().into_owned()) + .collect::>() + .join("/"); + if EXT_DIR_METADATA.contains(&rel.as_str()) || rel == MARKER_FILE { + continue; + } + out.push(rel); + } + out +} + +/// An extension-dir artifact's copy in the gem's require dir, relative to +/// the gem dir (`a/a.so` → `lib/a/a.so`). `None` when the require dir is +/// not inside the gem dir. +fn lib_rel(probe: &Probe, rel: &str) -> Option { + let lib = probe.lib_dir.strip_prefix(&probe.gem_dir).ok()?; + let s = lib + .join(rel) + .components() + .map(|c| c.as_os_str().to_string_lossy().into_owned()) + .collect::>() + .join("/"); + is_safe_relative_subpath(&s).then_some(s) +} + +/// A fresh copy of the tree `from` at `to` (replacing it). +fn copy_tree_sync(from: &Path, to: &Path) -> std::io::Result<()> { + crate::patch::copy_tree::copy_tree_blocking(from, to, None) +} + +/// Replace `dst` through a stage beside it and a rename, never rewriting +/// its inode: a running Ruby may have the old `.so` / `.bundle` mapped, and +/// rewriting those pages under it crashes it. `fill` writes the stage. +fn replace_file( + dst: &Path, + fill: impl FnOnce(&Path) -> std::io::Result<()>, +) -> std::io::Result<()> { + let leaf = dst + .file_name() + .map(|n| n.to_string_lossy().into_owned()) + .unwrap_or_default(); + let stage = dst.with_file_name(format!(".{leaf}.socket-patch-stage")); + if let Some(parent) = dst.parent() { + std::fs::create_dir_all(parent)?; + } + let _ = std::fs::remove_file(&stage); + let filled = fill(&stage).and_then(|()| std::fs::rename(&stage, dst)); + if filled.is_err() { + let _ = std::fs::remove_file(&stage); + } + filled +} + +fn remove_dir_if_exists(dir: &Path) -> std::io::Result<()> { + match std::fs::symlink_metadata(dir) { + Ok(_) => crate::patch::copy_tree::force_remove_dir_all(dir), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(e) => Err(e), + } +} + +/// The backup's description. +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +struct BackupInfo { + /// As [`BuildMarker::extension_dir`]. + extension_dir: String, + /// The extension dir existed (its tree is under `ext/`). + had_extension_dir: bool, + /// The lib copies backed up under `lib/` (paths relative to the gem dir). + lib_files: Vec, +} + +/// [`BuildMarker::extension_dir`] for `ext_dir` (one of the copy's +/// installed extension dirs, [`find_ruby`]): relative to `home`, compared +/// canonically too (Ruby and the crawler may spell the same dir +/// differently: a relative crawl root, `/tmp` vs `/private/tmp`), so the +/// record survives the project moving. +fn ext_dir_label(home: &Path, ext_dir: &Path) -> String { + let canonical_rel = || { + let (home, ext) = (canonical(home)?, canonical(ext_dir)?); + ext.strip_prefix(&home).ok().map(Path::to_path_buf) + }; + let rel = ext_dir + .strip_prefix(home) + .ok() + .map(Path::to_path_buf) + .or_else(canonical_rel); + match rel.as_deref().ok_or(()) { + Ok(rel) => rel + .components() + .map(|c| c.as_os_str().to_string_lossy().into_owned()) + .collect::>() + .join("/"), + Err(_) => ext_dir.display().to_string(), + } +} + +/// Back up the original build of `copy` (its extension dir and the lib +/// copies of its artifacts) to [`BACKUP_DIR`], unless a backup exists or the +/// current build is one socket-patch made (a marker exists: its original is +/// gone). Staged and renamed into place, so a backup dir is always whole. +/// `Ok(true)` when this call made the backup. +fn back_up_original(copy: &GemCopy, probe: &Probe) -> std::io::Result { + let backup = copy.gem_dir.join(BACKUP_DIR); + if std::fs::symlink_metadata(&backup).is_ok() + || std::fs::symlink_metadata(copy.gem_dir.join(MARKER_FILE)).is_ok() + { + return Ok(false); + } + let stage = copy.gem_dir.join(format!("{BACKUP_DIR}.tmp")); + remove_dir_if_exists(&stage)?; + std::fs::create_dir_all(&stage)?; + let had_extension_dir = probe.extension_dir.is_dir(); + let mut lib_files = Vec::new(); + if had_extension_dir { + copy_tree_sync(&probe.extension_dir, &stage.join("ext"))?; + for rel in artifact_rels(&probe.extension_dir) { + let Some(lib) = lib_rel(probe, &rel) else { + continue; + }; + // Only a regular file: a FIFO there would block the read. + if let Ok(bytes) = read_regular_to_bytes_sync(©.gem_dir.join(&lib)) { + let dst = stage.join("lib").join(&lib); + if let Some(parent) = dst.parent() { + std::fs::create_dir_all(parent)?; + } + std::fs::write(&dst, bytes)?; + lib_files.push(lib); + } + } + } + let info = BackupInfo { + extension_dir: ext_dir_label(©.home, &probe.extension_dir), + had_extension_dir, + lib_files, + }; + std::fs::write( + stage.join(BACKUP_INFO), + serde_json::to_vec_pretty(&info).map_err(std::io::Error::other)?, + )?; + std::fs::rename(&stage, &backup)?; + Ok(true) +} + +/// One lib copy read into memory: gem-dir-relative path, bytes, mode. +type LibSnapshot = (String, Vec, std::fs::Permissions); + +/// The lib copies of an extension dir's artifacts, read into memory. +fn snapshot_lib_copies(copy: &GemCopy, probe: &Probe, rels: &[String]) -> Vec { + rels.iter() + .filter_map(|rel| lib_rel(probe, rel)) + .filter_map(|lib| { + let path = copy.gem_dir.join(&lib); + let perms = std::fs::metadata(&path).ok()?.permissions(); + let bytes = read_regular_to_bytes_sync(&path).ok()?; + Some((lib, bytes, perms)) + }) + .collect() +} + +/// State a build moved aside, to put back if it fails. +struct Aside { + /// The previous extension dir, renamed beside itself. + old_ext: Option, + /// The previous build's lib copies. + lib: Vec, + /// The previous marker's bytes. + marker: Option>, +} + +/// Move the current build out of the way of a fresh one. +fn set_aside(copy: &GemCopy, probe: &Probe) -> std::io::Result