diff --git a/.gitattributes b/.gitattributes index e27b01a69..089b20f00 100644 --- a/.gitattributes +++ b/.gitattributes @@ -13,6 +13,10 @@ crates/socket-patch-core/tests/fixtures/poetry/** -text crates/socket-patch-core/tests/fixtures/pipenv/** -text crates/socket-patch-core/tests/fixtures/pipenv-shapes/** -text +# The native uv fixtures under upstream/ are real `uv lock` output: the +# restore tests derive their CRLF variants from the LF bytes themselves. +crates/socket-patch-core/tests/fixtures/upstream/** -text + # The captured pnpm 1-12 locks are byte-real: the hosted/vendored rewriters # refuse CRLF by design (vendor_lockfile_crlf_unsupported), and the tests # derive their CRLF variants from the LF bytes themselves. diff --git a/crates/socket-patch-cli/tests/e2e_redirect_uv_build.rs b/crates/socket-patch-cli/tests/e2e_redirect_uv_build.rs index 0d6912034..7729c6941 100644 --- a/crates/socket-patch-cli/tests/e2e_redirect_uv_build.rs +++ b/crates/socket-patch-cli/tests/e2e_redirect_uv_build.rs @@ -12,6 +12,10 @@ //! wired through `[tool.uv] override-dependencies` + `[tool.uv.sources]` — //! the uv 0.5.6 boundary (older uv re-resolves the override against the //! registry on a plain `uv sync`, and VEX must stop attesting); +//! * the same with `six` also in an extra under a different specifier, and +//! with `six` only in a PEP 735 group reached through `include-group` +//! (the hosted unwind re-derives each entry's specifier the way uv +//! lowered it, #606 / #473); //! * a PEP 723 script lock (`uv lock --script`), installed by `uv run //! --frozen --script` into uv's own env — so VEX attests it from the lock's //! sha256 pin, the not-installed hosted basis; @@ -70,6 +74,18 @@ fn hosted_uv_transitive_override_manifestless_vex() { hosted(Lane::Transitive); } +#[test] +#[ignore = "real uv + PyPI; run with --ignored"] +fn hosted_uv_extras_manifestless_vex() { + hosted(Lane::Extras); +} + +#[test] +#[ignore = "real uv + PyPI; run with --ignored"] +fn hosted_uv_include_group_manifestless_vex() { + hosted(Lane::IncludeGroup); +} + #[test] #[ignore = "real uv + PyPI; run with --ignored"] fn hosted_uv_script_lock_manifestless_vex() { diff --git a/crates/socket-patch-cli/tests/vex_e2e_common/uv.rs b/crates/socket-patch-cli/tests/vex_e2e_common/uv.rs index 369b52a45..a25b5e858 100644 --- a/crates/socket-patch-cli/tests/vex_e2e_common/uv.rs +++ b/crates/socket-patch-cli/tests/vex_e2e_common/uv.rs @@ -305,6 +305,15 @@ pub enum Lane { /// `six` only as `python-dateutil`'s dependency: wired through `[tool.uv] /// override-dependencies` + `[tool.uv.sources]` (the 0.5.6 boundary). Transitive, + /// `six==1.16.0` in `dependencies` and `six>=1.15` in an extra: two + /// `requires-dist` entries told apart only by their `extra` marker, so + /// the hosted unwind must follow uv's lowering to put each specifier + /// back (#606). `idna==3.7` is the registry sibling the unwind needs. + Extras, + /// `six` only in a PEP 735 group that another group pulls in with + /// `{ include-group = … }`: uv expands it into both groups' + /// `requires-dev` entries (#473). + IncludeGroup, /// PEP 723 `tool.py` + `uv lock --script` → `tool.py.lock`. Script, /// `uv export --format pylock.toml` (a pylock-only consumer checkout). @@ -316,10 +325,12 @@ pub enum Lane { } impl Lane { - pub const ALL: [Lane; 7] = [ + pub const ALL: [Lane; 9] = [ Lane::Project, Lane::Constraints, Lane::Transitive, + Lane::Extras, + Lane::IncludeGroup, Lane::Script, Lane::ExportPylock, Lane::CompilePylock, @@ -331,6 +342,8 @@ impl Lane { Lane::Project => "project", Lane::Constraints => "constraints", Lane::Transitive => "transitive", + Lane::Extras => "extras", + Lane::IncludeGroup => "include-group", Lane::Script => "script", Lane::ExportPylock => "export-pylock", Lane::CompilePylock => "compile-pylock", @@ -341,7 +354,11 @@ impl Lane { /// The files the writers wire (and the fresh checkout commits). fn wiring(self) -> &'static [&'static str] { match self { - Lane::Project | Lane::Constraints | Lane::Transitive => &["pyproject.toml", "uv.lock"], + Lane::Project + | Lane::Constraints + | Lane::Transitive + | Lane::Extras + | Lane::IncludeGroup => &["pyproject.toml", "uv.lock"], Lane::Script => &[SCRIPT, "tool.py.lock"], _ => &["pylock.toml"], } @@ -350,7 +367,11 @@ impl Lane { /// The lock file among [`Self::wiring`]. fn lock(self) -> &'static str { match self { - Lane::Project | Lane::Constraints | Lane::Transitive => "uv.lock", + Lane::Project + | Lane::Constraints + | Lane::Transitive + | Lane::Extras + | Lane::IncludeGroup => "uv.lock", Lane::Script => "tool.py.lock", _ => "pylock.toml", } @@ -358,7 +379,14 @@ impl Lane { /// A `pyproject.toml` + `uv.lock` project lane. fn has_project(self) -> bool { - matches!(self, Lane::Project | Lane::Constraints | Lane::Transitive) + matches!( + self, + Lane::Project + | Lane::Constraints + | Lane::Transitive + | Lane::Extras + | Lane::IncludeGroup + ) } /// `Err(why)` when this uv release has no such flow (reported `n/a`). @@ -385,6 +413,15 @@ impl Lane { Err("no `[tool.uv] override-dependencies` + sources before uv 0.2.35".into()) } Lane::Transitive => Ok(()), + // The hosted unwind's declaration matching; dependency groups + // (and `include-group`) arrived in uv 0.4.27. + Lane::Extras | Lane::IncludeGroup if mode == Mode::Vendored => { + Err("a hosted-unwind lane".into()) + } + Lane::Extras | Lane::IncludeGroup if !uv.at_least((0, 4, 27)) => { + Err("no PEP 735 dependency groups in uv.lock before uv 0.4.27".into()) + } + Lane::Extras | Lane::IncludeGroup => Ok(()), Lane::Script if !uv.help_has(&["lock"], "--script") => { Err("no `uv lock --script` before uv 0.5.17".into()) } @@ -721,11 +758,31 @@ fn build(uv: &Uv, lane: Lane, mode: Mode, tmp: &Path) -> Result { Ok(()) }; match lane { - Lane::Project | Lane::Constraints | Lane::Transitive => { + Lane::Project + | Lane::Constraints + | Lane::Transitive + | Lane::Extras + | Lane::IncludeGroup => { project_deps(match lane { Lane::Transitive => "\"python-dateutil==2.9.0.post0\"", + Lane::Extras => "\"six==1.16.0\", \"idna==3.7\"", + Lane::IncludeGroup => "\"idna==3.7\"", _ => "\"six==1.16.0\"", }); + let tail = match lane { + Lane::Extras => "\n[project.optional-dependencies]\nextra = [\"six>=1.15\"]\n", + Lane::IncludeGroup => { + "\n[dependency-groups]\ntest = [\"six==1.16.0\"]\n\ + dev = [{ include-group = \"test\" }]\n" + } + _ => "", + }; + if !tail.is_empty() { + let path = proj.join("pyproject.toml"); + let mut text = std::fs::read_to_string(&path).unwrap(); + text.push_str(tail); + std::fs::write(&path, text).unwrap(); + } if lane == Lane::Constraints { let path = proj.join("pyproject.toml"); let mut text = std::fs::read_to_string(&path).unwrap(); @@ -929,7 +986,11 @@ fn stage_manifest(proj: &Path, purl: &str, uuid: &str, orig: &[u8], patched: &[u fn install(uv: &Uv, lane: Lane, dir: &Path, cache: &Path, offline: bool, frozen: bool) -> String { let mut args: Vec<&str> = Vec::new(); match lane { - Lane::Project | Lane::Constraints | Lane::Transitive => { + Lane::Project + | Lane::Constraints + | Lane::Transitive + | Lane::Extras + | Lane::IncludeGroup => { args.push("sync"); if frozen && uv.help_has(&["sync"], "--frozen") { args.push("--frozen"); @@ -1635,7 +1696,14 @@ pub fn run_lane(suite: &str, uv: &Uv, mode: Mode, lane: Lane) { // (an `index`, or for `uv pip compile` PyPI files with none, #407) // and the artifact shape, so they restore to the bytes uv wrote // (#408). - let byte_exact = matches!(lane, Lane::ExportPylock | Lane::CompilePylock); + // The extras / include-group lanes lock an `idna` sibling too, so + // their unwind runs and must re-derive every `requires-dist` / + // `requires-dev` specifier from the declaration uv lowered it from + // (#606, #473). + let byte_exact = matches!( + lane, + Lane::ExportPylock | Lane::CompilePylock | Lane::Extras | Lane::IncludeGroup + ); let env: Value = serde_json::from_slice(&out.stdout) .unwrap_or_else(|e| panic!("{}: ({e})\n{}", report.what("revert"), dump(&out))); let still_wired = diff --git a/crates/socket-patch-core/src/patch/redirect/upstream/uv.rs b/crates/socket-patch-core/src/patch/redirect/upstream/uv.rs index 6b853d3e8..3d6cd921d 100644 --- a/crates/socket-patch-core/src/patch/redirect/upstream/uv.rs +++ b/crates/socket-patch-core/src/patch/redirect/upstream/uv.rs @@ -30,7 +30,11 @@ //! `overrides` entries lost their `specifier` for the url — re-derived //! from the paired metadata's declarations in uv's spelling (a //! multi-clause specifier only when another entry of the lock shows how -//! this uv joins clauses), and an `overrides` entry the rewrite added for +//! this uv joins clauses). The declaration is the one uv lowered the +//! entry from: PEP 735 `include-group` members are expanded, and when one +//! name has several specifiers the entry's marker picks one (its +//! `extra == ''` terms name the extra, the rest is the declaration's +//! own marker). An `overrides` entry the rewrite added for //! a transitive dependency is removed with its `override-dependencies` //! line; //! * the metadata's `[tool.uv.sources]. = { url }` is removed. @@ -212,6 +216,9 @@ async fn restore_lock( continue; } }; + // A refused hit leaves the lock as it was: its entry and every + // requirement array are restored together or not at all. + let before = doc.clone(); let restore = if pep751 { restore_pylock_entry(&mut doc, hit, &shape, artifacts) } else { @@ -219,6 +226,7 @@ async fn restore_lock( .and_then(|()| restore_requirements(&mut doc, hit, metadata.as_ref(), &styles, ctx)) }; if let Err(why) = restore { + doc = before; result.refuse(&hit.uuid, format!("{rel}: {why}")); continue; } @@ -793,36 +801,57 @@ enum Declared<'a> { Manifest(&'a str), } -/// Every declaration string `declared` covers in the metadata. -fn declarations<'d>(meta: &'d Metadata, declared: Declared<'_>) -> Vec<&'d str> { +/// One declaration a lock requirement entry can mirror: the PEP 508 string +/// and, for a `[project.optional-dependencies]` member, its extra (PEP 685 +/// normalized) — uv lowers that into the entry's marker as `extra == ''`. +struct Declaration<'d> { + spec: &'d str, + extra: Option, +} + +/// Every declaration `declared` covers in the metadata. +fn declarations<'d>(meta: &'d Metadata, declared: Declared<'_>) -> Vec> { let doc = &meta.doc; let uv = tool_uv(doc); + let plain = |specs: Vec<&'d str>| { + specs + .into_iter() + .map(|spec| Declaration { spec, extra: None }) + .collect() + }; match declared { Declared::Dist => { let project = doc.get("project"); - let mut out = strings(project.and_then(|p| p.get("dependencies"))); + let mut out: Vec> = + plain(strings(project.and_then(|p| p.get("dependencies")))); if let Some(extras) = project .and_then(|p| p.get("optional-dependencies")) .and_then(Item::as_table_like) { - for (_, group) in extras.iter() { - out.extend(strings(Some(group))); + for (extra, group) in extras.iter() { + let extra = canonicalize_pypi_name(extra); + out.extend(strings(Some(group)).into_iter().map(|spec| Declaration { + spec, + extra: Some(extra.clone()), + })); } } out } Declared::Dev(group) => { - let mut out = strings( - doc.get("dependency-groups") - .and_then(Item::as_table_like) - .and_then(|g| g.get(group)), + let mut out = Vec::new(); + group_members( + doc.get("dependency-groups").and_then(Item::as_table_like), + group, + &mut Vec::new(), + &mut out, ); if group == "dev" { out.extend(strings(uv.and_then(|u| u.get("dev-dependencies")))); } - out + plain(out) } - Declared::Manifest("requirements") => strings(doc.get("dependencies")), + Declared::Manifest("requirements") => plain(strings(doc.get("dependencies"))), Declared::Manifest(key) => { let key = match key { "constraints" => "constraint-dependencies", @@ -830,11 +859,116 @@ fn declarations<'d>(meta: &'d Metadata, declared: Declared<'_>) -> Vec<&'d str> "overrides" => "override-dependencies", other => other, }; - strings(uv.and_then(|u| u.get(key))) + plain(strings(uv.and_then(|u| u.get(key)))) } } } +/// A PEP 735 group's requirement strings, with its `{ include-group = … }` +/// members expanded the way uv expands them into the lock (group names +/// compare normalized; a group already being expanded is not re-entered). +fn group_members<'d>( + groups: Option<&'d dyn TableLike>, + group: &str, + expanding: &mut Vec, + out: &mut Vec<&'d str>, +) { + let canon = canonicalize_pypi_name(group); + if expanding.contains(&canon) { + return; + } + let Some(members) = groups + .into_iter() + .flat_map(|g| g.iter()) + .find(|(name, _)| canonicalize_pypi_name(name) == canon) + .and_then(|(_, item)| item.as_array()) + else { + return; + }; + expanding.push(canon); + for member in members.iter() { + if let Some(spec) = member.as_str() { + out.push(spec); + } else if let Some(included) = member + .as_inline_table() + .and_then(|t| t.get("include-group")) + .and_then(Value::as_str) + { + group_members(groups, included, expanding, out); + } + } + expanding.pop(); +} + +/// The extras an entry's marker names (`extra == ''`), normalized. +fn marker_extras(marker: &str) -> BTreeSet { + static EXTRA: std::sync::LazyLock = std::sync::LazyLock::new(|| { + regex::Regex::new(r#"\bextra\s*==\s*['"]([^'"]+)['"]"#).expect("static extra regex") + }); + EXTRA + .captures_iter(marker) + .map(|c| canonicalize_pypi_name(&c[1])) + .collect() +} + +/// A comparison key for a PEP 508 marker as uv records it in the lock: +/// `and`-joined atoms with `extra` terms dropped, quotes and spacing +/// normalized, sorted, and `python_version` comparisons spelled as the +/// `python_full_version` bounds uv rewrites them into. A marker with `or` +/// or parentheses is compared as normalized text. +fn marker_key(marker: &str) -> String { + static ATOM: std::sync::LazyLock = std::sync::LazyLock::new(|| { + regex::Regex::new( + r#"^([A-Za-z_][A-Za-z0-9_.]*)\s*(===|==|!=|~=|<=|>=|<|>)\s*(?:'([^']*)'|"([^"]*)")$"#, + ) + .expect("static marker atom regex") + }); + static AND: std::sync::LazyLock = + std::sync::LazyLock::new(|| regex::Regex::new(r"\s+and\s+").expect("static and regex")); + let text = marker.trim(); + if text.contains('(') || text.split_whitespace().any(|w| w == "or") { + return text + .replace('"', "'") + .split_whitespace() + .collect::>() + .join(" "); + } + let mut atoms: Vec = AND + .split(text) + .filter(|atom| !atom.trim().is_empty()) + .filter_map(|atom| { + let atom = atom.trim(); + let Some(c) = ATOM.captures(atom) else { + return Some(atom.replace('"', "'").split_whitespace().collect()); + }; + let (var, op) = (&c[1], &c[2]); + let value = c.get(3).or_else(|| c.get(4)).map_or("", |m| m.as_str()); + if var == "extra" { + return None; + } + if var == "python_version" { + if let Some((major, minor)) = value + .split_once('.') + .and_then(|(a, b)| Some((a.parse::().ok()?, b.parse::().ok()?))) + { + let next = format!("{major}.{}", minor + 1); + let full = |op: &str, v: &str| format!("python_full_version {op} '{v}'"); + return Some(match op { + "<" | ">=" => full(op, value), + "<=" => full("<", &next), + ">" => full(">=", &next), + "==" | "!=" => full(op, &format!("{value}.*")), + _ => format!("{var} {op} '{value}'"), + }); + } + } + Some(format!("{var} {op} '{value}'")) + }) + .collect(); + atoms.sort(); + atoms.join(" and ") +} + /// The normalized version clauses of a PEP 508 registry requirement (`[]` /// when unconstrained), or why uv's spelling of them is not derivable. fn spec_clauses(spec: &str) -> Result, String> { @@ -931,7 +1065,8 @@ impl SpecStyle { if !specifier.contains(',') { continue; } - if let Ok(Some(clauses)) = declared_clauses(meta, declared, name) { + let marker = entry.get("marker").and_then(Value::as_str); + if let Ok(Some(clauses)) = declared_clauses(meta, declared, name, marker) { evidence.push((specifier.to_string(), clauses)); } } @@ -946,32 +1081,91 @@ impl SpecStyle { } } -/// The one clause list every declaration of `name` in `declared` agrees -/// on; `Ok(None)` when nothing declares it. +/// The clause list of the declaration of `name` in `declared` that a lock +/// entry with `marker` mirrors; `Ok(None)` when nothing declares it. +/// +/// When every declaration agrees, that is the answer whatever the marker. +/// Otherwise uv's lowering picks one: the marker's `extra == ''` terms +/// name the extras it came from (or a declaration-owned simple equality), +/// and the rest of the marker is the declaration's own. More complex +/// declaration-owned extra predicates can lower to the same marker, so +/// differing clauses remain ambiguous and are refused for those shapes. fn declared_clauses( meta: &Metadata, declared: Declared<'_>, name: &str, + marker: Option<&str>, ) -> Result>, String> { let canon = canonicalize_pypi_name(name); - let mut found: Option> = None; - for spec in declarations(meta, declared) { - if canonicalize_pypi_name(pep508_name(spec)) != canon { - continue; - } - let clauses = spec_clauses(spec)?; - match &found { - Some(prior) if *prior != clauses => { - return Err(format!( - "{} declares {name} with different specifiers; which one each lock entry \ - mirrors is not derivable", - meta.rel - )) + let named: Vec> = declarations(meta, declared) + .into_iter() + .filter(|d| canonicalize_pypi_name(pep508_name(d.spec)) == canon) + .collect(); + if named.is_empty() { + return Ok(None); + } + let agreed = |set: &[&Declaration<'_>]| -> Result>, String> { + let mut found: Option> = None; + for d in set { + let clauses = spec_clauses(d.spec)?; + match &found { + Some(prior) if *prior != clauses => return Ok(None), + _ => found = Some(clauses), } - _ => found = Some(clauses), } + Ok(found) + }; + let mut set: Vec<&Declaration<'_>> = named.iter().collect(); + // The matcher understands a declaration-owned `extra == ''`, + // but uv can lower other predicates (including reversed equality) to + // that same marker. Without their erased specifiers, keep differing + // clauses ambiguous rather than discard a possible declaration. + static SIMPLE_EXTRA: std::sync::LazyLock = std::sync::LazyLock::new(|| { + regex::Regex::new(r#"^\s*extra\s*==\s*(?:'[A-Za-z0-9._-]+'|"[A-Za-z0-9._-]+")\s*$"#) + .expect("static simple extra regex") + }); + let unsupported_extra = named.iter().any(|d| { + d.spec.split_once(';').is_some_and(|(_, marker)| { + marker + .split(|c: char| !c.is_ascii_alphanumeric() && c != '_') + .any(|token| token == "extra") + && !SIMPLE_EXTRA.is_match(marker) + }) + }); + let marker = marker.unwrap_or(""); + let extras = marker_extras(marker); + let narrowings: [&dyn Fn(&Declaration<'_>) -> bool; 2] = [ + // A `dependencies` line can carry its own `extra == ''` marker + // (uv accepts it), so its lock entry looks like one from extra `x`. + &|d| match &d.extra { + Some(extra) => extras.contains(extra), + None => marker_extras(d.spec.split_once(';').map_or("", |(_, m)| m)) == extras, + }, + &|d| { + let own = d.spec.split_once(';').map_or("", |(_, m)| m); + marker_key(own) == marker_key(marker) + }, + ]; + for narrow in narrowings { + if let Ok(Some(clauses)) = agreed(&set) { + return Ok(Some(clauses)); + } + if unsupported_extra { + break; + } + let narrowed: Vec<&Declaration<'_>> = set.iter().copied().filter(|d| narrow(d)).collect(); + if narrowed.is_empty() { + break; + } + set = narrowed; } - Ok(found) + agreed(&set)?.map(Some).ok_or_else(|| { + format!( + "{} declares {name} with different specifiers; which one each lock entry \ + mirrors is not derivable", + meta.rel + ) + }) } /// Every lock requirement array with the declarations it mirrors @@ -1050,7 +1244,8 @@ fn restore_requirement_array( "the lock's requirement entries name the hosted artifact but its paired metadata \ file is missing, so their specifiers are not derivable", )?; - let clauses = declared_clauses(meta, declared, &hit.name)?.ok_or_else(|| { + let marker = entry.get("marker").and_then(Value::as_str); + let clauses = declared_clauses(meta, declared, &hit.name, marker)?.ok_or_else(|| { format!( "{} no longer declares {}, so the lock entry's specifier is not derivable", meta.rel, hit.name @@ -1411,3 +1606,337 @@ mod tests { assert_eq!(SpecStyle::CANDIDATES[3].join(&clauses), "<2,>=1"); } } + +/// The hosted unwind re-derives each requirement entry's specifier from the +/// declaration uv lowered it from: by extra and marker when one name has +/// several specifiers (#606), through PEP 735 `include-group` (#473). +#[cfg(test)] +mod declaration_tests { + use super::*; + + const UUID: &str = "e828efa5-5c6d-43f3-9909-03f5ac232b98"; + const HOSTED: &str = "https://patch.socket.dev/patch/pypi/six/1.16.0/g/e828efa5-5c6d-43f3-9909-03f5ac232b98/six-1.16.0-py2.py3-none-any.whl"; + + /// A hosted entry for six with an optional `marker`. + fn six(marker: Option<&str>) -> String { + match marker { + Some(m) => format!("{{ name = \"six\", marker = \"{m}\", url = \"{HOSTED}\" }}"), + None => format!("{{ name = \"six\", url = \"{HOSTED}\" }}"), + } + } + + /// The registry entry the unwind should write back. + fn spec(specifier: &str, marker: Option<&str>) -> String { + match marker { + Some(m) => { + format!("{{ name = \"six\", marker = \"{m}\", specifier = \"{specifier}\" }}") + } + None => format!("{{ name = \"six\", specifier = \"{specifier}\" }}"), + } + } + + fn lock(requires_dist: &[String], requires_dev: &[(&str, Vec)]) -> String { + let mut out = String::from( + "version = 1\nrequires-python = \">=3.9\"\n\n[[package]]\nname = \"uvp\"\n\ + version = \"0.1.0\"\nsource = { editable = \".\" }\n\n[package.metadata]\n", + ); + out.push_str(&format!("requires-dist = [{}]\n", requires_dist.join(", "))); + if !requires_dev.is_empty() { + out.push_str("\n[package.metadata.requires-dev]\n"); + for (group, entries) in requires_dev { + out.push_str(&format!("{group} = [{}]\n", entries.join(", "))); + } + } + out + } + + /// Run the requirement unwind for six over `lock_text` with `pyproject`. + fn unwind(pyproject: &str, lock_text: &str) -> Result { + let mut doc: DocumentMut = lock_text.parse().unwrap(); + let meta = Metadata { + rel: "pyproject.toml".into(), + text: pyproject.into(), + script: false, + doc: pyproject.parse().unwrap(), + }; + let hit = Hit { + index: 0, + uuid: UUID.into(), + name: "six".into(), + version: "1.16.0".into(), + }; + let client = super::super::UpstreamClient::new(true); + let ctx = Ctx { + client: &client, + origins: &[], + bun_lockb: false, + }; + restore_requirements(&mut doc, &hit, Some(&meta), &[], &ctx)?; + Ok(doc.to_string()) + } + + const HEAD: &str = + "[project]\nname = \"uvp\"\nversion = \"0.1.0\"\nrequires-python = \">=3.9\"\n"; + + /// #606 (a): a pin in `dependencies` and a floor in an extra. + #[test] + fn dependencies_and_extra_with_different_specifiers() { + let pyproject = format!( + "{HEAD}dependencies = [\"six==1.16.0\", \"idna==3.7\"]\n\n\ + [project.optional-dependencies]\nextra = [\"six>=1.15\"]\n" + ); + let idna = "{ name = \"idna\", specifier = \"==3.7\" }".to_string(); + let hosted = lock( + &[idna.clone(), six(None), six(Some("extra == 'extra'"))], + &[], + ); + assert_eq!( + unwind(&pyproject, &hosted).unwrap(), + lock( + &[ + idna, + spec("==1.16.0", None), + spec(">=1.15", Some("extra == 'extra'")) + ], + &[] + ) + ); + } + + /// A declaration-owned extra predicate can collide with uv's lowering + /// of optional group membership. Different clauses must remain refused. + #[test] + fn declaration_owned_extra_predicates_keep_ambiguity() { + for own in [ + "extra == 'x'", + "'x' == extra", + "extra != 'y'", + "extra in 'x,y'", + "extra not in 'y'", + "(extra == 'x' or extra == 'y')", + ] { + let pyproject = format!( + "{HEAD}dependencies = [\"six>=1.10; {own}\"]\n\n\ + [project.optional-dependencies]\nx = [\"six==1.16.0\"]\n" + ); + let marker = "extra == 'x'"; + let hosted = lock(&[six(Some(marker)), six(Some(marker))], &[]); + let err = unwind(&pyproject, &hosted).unwrap_err(); + assert!(err.contains("different specifiers"), "{own}: {err}"); + } + } + + /// Matching version clauses need no provenance inference, even when + /// an explicit extra predicate and a lowered group have the same marker. + #[test] + fn declaration_owned_extra_with_agreed_clauses_restores() { + let pyproject = format!( + "{HEAD}dependencies = [\"six==1.16.0; 'x' == extra\"]\n\n\ + [project.optional-dependencies]\nx = [\"six==1.16.0\"]\n" + ); + let marker = "extra == 'x'"; + let hosted = lock(&[six(Some(marker))], &[]); + assert_eq!( + unwind(&pyproject, &hosted).unwrap(), + lock(&[spec("==1.16.0", Some(marker))], &[]) + ); + } + + /// #606 (c): two extras with different floors. + #[test] + fn two_extras_with_different_specifiers() { + let pyproject = format!( + "{HEAD}dependencies = [\"idna==3.7\"]\n\n[project.optional-dependencies]\n\ + a = [\"six==1.16.0\"]\nb = [\"six>=1.10\"]\n" + ); + let hosted = lock(&[six(Some("extra == 'a'")), six(Some("extra == 'b'"))], &[]); + assert_eq!( + unwind(&pyproject, &hosted).unwrap(), + lock( + &[ + spec("==1.16.0", Some("extra == 'a'")), + spec(">=1.10", Some("extra == 'b'")) + ], + &[] + ) + ); + } + + /// Extras sharing one specifier lower to one entry naming both. + #[test] + fn extras_merged_into_one_entry() { + let pyproject = format!( + "{HEAD}dependencies = [\"six>=1.10\"]\n\n[project.optional-dependencies]\n\ + a = [\"six==1.16.0\"]\nc = [\"six==1.16.0\"]\n" + ); + let marker = "extra == 'a' or extra == 'c'"; + let hosted = lock(&[six(None), six(Some(marker))], &[]); + assert_eq!( + unwind(&pyproject, &hosted).unwrap(), + lock(&[spec(">=1.10", None), spec("==1.16.0", Some(marker))], &[]) + ); + } + + /// #606 (e): marker-split specifiers in `dependencies`, in both of + /// uv's spellings of a `python_version` marker. + #[test] + fn marker_split_dependencies() { + let pyproject = format!( + "{HEAD}dependencies = [\"idna==3.7\", \"six>=1.10; python_version < \\\"3.10\\\"\", \ + \"six==1.16.0; python_version >= \\\"3.10\\\"\"]\n" + ); + for (lt, ge) in [ + ( + "python_full_version < '3.10'", + "python_full_version >= '3.10'", + ), + ("python_version < '3.10'", "python_version >= '3.10'"), + ] { + let hosted = lock(&[six(Some(lt)), six(Some(ge))], &[]); + assert_eq!( + unwind(&pyproject, &hosted).unwrap(), + lock(&[spec(">=1.10", Some(lt)), spec("==1.16.0", Some(ge))], &[]), + "{lt} / {ge}" + ); + } + } + + /// uv rewrites `<=` / `>` / `==` on `python_version` into + /// `python_full_version` bounds. + #[test] + fn python_version_operators_match_uvs_rewrite() { + let pyproject = format!( + "{HEAD}dependencies = [\"six>=1.10; python_version <= '3.9'\", \ + \"six==1.16.0; python_version > '3.9' and sys_platform == 'linux'\", \ + \"six>=1.12; python_version == '3.12' and sys_platform != 'linux'\"]\n" + ); + let le = "python_full_version < '3.10'"; + let gt = "python_full_version >= '3.10' and sys_platform == 'linux'"; + let eq = "python_full_version == '3.12.*' and sys_platform != 'linux'"; + let hosted = lock(&[six(Some(le)), six(Some(gt)), six(Some(eq))], &[]); + assert_eq!( + unwind(&pyproject, &hosted).unwrap(), + lock( + &[ + spec(">=1.10", Some(le)), + spec("==1.16.0", Some(gt)), + spec(">=1.12", Some(eq)) + ], + &[] + ) + ); + } + + /// A marker inside an extra lowers to ` and extra == ''`. + #[test] + fn marker_inside_an_extra() { + let pyproject = format!( + "{HEAD}dependencies = [\"six==1.16.0\"]\n\n[project.optional-dependencies]\n\ + win = [\"six>=1.15; sys_platform == 'win32'\"]\n" + ); + let marker = "sys_platform == 'win32' and extra == 'win'"; + let hosted = lock(&[six(None), six(Some(marker))], &[]); + assert_eq!( + unwind(&pyproject, &hosted).unwrap(), + lock(&[spec("==1.16.0", None), spec(">=1.15", Some(marker))], &[]) + ); + } + + /// An entry no declaration lowers to is still refused. + #[test] + fn unmatched_marker_still_refuses() { + let pyproject = format!( + "{HEAD}dependencies = [\"six>=1.10; python_version < '3.10'\", \ + \"six==1.16.0; python_version >= '3.10'\"]\n" + ); + let hosted = lock(&[six(Some("sys_platform == 'linux'"))], &[]); + let err = unwind(&pyproject, &hosted).unwrap_err(); + assert!(err.contains("different specifiers"), "{err}"); + } + + /// A `dependencies` line with its own `extra == 'x'` marker lowers to + /// the same marker as extra `x`'s member: with different specifiers, + /// which entry mirrors which is not derivable, so the unwind refuses + /// rather than restore both from one declaration. + #[test] + fn dependency_with_its_own_extra_marker_is_ambiguous() { + let pyproject = format!( + "{HEAD}dependencies = [\"six>=1.10; extra == 'x'\"]\n\n\ + [project.optional-dependencies]\nx = [\"six==1.16.0\"]\n" + ); + let hosted = lock(&[six(Some("extra == 'x'")), six(Some("extra == 'x'"))], &[]); + let err = unwind(&pyproject, &hosted).unwrap_err(); + assert!(err.contains("different specifiers"), "{err}"); + } + + /// Unambiguous when the dependency's own extra is not also declared. + #[test] + fn dependency_with_its_own_extra_marker() { + let pyproject = format!( + "{HEAD}dependencies = [\"six>=1.10; extra == 'x'\"]\n\n\ + [project.optional-dependencies]\ny = [\"six==1.16.0\"]\n" + ); + let hosted = lock(&[six(Some("extra == 'x'")), six(Some("extra == 'y'"))], &[]); + assert_eq!( + unwind(&pyproject, &hosted).unwrap(), + lock( + &[ + spec(">=1.10", Some("extra == 'x'")), + spec("==1.16.0", Some("extra == 'y'")) + ], + &[] + ) + ); + } + + /// #473: a group reaching six through `include-group`. + #[test] + fn include_group_member() { + let pyproject = format!( + "{HEAD}dependencies = [\"python-dateutil==2.8.2\"]\n\n[dependency-groups]\n\ + test = [\"six==1.16.0\"]\ndev = [\"idna==3.7\", {{include-group = \"test\"}}]\n" + ); + let idna = "{ name = \"idna\", specifier = \"==3.7\" }".to_string(); + let dateutil = "{ name = \"python-dateutil\", specifier = \"==2.8.2\" }".to_string(); + let hosted = lock( + &[dateutil.clone()], + &[ + ("dev", vec![idna.clone(), six(None)]), + ("test", vec![six(None)]), + ], + ); + assert_eq!( + unwind(&pyproject, &hosted).unwrap(), + lock( + &[dateutil], + &[ + ("dev", vec![idna, spec("==1.16.0", None)]), + ("test", vec![spec("==1.16.0", None)]), + ] + ) + ); + } + + /// Nested and cyclic `include-group`s (uv rejects a cycle, but the + /// unwind must not loop on one) and PEP 735 group-name normalization. + #[test] + fn nested_and_cyclic_include_groups() { + let pyproject = format!( + "{HEAD}dependencies = []\n\n[dependency-groups]\n\ + Unit_Tests = [\"six==1.16.0\", {{include-group = \"all\"}}]\n\ + qa = [{{include-group = \"unit-tests\"}}]\n\ + all = [{{include-group = \"qa\"}}]\n" + ); + let hosted = lock(&[], &[("all", vec![six(None)]), ("qa", vec![six(None)])]); + assert_eq!( + unwind(&pyproject, &hosted).unwrap(), + lock( + &[], + &[ + ("all", vec![spec("==1.16.0", None)]), + ("qa", vec![spec("==1.16.0", None)]), + ] + ) + ); + } +} diff --git a/crates/socket-patch-core/tests/fixtures/upstream/uv-explicit-extra/pyproject.toml b/crates/socket-patch-core/tests/fixtures/upstream/uv-explicit-extra/pyproject.toml new file mode 100644 index 000000000..31b38b20b --- /dev/null +++ b/crates/socket-patch-core/tests/fixtures/upstream/uv-explicit-extra/pyproject.toml @@ -0,0 +1,7 @@ +[project] +name = "uvp" +version = "0.1.0" +requires-python = ">=3.9" +dependencies = ["idna==3.7", "six>=1.10; extra == 'x'"] +[project.optional-dependencies] +x = ["six==1.16.0"] diff --git a/crates/socket-patch-core/tests/fixtures/upstream/uv-explicit-extra/uv.lock b/crates/socket-patch-core/tests/fixtures/upstream/uv-explicit-extra/uv.lock new file mode 100644 index 000000000..50fb4bcef --- /dev/null +++ b/crates/socket-patch-core/tests/fixtures/upstream/uv-explicit-extra/uv.lock @@ -0,0 +1,42 @@ +version = 1 +revision = 3 +requires-python = ">=3.9" + +[[package]] +name = "idna" +version = "3.7" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/21/ed/f86a79a07470cb07819390452f178b3bef1d375f2ec021ecfc709fc7cf07/idna-3.7.tar.gz", hash = "sha256:028ff3aadf0609c1fd278d8ea3089299412a7a8b9bd005dd08b9f8285bcb5cfc", size = 189575, upload-time = "2024-04-11T03:34:43.276Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e5/3e/741d8c82801c347547f8a2a06aa57dbb1992be9e948df2ea0eda2c8b79e8/idna-3.7-py3-none-any.whl", hash = "sha256:82fee1fc78add43492d3a1898bfa6d8a904cc97d8427f683ed8e798d07761aa0", size = 66836, upload-time = "2024-04-11T03:34:41.447Z" }, +] + +[[package]] +name = "six" +version = "1.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/71/39/171f1c67cd00715f190ba0b100d606d440a28c93c7714febeca8b79af85e/six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926", size = 34041, upload-time = "2021-05-05T14:18:18.379Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d9/5a/e7c31adbe875f2abbb91bd84cf2dc52d792b5a01506781dbcf25c91daf11/six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254", size = 11053, upload-time = "2021-05-05T14:18:17.237Z" }, +] + +[[package]] +name = "uvp" +version = "0.1.0" +source = { virtual = "." } +dependencies = [ + { name = "idna" }, +] + +[package.optional-dependencies] +x = [ + { name = "six" }, +] + +[package.metadata] +requires-dist = [ + { name = "idna", specifier = "==3.7" }, + { name = "six", marker = "extra == 'x'", specifier = "==1.16.0" }, + { name = "six", marker = "extra == 'x'", specifier = ">=1.10" }, +] +provides-extras = ["x"] diff --git a/crates/socket-patch-core/tests/fixtures/vex-discover-golden/upstream.json b/crates/socket-patch-core/tests/fixtures/vex-discover-golden/upstream.json new file mode 100644 index 000000000..0915c1e47 --- /dev/null +++ b/crates/socket-patch-core/tests/fixtures/vex-discover-golden/upstream.json @@ -0,0 +1,19 @@ +{ + "upstream/uv-explicit-extra": { + "refs": [], + "diagnostics": [], + "recognized": [], + "unlocked_pins": [], + "elsewhere": [ + { + "purl": "pkg:pypi/idna@3.7", + "file": "uv.lock" + }, + { + "purl": "pkg:pypi/six@1.16.0", + "file": "uv.lock" + } + ], + "live_claims": [] + } +} diff --git a/crates/socket-patch-core/tests/upstream_restore_golden.rs b/crates/socket-patch-core/tests/upstream_restore_golden.rs index 231d221ba..89cf0e5c6 100644 --- a/crates/socket-patch-core/tests/upstream_restore_golden.rs +++ b/crates/socket-patch-core/tests/upstream_restore_golden.rs @@ -1762,6 +1762,66 @@ async fn pylock_whole_second_upload_times_round_trip() { assert_pypi_round_trip("uv export pylock", &input, &[urllib3_dep()], None).await; } +/// Native uv 0.11.19 gives these two different declarations the same +/// `extra == 'x'` marker. Once hosted URLs replace their specifiers, their +/// provenance is ambiguous: refusing must retain both files byte for byte. +#[tokio::test] +#[serial] +async fn uv_explicit_extra_collision_refuses_without_writing() { + let pyproject = include_str!("fixtures/upstream/uv-explicit-extra/pyproject.toml"); + let lock = include_str!("fixtures/upstream/uv-explicit-extra/uv.lock"); + let wheel = "six-1.16.0-py2.py3-none-any.whl"; + let dep = pypi_dep("six", "1.16.0", wheel, PYPI_UUID); + let (_server, _env) = pypi_mock(&[( + "six", + "1.16.0", + vec![ + ( + wheel, + "8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254", + 11053, + "2021-05-05T14:18:17.237Z", + ), + ( + "six-1.16.0.tar.gz", + "1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926", + 34041, + "2021-05-05T14:18:18.379Z", + ), + ], + )]) + .await; + for own_marker in ["extra == 'x'", "'x' == extra"] { + let pyproject = pyproject.replace("extra == 'x'", own_marker); + for eol in ["\n", "\r\n"] { + let input = tree(&[ + ("uv.lock", lock.replace('\n', eol)), + ("pyproject.toml", pyproject.replace('\n', eol)), + ]); + for dry_run in [false, true] { + println!("uv extra-marker refusal: {own_marker:?}, eol={eol:?}, dry_run={dry_run}"); + let (why, rewritten, after) = pypi_refusal( + &input, + std::slice::from_ref(&dep), + &RestoreOptions { + dry_run, + ..Default::default() + }, + ) + .await; + assert!(why.contains("different specifiers"), "{why}"); + assert!(why.contains("git checkout -- uv.lock"), "{why}"); + assert!(rewritten["uv.lock"].contains("patch.socket.dev")); + assert!(rewritten["pyproject.toml"].contains("patch.socket.dev")); + assert_eq!( + after, rewritten, + "refused unwind changed files ({eol:?}, dry_run={dry_run})" + ); + } + } + } +} + #[tokio::test] #[serial] async fn uv_refusals() { diff --git a/docs/testing/uv-compatibility.md b/docs/testing/uv-compatibility.md index 42d248742..d415ffbad 100644 --- a/docs/testing/uv-compatibility.md +++ b/docs/testing/uv-compatibility.md @@ -92,7 +92,18 @@ frozen, locked, and ordinary installation outcomes separately where supported. - `[tool.uv] dev-dependencies` (the pre-PEP 735 dev group) is classified as a direct dependency, and every duplicate `requires-dist` / `requires-dev` entry for the package (extras, markers) is repointed, so `uv sync --locked` - accepts the lock. `[tool.uv] constraint-dependencies` / + accepts the lock. The hosted unwind (`rollback`, `remove`, the hosted → + vendored takeover) puts each entry's specifier back from the declaration + uv lowered it from, so one package declared with different specifiers in + `dependencies`, extras or marker-split lines, or reached through a PEP 735 + `include-group`, rolls back byte for byte (the `extras` and + `include-group` lanes of `e2e_redirect_uv_build`, uv ≥ 0.4.27). An entry + whose marker matches no declaration is still refused. Declaration-owned + simple equality markers (`extra == 'name'`) are matched explicitly. More + complex `extra` predicates with differing version clauses remain refused, + as do declarations whose lowered markers are indistinguishable: hosted + URLs erase the specifiers needed to recover their provenance. Refusals + leave the lock and paired metadata unchanged. `[tool.uv] constraint-dependencies` / `build-constraint-dependencies` naming the package are repointed in the lock's `[manifest]` `constraints` / `build-constraints` entries, which uv ≥ 0.5.6 serializes with the package's source. uv 0.2.37–0.5.3 serialize