From afdb4227003f4c31d931fd4e8fb66a7820d775b6 Mon Sep 17 00:00:00 2001 From: Vincent Date: Sat, 7 Nov 2026 18:05:44 -0500 Subject: [PATCH 1/3] perf: share one span render context per transaction Every caller of retrieve_span_if_present built a fresh SpanRenderContext, which verifies and decodes the whole visible closure, for every file it rendered. On a span import of hyperfine (1,097 commits, 75 files) one `atomic view switch` built the context 312 times, `atomic record -a` 466 times and `atomic restore -f` 157 times, at roughly 55 ms each, while the materialization itself took milliseconds. - Add SharedSpanRender: a lazily built context shared by every file read through one transaction and visibility closure. The per-file root, capability and visibility checks are unchanged; only the closure decode is shared. - Use it in project_tree, project_change_closure_with_txn and plan_file_moves, which render every projected file in one transaction. - Add Repository::get_file_contents_on_view and recorded_presence_on_view so callers that read many paths get one transaction, one projection and one context, with the absent-liveness check scoped to exactly those paths. Use them in the shim checkpoint observation, working-copy verification and the CLI and daemon restore paths, which previously rendered (and discarded) every file one at a time. Batched reads are tested byte for byte against the single-path reads for graph-origin, span-rooted, empty, directory and absent paths on two views. On the hyperfine span import, `atomic view switch` drops from 25-34 s to 4-7 s (12 context builds) and `atomic restore -f` of all 75 files from 15 s to 4.8 s (8 builds); graph-only repositories are unaffected. --- atomic-cli/src/commands/git/checkpoint.rs | 20 +- atomic-cli/src/commands/restore.rs | 123 +++++++++- atomic-repository/src/repository/content.rs | 221 ++++++++++++++++-- .../src/repository/deferred_tree.rs | 27 ++- atomic-repository/src/repository/mod.rs | 1 + .../src/repository/project_tree.rs | 22 +- atomic-repository/src/repository/record.rs | 8 +- .../repository/tests/batch_content_tests.rs | 123 ++++++++++ atomic-repository/src/repository/tests/mod.rs | 1 + atomic-repository/src/repository/verify.rs | 23 +- libatomic/src/daemon/services.rs | 36 ++- 11 files changed, 546 insertions(+), 59 deletions(-) create mode 100644 atomic-repository/src/repository/tests/batch_content_tests.rs diff --git a/atomic-cli/src/commands/git/checkpoint.rs b/atomic-cli/src/commands/git/checkpoint.rs index 26b675e4..df7db7af 100644 --- a/atomic-cli/src/commands/git/checkpoint.rs +++ b/atomic-cli/src/commands/git/checkpoint.rs @@ -344,21 +344,23 @@ fn observe_atomic_on_handle( .map_err(|error| CheckpointError::Atomic(error.to_string()))? .map(|(object, markers)| (object.hash().ok(), markers)); + // One read for every path: the view is projected and its span history + // decoded once instead of once per file. + let contents = repo + .get_file_contents_on_view(&paths, view) + .map_err(|error| CheckpointError::Atomic(error.to_string()))?; let mut hasher = blake3::Hasher::new(); hasher.update(ATOMIC_MANIFEST_DOMAIN); - for path in paths { + for (path, graph_content) in paths.into_iter().zip(contents) { let content = match &conflict_markers { Some((_, markers)) if markers.contains_key(&path) => { markers.get(&path).cloned().expect("marker bytes") } - _ => repo - .get_file_content_on_view(&path, view) - .map_err(|error| CheckpointError::Atomic(error.to_string()))? - .ok_or_else(|| { - CheckpointError::Atomic(format!( - "visible path '{path}' has no graph-derived content on view '{view}'" - )) - })?, + _ => graph_content.ok_or_else(|| { + CheckpointError::Atomic(format!( + "visible path '{path}' has no graph-derived content on view '{view}'" + )) + })?, }; hasher.update(&(path.len() as u64).to_le_bytes()); hasher.update(path.as_bytes()); diff --git a/atomic-cli/src/commands/restore.rs b/atomic-cli/src/commands/restore.rs index 544db13e..75c5cded 100644 --- a/atomic-cli/src/commands/restore.rs +++ b/atomic-cli/src/commands/restore.rs @@ -59,7 +59,7 @@ use std::path::{Path, PathBuf}; use atomic_repository::tracking::TrackingOptions; use atomic_repository::{ - FileStatus, Repository, RepositoryStatus, StatusOptions, WorkspaceTxnMode, + FileStatus, RecordedPresence, Repository, RepositoryStatus, StatusOptions, WorkspaceTxnMode, }; use clap::Parser; @@ -247,6 +247,106 @@ impl Restore { } } + /// Restore every path that the view projects as a file, in one + /// materialization. Returns `(restored, failed)` counts. + /// + /// Paths the view does not project (or projects as a directory) are + /// skipped silently, and a path with an unresolved name conflict is a + /// per-file failure, exactly as the single-file path reports them. If the + /// batched materialization itself fails, each path falls back to the + /// single-file restore so the per-file outcome stays accurate. + fn restore_contents( + &self, + repo: &Repository, + working_copy: atomic_core::WorkingCopyId, + view: &str, + paths: &[PathBuf], + ) -> (usize, usize) { + if paths.is_empty() { + return (0, 0); + } + let names: Vec = paths + .iter() + .map(|path| path.to_string_lossy().into_owned()) + .collect(); + let presence = match repo.recorded_presence_on_view(&names, view) { + Ok(presence) => presence, + Err(error) => { + print_warning(&format!("Failed to project the working copy: {}", error)); + return self.restore_contents_one_by_one(repo, working_copy, view, paths); + } + }; + let mut failed = 0; + let mut selected = Vec::new(); + for (name, presence) in names.iter().zip(presence) { + match presence { + RecordedPresence::Present => selected.push(name.clone()), + RecordedPresence::Absent => {} + RecordedPresence::NameConflict => { + print_warning(&format!( + "Failed to restore '{}': path has an unresolved name conflict; refusing to choose content", + name + )); + failed += 1; + } + } + } + if selected.is_empty() { + return (0, failed); + } + match repo.materialize_paths(working_copy, selected.iter().cloned().collect()) { + Ok(_) => { + for name in &selected { + println!(" Restored: {}", name); + } + (selected.len(), failed) + } + Err(error) => { + print_warning(&format!( + "Batched restore failed ({}); retrying file by file", + error + )); + let retry: Vec = selected.iter().map(PathBuf::from).collect(); + let (restored, retry_failed) = + self.restore_contents_one_by_one(repo, working_copy, view, &retry); + (restored, failed + retry_failed) + } + } + } + + fn restore_contents_one_by_one( + &self, + repo: &Repository, + working_copy: atomic_core::WorkingCopyId, + view: &str, + paths: &[PathBuf], + ) -> (usize, usize) { + let mut restored = 0; + let mut failed = 0; + for path in paths { + let path_display = path.display(); + match self.restore_file( + repo, + working_copy, + view, + Path::new(""), + path, + FileStatus::Modified, + ) { + Ok(RestoreOutcome::Restored) => { + println!(" Restored: {}", path_display); + restored += 1; + } + Ok(RestoreOutcome::Untracked | RestoreOutcome::Skipped) => {} + Err(e) => { + print_warning(&format!("Failed to restore '{}': {}", path_display, e)); + failed += 1; + } + } + } + (restored, failed) + } + /// Restore a single file according to its status. /// /// - `Added` (tracked but not recorded): undo tracking. The file stays @@ -455,9 +555,16 @@ impl Command for Restore { let mut restored_count = 0; let mut error_count = 0; + // Added files are untracked one by one; every other dirty file is + // restored in a single materialization so the view is projected and + // its span history decoded once, not once per file. + let mut content_paths = Vec::new(); for (path, file_status) in &files_to_restore { + if *file_status != FileStatus::Added { + content_paths.push(path.clone()); + continue; + } let path_display = path.display(); - match self.restore_file( &repo, working_copy, @@ -466,23 +573,21 @@ impl Command for Restore { path, *file_status, ) { - Ok(RestoreOutcome::Restored) => { - println!(" Restored: {}", path_display); - restored_count += 1; - } Ok(RestoreOutcome::Untracked) => { println!(" Untracked: {} (kept on disk)", path_display); restored_count += 1; } - Ok(RestoreOutcome::Skipped) => { - // No pristine content to restore; nothing to do. - } + Ok(RestoreOutcome::Restored | RestoreOutcome::Skipped) => {} Err(e) => { print_warning(&format!("Failed to restore '{}': {}", path_display, e)); error_count += 1; } } } + let (restored, failed) = + self.restore_contents(&repo, working_copy, &workspace_view, &content_paths); + restored_count += restored; + error_count += failed; // Summary println!(); diff --git a/atomic-repository/src/repository/content.rs b/atomic-repository/src/repository/content.rs index fa03ff60..b0e93b14 100644 --- a/atomic-repository/src/repository/content.rs +++ b/atomic-repository/src/repository/content.rs @@ -23,6 +23,17 @@ fn graph_visibility_at_change( graph_visibility_from_membership(txn, &membership).map(Some) } +/// Whether a view projects a file at a path, without rendering its content. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum RecordedPresence { + /// The view projects a non-directory entry at this path. + Present, + /// The path is absent on the view, or projects a directory. + Absent, + /// The path has an unresolved name conflict; no single content applies. + NameConflict, +} + impl Repository { /// Get the recorded content for a tracked file. /// @@ -289,6 +300,49 @@ impl Repository { } } + /// Decide, for many paths at once, whether a view projects a file there. + /// + /// One transaction and one tree projection answer every path; nothing is + /// rendered. The rules match [`Self::get_materialized_entry_on_view`]: + /// a visible name conflict is reported instead of choosing content, and a + /// directory or an absent path is `Absent`. + pub fn recorded_presence_on_view( + &self, + paths: &[String], + view_name: &str, + ) -> Result, RepositoryError> { + let txn = self + .pristine + .read_txn() + .map_err(|e| RepositoryError::Database(e.to_string()))?; + let view = txn + .get_view(view_name) + .map_err(|e| RepositoryError::Database(e.to_string()))? + .ok_or_else(|| RepositoryError::ViewNotFound { + name: view_name.to_string(), + })?; + let visibility = graph_visibility_closure(&txn, &view)?; + let wanted: std::collections::HashSet = paths + .iter() + .map(|path| normalize_path(Path::new(path))) + .collect(); + let projection = + self.project_tree_for_visibility_wanted(&txn, &visibility, Some(&wanted))?; + Ok(paths + .iter() + .map(|path| { + let normalized = normalize_path(Path::new(path)); + if projection.name_conflicts.contains_key(&normalized) { + return RecordedPresence::NameConflict; + } + match projection.present.get(&normalized) { + Some(item) if !item.is_directory => RecordedPresence::Present, + _ => RecordedPresence::Absent, + } + }) + .collect()) + } + /// Resolve a file as an explicit present/absent materialization entry on a view. /// /// Presence comes from the operation-aware path lifecycle projection. Empty @@ -335,22 +389,66 @@ impl Repository { if item.is_directory { return Ok(MaterializedEntry::absent(normalized, Some(item.inode))); } - let bytes = - match retrieve_span_if_present(&txn, &self.change_store, item.inode, &visibility)? { - Some(bytes) => bytes, - None => retrieve_content_with_filter_fast( - &txn, - &self.change_store, - item.inode, - item.position, - atomic_core::output::alive::RetrieveOptions::new() - .with_graph_visibility(visibility), - ) - .map_err(|error| RepositoryError::Database(error.to_string()))?, - }; + let spans = SharedSpanRender::new(&txn, &self.change_store, &visibility); + let bytes = render_projected_file(&txn, &self.change_store, &spans, item, &visibility)?; Ok(MaterializedEntry::present(normalized, item.inode, bytes)) } + /// Render many files on a view through one transaction, one tree + /// projection and one shared span context. + /// + /// Each entry is exactly what [`Self::get_file_content_on_view`] returns + /// for that path: `None` for an absent path or a directory. A requested + /// path with an unresolved name conflict fails the whole call, as the + /// single-path read refuses to choose content for it. + pub fn get_file_contents_on_view( + &self, + paths: &[String], + view_name: &str, + ) -> Result>>, RepositoryError> { + let txn = self + .pristine + .read_txn() + .map_err(|e| RepositoryError::Database(e.to_string()))?; + let view = txn + .get_view(view_name) + .map_err(|e| RepositoryError::Database(e.to_string()))? + .ok_or_else(|| RepositoryError::ViewNotFound { + name: view_name.to_string(), + })?; + let visibility = graph_visibility_closure(&txn, &view)?; + let wanted: std::collections::HashSet = paths + .iter() + .map(|path| normalize_path(Path::new(path))) + .collect(); + let projection = + self.project_tree_for_visibility_wanted(&txn, &visibility, Some(&wanted))?; + let spans = SharedSpanRender::new(&txn, &self.change_store, &visibility); + paths + .iter() + .map(|path| { + let normalized = normalize_path(Path::new(path)); + if let Some(conflict) = projection.name_conflicts.get(&normalized) { + return Err(RepositoryError::InvalidOperation { + message: format!( + "path '{}' has an unresolved name conflict ({} path(s), {} claimant(s)); refusing to choose content", + normalized, + conflict.paths.len(), + conflict.sides.len() + ), + }); + } + match projection.present.get(&normalized) { + Some(item) if !item.is_directory => { + render_projected_file(&txn, &self.change_store, &spans, item, &visibility) + .map(Some) + } + _ => Ok(None), + } + }) + .collect() + } + /// Read a path through an exact assembly visibility: `view`'s validated /// closure minus `excluded` changes (CB-9B review R2). /// @@ -907,6 +1005,37 @@ impl Repository { } } +/// Recorded bytes of one projected file: its span rendering when the file is +/// span-rooted on this view, otherwise the filtered graph rendering. +fn render_projected_file( + txn: &T, + store: &C, + spans: &SharedSpanRender<'_, T, C>, + item: &atomic_core::output::repo::OutputItem, + visibility: &GraphVisibilityClosure, +) -> Result, RepositoryError> +where + T: atomic_core::pristine::span_patch_storage::SpanPatchReadTxn + + atomic_core::pristine::GraphTxnT + + atomic_core::pristine::InodeGraphOps + + atomic_core::pristine::TreeTxnT + + atomic_core::pristine::CapabilityTxnT, + C: atomic_core::change::ChangeStore, +{ + if let Some(bytes) = spans.render_if_present(item.inode)? { + return Ok(bytes); + } + retrieve_content_with_filter_fast( + txn, + store, + item.inode, + item.position, + atomic_core::output::alive::RetrieveOptions::new() + .with_graph_visibility(visibility.clone()), + ) + .map_err(|error| RepositoryError::Database(error.to_string())) +} + pub(super) fn retrieve_span_if_present( txn: &T, store: &C, @@ -928,6 +1057,72 @@ where .map_err(|error| RepositoryError::Database(error.to_string())) } +/// A span renderer shared by every file read through one transaction and +/// visibility closure. +/// +/// Building a [`SpanRenderContext`] verifies and decodes the whole visible +/// closure, so callers that render many files in one transaction must not +/// rebuild it per file. The context is built lazily on the first span-rooted +/// file and reused for the rest; graph-origin files never trigger a build. +/// The per-file root, capability and visibility checks are unchanged. +pub(super) struct SharedSpanRender< + 'a, + T: atomic_core::pristine::span_patch_storage::SpanPatchReadTxn, + C, +> { + txn: &'a T, + store: &'a C, + visibility: &'a GraphVisibilityClosure, + context: + std::cell::OnceCell>, +} + +impl<'a, T, C> SharedSpanRender<'a, T, C> +where + T: atomic_core::pristine::span_patch_storage::SpanPatchReadTxn + + atomic_core::pristine::TreeTxnT + + atomic_core::pristine::CapabilityTxnT, + C: atomic_core::change::ChangeStore, +{ + pub(super) fn new(txn: &'a T, store: &'a C, visibility: &'a GraphVisibilityClosure) -> Self { + Self { + txn, + store, + visibility, + context: std::cell::OnceCell::new(), + } + } + + /// Render `inode` through the shared context when its span root is visible; + /// `None` means the file is graph-origin and must be read from the graph. + pub(super) fn render_if_present( + &self, + inode: Inode, + ) -> Result>, RepositoryError> { + if !has_visible_span_root(self.txn, self.store, inode, self.visibility)? { + return Ok(None); + } + let context = match self.context.get() { + Some(context) => context, + None => { + let built = atomic_core::pristine::span_patch_storage::SpanRenderContext::new( + self.txn, + self.store, + self.visibility, + ) + .map_err(|error| RepositoryError::Database(error.to_string()))?; + // A concurrent `set` is impossible: `OnceCell` is single-threaded. + let _ = self.context.set(built); + self.context.get().expect("span context was just set") + } + }; + context + .render(inode) + .map(Some) + .map_err(|error| RepositoryError::Database(error.to_string())) + } +} + pub(super) fn retrieve_span_bounded_if_present( txn: &T, store: &C, diff --git a/atomic-repository/src/repository/deferred_tree.rs b/atomic-repository/src/repository/deferred_tree.rs index cc2482ae..3b7725e6 100644 --- a/atomic-repository/src/repository/deferred_tree.rs +++ b/atomic-repository/src/repository/deferred_tree.rs @@ -1051,6 +1051,31 @@ impl Repository { visibility: &GraphVisibilityClosure, wanted: Option<&str>, ) -> Result + where + T: GraphTxnT + + TreeTxnT + + PathClaimTxnT + + atomic_core::pristine::InodeGraphOps, + { + match wanted { + Some(path) => self.project_tree_for_visibility_wanted( + txn, + visibility, + Some(&HashSet::from([path.to_string()])), + ), + None => self.project_tree_for_visibility_wanted(txn, visibility, None), + } + } + + /// Set-scoped variant of [`Self::project_tree_for_visibility_scoped`]: + /// the absent-liveness check runs only for the `wanted` paths, so a + /// batch of path reads costs what the same reads cost one by one. + pub(super) fn project_tree_for_visibility_wanted( + &self, + txn: &T, + visibility: &GraphVisibilityClosure, + wanted: Option<&HashSet>, + ) -> Result where T: GraphTxnT + TreeTxnT @@ -1105,7 +1130,7 @@ impl Repository { let mut absent_check_ms = 0u128; let visible_changes = visibility.iter_dependency_first().copied().collect(); for absent in reduced.absent { - let wanted_here = wanted.is_none_or(|path| path == absent.path.as_str()); + let wanted_here = wanted.is_none_or(|paths| paths.contains(absent.path.as_str())); if wanted_here && !absent.directory && !alive_inodes.contains(&absent.inode) diff --git a/atomic-repository/src/repository/mod.rs b/atomic-repository/src/repository/mod.rs index 51f727eb..863a0c94 100644 --- a/atomic-repository/src/repository/mod.rs +++ b/atomic-repository/src/repository/mod.rs @@ -44,6 +44,7 @@ use std::sync::Arc; use atomic_core::change::{Change, ChangeHeader, GraphOp}; pub use atomic_core::output::repo::MaterializedEntry; use atomic_core::output::repo::{MaterializeResult, OutputItem}; +pub use content::RecordedPresence; use atomic_core::pristine::{ GraphTxnT, GraphVisibilityClosure, MutTxnT, Pristine, TreeTxnT, ViewMembershipSet, ViewScope, diff --git a/atomic-repository/src/repository/project_tree.rs b/atomic-repository/src/repository/project_tree.rs index 1a945254..62a54fd6 100644 --- a/atomic-repository/src/repository/project_tree.rs +++ b/atomic-repository/src/repository/project_tree.rs @@ -1229,6 +1229,7 @@ impl Repository { ))); } let mut entries = Vec::new(); + let spans = super::content::SharedSpanRender::new(&txn, &self.change_store, &closure); for item in projection .present .values() @@ -1261,13 +1262,9 @@ impl Repository { ))); } let materialization = attrs.materialization; - let bytes = match super::content::retrieve_span_if_present( - &txn, - &self.change_store, - item.inode, - &closure, - ) - .map_err(ProjectTreeError::from)? + let bytes = match spans + .render_if_present(item.inode) + .map_err(ProjectTreeError::from)? { Some(content) => content, None => super::content::retrieve_content_with_filter_fast( @@ -1445,6 +1442,7 @@ impl Repository { } let mut entries = Vec::new(); let mut projected_paths = std::collections::BTreeSet::new(); + let spans = super::content::SharedSpanRender::new(txn, &self.change_store, &closure); for item in projection .present .values() @@ -1468,13 +1466,9 @@ impl Repository { let materialization = attrs.materialization; let bytes = match marker_bytes.get(item.path.as_str()) { Some(override_bytes) => override_bytes.clone(), - None => match super::content::retrieve_span_if_present( - txn, - &self.change_store, - item.inode, - &closure, - ) - .map_err(ProjectTreeError::from)? + None => match spans + .render_if_present(item.inode) + .map_err(ProjectTreeError::from)? { Some(content) => content, None => super::content::retrieve_content_with_filter_fast( diff --git a/atomic-repository/src/repository/record.rs b/atomic-repository/src/repository/record.rs index c4e13d0b..f968b06c 100644 --- a/atomic-repository/src/repository/record.rs +++ b/atomic-repository/src/repository/record.rs @@ -4694,11 +4694,13 @@ fn plan_file_moves( basis: MoveBasis, } + // One span context serves every deleted candidate in this planning pass. + let spans = super::content::SharedSpanRender::new(cached, store, visibility); let load_old_content = |inode: Inode, position: Position| -> Result, RecordError> { - if let Some(bytes) = - super::content::retrieve_span_if_present(cached, store, inode, visibility) - .map_err(RecordError::Repository)? + if let Some(bytes) = spans + .render_if_present(inode) + .map_err(RecordError::Repository)? { return Ok(bytes); } diff --git a/atomic-repository/src/repository/tests/batch_content_tests.rs b/atomic-repository/src/repository/tests/batch_content_tests.rs new file mode 100644 index 00000000..0fc023de --- /dev/null +++ b/atomic-repository/src/repository/tests/batch_content_tests.rs @@ -0,0 +1,123 @@ +use super::*; +use atomic_core::pristine::{CapabilityMutTxnT, MutTxnT, SPAN_RANGE_CAPABILITY}; + +fn record_all(repo: &TestRepository, message: &str) { + repo.record( + ChangeHeader::new(message), + RecordOptions::new() + .with_all(true) + .save_to_store(true) + .apply_after_record(true) + .sync_vault(false) + .enrich_kg(false), + ) + .unwrap(); +} + +fn enable_spans(repo: &TestRepository) { + let mut txn = repo.pristine().write_txn().unwrap(); + txn.put_required_capability(SPAN_RANGE_CAPABILITY).unwrap(); + txn.commit().unwrap(); +} + +fn write_and_add(temp: &tempfile::TempDir, repo: &mut TestRepository, path: &str, bytes: &[u8]) { + let full = temp.path().join(path); + if let Some(parent) = full.parent() { + std::fs::create_dir_all(parent).unwrap(); + } + std::fs::write(full, bytes).unwrap(); + repo.add(path, TrackingOptions::default()).unwrap(); +} + +/// Batched reads must be byte-identical to the single-path reads they replace, +/// for graph-origin files, span-rooted files, empty files, directories and +/// absent paths, on every view they are asked about. +#[test] +fn batch_reads_and_presence_match_single_path_reads() { + for spans in [false, true] { + let (temp, mut repo) = create_temp_repo(); + write_and_add(&temp, &mut repo, "legacy.txt", b"legacy\n"); + record_all(&repo, "graph origin"); + if spans { + enable_spans(&repo); + } + write_and_add(&temp, &mut repo, "a.txt", b"alpha\n"); + write_and_add(&temp, &mut repo, "b.txt", b"beta\n"); + write_and_add(&temp, &mut repo, "empty.txt", b""); + write_and_add(&temp, &mut repo, "dir/c.txt", b"gamma\n"); + record_all(&repo, "new files"); + std::fs::write(temp.path().join("b.txt"), b"beta edited\n").unwrap(); + record_all(&repo, "edit b"); + + repo.create_view_from("feature", "dev").unwrap(); + repo.switch_view("feature").unwrap(); + std::fs::write(temp.path().join("a.txt"), b"alpha on feature\n").unwrap(); + record_all(&repo, "feature edit"); + + let paths: Vec = [ + "a.txt", + "b.txt", + "empty.txt", + "legacy.txt", + "dir/c.txt", + "dir", + "missing.txt", + ] + .map(str::to_string) + .into(); + for view in ["dev", "feature"] { + let batch = repo.get_file_contents_on_view(&paths, view).unwrap(); + assert_eq!(batch.len(), paths.len()); + for (path, bytes) in paths.iter().zip(&batch) { + assert_eq!( + bytes, + &repo.get_file_content_on_view(path, view).unwrap(), + "spans={spans} view={view} path={path}" + ); + } + let expected_a: &[u8] = if view == "feature" { + b"alpha on feature\n" + } else { + b"alpha\n" + }; + assert_eq!(batch[0].as_deref(), Some(expected_a)); + assert_eq!(batch[1].as_deref(), Some(b"beta edited\n".as_slice())); + assert_eq!(batch[2].as_deref(), Some(b"".as_slice())); + assert_eq!(batch[3].as_deref(), Some(b"legacy\n".as_slice())); + assert_eq!(batch[4].as_deref(), Some(b"gamma\n".as_slice())); + assert_eq!(batch[5], None, "directories have no content"); + assert_eq!(batch[6], None, "absent paths have no content"); + + let presence = repo.recorded_presence_on_view(&paths, view).unwrap(); + let expected: Vec = batch + .iter() + .map(|bytes| { + if bytes.is_some() { + RecordedPresence::Present + } else { + RecordedPresence::Absent + } + }) + .collect(); + assert_eq!(presence, expected, "spans={spans} view={view}"); + } + } +} + +/// An unknown view fails the batch exactly like the single-path read. +#[test] +fn batch_reads_reject_unknown_views() { + let (temp, mut repo) = create_temp_repo(); + write_and_add(&temp, &mut repo, "a.txt", b"alpha\n"); + record_all(&repo, "one file"); + let paths = vec!["a.txt".to_string()]; + assert!(matches!( + repo.get_file_contents_on_view(&paths, "nope"), + Err(RepositoryError::ViewNotFound { .. }) + )); + assert!(matches!( + repo.recorded_presence_on_view(&paths, "nope"), + Err(RepositoryError::ViewNotFound { .. }) + )); + assert_eq!(repo.get_file_contents_on_view(&[], "dev").unwrap(), vec![]); +} diff --git a/atomic-repository/src/repository/tests/mod.rs b/atomic-repository/src/repository/tests/mod.rs index 80d3a470..0a505b6f 100644 --- a/atomic-repository/src/repository/tests/mod.rs +++ b/atomic-repository/src/repository/tests/mod.rs @@ -9,6 +9,7 @@ use std::path::Path; use tempfile::TempDir; mod attribute_tests; +mod batch_content_tests; mod binding_fetch_tests; mod binding_store_tests; mod binding_verify_tests; diff --git a/atomic-repository/src/repository/verify.rs b/atomic-repository/src/repository/verify.rs index b41df533..18554ba5 100644 --- a/atomic-repository/src/repository/verify.rs +++ b/atomic-repository/src/repository/verify.rs @@ -138,6 +138,27 @@ impl Repository { // Files visible (tracked + recorded) on the current view. let visible = self.visible_file_paths(&desired_view)?; + // Clean files are compared against the graph; read them all through + // one transaction so the view's span history is decoded once. + let is_uncommitted_or_conflicted = |path: &String| { + matches!( + status_by_path.get(path), + Some(FileStatus::Modified) + | Some(FileStatus::Added) + | Some(FileStatus::Deleted) + | Some(FileStatus::Conflicted) + ) + }; + let clean_paths: Vec = visible + .iter() + .filter(|path| !is_uncommitted_or_conflicted(path)) + .cloned() + .collect(); + let graph_contents: std::collections::HashMap<&String, Option>> = clean_paths + .iter() + .zip(self.get_file_contents_on_view(&clean_paths, &desired_view)?) + .collect(); + for path in &visible { let st = status_by_path.get(path).copied(); @@ -156,7 +177,7 @@ impl Repository { _ => { let abs = self.root.join(path); let disk = std::fs::read(&abs).ok(); - let graph = self.get_file_content_on_view(path, &desired_view)?; + let graph = graph_contents.get(path).cloned().flatten(); if let (Some(disk), Some(graph)) = (disk.as_ref(), graph.as_ref()) { report.clean_files_checked += 1; if disk != graph { diff --git a/libatomic/src/daemon/services.rs b/libatomic/src/daemon/services.rs index e7c69040..1431ed23 100644 --- a/libatomic/src/daemon/services.rs +++ b/libatomic/src/daemon/services.rs @@ -2045,6 +2045,7 @@ impl repository_mutation_service_server::RepositoryMutationService for MutationI } let mut restored: Vec = Vec::new(); let mut materialize = std::collections::HashSet::new(); + let mut content_paths: Vec = Vec::new(); for (path, file_status) in dirty { match file_status { // Added: undo the add — untrack, keep on disk. @@ -2058,15 +2059,32 @@ impl repository_mutation_service_server::RepositoryMutationService for MutationI restored.push(path); } // Modified/Deleted: pristine content back to disk. - _ => { - let content = repo - .get_file_content(std::path::Path::new(&path)) - .map_err(repository_error)?; - if content.is_some() { - materialize.insert(path.clone()); - restored.push(path); - } - // No pristine content: skip (nothing safe to do). + _ => content_paths.push(path), + } + } + // One projection decides which dirty paths the view still records; + // rendering every file here only to discard the bytes would decode + // the view's history once per file. + let presence = if content_paths.is_empty() { + Vec::new() + } else { + repo.recorded_presence_on_view(&content_paths, repo.current_view()) + .map_err(repository_error)? + }; + for (path, presence) in content_paths.into_iter().zip(presence) { + match presence { + atomic_repository::RecordedPresence::Present => { + materialize.insert(path.clone()); + restored.push(path); + } + // No pristine content: skip (nothing safe to do). + atomic_repository::RecordedPresence::Absent => {} + atomic_repository::RecordedPresence::NameConflict => { + return Err(repository_error(atomic_repository::RepositoryError::InvalidOperation { + message: format!( + "path '{path}' has an unresolved name conflict; refusing to choose content" + ), + })); } } } From b15406c2f6f98d3a267b6b07099087b66055b5c0 Mon Sep 17 00:00:00 2001 From: Vincent Date: Sat, 7 Nov 2026 19:45:44 -0500 Subject: [PATCH 2/3] fix: keep batched restore and checkpoint reads behaviour-identical Checkpoint observation no longer reads graph content for paths that carry conflict-marker bytes. Name-conflict errors carry the same path and claimant counts as the single-path reader. The daemon and CLI restore paths classify every path up front but still walk and report them in status order, so untracking, failures and output lines happen exactly where they did before. The batch-read test now asserts that the fixture really records span-rooted files. --- atomic-cli/src/commands/git/checkpoint.rs | 39 ++-- atomic-cli/src/commands/restore.rs | 185 ++++++++++-------- atomic-repository/src/repository/content.rs | 46 +++-- atomic-repository/src/repository/mod.rs | 2 +- .../repository/tests/batch_content_tests.rs | 16 ++ libatomic/src/daemon/services.rs | 76 ++++--- 6 files changed, 227 insertions(+), 137 deletions(-) diff --git a/atomic-cli/src/commands/git/checkpoint.rs b/atomic-cli/src/commands/git/checkpoint.rs index df7db7af..d16ba60b 100644 --- a/atomic-cli/src/commands/git/checkpoint.rs +++ b/atomic-cli/src/commands/git/checkpoint.rs @@ -344,19 +344,36 @@ fn observe_atomic_on_handle( .map_err(|error| CheckpointError::Atomic(error.to_string()))? .map(|(object, markers)| (object.hash().ok(), markers)); - // One read for every path: the view is projected and its span history - // decoded once instead of once per file. - let contents = repo - .get_file_contents_on_view(&paths, view) - .map_err(|error| CheckpointError::Atomic(error.to_string()))?; + // Paths with marker bytes never consult the graph. Every other path is + // read in one call so the view is projected and its span history decoded + // once instead of once per file. + let has_marker = |path: &String| -> bool { + conflict_markers + .as_ref() + .is_some_and(|(_, markers)| markers.contains_key(path)) + }; + let graph_paths: Vec = paths + .iter() + .filter(|path| !has_marker(path)) + .cloned() + .collect(); + let mut graph_contents: std::collections::HashMap>> = graph_paths + .iter() + .cloned() + .zip( + repo.get_file_contents_on_view(&graph_paths, view) + .map_err(|error| CheckpointError::Atomic(error.to_string()))?, + ) + .collect(); let mut hasher = blake3::Hasher::new(); hasher.update(ATOMIC_MANIFEST_DOMAIN); - for (path, graph_content) in paths.into_iter().zip(contents) { - let content = match &conflict_markers { - Some((_, markers)) if markers.contains_key(&path) => { - markers.get(&path).cloned().expect("marker bytes") - } - _ => graph_content.ok_or_else(|| { + for path in paths { + let marker_bytes = conflict_markers + .as_ref() + .and_then(|(_, markers)| markers.get(&path).cloned()); + let content = match marker_bytes { + Some(marker_bytes) => marker_bytes, + None => graph_contents.remove(&path).flatten().ok_or_else(|| { CheckpointError::Atomic(format!( "visible path '{path}' has no graph-derived content on view '{view}'" )) diff --git a/atomic-cli/src/commands/restore.rs b/atomic-cli/src/commands/restore.rs index 75c5cded..97fb0dad 100644 --- a/atomic-cli/src/commands/restore.rs +++ b/atomic-cli/src/commands/restore.rs @@ -55,6 +55,7 @@ //! Would restore: src/lib.rs //! ``` +use std::collections::HashMap; use std::path::{Path, PathBuf}; use atomic_repository::tracking::TrackingOptions; @@ -247,23 +248,27 @@ impl Restore { } } - /// Restore every path that the view projects as a file, in one - /// materialization. Returns `(restored, failed)` counts. + /// Classify and restore every Modified/Deleted path in one pass, returning + /// the outcome for each path so the caller can report them in status order. /// - /// Paths the view does not project (or projects as a directory) are - /// skipped silently, and a path with an unresolved name conflict is a - /// per-file failure, exactly as the single-file path reports them. If the - /// batched materialization itself fails, each path falls back to the - /// single-file restore so the per-file outcome stays accurate. + /// One projection decides which paths the view still records, and one + /// materialization writes all of them, instead of rendering and + /// materializing file by file. Outcomes match the single-file path: a path + /// the view does not project (or projects as a directory) is `Skipped`, + /// and a path with an unresolved name conflict fails with the same error. + /// A projection failure is reported for every path as the single-file read + /// would have reported it; if the batched materialization fails, every + /// path falls back to the single-file restore so outcomes stay per file. fn restore_contents( &self, repo: &Repository, working_copy: atomic_core::WorkingCopyId, view: &str, paths: &[PathBuf], - ) -> (usize, usize) { + ) -> HashMap> { + let mut outcomes = HashMap::new(); if paths.is_empty() { - return (0, 0); + return outcomes; } let names: Vec = paths .iter() @@ -272,46 +277,63 @@ impl Restore { let presence = match repo.recorded_presence_on_view(&names, view) { Ok(presence) => presence, Err(error) => { - print_warning(&format!("Failed to project the working copy: {}", error)); - return self.restore_contents_one_by_one(repo, working_copy, view, paths); + // A view or projection error would fail every single-path read + // the same way; report it per file without re-projecting. + for path in paths { + outcomes.insert( + path.clone(), + Err(CliError::Internal(anyhow::anyhow!( + "Failed to get file content: {}", + error + ))), + ); + } + return outcomes; } }; - let mut failed = 0; - let mut selected = Vec::new(); - for (name, presence) in names.iter().zip(presence) { + let mut selected: Vec<(PathBuf, String)> = Vec::new(); + for ((path, name), presence) in paths.iter().zip(&names).zip(presence) { match presence { - RecordedPresence::Present => selected.push(name.clone()), - RecordedPresence::Absent => {} - RecordedPresence::NameConflict => { - print_warning(&format!( - "Failed to restore '{}': path has an unresolved name conflict; refusing to choose content", - name - )); - failed += 1; + RecordedPresence::Present => selected.push((path.clone(), name.clone())), + RecordedPresence::Absent => { + outcomes.insert(path.clone(), Ok(RestoreOutcome::Skipped)); + } + RecordedPresence::NameConflict { paths, claimants } => { + let error = atomic_repository::name_conflict_error( + &atomic_repository::normalize_path(path), + paths, + claimants, + ); + outcomes.insert( + path.clone(), + Err(CliError::Internal(anyhow::anyhow!( + "Failed to get file content: {}", + error + ))), + ); } } } if selected.is_empty() { - return (0, failed); + return outcomes; } - match repo.materialize_paths(working_copy, selected.iter().cloned().collect()) { + let batch = selected.iter().map(|(_, name)| name.clone()).collect(); + match repo.materialize_paths(working_copy, batch) { Ok(_) => { - for name in &selected { - println!(" Restored: {}", name); + for (path, _) in selected { + outcomes.insert(path, Ok(RestoreOutcome::Restored)); } - (selected.len(), failed) } Err(error) => { print_warning(&format!( - "Batched restore failed ({}); retrying file by file", + "Batched restore failed ({}); restoring file by file", error )); - let retry: Vec = selected.iter().map(PathBuf::from).collect(); - let (restored, retry_failed) = - self.restore_contents_one_by_one(repo, working_copy, view, &retry); - (restored, failed + retry_failed) + let retry: Vec = selected.into_iter().map(|(path, _)| path).collect(); + outcomes.extend(self.restore_contents_one_by_one(repo, working_copy, view, &retry)); } } + outcomes } fn restore_contents_one_by_one( @@ -320,31 +342,21 @@ impl Restore { working_copy: atomic_core::WorkingCopyId, view: &str, paths: &[PathBuf], - ) -> (usize, usize) { - let mut restored = 0; - let mut failed = 0; - for path in paths { - let path_display = path.display(); - match self.restore_file( - repo, - working_copy, - view, - Path::new(""), - path, - FileStatus::Modified, - ) { - Ok(RestoreOutcome::Restored) => { - println!(" Restored: {}", path_display); - restored += 1; - } - Ok(RestoreOutcome::Untracked | RestoreOutcome::Skipped) => {} - Err(e) => { - print_warning(&format!("Failed to restore '{}': {}", path_display, e)); - failed += 1; - } - } - } - (restored, failed) + ) -> HashMap> { + paths + .iter() + .map(|path| { + let outcome = self.restore_file( + repo, + working_copy, + view, + Path::new(""), + path, + FileStatus::Modified, + ); + (path.clone(), outcome) + }) + .collect() } /// Restore a single file according to its status. @@ -555,39 +567,56 @@ impl Command for Restore { let mut restored_count = 0; let mut error_count = 0; - // Added files are untracked one by one; every other dirty file is - // restored in a single materialization so the view is projected and - // its span history decoded once, not once per file. - let mut content_paths = Vec::new(); + // Modified/Deleted files are classified and materialized together so + // the view is projected and its span history decoded once, not once + // per file. Added files are untracked one by one. Outcomes are + // reported in status order, as before. + let content_paths: Vec = files_to_restore + .iter() + .filter(|(_, file_status)| *file_status != FileStatus::Added) + .map(|(path, _)| path.clone()) + .collect(); + let mut content_outcomes = + self.restore_contents(&repo, working_copy, &workspace_view, &content_paths); + for (path, file_status) in &files_to_restore { - if *file_status != FileStatus::Added { - content_paths.push(path.clone()); - continue; - } let path_display = path.display(); - match self.restore_file( - &repo, - working_copy, - &workspace_view, - &repo_root, - path, - *file_status, - ) { + + let outcome = if *file_status == FileStatus::Added { + self.restore_file( + &repo, + working_copy, + &workspace_view, + &repo_root, + path, + *file_status, + ) + } else { + content_outcomes.remove(path).unwrap_or_else(|| { + Err(CliError::Internal(anyhow::anyhow!( + "no restore outcome recorded for '{}'", + path_display + ))) + }) + }; + match outcome { + Ok(RestoreOutcome::Restored) => { + println!(" Restored: {}", path_display); + restored_count += 1; + } Ok(RestoreOutcome::Untracked) => { println!(" Untracked: {} (kept on disk)", path_display); restored_count += 1; } - Ok(RestoreOutcome::Restored | RestoreOutcome::Skipped) => {} + Ok(RestoreOutcome::Skipped) => { + // No pristine content to restore; nothing to do. + } Err(e) => { print_warning(&format!("Failed to restore '{}': {}", path_display, e)); error_count += 1; } } } - let (restored, failed) = - self.restore_contents(&repo, working_copy, &workspace_view, &content_paths); - restored_count += restored; - error_count += failed; // Summary println!(); diff --git a/atomic-repository/src/repository/content.rs b/atomic-repository/src/repository/content.rs index b0e93b14..ecc4f1c0 100644 --- a/atomic-repository/src/repository/content.rs +++ b/atomic-repository/src/repository/content.rs @@ -31,7 +31,18 @@ pub enum RecordedPresence { /// The path is absent on the view, or projects a directory. Absent, /// The path has an unresolved name conflict; no single content applies. - NameConflict, + NameConflict { paths: usize, claimants: usize }, +} + +/// The error every content read returns for a path with an unresolved name +/// conflict. Callers that classify paths first must report the same error. +pub fn name_conflict_error(path: &str, paths: usize, claimants: usize) -> RepositoryError { + RepositoryError::InvalidOperation { + message: format!( + "path '{}' has an unresolved name conflict ({} path(s), {} claimant(s)); refusing to choose content", + path, paths, claimants + ), + } } impl Repository { @@ -332,8 +343,11 @@ impl Repository { .iter() .map(|path| { let normalized = normalize_path(Path::new(path)); - if projection.name_conflicts.contains_key(&normalized) { - return RecordedPresence::NameConflict; + if let Some(conflict) = projection.name_conflicts.get(&normalized) { + return RecordedPresence::NameConflict { + paths: conflict.paths.len(), + claimants: conflict.sides.len(), + }; } match projection.present.get(&normalized) { Some(item) if !item.is_directory => RecordedPresence::Present, @@ -369,14 +383,11 @@ impl Repository { let projection = self.project_tree_for_visibility_scoped(&txn, &visibility, Some(normalized.as_str()))?; if let Some(conflict) = projection.name_conflicts.get(&normalized) { - return Err(RepositoryError::InvalidOperation { - message: format!( - "path '{}' has an unresolved name conflict ({} path(s), {} claimant(s)); refusing to choose content", - normalized, - conflict.paths.len(), - conflict.sides.len() - ), - }); + return Err(name_conflict_error( + &normalized, + conflict.paths.len(), + conflict.sides.len(), + )); } let Some(item) = projection.present.get(&normalized) else { let inode = projection @@ -429,14 +440,11 @@ impl Repository { .map(|path| { let normalized = normalize_path(Path::new(path)); if let Some(conflict) = projection.name_conflicts.get(&normalized) { - return Err(RepositoryError::InvalidOperation { - message: format!( - "path '{}' has an unresolved name conflict ({} path(s), {} claimant(s)); refusing to choose content", - normalized, - conflict.paths.len(), - conflict.sides.len() - ), - }); + return Err(name_conflict_error( + &normalized, + conflict.paths.len(), + conflict.sides.len(), + )); } match projection.present.get(&normalized) { Some(item) if !item.is_directory => { diff --git a/atomic-repository/src/repository/mod.rs b/atomic-repository/src/repository/mod.rs index 863a0c94..f5cb2fef 100644 --- a/atomic-repository/src/repository/mod.rs +++ b/atomic-repository/src/repository/mod.rs @@ -44,7 +44,7 @@ use std::sync::Arc; use atomic_core::change::{Change, ChangeHeader, GraphOp}; pub use atomic_core::output::repo::MaterializedEntry; use atomic_core::output::repo::{MaterializeResult, OutputItem}; -pub use content::RecordedPresence; +pub use content::{name_conflict_error, RecordedPresence}; use atomic_core::pristine::{ GraphTxnT, GraphVisibilityClosure, MutTxnT, Pristine, TreeTxnT, ViewMembershipSet, ViewScope, diff --git a/atomic-repository/src/repository/tests/batch_content_tests.rs b/atomic-repository/src/repository/tests/batch_content_tests.rs index 0fc023de..7bca2d49 100644 --- a/atomic-repository/src/repository/tests/batch_content_tests.rs +++ b/atomic-repository/src/repository/tests/batch_content_tests.rs @@ -54,6 +54,22 @@ fn batch_reads_and_presence_match_single_path_reads() { std::fs::write(temp.path().join("a.txt"), b"alpha on feature\n").unwrap(); record_all(&repo, "feature edit"); + if spans { + use atomic_core::pristine::span_patch_storage::SpanPatchReadTxn; + use atomic_core::pristine::TreeTxnT; + let txn = repo.pristine().read_txn().unwrap(); + let inode = txn.get_inode("a.txt").unwrap().unwrap(); + assert!( + txn.span_root(inode).unwrap().is_some(), + "fixture must record a.txt as a span-rooted file" + ); + let legacy = txn.get_inode("legacy.txt").unwrap().unwrap(); + assert!( + txn.span_root(legacy).unwrap().is_none(), + "fixture must keep legacy.txt graph-origin" + ); + } + let paths: Vec = [ "a.txt", "b.txt", diff --git a/libatomic/src/daemon/services.rs b/libatomic/src/daemon/services.rs index 1431ed23..ea937f63 100644 --- a/libatomic/src/daemon/services.rs +++ b/libatomic/src/daemon/services.rs @@ -2043,9 +2043,36 @@ impl repository_mutation_service_server::RepositoryMutationService for MutationI name paths explicitly or pass force", )); } + // One projection decides which Modified/Deleted paths the view + // still records; rendering every file here only to discard the + // bytes would decode the view's history once per file. A + // projection error therefore fails the request before any path is + // untracked, and a render error now surfaces from materialization + // (which plans before it writes). The walk below keeps status + // order, so untracking and a name-conflict failure happen at the + // same position as before. + let content_paths: Vec = dirty + .iter() + .filter(|(_, file_status)| { + *file_status != atomic_repository::status::FileStatus::Added + }) + .map(|(path, _)| path.clone()) + .collect(); + let presence: std::collections::HashMap = + if content_paths.is_empty() { + Default::default() + } else { + content_paths + .iter() + .cloned() + .zip( + repo.recorded_presence_on_view(&content_paths, repo.current_view()) + .map_err(repository_error)?, + ) + .collect() + }; let mut restored: Vec = Vec::new(); let mut materialize = std::collections::HashSet::new(); - let mut content_paths: Vec = Vec::new(); for (path, file_status) in dirty { match file_status { // Added: undo the add — untrack, keep on disk. @@ -2059,33 +2086,26 @@ impl repository_mutation_service_server::RepositoryMutationService for MutationI restored.push(path); } // Modified/Deleted: pristine content back to disk. - _ => content_paths.push(path), - } - } - // One projection decides which dirty paths the view still records; - // rendering every file here only to discard the bytes would decode - // the view's history once per file. - let presence = if content_paths.is_empty() { - Vec::new() - } else { - repo.recorded_presence_on_view(&content_paths, repo.current_view()) - .map_err(repository_error)? - }; - for (path, presence) in content_paths.into_iter().zip(presence) { - match presence { - atomic_repository::RecordedPresence::Present => { - materialize.insert(path.clone()); - restored.push(path); - } - // No pristine content: skip (nothing safe to do). - atomic_repository::RecordedPresence::Absent => {} - atomic_repository::RecordedPresence::NameConflict => { - return Err(repository_error(atomic_repository::RepositoryError::InvalidOperation { - message: format!( - "path '{path}' has an unresolved name conflict; refusing to choose content" - ), - })); - } + _ => match presence.get(&path) { + Some(atomic_repository::RecordedPresence::Present) => { + materialize.insert(path.clone()); + restored.push(path); + } + // No pristine content: skip (nothing safe to do). + Some(atomic_repository::RecordedPresence::Absent) | None => {} + Some(atomic_repository::RecordedPresence::NameConflict { + paths, + claimants, + }) => { + return Err(repository_error(atomic_repository::name_conflict_error( + &atomic_repository::tracking::normalize_path(std::path::Path::new( + &path, + )), + *paths, + *claimants, + ))); + } + }, } } if !materialize.is_empty() { From 84729e29c5fdcf60a261d998c2b7e245dcc62a2d Mon Sep 17 00:00:00 2001 From: Vincent Date: Mon, 9 Nov 2026 09:10:50 -0500 Subject: [PATCH 3/3] fix: stream checkpoint and verify file contents --- atomic-cli/src/commands/git/checkpoint.rs | 78 ++++++++++++------ atomic-repository/src/repository/content.rs | 81 ++++++++++++------- .../repository/tests/batch_content_tests.rs | 50 ++++++++++++ atomic-repository/src/repository/verify.rs | 41 +++++----- 4 files changed, 177 insertions(+), 73 deletions(-) diff --git a/atomic-cli/src/commands/git/checkpoint.rs b/atomic-cli/src/commands/git/checkpoint.rs index d16ba60b..02d98c25 100644 --- a/atomic-cli/src/commands/git/checkpoint.rs +++ b/atomic-cli/src/commands/git/checkpoint.rs @@ -345,44 +345,70 @@ fn observe_atomic_on_handle( .map(|(object, markers)| (object.hash().ok(), markers)); // Paths with marker bytes never consult the graph. Every other path is - // read in one call so the view is projected and its span history decoded - // once instead of once per file. - let has_marker = |path: &String| -> bool { + // streamed from one read (one projection, one span context) in the same + // sorted order, and hashed as it arrives so only one file is held at a + // time; marker paths are hashed from their marker bytes in between. + let marker_bytes = |path: &str| -> Option<&[u8]> { conflict_markers .as_ref() - .is_some_and(|(_, markers)| markers.contains_key(path)) + .and_then(|(_, markers)| markers.get(path)) + .map(Vec::as_slice) }; let graph_paths: Vec = paths .iter() - .filter(|path| !has_marker(path)) + .filter(|path| marker_bytes(path).is_none()) .cloned() .collect(); - let mut graph_contents: std::collections::HashMap>> = graph_paths - .iter() - .cloned() - .zip( - repo.get_file_contents_on_view(&graph_paths, view) - .map_err(|error| CheckpointError::Atomic(error.to_string()))?, - ) - .collect(); let mut hasher = blake3::Hasher::new(); hasher.update(ATOMIC_MANIFEST_DOMAIN); - for path in paths { - let marker_bytes = conflict_markers - .as_ref() - .and_then(|(_, markers)| markers.get(&path).cloned()); - let content = match marker_bytes { - Some(marker_bytes) => marker_bytes, - None => graph_contents.remove(&path).flatten().ok_or_else(|| { - CheckpointError::Atomic(format!( - "visible path '{path}' has no graph-derived content on view '{view}'" - )) - })?, - }; + let hash_entry = |hasher: &mut blake3::Hasher, path: &str, content: &[u8]| { hasher.update(&(path.len() as u64).to_le_bytes()); hasher.update(path.as_bytes()); hasher.update(&(content.len() as u64).to_le_bytes()); - hasher.update(&content); + hasher.update(content); + }; + let mut next = 0usize; + let mut missing: Option = None; + let streamed = repo.for_each_file_content_on_view(&graph_paths, view, |path, bytes| { + while paths.get(next).is_some_and(|pending| pending != path) { + let pending = &paths[next]; + match marker_bytes(pending) { + Some(marker) => hash_entry(&mut hasher, pending, marker), + None => { + missing = Some(pending.clone()); + return Err(atomic_repository::RepositoryError::InvalidOperation { + message: format!("visible path '{pending}' has no content"), + }); + } + } + next += 1; + } + let Some(content) = bytes else { + missing = Some(path.to_string()); + return Err(atomic_repository::RepositoryError::InvalidOperation { + message: format!("visible path '{path}' has no content"), + }); + }; + hash_entry(&mut hasher, path, &content); + next += 1; + Ok(()) + }); + if let Some(path) = missing { + return Err(CheckpointError::Atomic(format!( + "visible path '{path}' has no graph-derived content on view '{view}'" + ))); + } + streamed.map_err(|error| CheckpointError::Atomic(error.to_string()))?; + while let Some(pending) = paths.get(next) { + match marker_bytes(pending) { + Some(marker) => hash_entry(&mut hasher, pending, marker), + None => { + return Err(CheckpointError::Atomic(format!( + "visible path '{pending}' has no graph-derived content on view '{view}'" + ))) + } + } + next += 1; } Ok(CheckpointAtomicState { diff --git a/atomic-repository/src/repository/content.rs b/atomic-repository/src/repository/content.rs index ecc4f1c0..34bee428 100644 --- a/atomic-repository/src/repository/content.rs +++ b/atomic-repository/src/repository/content.rs @@ -405,18 +405,23 @@ impl Repository { Ok(MaterializedEntry::present(normalized, item.inode, bytes)) } - /// Render many files on a view through one transaction, one tree - /// projection and one shared span context. - /// - /// Each entry is exactly what [`Self::get_file_content_on_view`] returns - /// for that path: `None` for an absent path or a directory. A requested - /// path with an unresolved name conflict fails the whole call, as the - /// single-path read refuses to choose content for it. - pub fn get_file_contents_on_view( + /// Stream the recorded bytes of many files on a view through one + /// transaction, one tree projection and one shared span context. + /// + /// `visit` receives each path in the order given, with exactly what + /// [`Self::get_file_content_on_view`] returns for it: `None` for an absent + /// path or a directory. Bytes are handed over one file at a time and + /// dropped by the visitor, so peak memory stays at one file, not the whole + /// set. A requested path with an unresolved name conflict fails the call, + /// as the single-path read refuses to choose content for it; a visitor + /// error stops the walk and is returned unchanged. Nothing survives this + /// call's transaction; the visitor must not mutate repository state. + pub fn for_each_file_content_on_view( &self, paths: &[String], view_name: &str, - ) -> Result>>, RepositoryError> { + mut visit: impl FnMut(&str, Option>) -> Result<(), RepositoryError>, + ) -> Result<(), RepositoryError> { let txn = self .pristine .read_txn() @@ -435,26 +440,44 @@ impl Repository { let projection = self.project_tree_for_visibility_wanted(&txn, &visibility, Some(&wanted))?; let spans = SharedSpanRender::new(&txn, &self.change_store, &visibility); - paths - .iter() - .map(|path| { - let normalized = normalize_path(Path::new(path)); - if let Some(conflict) = projection.name_conflicts.get(&normalized) { - return Err(name_conflict_error( - &normalized, - conflict.paths.len(), - conflict.sides.len(), - )); - } - match projection.present.get(&normalized) { - Some(item) if !item.is_directory => { - render_projected_file(&txn, &self.change_store, &spans, item, &visibility) - .map(Some) - } - _ => Ok(None), - } - }) - .collect() + for path in paths { + let normalized = normalize_path(Path::new(path)); + if let Some(conflict) = projection.name_conflicts.get(&normalized) { + return Err(name_conflict_error( + &normalized, + conflict.paths.len(), + conflict.sides.len(), + )); + } + let bytes = match projection.present.get(&normalized) { + Some(item) if !item.is_directory => Some(render_projected_file( + &txn, + &self.change_store, + &spans, + item, + &visibility, + )?), + _ => None, + }; + visit(path, bytes)?; + } + Ok(()) + } + + /// Collected form of [`Self::for_each_file_content_on_view`]: one entry per + /// requested path, in order. This holds every rendered file at once, so + /// callers that only need to consume each file should stream instead. + pub fn get_file_contents_on_view( + &self, + paths: &[String], + view_name: &str, + ) -> Result>>, RepositoryError> { + let mut contents = Vec::with_capacity(paths.len()); + self.for_each_file_content_on_view(paths, view_name, |_, bytes| { + contents.push(bytes); + Ok(()) + })?; + Ok(contents) } /// Read a path through an exact assembly visibility: `view`'s validated diff --git a/atomic-repository/src/repository/tests/batch_content_tests.rs b/atomic-repository/src/repository/tests/batch_content_tests.rs index 7bca2d49..18903e9a 100644 --- a/atomic-repository/src/repository/tests/batch_content_tests.rs +++ b/atomic-repository/src/repository/tests/batch_content_tests.rs @@ -137,3 +137,53 @@ fn batch_reads_reject_unknown_views() { )); assert_eq!(repo.get_file_contents_on_view(&[], "dev").unwrap(), vec![]); } + +/// The streaming read visits every path in the requested order with the same +/// bytes the collected read returns, and a visitor error stops the walk and +/// comes back unchanged. +#[test] +fn streaming_reads_visit_in_order_and_propagate_visitor_errors() { + for spans in [false, true] { + let (temp, mut repo) = create_temp_repo(); + if spans { + enable_spans(&repo); + } + write_and_add(&temp, &mut repo, "b.txt", b"beta\n"); + write_and_add(&temp, &mut repo, "a.txt", b"alpha\n"); + write_and_add(&temp, &mut repo, "dir/c.txt", b"gamma\n"); + record_all(&repo, "files"); + let paths: Vec = ["dir/c.txt", "missing.txt", "a.txt", "dir", "b.txt"] + .map(str::to_string) + .into(); + let collected = repo.get_file_contents_on_view(&paths, "dev").unwrap(); + let mut visited = Vec::new(); + repo.for_each_file_content_on_view(&paths, "dev", |path, bytes| { + visited.push((path.to_string(), bytes)); + Ok(()) + }) + .unwrap(); + let visited_paths: Vec<&str> = visited.iter().map(|(path, _)| path.as_str()).collect(); + assert_eq!( + visited_paths, + paths.iter().map(String::as_str).collect::>() + ); + let visited_bytes: Vec>> = + visited.into_iter().map(|(_, bytes)| bytes).collect(); + assert_eq!(visited_bytes, collected, "spans={spans}"); + + let mut seen = Vec::new(); + let error = repo + .for_each_file_content_on_view(&paths, "dev", |path, _| { + seen.push(path.to_string()); + if path == "a.txt" { + return Err(RepositoryError::InvalidOperation { + message: "visitor refused".into(), + }); + } + Ok(()) + }) + .unwrap_err(); + assert!(error.to_string().contains("visitor refused"), "{error}"); + assert_eq!(seen, ["dir/c.txt", "missing.txt", "a.txt"], "spans={spans}"); + } +} diff --git a/atomic-repository/src/repository/verify.rs b/atomic-repository/src/repository/verify.rs index 18554ba5..67f6c92f 100644 --- a/atomic-repository/src/repository/verify.rs +++ b/atomic-repository/src/repository/verify.rs @@ -138,8 +138,10 @@ impl Repository { // Files visible (tracked + recorded) on the current view. let visible = self.visible_file_paths(&desired_view)?; - // Clean files are compared against the graph; read them all through - // one transaction so the view's span history is decoded once. + // Clean files are compared against the graph. Stream them through one + // transaction so the view's span history is decoded once, comparing + // each file as it arrives; only the mismatching lengths are kept and + // reported below in the visible-path order. let is_uncommitted_or_conflicted = |path: &String| { matches!( status_by_path.get(path), @@ -154,10 +156,19 @@ impl Repository { .filter(|path| !is_uncommitted_or_conflicted(path)) .cloned() .collect(); - let graph_contents: std::collections::HashMap<&String, Option>> = clean_paths - .iter() - .zip(self.get_file_contents_on_view(&clean_paths, &desired_view)?) - .collect(); + let mut drift: HashMap = HashMap::new(); + let mut clean_files_checked = 0usize; + self.for_each_file_content_on_view(&clean_paths, &desired_view, |path, graph| { + let disk = std::fs::read(self.root.join(path)).ok(); + if let (Some(disk), Some(graph)) = (disk, graph) { + clean_files_checked += 1; + if disk != graph { + drift.insert(path.to_string(), (disk.len(), graph.len())); + } + } + Ok(()) + })?; + report.clean_files_checked += clean_files_checked; for path in &visible { let st = status_by_path.get(path).copied(); @@ -175,18 +186,12 @@ impl Repository { // Clean (absent from status) or other benign states: the disk // must match the graph exactly. _ => { - let abs = self.root.join(path); - let disk = std::fs::read(&abs).ok(); - let graph = graph_contents.get(path).cloned().flatten(); - if let (Some(disk), Some(graph)) = (disk.as_ref(), graph.as_ref()) { - report.clean_files_checked += 1; - if disk != graph { - report.problems.push(VerifyProblem::MaterializationDrift { - path: path.clone(), - disk_len: disk.len(), - graph_len: graph.len(), - }); - } + if let Some((disk_len, graph_len)) = drift.remove(path.as_str()) { + report.problems.push(VerifyProblem::MaterializationDrift { + path: path.clone(), + disk_len, + graph_len, + }); } } }