From 9a5dec94e29d381a8848a5424dc18e5482dfdfa8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20Og=C3=B3rek?= Date: Tue, 6 Jul 2021 15:49:28 +0200 Subject: [PATCH 1/4] feat: Print sourcemaps/files operations timings information --- src/utils/chunks.rs | 5 ++++- src/utils/file_upload.rs | 9 ++++++--- src/utils/progress.rs | 11 ++++++++++- src/utils/sourcemaps.rs | 7 +++++-- 4 files changed, 25 insertions(+), 7 deletions(-) diff --git a/src/utils/chunks.rs b/src/utils/chunks.rs index 1b1ab284ef..6ee5e910c4 100644 --- a/src/utils/chunks.rs +++ b/src/utils/chunks.rs @@ -7,6 +7,7 @@ use std::sync::Arc; use std::time::Duration; +use std::time::Instant; use failure::Error; use log::warn; @@ -175,6 +176,7 @@ pub fn upload_chunks( // `Api::upload_chunks`, the progress bar is created in an Arc. let progress = Arc::new(ProgressBar::new(total_bytes)); progress.set_style(progress_style); + let start = Instant::now(); // Select the best available compression mechanism. We assume that every // compression algorithm has been implemented for uploading, except `Other` @@ -224,6 +226,7 @@ pub fn upload_chunks( .collect::>() })?; - progress.finish_and_clear(); + progress.finish_with_duration("Uploading", start.elapsed()); + Ok(()) } diff --git a/src/utils/file_upload.rs b/src/utils/file_upload.rs index 2c56112638..3fa57b6511 100644 --- a/src/utils/file_upload.rs +++ b/src/utils/file_upload.rs @@ -216,6 +216,7 @@ fn upload_files_chunked( let progress = ProgressBar::new_spinner(); progress.enable_steady_tick(100); progress.set_style(progress_style); + let start = Instant::now(); let view = ByteView::open(archive.path())?; let (checksum, checksums) = get_sha1_checksums(&view, options.chunk_size)?; @@ -225,7 +226,7 @@ fn upload_files_chunked( .map(|(data, checksum)| Chunk((*checksum, data))) .collect::>(); - progress.finish_and_clear(); + progress.finish_with_duration("Optimizing", start.elapsed()); let progress_style = ProgressStyle::default_bar().template(&format!( "{} Uploading release files...\ @@ -272,7 +273,7 @@ fn upload_files_chunked( bail!("Failed to process uploaded files: {}", message); } - progress.finish_and_clear(); + progress.finish_with_duration("Processing", assemble_start.elapsed()); if response.state.is_pending() { if context.wait { @@ -299,6 +300,7 @@ fn build_artifact_bundle(context: &UploadContext, files: &ReleaseFiles) -> Resul let progress = ProgressBar::new(files.len() as u64); progress.set_style(progress_style); progress.set_prefix(">"); + let start = Instant::now(); let archive = TempFile::create()?; let mut bundle = SourceBundleWriter::start(BufWriter::new(archive.open()?))?; @@ -329,7 +331,8 @@ fn build_artifact_bundle(context: &UploadContext, files: &ReleaseFiles) -> Resul bundle.finish()?; - progress.finish_and_clear(); + progress.finish_with_duration("Bundling", start.elapsed()); + println!( "{} Bundled {} {} for upload", style(">").dim(), diff --git a/src/utils/progress.rs b/src/utils/progress.rs index d3bb655307..d38b920eab 100644 --- a/src/utils/progress.rs +++ b/src/utils/progress.rs @@ -1,7 +1,7 @@ use console::{style, Term}; -use std::env; use std::ops::Deref; use std::sync::Arc; +use std::{env, time::Duration}; use crate::utils::logging; @@ -46,6 +46,15 @@ impl ProgressBar { logging::set_progress_bar(None); } + pub fn finish_with_duration(&self, op: &str, dur: Duration) { + let progress_style = ProgressStyle::default_bar().template("{prefix:.dim} {msg}"); + let msg = format!("{} completed in {}.{}s", op, dur.as_secs(), dur.as_millis()); + self.inner.set_style(progress_style); + self.inner.set_prefix(">"); + self.inner.finish_with_message(&msg); + logging::set_progress_bar(None); + } + pub fn finish_and_clear(&self) { self.inner.finish_and_clear(); logging::set_progress_bar(None); diff --git a/src/utils/sourcemaps.rs b/src/utils/sourcemaps.rs index 93b1424e93..d60544aa8f 100644 --- a/src/utils/sourcemaps.rs +++ b/src/utils/sourcemaps.rs @@ -4,6 +4,7 @@ use std::ffi::OsStr; use std::mem; use std::path::PathBuf; use std::str; +use std::time::Instant; use console::style; use failure::{bail, Error}; @@ -192,6 +193,7 @@ impl SourceMapProcessor { } let pb = make_progress_bar(self.pending_sources.len() as u64); + let start = Instant::now(); println!( "{} Analyzing {} sources", @@ -246,7 +248,7 @@ impl SourceMapProcessor { pb.inc(1); } - pb.finish_and_clear(); + pb.finish_with_duration("Analyzing", start.elapsed()); } pub fn dump_log(&self, title: &str) { @@ -461,6 +463,7 @@ impl SourceMapProcessor { self.unpack_indexed_ram_bundles()?; let pb = make_progress_bar(self.sources.len() as u64); + let start = Instant::now(); for source in self.sources.values_mut() { pb.set_message(&source.url); if source.ty != SourceFileType::SourceMap { @@ -487,7 +490,7 @@ impl SourceMapProcessor { source.contents = new_source; pb.inc(1); } - pb.finish_and_clear(); + pb.finish_with_duration("Rewriting", start.elapsed()); Ok(()) } From 929d2d496f7e4287279a48d26892e207031e8fb3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20Og=C3=B3rek?= Date: Tue, 6 Jul 2021 16:22:34 +0200 Subject: [PATCH 2/4] Update to indicatiff 0.16 --- Cargo.lock | 29 +++++++++++++++++++++++------ Cargo.toml | 4 ++-- src/api.rs | 2 +- src/commands/difutil_find.rs | 4 ++-- src/commands/upload_dif.rs | 2 +- src/utils/dif_upload.rs | 10 +++++----- src/utils/file_search.rs | 4 ++-- src/utils/file_upload.rs | 2 +- src/utils/progress.rs | 6 +++--- src/utils/sourcemaps.rs | 6 +++--- 10 files changed, 43 insertions(+), 26 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 21e91baf7f..a79190a8c1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -101,7 +101,7 @@ checksum = "a88b6bd5df287567ffdf4ddf4d33060048e1068308e5f62d81c6f9824a045a48" dependencies = [ "bstr", "doc-comment", - "predicates", + "predicates 1.0.8", "predicates-core", "predicates-tree", "wait-timeout", @@ -478,6 +478,12 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "524cbf6897b527295dff137cec09ecf3a05f4fddffd7dfcd1585403449e74198" +[[package]] +name = "difflib" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6184e33543162437515c2e2b48714794e37845ec9851711914eec9d308f6ebe8" + [[package]] name = "digest" version = "0.8.1" @@ -932,9 +938,9 @@ dependencies = [ [[package]] name = "indicatif" -version = "0.14.0" +version = "0.16.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49a68371cf417889c9d7f98235b7102ea7c54fc59bcbd22f3dea785be9d27e40" +checksum = "2d207dc617c7a380ab07ff572a6e52fa202a2a8f355860ac9c38e23f8196be1b" dependencies = [ "console", "lazy_static", @@ -1217,9 +1223,9 @@ dependencies = [ [[package]] name = "number_prefix" -version = "0.3.0" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17b02fc0ff9a9e4b35b3342880f48e896ebf69f2967921fe8646bf5b7125956a" +checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" [[package]] name = "object" @@ -1457,7 +1463,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f49cfaf7fdaa3bfacc6fa3e7054e65148878354a5cfddcf661df4c851f8021df" dependencies = [ "difference", + "predicates-core", +] + +[[package]] +name = "predicates" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6e46ca79eb4e21e2ec14430340c71250ab69332abf85521c95d3a8bc336aa76" +dependencies = [ + "difflib", "float-cmp", + "itertools", "normalize-line-endings", "predicates-core", "regex", @@ -1954,7 +1971,7 @@ dependencies = [ "parking_lot", "percent-encoding 2.1.0", "plist", - "predicates", + "predicates 2.0.0", "prettytable-rs", "proguard", "r2d2", diff --git a/Cargo.toml b/Cargo.toml index 497f821bfc..927fdb558b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,7 +26,7 @@ git2 = { version = "0.13.6", default-features = false } glob = "0.3.0" if_chain = "1.0.0" ignore = "0.4.16" -indicatif = "0.14.0" +indicatif = "0.16.2" itertools = "0.10.0" java-properties = "1.2.0" lazy_static = "1.4.0" @@ -62,7 +62,7 @@ zip = "0.5.6" assert_cmd = "1.0.1" insta = { version = "1.7.1", features = ["redactions"] } mockito = "0.30.0" -predicates = "1.0.4" +predicates = "2.0.0" tempfile = "3.1.0" [features] diff --git a/src/api.rs b/src/api.rs index 09b4d8185b..1772d1efb9 100644 --- a/src/api.rs +++ b/src/api.rs @@ -669,7 +669,7 @@ impl Api { let pb = make_progress_bar(sources.len() as u64); for (url, path) in sources { - pb.set_message(&path.to_str().unwrap()); + pb.set_message(path.to_str().unwrap().to_owned()); let upload_result = self.upload_release_file( org, diff --git a/src/commands/difutil_find.rs b/src/commands/difutil_find.rs index 4f797156ee..d61f2f54b2 100644 --- a/src/commands/difutil_find.rs +++ b/src/commands/difutil_find.rs @@ -141,10 +141,10 @@ fn find_ids( } if let Some(p) = dirent.file_name().to_str() { - pb.set_message(&p); + pb.set_message(p.to_owned()); } pb.tick(); - pb.set_prefix(&format!("{}", found_files.len())); + pb.set_prefix(found_files.len().to_string()); let mut found = vec![]; diff --git a/src/commands/upload_dif.rs b/src/commands/upload_dif.rs index 56f534f768..289c09506b 100644 --- a/src/commands/upload_dif.rs +++ b/src/commands/upload_dif.rs @@ -290,7 +290,7 @@ fn execute_internal(matches: &ArgMatches<'_>, legacy: bool) -> Result<(), Error> let progress = ProgressBar::new_spinner(); progress.enable_steady_tick(100); progress.set_style(progress_style); - progress.set_message(&info_plist.to_string()); + progress.set_message(info_plist.to_string()); let checksums = uploaded.iter().map(|dif| dif.checksum.clone()).collect(); let response = api.associate_apple_dsyms(&org, &project, info_plist, checksums)?; diff --git a/src/utils/dif_upload.rs b/src/utils/dif_upload.rs index 88db088daa..33b203eeda 100644 --- a/src/utils/dif_upload.rs +++ b/src/utils/dif_upload.rs @@ -655,7 +655,7 @@ fn search_difs(options: &DifUpload) -> Result>, Error> { } walk_difs_directory(base_path, options, |source, name, buffer| { debug!("trying to process {}", name); - progress.set_message(&name); + progress.set_message(name.to_owned()); if Archive::peek(&buffer) != FileFormat::Unknown { let mut difs = @@ -671,7 +671,7 @@ fn search_difs(options: &DifUpload) -> Result>, Error> { } }; - progress.set_prefix(&collected.len().to_string()); + progress.set_prefix(collected.len().to_string()); Ok(()) })?; } @@ -941,7 +941,7 @@ where let mut calculated = Vec::new(); for item in items { progress.inc(1); - progress.set_message(item.path()); + progress.set_message(item.path().to_owned()); calculated.push(func(item)?); } @@ -1001,7 +1001,7 @@ fn process_symbol_maps<'a>( for dif in with_hidden { progress.inc(1); - progress.set_message(dif.path()); + progress.set_message(dif.path().to_owned()); without_hidden.push(resolve_hidden_symbols(dif, symbol_map)?); } @@ -1052,7 +1052,7 @@ fn create_source_bundles<'a>(difs: &[DifMatch<'a>]) -> Result>, for dif in difs { progress.inc(1); - progress.set_message(dif.path()); + progress.set_message(dif.path().to_owned()); let object = match dif.object() { Some(object) => object, diff --git a/src/utils/file_search.rs b/src/utils/file_search.rs index 1114eaf68e..13caa67c08 100644 --- a/src/utils/file_search.rs +++ b/src/utils/file_search.rs @@ -139,7 +139,7 @@ impl ReleaseFileSearch { if file.file_type().map_or(false, |t| t.is_dir()) { continue; } - progress.set_message(&format!("{}", file.path().display())); + progress.set_message(format!("{}", file.path().display())); info!( "found: {} ({} bytes)", @@ -158,7 +158,7 @@ impl ReleaseFileSearch { }; collected.push(file_match); - progress.set_prefix(&collected.len().to_string()); + progress.set_prefix(collected.len().to_string()); } progress.finish_and_clear(); diff --git a/src/utils/file_upload.rs b/src/utils/file_upload.rs index 3fa57b6511..417ee51127 100644 --- a/src/utils/file_upload.rs +++ b/src/utils/file_upload.rs @@ -316,7 +316,7 @@ fn build_artifact_bundle(context: &UploadContext, files: &ReleaseFiles) -> Resul for file in files.values() { progress.inc(1); - progress.set_message(&file.url); + progress.set_message(file.url.to_owned()); let mut info = SourceFileInfo::new(); info.set_ty(file.ty); diff --git a/src/utils/progress.rs b/src/utils/progress.rs index d38b920eab..2fda938dd4 100644 --- a/src/utils/progress.rs +++ b/src/utils/progress.rs @@ -42,7 +42,7 @@ impl ProgressBar { } pub fn finish_with_message(&self, msg: &str) { - self.inner.finish_with_message(msg); + self.inner.finish_with_message(msg.to_owned()); logging::set_progress_bar(None); } @@ -51,7 +51,7 @@ impl ProgressBar { let msg = format!("{} completed in {}.{}s", op, dur.as_secs(), dur.as_millis()); self.inner.set_style(progress_style); self.inner.set_prefix(">"); - self.inner.finish_with_message(&msg); + self.inner.finish_with_message(msg); logging::set_progress_bar(None); } @@ -85,7 +85,7 @@ impl Deref for ProgressBar { pub fn make_progress_bar(len: u64) -> ProgressBar { let pb = ProgressBar::new(len); - pb.set_draw_target(ProgressDrawTarget::to_term(Term::stdout(), None)); + pb.set_draw_target(ProgressDrawTarget::term(Term::stdout(), None)); pb.set_style(ProgressStyle::default_bar().template(&format!( "{} {{msg}}\n{{wide_bar}} {{pos}}/{{len}}", style(">").cyan() diff --git a/src/utils/sourcemaps.rs b/src/utils/sourcemaps.rs index d60544aa8f..b24e879967 100644 --- a/src/utils/sourcemaps.rs +++ b/src/utils/sourcemaps.rs @@ -201,7 +201,7 @@ impl SourceMapProcessor { style(self.pending_sources.len()).yellow() ); for (url, mut file) in self.pending_sources.drain() { - pb.set_message(&url); + pb.set_message(url.to_owned()); let ty = if sourcemap::is_sourcemap_slice(&file.contents) { SourceFileType::SourceMap } else if file @@ -307,7 +307,7 @@ impl SourceMapProcessor { println!("{} Validating sources", style(">").dim()); let pb = make_progress_bar(sources.len() as u64); for source in sources { - pb.set_message(&source.url); + pb.set_message(source.url.clone()); match source.ty { SourceFileType::Source | SourceFileType::MinifiedSource => { if let Err(err) = validate_script(source) { @@ -465,7 +465,7 @@ impl SourceMapProcessor { let pb = make_progress_bar(self.sources.len() as u64); let start = Instant::now(); for source in self.sources.values_mut() { - pb.set_message(&source.url); + pb.set_message(source.url.clone()); if source.ty != SourceFileType::SourceMap { pb.inc(1); continue; From 218d0c8108f061bb47ae865bbb5682508a60fbe3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20Og=C3=B3rek?= Date: Tue, 6 Jul 2021 16:26:02 +0200 Subject: [PATCH 3/4] Use indicatiff built-in timers for elapsed logs --- src/utils/chunks.rs | 4 +--- src/utils/file_upload.rs | 8 +++----- src/utils/progress.rs | 7 ++++--- src/utils/sourcemaps.rs | 7 ++----- 4 files changed, 10 insertions(+), 16 deletions(-) diff --git a/src/utils/chunks.rs b/src/utils/chunks.rs index 6ee5e910c4..ec19ebb590 100644 --- a/src/utils/chunks.rs +++ b/src/utils/chunks.rs @@ -7,7 +7,6 @@ use std::sync::Arc; use std::time::Duration; -use std::time::Instant; use failure::Error; use log::warn; @@ -176,7 +175,6 @@ pub fn upload_chunks( // `Api::upload_chunks`, the progress bar is created in an Arc. let progress = Arc::new(ProgressBar::new(total_bytes)); progress.set_style(progress_style); - let start = Instant::now(); // Select the best available compression mechanism. We assume that every // compression algorithm has been implemented for uploading, except `Other` @@ -226,7 +224,7 @@ pub fn upload_chunks( .collect::>() })?; - progress.finish_with_duration("Uploading", start.elapsed()); + progress.finish_with_duration("Uploading"); Ok(()) } diff --git a/src/utils/file_upload.rs b/src/utils/file_upload.rs index 417ee51127..6a8de15a3c 100644 --- a/src/utils/file_upload.rs +++ b/src/utils/file_upload.rs @@ -216,7 +216,6 @@ fn upload_files_chunked( let progress = ProgressBar::new_spinner(); progress.enable_steady_tick(100); progress.set_style(progress_style); - let start = Instant::now(); let view = ByteView::open(archive.path())?; let (checksum, checksums) = get_sha1_checksums(&view, options.chunk_size)?; @@ -226,7 +225,7 @@ fn upload_files_chunked( .map(|(data, checksum)| Chunk((*checksum, data))) .collect::>(); - progress.finish_with_duration("Optimizing", start.elapsed()); + progress.finish_with_duration("Optimizing"); let progress_style = ProgressStyle::default_bar().template(&format!( "{} Uploading release files...\ @@ -273,7 +272,7 @@ fn upload_files_chunked( bail!("Failed to process uploaded files: {}", message); } - progress.finish_with_duration("Processing", assemble_start.elapsed()); + progress.finish_with_duration("Processing"); if response.state.is_pending() { if context.wait { @@ -300,7 +299,6 @@ fn build_artifact_bundle(context: &UploadContext, files: &ReleaseFiles) -> Resul let progress = ProgressBar::new(files.len() as u64); progress.set_style(progress_style); progress.set_prefix(">"); - let start = Instant::now(); let archive = TempFile::create()?; let mut bundle = SourceBundleWriter::start(BufWriter::new(archive.open()?))?; @@ -331,7 +329,7 @@ fn build_artifact_bundle(context: &UploadContext, files: &ReleaseFiles) -> Resul bundle.finish()?; - progress.finish_with_duration("Bundling", start.elapsed()); + progress.finish_with_duration("Bundling"); println!( "{} Bundled {} {} for upload", diff --git a/src/utils/progress.rs b/src/utils/progress.rs index 2fda938dd4..49b492a250 100644 --- a/src/utils/progress.rs +++ b/src/utils/progress.rs @@ -1,7 +1,7 @@ use console::{style, Term}; +use std::env; use std::ops::Deref; use std::sync::Arc; -use std::{env, time::Duration}; use crate::utils::logging; @@ -46,9 +46,10 @@ impl ProgressBar { logging::set_progress_bar(None); } - pub fn finish_with_duration(&self, op: &str, dur: Duration) { + pub fn finish_with_duration(&self, op: &str) { + let e = self.inner.elapsed(); + let msg = format!("{} completed in {}.{}s", op, e.as_secs(), e.as_millis()); let progress_style = ProgressStyle::default_bar().template("{prefix:.dim} {msg}"); - let msg = format!("{} completed in {}.{}s", op, dur.as_secs(), dur.as_millis()); self.inner.set_style(progress_style); self.inner.set_prefix(">"); self.inner.finish_with_message(msg); diff --git a/src/utils/sourcemaps.rs b/src/utils/sourcemaps.rs index b24e879967..8dbc40f053 100644 --- a/src/utils/sourcemaps.rs +++ b/src/utils/sourcemaps.rs @@ -4,7 +4,6 @@ use std::ffi::OsStr; use std::mem; use std::path::PathBuf; use std::str; -use std::time::Instant; use console::style; use failure::{bail, Error}; @@ -193,7 +192,6 @@ impl SourceMapProcessor { } let pb = make_progress_bar(self.pending_sources.len() as u64); - let start = Instant::now(); println!( "{} Analyzing {} sources", @@ -248,7 +246,7 @@ impl SourceMapProcessor { pb.inc(1); } - pb.finish_with_duration("Analyzing", start.elapsed()); + pb.finish_with_duration("Analyzing"); } pub fn dump_log(&self, title: &str) { @@ -463,7 +461,6 @@ impl SourceMapProcessor { self.unpack_indexed_ram_bundles()?; let pb = make_progress_bar(self.sources.len() as u64); - let start = Instant::now(); for source in self.sources.values_mut() { pb.set_message(source.url.clone()); if source.ty != SourceFileType::SourceMap { @@ -490,7 +487,7 @@ impl SourceMapProcessor { source.contents = new_source; pb.inc(1); } - pb.finish_with_duration("Rewriting", start.elapsed()); + pb.finish_with_duration("Rewriting"); Ok(()) } From 0c196bbde2a46a1b88f560423f5e0165abb74930 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20Og=C3=B3rek?= Date: Wed, 7 Jul 2021 11:53:58 +0200 Subject: [PATCH 4/4] Fix correct timing calculations --- src/utils/file_upload.rs | 4 ++-- src/utils/progress.rs | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/utils/file_upload.rs b/src/utils/file_upload.rs index 6a8de15a3c..d172f2a9c5 100644 --- a/src/utils/file_upload.rs +++ b/src/utils/file_upload.rs @@ -329,8 +329,6 @@ fn build_artifact_bundle(context: &UploadContext, files: &ReleaseFiles) -> Resul bundle.finish()?; - progress.finish_with_duration("Bundling"); - println!( "{} Bundled {} {} for upload", style(">").dim(), @@ -341,6 +339,8 @@ fn build_artifact_bundle(context: &UploadContext, files: &ReleaseFiles) -> Resul } ); + progress.finish_with_duration("Bundling"); + Ok(archive) } diff --git a/src/utils/progress.rs b/src/utils/progress.rs index 49b492a250..4e5ee27a6c 100644 --- a/src/utils/progress.rs +++ b/src/utils/progress.rs @@ -47,8 +47,9 @@ impl ProgressBar { } pub fn finish_with_duration(&self, op: &str) { - let e = self.inner.elapsed(); - let msg = format!("{} completed in {}.{}s", op, e.as_secs(), e.as_millis()); + let dur = self.inner.elapsed(); + // We could use `dur.as_secs_f64()`, but its unnecessarily precise (micros). Millis are enough for our purpose. + let msg = format!("{} completed in {}s", op, dur.as_millis() as f64 / 1000.0); let progress_style = ProgressStyle::default_bar().template("{prefix:.dim} {msg}"); self.inner.set_style(progress_style); self.inner.set_prefix(">");