diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index de12fb36..fe3022e0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -200,7 +200,7 @@ jobs: mode: ${{ matrix.mode }} walltime-integration-test: - runs-on: codspeed-macro + runs-on: ${{ matrix.runner || 'codspeed-macro' }} strategy: matrix: include: @@ -218,6 +218,8 @@ jobs: bench: criterion_integration_with_inputs - package: codspeed-criterion-compat bench: criterion_integration_measurement_overhead + - package: codspeed-divan-compat + runner: macos-latest steps: - uses: actions/checkout@v4 with: @@ -234,47 +236,18 @@ jobs: run: | # Remove the cargo config else it forces instrumentation mode rm -f .cargo/config.toml - cargo codspeed build -p ${{ matrix.package }} ${{ matrix.bench && format('--bench {0}', matrix.bench) || '' }} + cargo codspeed build -m walltime -p ${{ matrix.package }} ${{ matrix.bench && format('--bench {0}', matrix.bench) || '' }} - name: Run the benchmarks uses: CodSpeedHQ/action@main env: MY_ENV_VAR: "YES" + # Keep the macOS results out of the uploaded Linux walltime results + CODSPEED_SKIP_UPLOAD: ${{ matrix.runner == 'macos-latest' }} with: run: cargo codspeed run ${{ matrix.bench && format('--bench {0}', matrix.bench) || '' }} mode: walltime - walltime-macos-test: - runs-on: macos-latest - steps: - - uses: actions/checkout@v4 - with: - submodules: true - - uses: moonrepo/setup-rust@v1 - with: - cache-target: release - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - run: cargo install --path crates/cargo-codspeed --locked - - - name: Build the benchmarks - run: | - # Remove the cargo config else it forces instrumentation mode - rm -f .cargo/config.toml - cargo codspeed build -p codspeed-divan-compat -m walltime - - - name: Run the benchmarks - uses: CodSpeedHQ/action@main - env: - MY_ENV_VAR: "YES" - CODSPEED_SKIP_UPLOAD: "true" - with: - run: cargo codspeed run -p codspeed-divan-compat - mode: walltime - # TODO: Remove this once the runner has been released with macos support - runner-version: branch:main - musl-build-check: strategy: matrix: @@ -312,7 +285,6 @@ jobs: - minimal-versions - analysis-integration-test - walltime-integration-test - - walltime-macos-test - musl-build-check steps: - uses: re-actors/alls-green@release/v1 diff --git a/Cargo.lock b/Cargo.lock index 16f9e437..c495c0c2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -608,6 +608,8 @@ version = "6.0.0" dependencies = [ "bencher", "codspeed", + "log", + "tracing", ] [[package]] @@ -620,9 +622,11 @@ dependencies = [ "codspeed-criterion-compat-walltime", "colored", "futures", + "log", "regex", "smol 2.0.0", "tokio", + "tracing", ] [[package]] diff --git a/crates/bencher_compat/Cargo.toml b/crates/bencher_compat/Cargo.toml index 2e3c1cca..9fa23864 100644 --- a/crates/bencher_compat/Cargo.toml +++ b/crates/bencher_compat/Cargo.toml @@ -21,6 +21,19 @@ keywords = ["codspeed", "benchmark", "bencher"] bencher = "0.1.5" codspeed = { path = "../codspeed", version = "=6.0.0" } +[features] +# Export the `tracing` spans and events of benchmarks over OTLP to `CODSPEED_OTEL_ENDPOINT` +telemetry = ["codspeed/telemetry"] + +[dev-dependencies] +log = "0.4" +tracing = "0.1" + [[bench]] name = "bencher_example" harness = false + +[[bench]] +name = "telemetry" +harness = false +required-features = ["telemetry"] diff --git a/crates/bencher_compat/benches/telemetry.rs b/crates/bencher_compat/benches/telemetry.rs new file mode 100644 index 00000000..a02fbbf9 --- /dev/null +++ b/crates/bencher_compat/benches/telemetry.rs @@ -0,0 +1,68 @@ +//! Benchmarks emitting telemetry in different ways: `tracing` spans and events, records of the +//! `log` crate, or nothing at all. + +use codspeed::codspeed::black_box; +use codspeed_bencher_compat::{benchmark_group, benchmark_main, Bencher}; +use tracing::{debug, debug_span, info, info_span, warn}; + +fn parse(input: &str) -> Vec { + let _span = info_span!("parse", input_len = input.len()).entered(); + let values: Vec = input + .split(',') + .filter_map(|field| field.parse().ok()) + .collect(); + info!(count = values.len(), "parsed values"); + values +} + +fn sum(values: &[u32]) -> u32 { + let _span = info_span!("sum").entered(); + let _filtered_out = debug_span!("sum_inner").entered(); + debug!("filtered out by the default filter"); + values.iter().sum() +} + +/// Nested `tracing` spans, with events in them. +pub fn parse_and_sum(bench: &mut Bencher) { + bench.iter(|| { + let _span = info_span!("parse_and_sum").entered(); + let values = parse("1,2,x,3"); + if values.len() < 4 { + warn!(skipped = 4 - values.len(), "skipped invalid fields"); + } + sum(&values) + }) +} + +/// A `tracing` event outside of any span. +pub fn event_without_span(bench: &mut Bencher) { + bench.iter(|| { + let values = black_box([1, 2, 3]); + info!(count = values.len(), "summed without a span"); + values.iter().sum::() + }) +} + +/// Records of the `log` crate, without `tracing`. +pub fn log_records(bench: &mut Bencher) { + bench.iter(|| { + let values = black_box([1, 2, 3]); + log::info!("summing {} values, logged with the log crate", values.len()); + log::debug!("filtered out by the default filter"); + values.iter().sum::() + }) +} + +/// No telemetry at all. +pub fn no_telemetry(bench: &mut Bencher) { + bench.iter(|| black_box([1, 2, 3]).iter().sum::()) +} + +benchmark_group!( + benches, + parse_and_sum, + event_without_span, + log_records, + no_telemetry +); +benchmark_main!(benches); diff --git a/crates/bencher_compat/src/compat.rs b/crates/bencher_compat/src/compat.rs index 35147788..82c27567 100644 --- a/crates/bencher_compat/src/compat.rs +++ b/crates/bencher_compat/src/compat.rs @@ -81,6 +81,8 @@ macro_rules! benchmark_group { macro_rules! benchmark_main { ($($group_name:path),+) => { pub fn main() { + // Exports spans and events for the duration of the run, with the `telemetry` feature. + let _telemetry_guard = $crate::__init_telemetry(); let mut bencher = $crate::Bencher::default(); $( $group_name(&mut bencher); diff --git a/crates/bencher_compat/src/lib.rs b/crates/bencher_compat/src/lib.rs index 628c23cd..ddcb8ac4 100644 --- a/crates/bencher_compat/src/lib.rs +++ b/crates/bencher_compat/src/lib.rs @@ -1,3 +1,6 @@ +#[cfg(feature = "telemetry")] +pub use codspeed::telemetry; + #[cfg(not(codspeed))] mod compat_bencher { pub use bencher::*; @@ -13,3 +16,19 @@ mod compat_bencher { } pub use compat_bencher::*; + +/// Installs the telemetry export for the `main` of `benchmark_main!`, with the `telemetry` +/// feature. The macro expands in the user's crate, so it can't check the feature itself. +/// +/// Keep the returned guard alive until the benchmarks are done. +#[doc(hidden)] +#[cfg(feature = "telemetry")] +pub fn __init_telemetry() -> Option { + telemetry::init() +} + +#[doc(hidden)] +#[cfg(not(feature = "telemetry"))] +pub fn __init_telemetry() -> Option<()> { + None +} diff --git a/crates/criterion_compat/Cargo.toml b/crates/criterion_compat/Cargo.toml index 1eacd8fb..7af5789b 100644 --- a/crates/criterion_compat/Cargo.toml +++ b/crates/criterion_compat/Cargo.toml @@ -45,6 +45,13 @@ csv_output = ["criterion/csv_output"] rayon = ["criterion/rayon"] plotters = ["criterion/plotters"] +# Export the `tracing` spans and events of benchmarks over OTLP to `CODSPEED_OTEL_ENDPOINT` +telemetry = ["codspeed/telemetry"] + +[dev-dependencies] +log = "0.4" +tracing = "0.1" + [[bench]] name = "criterion_example" harness = false @@ -72,3 +79,8 @@ harness = false [[bench]] name = "custom_main" harness = false + +[[bench]] +name = "telemetry" +harness = false +required-features = ["telemetry"] diff --git a/crates/criterion_compat/benches/telemetry.rs b/crates/criterion_compat/benches/telemetry.rs new file mode 100644 index 00000000..1c0a0f7b --- /dev/null +++ b/crates/criterion_compat/benches/telemetry.rs @@ -0,0 +1,62 @@ +//! Benchmarks emitting telemetry in different ways: `tracing` spans and events, records of the +//! `log` crate, or nothing at all. + +use codspeed_criterion_compat::{black_box, criterion_group, criterion_main, Criterion}; +use tracing::{debug, debug_span, info, info_span, warn}; + +fn parse(input: &str) -> Vec { + let _span = info_span!("parse", input_len = input.len()).entered(); + let values: Vec = input + .split(',') + .filter_map(|field| field.parse().ok()) + .collect(); + info!(count = values.len(), "parsed values"); + values +} + +fn sum(values: &[u32]) -> u32 { + let _span = info_span!("sum").entered(); + let _filtered_out = debug_span!("sum_inner").entered(); + debug!("filtered out by the default filter"); + values.iter().sum() +} + +/// Nested `tracing` spans, with events in them. +fn parse_and_sum() -> u32 { + let _span = info_span!("parse_and_sum").entered(); + let values = parse("1,2,x,3"); + if values.len() < 4 { + warn!(skipped = 4 - values.len(), "skipped invalid fields"); + } + sum(&values) +} + +/// A `tracing` event outside of any span. +fn event_without_span() -> u32 { + let values = black_box([1, 2, 3]); + info!(count = values.len(), "summed without a span"); + values.iter().sum() +} + +/// Records of the `log` crate, without `tracing`. +fn log_records() -> u32 { + let values = black_box([1, 2, 3]); + log::info!("summing {} values, logged with the log crate", values.len()); + log::debug!("filtered out by the default filter"); + values.iter().sum() +} + +/// No telemetry at all. +fn no_telemetry() -> u32 { + black_box([1, 2, 3]).iter().sum() +} + +fn telemetry(c: &mut Criterion) { + c.bench_function("parse_and_sum", |b| b.iter(parse_and_sum)); + c.bench_function("event_without_span", |b| b.iter(event_without_span)); + c.bench_function("log_records", |b| b.iter(log_records)); + c.bench_function("no_telemetry", |b| b.iter(no_telemetry)); +} + +criterion_group!(benches, telemetry); +criterion_main!(benches); diff --git a/crates/criterion_compat/src/compat/macros.rs b/crates/criterion_compat/src/compat/macros.rs index 495dea4c..3fa67588 100644 --- a/crates/criterion_compat/src/compat/macros.rs +++ b/crates/criterion_compat/src/compat/macros.rs @@ -23,6 +23,8 @@ macro_rules! criterion_group { macro_rules! criterion_main { ( $( $group:path ),+ $(,)* ) => { pub fn main() { + // Exports spans and events for the duration of the run, with the `telemetry` feature. + let _telemetry_guard = $crate::__init_telemetry(); let mut criterion = $crate::Criterion::new_instrumented(); $( $group(&mut criterion); diff --git a/crates/criterion_compat/src/lib.rs b/crates/criterion_compat/src/lib.rs index 8e7e03dd..3f57a1bf 100644 --- a/crates/criterion_compat/src/lib.rs +++ b/crates/criterion_compat/src/lib.rs @@ -1,4 +1,6 @@ pub use codspeed::codspeed_uri; +#[cfg(feature = "telemetry")] +pub use codspeed::telemetry; #[cfg(not(codspeed))] mod compat_criterion { @@ -21,3 +23,19 @@ mod compat_criterion { } pub use compat_criterion::*; + +/// Installs the telemetry export for the `main` of `criterion_main!`, with the `telemetry` +/// feature. The macro expands in the user's crate, so it can't check the feature itself. +/// +/// Keep the returned guard alive until the benchmarks are done. +#[doc(hidden)] +#[cfg(feature = "telemetry")] +pub fn __init_telemetry() -> Option { + telemetry::init() +} + +#[doc(hidden)] +#[cfg(not(feature = "telemetry"))] +pub fn __init_telemetry() -> Option<()> { + None +}