From 6084d6b3138cc29e9166f5a2c82a8f8dea4eca9e Mon Sep 17 00:00:00 2001 From: Luc Varoqui Date: Mon, 16 Nov 2026 17:49:10 +0100 Subject: [PATCH 1/3] feat(criterion): export the telemetry of benchmarks over OTLP (#204) Add a `telemetry` feature to `codspeed-criterion-compat`, as `codspeed-divan-compat` has. When `CODSPEED_OTEL_ENDPOINT` is set, the `main` of `criterion_main!` installs the `tracing` subscriber of `codspeed`, which exports spans as traces, and `tracing` events and `log` records as logs, and keeps it until the groups are done. The export is the one of the `codspeed` crate, so only the telemetry build of `cargo-codspeed` compiles it. The macro expands in the user's crate, where it can't check the feature of the compat crate, so it calls a hidden function that is a no-op without the feature. `criterion::telemetry` re-exports the module, for custom `main` functions and benchmarks installing their own subscriber. `benches/telemetry.rs` emits telemetry in each way the divan example does. It needs the feature, so builds without it skip it. Closes COD-4287 Co-Authored-By: Claude Fable 5.1 --- Cargo.lock | 2 + crates/criterion_compat/Cargo.toml | 12 ++++ crates/criterion_compat/benches/telemetry.rs | 62 ++++++++++++++++++++ crates/criterion_compat/src/compat/macros.rs | 2 + crates/criterion_compat/src/lib.rs | 18 ++++++ 5 files changed, 96 insertions(+) create mode 100644 crates/criterion_compat/benches/telemetry.rs diff --git a/Cargo.lock b/Cargo.lock index 16f9e437..d7b153a4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -620,9 +620,11 @@ dependencies = [ "codspeed-criterion-compat-walltime", "colored", "futures", + "log", "regex", "smol 2.0.0", "tokio", + "tracing", ] [[package]] diff --git a/crates/criterion_compat/Cargo.toml b/crates/criterion_compat/Cargo.toml index 1eacd8fb..7af5789b 100644 --- a/crates/criterion_compat/Cargo.toml +++ b/crates/criterion_compat/Cargo.toml @@ -45,6 +45,13 @@ csv_output = ["criterion/csv_output"] rayon = ["criterion/rayon"] plotters = ["criterion/plotters"] +# Export the `tracing` spans and events of benchmarks over OTLP to `CODSPEED_OTEL_ENDPOINT` +telemetry = ["codspeed/telemetry"] + +[dev-dependencies] +log = "0.4" +tracing = "0.1" + [[bench]] name = "criterion_example" harness = false @@ -72,3 +79,8 @@ harness = false [[bench]] name = "custom_main" harness = false + +[[bench]] +name = "telemetry" +harness = false +required-features = ["telemetry"] diff --git a/crates/criterion_compat/benches/telemetry.rs b/crates/criterion_compat/benches/telemetry.rs new file mode 100644 index 00000000..1c0a0f7b --- /dev/null +++ b/crates/criterion_compat/benches/telemetry.rs @@ -0,0 +1,62 @@ +//! Benchmarks emitting telemetry in different ways: `tracing` spans and events, records of the +//! `log` crate, or nothing at all. + +use codspeed_criterion_compat::{black_box, criterion_group, criterion_main, Criterion}; +use tracing::{debug, debug_span, info, info_span, warn}; + +fn parse(input: &str) -> Vec { + let _span = info_span!("parse", input_len = input.len()).entered(); + let values: Vec = input + .split(',') + .filter_map(|field| field.parse().ok()) + .collect(); + info!(count = values.len(), "parsed values"); + values +} + +fn sum(values: &[u32]) -> u32 { + let _span = info_span!("sum").entered(); + let _filtered_out = debug_span!("sum_inner").entered(); + debug!("filtered out by the default filter"); + values.iter().sum() +} + +/// Nested `tracing` spans, with events in them. +fn parse_and_sum() -> u32 { + let _span = info_span!("parse_and_sum").entered(); + let values = parse("1,2,x,3"); + if values.len() < 4 { + warn!(skipped = 4 - values.len(), "skipped invalid fields"); + } + sum(&values) +} + +/// A `tracing` event outside of any span. +fn event_without_span() -> u32 { + let values = black_box([1, 2, 3]); + info!(count = values.len(), "summed without a span"); + values.iter().sum() +} + +/// Records of the `log` crate, without `tracing`. +fn log_records() -> u32 { + let values = black_box([1, 2, 3]); + log::info!("summing {} values, logged with the log crate", values.len()); + log::debug!("filtered out by the default filter"); + values.iter().sum() +} + +/// No telemetry at all. +fn no_telemetry() -> u32 { + black_box([1, 2, 3]).iter().sum() +} + +fn telemetry(c: &mut Criterion) { + c.bench_function("parse_and_sum", |b| b.iter(parse_and_sum)); + c.bench_function("event_without_span", |b| b.iter(event_without_span)); + c.bench_function("log_records", |b| b.iter(log_records)); + c.bench_function("no_telemetry", |b| b.iter(no_telemetry)); +} + +criterion_group!(benches, telemetry); +criterion_main!(benches); diff --git a/crates/criterion_compat/src/compat/macros.rs b/crates/criterion_compat/src/compat/macros.rs index 495dea4c..3fa67588 100644 --- a/crates/criterion_compat/src/compat/macros.rs +++ b/crates/criterion_compat/src/compat/macros.rs @@ -23,6 +23,8 @@ macro_rules! criterion_group { macro_rules! criterion_main { ( $( $group:path ),+ $(,)* ) => { pub fn main() { + // Exports spans and events for the duration of the run, with the `telemetry` feature. + let _telemetry_guard = $crate::__init_telemetry(); let mut criterion = $crate::Criterion::new_instrumented(); $( $group(&mut criterion); diff --git a/crates/criterion_compat/src/lib.rs b/crates/criterion_compat/src/lib.rs index 8e7e03dd..3f57a1bf 100644 --- a/crates/criterion_compat/src/lib.rs +++ b/crates/criterion_compat/src/lib.rs @@ -1,4 +1,6 @@ pub use codspeed::codspeed_uri; +#[cfg(feature = "telemetry")] +pub use codspeed::telemetry; #[cfg(not(codspeed))] mod compat_criterion { @@ -21,3 +23,19 @@ mod compat_criterion { } pub use compat_criterion::*; + +/// Installs the telemetry export for the `main` of `criterion_main!`, with the `telemetry` +/// feature. The macro expands in the user's crate, so it can't check the feature itself. +/// +/// Keep the returned guard alive until the benchmarks are done. +#[doc(hidden)] +#[cfg(feature = "telemetry")] +pub fn __init_telemetry() -> Option { + telemetry::init() +} + +#[doc(hidden)] +#[cfg(not(feature = "telemetry"))] +pub fn __init_telemetry() -> Option<()> { + None +} From 619a83461f4c63200b98444a97ff1468557915b3 Mon Sep 17 00:00:00 2001 From: Luc Varoqui Date: Tue, 17 Nov 2026 11:31:56 +0100 Subject: [PATCH 2/3] feat(bencher): export the telemetry of benchmarks over OTLP (#204) Add the `telemetry` feature to `codspeed-bencher-compat` too, wired as in `codspeed-criterion-compat`: the `main` of `benchmark_main!` installs the `tracing` subscriber of `codspeed` when `CODSPEED_OTEL_ENDPOINT` is set, through a hidden function that is a no-op without the feature, and `bencher::telemetry` re-exports the module. `benches/telemetry.rs` emits telemetry in each way the other examples do, behind the feature. Closes COD-4287 Co-Authored-By: Claude Fable 5.1 --- Cargo.lock | 2 + crates/bencher_compat/Cargo.toml | 13 +++++ crates/bencher_compat/benches/telemetry.rs | 68 ++++++++++++++++++++++ crates/bencher_compat/src/compat.rs | 2 + crates/bencher_compat/src/lib.rs | 19 ++++++ 5 files changed, 104 insertions(+) create mode 100644 crates/bencher_compat/benches/telemetry.rs diff --git a/Cargo.lock b/Cargo.lock index d7b153a4..c495c0c2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -608,6 +608,8 @@ version = "6.0.0" dependencies = [ "bencher", "codspeed", + "log", + "tracing", ] [[package]] diff --git a/crates/bencher_compat/Cargo.toml b/crates/bencher_compat/Cargo.toml index 2e3c1cca..9fa23864 100644 --- a/crates/bencher_compat/Cargo.toml +++ b/crates/bencher_compat/Cargo.toml @@ -21,6 +21,19 @@ keywords = ["codspeed", "benchmark", "bencher"] bencher = "0.1.5" codspeed = { path = "../codspeed", version = "=6.0.0" } +[features] +# Export the `tracing` spans and events of benchmarks over OTLP to `CODSPEED_OTEL_ENDPOINT` +telemetry = ["codspeed/telemetry"] + +[dev-dependencies] +log = "0.4" +tracing = "0.1" + [[bench]] name = "bencher_example" harness = false + +[[bench]] +name = "telemetry" +harness = false +required-features = ["telemetry"] diff --git a/crates/bencher_compat/benches/telemetry.rs b/crates/bencher_compat/benches/telemetry.rs new file mode 100644 index 00000000..a02fbbf9 --- /dev/null +++ b/crates/bencher_compat/benches/telemetry.rs @@ -0,0 +1,68 @@ +//! Benchmarks emitting telemetry in different ways: `tracing` spans and events, records of the +//! `log` crate, or nothing at all. + +use codspeed::codspeed::black_box; +use codspeed_bencher_compat::{benchmark_group, benchmark_main, Bencher}; +use tracing::{debug, debug_span, info, info_span, warn}; + +fn parse(input: &str) -> Vec { + let _span = info_span!("parse", input_len = input.len()).entered(); + let values: Vec = input + .split(',') + .filter_map(|field| field.parse().ok()) + .collect(); + info!(count = values.len(), "parsed values"); + values +} + +fn sum(values: &[u32]) -> u32 { + let _span = info_span!("sum").entered(); + let _filtered_out = debug_span!("sum_inner").entered(); + debug!("filtered out by the default filter"); + values.iter().sum() +} + +/// Nested `tracing` spans, with events in them. +pub fn parse_and_sum(bench: &mut Bencher) { + bench.iter(|| { + let _span = info_span!("parse_and_sum").entered(); + let values = parse("1,2,x,3"); + if values.len() < 4 { + warn!(skipped = 4 - values.len(), "skipped invalid fields"); + } + sum(&values) + }) +} + +/// A `tracing` event outside of any span. +pub fn event_without_span(bench: &mut Bencher) { + bench.iter(|| { + let values = black_box([1, 2, 3]); + info!(count = values.len(), "summed without a span"); + values.iter().sum::() + }) +} + +/// Records of the `log` crate, without `tracing`. +pub fn log_records(bench: &mut Bencher) { + bench.iter(|| { + let values = black_box([1, 2, 3]); + log::info!("summing {} values, logged with the log crate", values.len()); + log::debug!("filtered out by the default filter"); + values.iter().sum::() + }) +} + +/// No telemetry at all. +pub fn no_telemetry(bench: &mut Bencher) { + bench.iter(|| black_box([1, 2, 3]).iter().sum::()) +} + +benchmark_group!( + benches, + parse_and_sum, + event_without_span, + log_records, + no_telemetry +); +benchmark_main!(benches); diff --git a/crates/bencher_compat/src/compat.rs b/crates/bencher_compat/src/compat.rs index 35147788..82c27567 100644 --- a/crates/bencher_compat/src/compat.rs +++ b/crates/bencher_compat/src/compat.rs @@ -81,6 +81,8 @@ macro_rules! benchmark_group { macro_rules! benchmark_main { ($($group_name:path),+) => { pub fn main() { + // Exports spans and events for the duration of the run, with the `telemetry` feature. + let _telemetry_guard = $crate::__init_telemetry(); let mut bencher = $crate::Bencher::default(); $( $group_name(&mut bencher); diff --git a/crates/bencher_compat/src/lib.rs b/crates/bencher_compat/src/lib.rs index 628c23cd..ddcb8ac4 100644 --- a/crates/bencher_compat/src/lib.rs +++ b/crates/bencher_compat/src/lib.rs @@ -1,3 +1,6 @@ +#[cfg(feature = "telemetry")] +pub use codspeed::telemetry; + #[cfg(not(codspeed))] mod compat_bencher { pub use bencher::*; @@ -13,3 +16,19 @@ mod compat_bencher { } pub use compat_bencher::*; + +/// Installs the telemetry export for the `main` of `benchmark_main!`, with the `telemetry` +/// feature. The macro expands in the user's crate, so it can't check the feature itself. +/// +/// Keep the returned guard alive until the benchmarks are done. +#[doc(hidden)] +#[cfg(feature = "telemetry")] +pub fn __init_telemetry() -> Option { + telemetry::init() +} + +#[doc(hidden)] +#[cfg(not(feature = "telemetry"))] +pub fn __init_telemetry() -> Option<()> { + None +} From 1e075ade15570d30090894516d95531dd2754f20 Mon Sep 17 00:00:00 2001 From: Luc Varoqui Date: Tue, 17 Nov 2026 14:44:48 +0100 Subject: [PATCH 3/3] ci: run the macOS walltime test in the walltime matrix (#204) The standalone macOS job pinned the runner to `branch:main`. The action installs from a branch by sourcing `~/.cargo/env`, which does not exist on `macos-latest`, so the job failed before running any benchmark. The released runner now supports macOS, so drop the pin and run the job as one more entry of the walltime integration matrix. Build with `-m walltime` for every entry: `macos-latest` does not set `CODSPEED_RUNNER_MODE`, so the build would otherwise use analysis mode. Skip the upload on macOS, as the old job did. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 40 ++++++---------------------------------- 1 file changed, 6 insertions(+), 34 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index de12fb36..fe3022e0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -200,7 +200,7 @@ jobs: mode: ${{ matrix.mode }} walltime-integration-test: - runs-on: codspeed-macro + runs-on: ${{ matrix.runner || 'codspeed-macro' }} strategy: matrix: include: @@ -218,6 +218,8 @@ jobs: bench: criterion_integration_with_inputs - package: codspeed-criterion-compat bench: criterion_integration_measurement_overhead + - package: codspeed-divan-compat + runner: macos-latest steps: - uses: actions/checkout@v4 with: @@ -234,47 +236,18 @@ jobs: run: | # Remove the cargo config else it forces instrumentation mode rm -f .cargo/config.toml - cargo codspeed build -p ${{ matrix.package }} ${{ matrix.bench && format('--bench {0}', matrix.bench) || '' }} + cargo codspeed build -m walltime -p ${{ matrix.package }} ${{ matrix.bench && format('--bench {0}', matrix.bench) || '' }} - name: Run the benchmarks uses: CodSpeedHQ/action@main env: MY_ENV_VAR: "YES" + # Keep the macOS results out of the uploaded Linux walltime results + CODSPEED_SKIP_UPLOAD: ${{ matrix.runner == 'macos-latest' }} with: run: cargo codspeed run ${{ matrix.bench && format('--bench {0}', matrix.bench) || '' }} mode: walltime - walltime-macos-test: - runs-on: macos-latest - steps: - - uses: actions/checkout@v4 - with: - submodules: true - - uses: moonrepo/setup-rust@v1 - with: - cache-target: release - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - run: cargo install --path crates/cargo-codspeed --locked - - - name: Build the benchmarks - run: | - # Remove the cargo config else it forces instrumentation mode - rm -f .cargo/config.toml - cargo codspeed build -p codspeed-divan-compat -m walltime - - - name: Run the benchmarks - uses: CodSpeedHQ/action@main - env: - MY_ENV_VAR: "YES" - CODSPEED_SKIP_UPLOAD: "true" - with: - run: cargo codspeed run -p codspeed-divan-compat - mode: walltime - # TODO: Remove this once the runner has been released with macos support - runner-version: branch:main - musl-build-check: strategy: matrix: @@ -312,7 +285,6 @@ jobs: - minimal-versions - analysis-integration-test - walltime-integration-test - - walltime-macos-test - musl-build-check steps: - uses: re-actors/alls-green@release/v1