Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 6 additions & 34 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -200,7 +200,7 @@ jobs:
mode: ${{ matrix.mode }}

walltime-integration-test:
runs-on: codspeed-macro
runs-on: ${{ matrix.runner || 'codspeed-macro' }}
strategy:
matrix:
include:
Expand All @@ -218,6 +218,8 @@ jobs:
bench: criterion_integration_with_inputs
- package: codspeed-criterion-compat
bench: criterion_integration_measurement_overhead
- package: codspeed-divan-compat
runner: macos-latest
steps:
- uses: actions/checkout@v4
with:
Expand All @@ -234,47 +236,18 @@ jobs:
run: |
# Remove the cargo config else it forces instrumentation mode
rm -f .cargo/config.toml
cargo codspeed build -p ${{ matrix.package }} ${{ matrix.bench && format('--bench {0}', matrix.bench) || '' }}
cargo codspeed build -m walltime -p ${{ matrix.package }} ${{ matrix.bench && format('--bench {0}', matrix.bench) || '' }}

- name: Run the benchmarks
uses: CodSpeedHQ/action@main
env:
MY_ENV_VAR: "YES"
# Keep the macOS results out of the uploaded Linux walltime results
CODSPEED_SKIP_UPLOAD: ${{ matrix.runner == 'macos-latest' }}
with:
run: cargo codspeed run ${{ matrix.bench && format('--bench {0}', matrix.bench) || '' }}
mode: walltime

walltime-macos-test:
runs-on: macos-latest
steps:
- uses: actions/checkout@v4
with:
submodules: true
- uses: moonrepo/setup-rust@v1
with:
cache-target: release
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

- run: cargo install --path crates/cargo-codspeed --locked

- name: Build the benchmarks
run: |
# Remove the cargo config else it forces instrumentation mode
rm -f .cargo/config.toml
cargo codspeed build -p codspeed-divan-compat -m walltime

- name: Run the benchmarks
uses: CodSpeedHQ/action@main
env:
MY_ENV_VAR: "YES"
CODSPEED_SKIP_UPLOAD: "true"
with:
run: cargo codspeed run -p codspeed-divan-compat
mode: walltime
# TODO: Remove this once the runner has been released with macos support
runner-version: branch:main

musl-build-check:
strategy:
matrix:
Expand Down Expand Up @@ -312,7 +285,6 @@ jobs:
- minimal-versions
- analysis-integration-test
- walltime-integration-test
- walltime-macos-test
- musl-build-check
steps:
- uses: re-actors/alls-green@release/v1
Expand Down
4 changes: 4 additions & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

13 changes: 13 additions & 0 deletions crates/bencher_compat/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,19 @@ keywords = ["codspeed", "benchmark", "bencher"]
bencher = "0.1.5"
codspeed = { path = "../codspeed", version = "=6.0.0" }

[features]
# Export the `tracing` spans and events of benchmarks over OTLP to `CODSPEED_OTEL_ENDPOINT`
telemetry = ["codspeed/telemetry"]

[dev-dependencies]
log = "0.4"
tracing = "0.1"

[[bench]]
name = "bencher_example"
harness = false

[[bench]]
name = "telemetry"
harness = false
required-features = ["telemetry"]
68 changes: 68 additions & 0 deletions crates/bencher_compat/benches/telemetry.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
//! Benchmarks emitting telemetry in different ways: `tracing` spans and events, records of the
//! `log` crate, or nothing at all.

use codspeed::codspeed::black_box;
use codspeed_bencher_compat::{benchmark_group, benchmark_main, Bencher};
use tracing::{debug, debug_span, info, info_span, warn};

fn parse(input: &str) -> Vec<u32> {
let _span = info_span!("parse", input_len = input.len()).entered();
let values: Vec<u32> = input
.split(',')
.filter_map(|field| field.parse().ok())
.collect();
info!(count = values.len(), "parsed values");
values
}

fn sum(values: &[u32]) -> u32 {
let _span = info_span!("sum").entered();
let _filtered_out = debug_span!("sum_inner").entered();
debug!("filtered out by the default filter");
values.iter().sum()
}

/// Nested `tracing` spans, with events in them.
pub fn parse_and_sum(bench: &mut Bencher) {
bench.iter(|| {
let _span = info_span!("parse_and_sum").entered();
let values = parse("1,2,x,3");
if values.len() < 4 {
warn!(skipped = 4 - values.len(), "skipped invalid fields");
}
sum(&values)
})
}

/// A `tracing` event outside of any span.
pub fn event_without_span(bench: &mut Bencher) {
bench.iter(|| {
let values = black_box([1, 2, 3]);
info!(count = values.len(), "summed without a span");
values.iter().sum::<u32>()
})
}

/// Records of the `log` crate, without `tracing`.
pub fn log_records(bench: &mut Bencher) {
bench.iter(|| {
let values = black_box([1, 2, 3]);
log::info!("summing {} values, logged with the log crate", values.len());
log::debug!("filtered out by the default filter");
values.iter().sum::<u32>()
})
}

/// No telemetry at all.
pub fn no_telemetry(bench: &mut Bencher) {
bench.iter(|| black_box([1, 2, 3]).iter().sum::<u32>())
}

benchmark_group!(
benches,
parse_and_sum,
event_without_span,
log_records,
no_telemetry
);
benchmark_main!(benches);
2 changes: 2 additions & 0 deletions crates/bencher_compat/src/compat.rs
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,8 @@ macro_rules! benchmark_group {
macro_rules! benchmark_main {
($($group_name:path),+) => {
pub fn main() {
// Exports spans and events for the duration of the run, with the `telemetry` feature.
let _telemetry_guard = $crate::__init_telemetry();
let mut bencher = $crate::Bencher::default();
$(
$group_name(&mut bencher);
Expand Down
19 changes: 19 additions & 0 deletions crates/bencher_compat/src/lib.rs
Original file line number Diff line number Diff line change
@@ -1,3 +1,6 @@
#[cfg(feature = "telemetry")]
pub use codspeed::telemetry;

#[cfg(not(codspeed))]
mod compat_bencher {
pub use bencher::*;
Expand All @@ -13,3 +16,19 @@ mod compat_bencher {
}

pub use compat_bencher::*;

/// Installs the telemetry export for the `main` of `benchmark_main!`, with the `telemetry`
/// feature. The macro expands in the user's crate, so it can't check the feature itself.
///
/// Keep the returned guard alive until the benchmarks are done.
#[doc(hidden)]
#[cfg(feature = "telemetry")]
pub fn __init_telemetry() -> Option<telemetry::TelemetryGuard> {
telemetry::init()
}

#[doc(hidden)]
#[cfg(not(feature = "telemetry"))]
pub fn __init_telemetry() -> Option<()> {
None
}
12 changes: 12 additions & 0 deletions crates/criterion_compat/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,13 @@ csv_output = ["criterion/csv_output"]
rayon = ["criterion/rayon"]
plotters = ["criterion/plotters"]

# Export the `tracing` spans and events of benchmarks over OTLP to `CODSPEED_OTEL_ENDPOINT`
telemetry = ["codspeed/telemetry"]

[dev-dependencies]
log = "0.4"
tracing = "0.1"

[[bench]]
name = "criterion_example"
harness = false
Expand Down Expand Up @@ -72,3 +79,8 @@ harness = false
[[bench]]
name = "custom_main"
harness = false

[[bench]]
name = "telemetry"
harness = false
required-features = ["telemetry"]
62 changes: 62 additions & 0 deletions crates/criterion_compat/benches/telemetry.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
//! Benchmarks emitting telemetry in different ways: `tracing` spans and events, records of the
//! `log` crate, or nothing at all.

use codspeed_criterion_compat::{black_box, criterion_group, criterion_main, Criterion};
use tracing::{debug, debug_span, info, info_span, warn};

fn parse(input: &str) -> Vec<u32> {
let _span = info_span!("parse", input_len = input.len()).entered();
let values: Vec<u32> = input
.split(',')
.filter_map(|field| field.parse().ok())
.collect();
info!(count = values.len(), "parsed values");
values
}

fn sum(values: &[u32]) -> u32 {
let _span = info_span!("sum").entered();
let _filtered_out = debug_span!("sum_inner").entered();
debug!("filtered out by the default filter");
values.iter().sum()
}

/// Nested `tracing` spans, with events in them.
fn parse_and_sum() -> u32 {
let _span = info_span!("parse_and_sum").entered();
let values = parse("1,2,x,3");
if values.len() < 4 {
warn!(skipped = 4 - values.len(), "skipped invalid fields");
}
sum(&values)
}

/// A `tracing` event outside of any span.
fn event_without_span() -> u32 {
let values = black_box([1, 2, 3]);
info!(count = values.len(), "summed without a span");
values.iter().sum()
}

/// Records of the `log` crate, without `tracing`.
fn log_records() -> u32 {
let values = black_box([1, 2, 3]);
log::info!("summing {} values, logged with the log crate", values.len());
log::debug!("filtered out by the default filter");
values.iter().sum()
}

/// No telemetry at all.
fn no_telemetry() -> u32 {
black_box([1, 2, 3]).iter().sum()
}

fn telemetry(c: &mut Criterion) {
c.bench_function("parse_and_sum", |b| b.iter(parse_and_sum));
c.bench_function("event_without_span", |b| b.iter(event_without_span));
c.bench_function("log_records", |b| b.iter(log_records));
c.bench_function("no_telemetry", |b| b.iter(no_telemetry));
}

criterion_group!(benches, telemetry);
criterion_main!(benches);
2 changes: 2 additions & 0 deletions crates/criterion_compat/src/compat/macros.rs
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@ macro_rules! criterion_group {
macro_rules! criterion_main {
( $( $group:path ),+ $(,)* ) => {
pub fn main() {
// Exports spans and events for the duration of the run, with the `telemetry` feature.
let _telemetry_guard = $crate::__init_telemetry();
let mut criterion = $crate::Criterion::new_instrumented();
$(
$group(&mut criterion);
Expand Down
18 changes: 18 additions & 0 deletions crates/criterion_compat/src/lib.rs
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
pub use codspeed::codspeed_uri;
#[cfg(feature = "telemetry")]
pub use codspeed::telemetry;

#[cfg(not(codspeed))]
mod compat_criterion {
Expand All @@ -21,3 +23,19 @@ mod compat_criterion {
}

pub use compat_criterion::*;

/// Installs the telemetry export for the `main` of `criterion_main!`, with the `telemetry`
/// feature. The macro expands in the user's crate, so it can't check the feature itself.
///
/// Keep the returned guard alive until the benchmarks are done.
#[doc(hidden)]
#[cfg(feature = "telemetry")]
pub fn __init_telemetry() -> Option<telemetry::TelemetryGuard> {
telemetry::init()
}

#[doc(hidden)]
#[cfg(not(feature = "telemetry"))]
pub fn __init_telemetry() -> Option<()> {
None
}
Loading