From 42313899166a203bb70ada762fd10d7c186335a9 Mon Sep 17 00:00:00 2001 From: Pascal Seitz Date: Sat, 12 Oct 2024 12:21:43 +0800 Subject: [PATCH] run benchmarks for 500ms, switch reporter to plugin system --- README.md | 1 - src/bench.rs | 4 +- src/bench_id.rs | 49 ------------ src/bench_input_group.rs | 11 +-- src/bench_runner.rs | 41 +++++----- src/lib.rs | 3 +- src/plugins/events.rs | 14 ++++ src/plugins/mod.rs | 8 ++ src/report/mod.rs | 92 +++++++++++++++++++--- src/report/plain_reporter.rs | 94 ++++++++++++++++------- src/report/table_reporter.rs | 145 ++++++++++++++++++++++++----------- 11 files changed, 296 insertions(+), 166 deletions(-) diff --git a/README.md b/README.md index 6b15692..426881a 100644 --- a/README.md +++ b/README.md @@ -103,7 +103,6 @@ Disabling the NMI watchdog should help: ### TODO -- [ ] Move reporter to plugin API - [ ] Improve the reporter api. Currently the reporter gets preaggregated data. #### Maybe Later Features: diff --git a/src/bench.rs b/src/bench.rs index 1735407..80852ab 100644 --- a/src/bench.rs +++ b/src/bench.rs @@ -178,8 +178,8 @@ impl<'a, I, O> NamedBench<'a, I, O> { #[inline] /// Each group has its own number of iterations. This is not the final num_iter pub fn sample_and_get_iter(&mut self, input: &'a I) -> usize { - // We want to run the benchmark for 100ms - const TARGET_MS_PER_BENCH: u64 = 100; + // We want to run the benchmark for 500ms + const TARGET_MS_PER_BENCH: u64 = 500; const TARGET_NS_PER_BENCH: u128 = TARGET_MS_PER_BENCH as u128 * 1_000_000; { // Preliminary test if function is very slow diff --git a/src/bench_id.rs b/src/bench_id.rs index e5147b5..d274b42 100644 --- a/src/bench_id.rs +++ b/src/bench_id.rs @@ -1,52 +1,3 @@ -use std::{ - ops::Deref, - sync::{Arc, Once}, -}; - -use yansi::Paint; - -/// The bench runners name is like a header and should only be printed if there are tests to be -/// run. Since this information is available at the time of creation, it will be handled when -/// executing the benches instead. -#[derive(Clone)] -pub struct PrintOnce { - inner: Arc, -} - -impl Deref for PrintOnce { - type Target = str; - fn deref(&self) -> &Self::Target { - &self.inner.name - } -} -struct PrintOnceInner { - name: String, - print_once: Once, -} - -impl PrintOnce { - /// Create a new PrintOnce instance - pub fn new(name: String) -> Self { - PrintOnce { - inner: Arc::new(PrintOnceInner { - name, - print_once: Once::new(), - }), - } - } - - /// Print the name. This will only print the name once. - pub fn print_name(&self) { - self.inner.print_once.call_once(|| { - println!("{}", self.get_name().black().on_red().invert().bold()); - }); - } - /// Get the name - pub fn get_name(&self) -> &str { - &self.inner.name - } -} - /// BenchId is a unique identifier for a benchmark. /// It has three components: /// - runner_name: The name of the runner that executed the benchmark. diff --git a/src/bench_input_group.rs b/src/bench_input_group.rs index 85a242a..c09882d 100644 --- a/src/bench_input_group.rs +++ b/src/bench_input_group.rs @@ -1,9 +1,9 @@ use std::{alloc::GlobalAlloc, mem}; use crate::output_value::OutputValue; +use crate::plugins::EventManager; use crate::{ - bench::NamedBench, bench_id::BenchId, bench_runner::BenchRunner, parse_args, report::Reporter, - BenchGroup, Config, + bench::NamedBench, bench_id::BenchId, bench_runner::BenchRunner, parse_args, BenchGroup, Config, }; use peakmem_alloc::*; @@ -152,9 +152,10 @@ impl InputGroup { &mut self.runner.config } - /// Set the reporter to be used for the benchmarks. See [Reporter] for more information. - pub fn set_reporter(&mut self, reporter: R) { - self.runner.set_reporter(reporter); + /// Returns the event manager, which can be used to add listeners to the benchmarks. + /// See [crate::plugins::EventManager] for more information. + pub fn get_event_manager(&mut self) -> &mut EventManager { + self.runner.get_event_manager() } } diff --git a/src/bench_runner.rs b/src/bench_runner.rs index 2a4658f..cf34b22 100644 --- a/src/bench_runner.rs +++ b/src/bench_runner.rs @@ -4,16 +4,16 @@ use std::{alloc::GlobalAlloc, cmp::Ordering}; use crate::output_value::OutputValue; use crate::plugins::alloc::AllocPerBench; use crate::plugins::{BingganEvents, EventManager}; +use crate::report::PlainReporter; use crate::{ bench::{Bench, InputWithBenchmark, NamedBench}, - bench_id::{BenchId, PrintOnce}, + bench_id::BenchId, black_box, parse_args, - report::{report_group, Reporter}, + report::report_group, BenchGroup, Config, }; use core::mem::size_of; use peakmem_alloc::*; -use yansi::Paint; /// The main struct to run benchmarks. /// @@ -25,9 +25,8 @@ pub struct BenchRunner { input_size_in_bytes: Option, /// Name of the test - pub(crate) name: Option, + pub(crate) name: Option, - reporter: Box, listeners: EventManager, } @@ -63,14 +62,15 @@ impl BenchRunner { use yansi::Condition; yansi::whenever(Condition::TTY_AND_COLOR); + let mut event_manager = EventManager::new(); + event_manager.add_listener_if_absent(PlainReporter::new()); + BenchRunner { cache_trasher: CacheTrasher::new(1024 * 1024 * 16), config: options, input_size_in_bytes: None, name: None, - //reporter: Box::new(crate::report::TableReporter {}), - reporter: Box::new(crate::report::PlainReporter::new()), - listeners: EventManager::new(), + listeners: event_manager, } } @@ -84,7 +84,7 @@ impl BenchRunner { /// runner. /// It is also used to distinguish when writing the results to disk. pub fn set_name>(&mut self, name: S) { - self.name = Some(PrintOnce::new(name.as_ref().to_string())); + self.name = Some(name.as_ref().to_string()); } /// Set the peak mem allocator to be used for the benchmarks. @@ -94,11 +94,6 @@ impl BenchRunner { self.listeners.add_listener_if_absent(alloc); } - /// Set the reporter to be used for the benchmarks. See [Reporter] for more information. - pub fn set_reporter(&mut self, reporter: R) { - self.reporter = Box::new(reporter); - } - /// Enables throughput reporting. The throughput will be valid for all inputs that are /// registered afterwards. pub fn set_input_size(&mut self, input_size: usize) { @@ -147,9 +142,6 @@ impl BenchRunner { if group.is_empty() { return; } - if let Some(runner_name) = &self.name { - runner_name.print_name(); - } #[cfg(target_os = "linux")] { use crate::plugins::profiler::PerfCounterPerBench; @@ -164,9 +156,6 @@ impl BenchRunner { group_name, output_value_column_title, }); - if let Some(name) = &group_name { - println!("{}", name.black().on_yellow().invert().bold()); - } const MAX_GROUP_SIZE: usize = 5; if self.config.verbose && group.len() > MAX_GROUP_SIZE { @@ -180,7 +169,7 @@ impl BenchRunner { // If the group is quite big, we don't want to create too big chunks, which causes // slow tests, therefore a chunk is at most 5 elements large. for group in group.chunks_mut(MAX_GROUP_SIZE) { - Self::detect_and_set_num_iter(group, self.config.verbose); + Self::detect_and_set_num_iter(group, self.config.verbose, &mut self.listeners); if self.config.interleave { Self::run_interleaved( @@ -198,7 +187,6 @@ impl BenchRunner { self.name.as_deref(), group_name, group, - &*self.reporter, output_value_column_title, &mut self.listeners, ); @@ -271,7 +259,11 @@ impl BenchRunner { } /// Detect how often each bench should be run if it is not set manually. - fn detect_and_set_num_iter<'b>(benches: &mut [Box + 'b>], verbose: bool) { + fn detect_and_set_num_iter<'b>( + benches: &mut [Box + 'b>], + verbose: bool, + events: &mut EventManager, + ) { if let Some(num_iter) = env::var("NUM_ITER_BENCH") .ok() .and_then(|val| val.parse::().ok()) @@ -305,6 +297,9 @@ impl BenchRunner { let max_num_iter = max_num_iter.min(min_num_iter * 10); // We round up, so that we may get the same number of iterations between runs let max_num_iter = round_up(max_num_iter as u64) as usize; + events.emit(BingganEvents::GroupNumIters { + num_iter: max_num_iter, + }); if verbose { println!("Set common iterations of {} for group", max_num_iter); } diff --git a/src/lib.rs b/src/lib.rs index 886cf38..6026bb1 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -33,8 +33,7 @@ //! See the [plugins] module for more information on how to register custom plugins. //! //! ## Reporting -//! See the [report] module for more information on how to customize the output. -//! Notice: This may be replaced by [plugins] later on. +//! See the [report] module for more information on how to customize the benchmark result reporting. //! //! # Perf Integration //! Binggan can integrate with perf to report hardware performance counters. diff --git a/src/plugins/events.rs b/src/plugins/events.rs index a0eadda..033d163 100644 --- a/src/plugins/events.rs +++ b/src/plugins/events.rs @@ -13,6 +13,13 @@ use std::any::Any; /// Events that can be emitted by the benchmark runner. #[derive(Debug, Clone, Copy)] pub enum BingganEvents<'a> { + /// The number of iterations for a group has been set. + GroupNumIters { + /// The number of iterations for each bench in the group. The whole group has the same + /// number of iterations to be a fair comparison between the benches in the group. + /// The previous event was `GroupStart`. + num_iter: usize, + }, /// Profiling of the group started GroupStart { /// The name of the runner @@ -79,6 +86,13 @@ impl EventManager { } } + /// Removes any listeners with the same name and sets the new listener. + pub fn replace_listener(&mut self, listener: L) { + self.remove_listener_by_name(listener.name()); + self.listeners + .push((listener.name().to_owned(), Box::new(listener))); + } + /// Add a new listener to the event manager if it is not already present by name. pub fn add_listener_if_absent(&mut self, listener: L) { if self.get_listener(listener.name()).is_some() { diff --git a/src/plugins/mod.rs b/src/plugins/mod.rs index 50e9eb9..3517a92 100644 --- a/src/plugins/mod.rs +++ b/src/plugins/mod.rs @@ -1,5 +1,13 @@ //! The plugin system works by registering to events. //! +//! The `BingganEvents` enum contains all the events that can be emitted. +//! The `EventListener` trait is used to listen to these events. +//! +//! The `BenchRunner` has an `EventManager` which can be used to add listeners. +//! The listeners can be used to track memory consumption, report results, etc. +//! +//! `name` is used to identify the listener. +//! //! # Example //! ```rust //! use binggan::*; diff --git a/src/report/mod.rs b/src/report/mod.rs index 0456a4a..cc32b8c 100644 --- a/src/report/mod.rs +++ b/src/report/mod.rs @@ -1,8 +1,10 @@ //! //! Module for reporting //! -//! The `report` module contains the [report::Reporter] trait and the [report::PlainReporter] struct. -//! You can set the reporter with the [BenchRunner::set_reporter] method. +//! The `report` module contains reporters that use the plugin system via the [EventListener](crate::plugins::EventListener) +//! trait. +//! You can set the reporter by registering at [BenchRunner::get_event_manager] . +//! Use [REPORTER_PLUGIN_NAME](crate::report::REPORTER_PLUGIN_NAME) as the name of a reporter, to overwrite the existing //! /// Helper methods to format benchmark results @@ -21,23 +23,20 @@ use yansi::Paint; use format::{bytes_to_string, format_duration_or_throughput}; use crate::{ - bench::{Bench, BenchResult}, + bench::Bench, plugins::{BingganEvents, EventManager}, stats::compute_diff, write_results::fetch_previous_run_and_write_results_to_disk, }; -/// The trait for reporting the results of a benchmark run. -pub trait Reporter { - /// Report the results from a group (can be a single bench) - fn report_results(&self, results: Vec, output_value_column_title: &'static str); -} +/// The default reporter name. Choose this in `EventListener` to make sure there's only one +/// reporter. +pub const REPORTER_PLUGIN_NAME: &str = "reporter"; pub(crate) fn report_group<'a>( runner_name: Option<&str>, group_name: Option<&str>, benches: &mut [Box + 'a>], - reporter: &dyn Reporter, output_value_column_title: &'static str, events: &mut EventManager, ) { @@ -57,7 +56,6 @@ pub(crate) fn report_group<'a>( results: &results, output_value_column_title, }); - reporter.report_results(results, output_value_column_title); } pub(crate) fn avg_median_str( @@ -68,7 +66,7 @@ pub(crate) fn avg_median_str( let avg_ns_diff = compute_diff(stats, input_size_in_bytes, other, |stats| stats.average_ns); let median_ns_diff = compute_diff(stats, input_size_in_bytes, other, |stats| stats.median_ns); - // if input_size_in_bytes is set report the throughput, otherwise just use format_duration + // if input_size_in_bytes is set, report the throughput, otherwise just use format_duration let avg_str = format!( "{} {}", format_duration_or_throughput(stats.average_ns, input_size_in_bytes), @@ -115,3 +113,75 @@ pub(crate) fn memory_str( mem_diff, ) } + +use std::{ + ops::Deref, + sync::{Arc, Once}, +}; + +/// The bench runners name is like a header and should only be printed if there are tests to be +/// run. Since this information is available at the time of creation, it will be handled when +/// executing the benches instead. +#[derive(Clone)] +pub struct PrintOnce { + inner: Arc, +} + +impl Deref for PrintOnce { + type Target = str; + fn deref(&self) -> &Self::Target { + &self.inner.name + } +} +struct PrintOnceInner { + name: String, + print_once: Once, +} + +/// Check and print the name. This will only print the name once. +/// If the past named differs (include None), sets the new name to be printed once +pub fn check_and_print(print_once: &mut Option, name: &str) { + if let Some(print_once) = print_once { + print_once.check_print(name); + return; + } + // set and print + *print_once = Some(PrintOnce::new(name.to_owned())); + print_once.as_ref().unwrap().print_name(); +} + +impl PrintOnce { + /// Create a new PrintOnce instance + pub fn new(name: String) -> Self { + PrintOnce { + inner: Arc::new(PrintOnceInner { + name, + print_once: Once::new(), + }), + } + } + + /// Check and print the name. This will only print the name once. + /// + /// If the past named differs, sets the new name to be printed once + pub fn check_print(&mut self, name: &str) { + if self.get_name() != name { + self.inner = Arc::new(PrintOnceInner { + name: name.to_owned(), + print_once: Once::new(), + }); + } + self.print_name(); + } + + /// Print the name. This will only print the name once. + pub fn print_name(&self) { + self.inner.print_once.call_once(|| { + println!("{}", self.get_name().black().on_red().invert().bold()); + }); + } + /// Get the name + pub fn get_name(&self) -> &str { + &self.inner.name + } +} diff --git a/src/report/plain_reporter.rs b/src/report/plain_reporter.rs index 982f87c..12671a6 100644 --- a/src/report/plain_reporter.rs +++ b/src/report/plain_reporter.rs @@ -1,9 +1,13 @@ +use std::any::Any; + use yansi::Paint; -use super::{avg_median_str, memory_str, min_max_str, BenchStats, Reporter}; -use crate::bench::BenchResult; +use super::{avg_median_str, memory_str, min_max_str, BenchStats, REPORTER_PLUGIN_NAME}; +use crate::{ + plugins::{BingganEvents, EventListener}, + report::{check_and_print, PrintOnce}, +}; -#[derive(Clone, Copy)] /// The PlainReporter prints the results in a plain text table. /// This is the default reporter. /// @@ -12,48 +16,80 @@ use crate::bench::BenchResult; /// factorial 100 Avg: 33ns Median: 32ns [32ns .. 45ns] /// factorial 400 Avg: 107ns Median: 107ns [107ns .. 109ns] /// ``` -pub struct PlainReporter {} - -impl Reporter for PlainReporter { - fn report_results(&self, results: Vec, output_value_column_title: &'static str) { - let mut table_data: Vec> = Vec::new(); - - for result in results { - let perf_counter = &result.perf_counter; +#[derive(Clone)] +pub struct PlainReporter { + print_runner_name_once: Option, +} - let mut stats_columns = self.to_columns( - result.stats, - result.old_stats, - result.input_size_in_bytes, - result.output_value, - result.tracked_memory, +impl EventListener for PlainReporter { + fn as_any(&mut self) -> &mut dyn Any { + self + } + fn name(&self) -> &'static str { + REPORTER_PLUGIN_NAME + } + fn on_event(&mut self, event: BingganEvents) { + match event { + BingganEvents::BenchStart { bench_id: _ } => {} + BingganEvents::GroupStart { + runner_name, + group_name: Some(group_name), + output_value_column_title: _, + } => { + if let Some(runner_name) = runner_name { + check_and_print(&mut self.print_runner_name_once, runner_name); + } + println!("{}", group_name.black().on_yellow().invert().bold()); + } + BingganEvents::GroupStop { + runner_name: _, + group_name: _, + results, output_value_column_title, - ); - stats_columns.insert(0, result.bench_id.bench_name.to_string()); - table_data.push(stats_columns); - - if let Some(perf_counter) = perf_counter.as_ref() { - let mut columns = perf_counter.to_columns(result.old_perf_counter); - columns.insert(0, "".to_string()); - table_data.push(columns); + } => { + let mut table_data: Vec> = Vec::new(); + + for result in results { + let perf_counter = &result.perf_counter; + + let mut stats_columns = self.to_columns( + result.stats, + result.old_stats, + result.input_size_in_bytes, + &result.output_value, + result.tracked_memory, + output_value_column_title, + ); + stats_columns.insert(0, result.bench_id.bench_name.to_string()); + table_data.push(stats_columns); + + if let Some(perf_counter) = perf_counter.as_ref() { + let mut columns = perf_counter.to_columns(result.old_perf_counter); + columns.insert(0, "".to_string()); + table_data.push(columns); + } + } + self.print_table(&table_data); } + _ => {} } - self.print_table(&table_data); } } impl PlainReporter { /// Create a new PlainReporter pub fn new() -> Self { - Self {} + Self { + print_runner_name_once: None, + } } pub(crate) fn to_columns( - self, + &self, stats: BenchStats, other: Option, input_size_in_bytes: Option, - output_value: Option, + output_value: &Option, report_memory: bool, output_value_column_title: &'static str, ) -> Vec { diff --git a/src/report/table_reporter.rs b/src/report/table_reporter.rs index cdf63e5..6f24d06 100644 --- a/src/report/table_reporter.rs +++ b/src/report/table_reporter.rs @@ -1,7 +1,13 @@ -use super::{avg_median_str, memory_str, min_max_str, Reporter}; -use crate::bench::BenchResult; +use std::any::Any; + +use yansi::Paint; + +use super::{avg_median_str, memory_str, min_max_str, REPORTER_PLUGIN_NAME}; +use crate::{ + plugins::{BingganEvents, EventListener}, + report::{check_and_print, PrintOnce}, +}; -#[derive(Clone, Copy)] /// The TableReporter prints the results using prettytable. /// /// It does not yet conver eveything, it does not report on OutputValue and perf stats. @@ -14,51 +20,102 @@ use crate::bench::BenchResult; /// | vec | Memory: 404 B | 8.6635 GiB/s (+1.16%) | 8.5639 GiB/s (-1.15%) | [8.7654 GiB/s .. 8.2784 GiB/s] | /// | hashmap | Memory: 84 B | 840.24 MiB/s (+1.54%) | 841.17 MiB/s (+0.33%) | [843.96 MiB/s .. 817.73 MiB/s] | /// ``` -pub struct TableReporter; -impl Reporter for TableReporter { - fn report_results(&self, results: Vec, output_value_column_title: &'static str) { - use prettytable::*; - let mut table = Table::new(); - let format = format::FormatBuilder::new() - .column_separator('|') - .borders('|') - .separators( - &[format::LinePosition::Title], - format::LineSeparator::new('-', '+', '+', '+'), - ) - .padding(1, 1) - .build(); - table.set_format(format); +#[derive(Clone)] +pub struct TableReporter { + print_runner_name_once: Option, +} - let mut row = prettytable::row!["Name", "Memory", "Avg", "Median", "Min .. Max"]; - if !results[0].tracked_memory { - row.remove_cell(1); - } - let has_output_value = results.iter().any(|r| r.output_value.is_some()); - if has_output_value { - row.add_cell(Cell::new(output_value_column_title)); +impl TableReporter { + /// Creates a new TableReporter + pub fn new() -> Self { + Self { + print_runner_name_once: None, } - table.set_titles(row); - for result in results { - let (avg_str, median_str) = - avg_median_str(&result.stats, result.input_size_in_bytes, result.old_stats); - let min_max = min_max_str(&result.stats, result.input_size_in_bytes); - let memory_string = memory_str(&result.stats, result.old_stats, result.tracked_memory); - let mut row = Row::new(vec![ - Cell::new(&result.bench_id.bench_name), - Cell::new(&memory_string), - Cell::new(&avg_str), - Cell::new(&median_str), - Cell::new(&min_max), - ]); - if has_output_value { - row.add_cell(Cell::new(&result.output_value.unwrap_or_default())); + } +} + +impl Default for TableReporter { + fn default() -> Self { + Self::new() + } +} + +impl EventListener for TableReporter { + fn as_any(&mut self) -> &mut dyn Any { + self + } + fn name(&self) -> &'static str { + REPORTER_PLUGIN_NAME + } + fn on_event(&mut self, event: BingganEvents) { + match event { + BingganEvents::BenchStart { bench_id: _ } => {} + BingganEvents::GroupStart { + runner_name, + group_name, + output_value_column_title: _, + } => { + if let Some(runner_name) = runner_name { + check_and_print(&mut self.print_runner_name_once, runner_name); + } + if let Some(group_name) = group_name { + println!("{}", group_name.black().on_yellow().invert().bold()); + } } - if !result.tracked_memory { - row.remove_cell(1); + BingganEvents::GroupStop { + runner_name: _, + group_name: _, + results, + output_value_column_title, + } => { + use prettytable::*; + let mut table = Table::new(); + let format = format::FormatBuilder::new() + .column_separator('|') + .borders('|') + .separators( + &[format::LinePosition::Title], + format::LineSeparator::new('-', '+', '+', '+'), + ) + .padding(1, 1) + .build(); + table.set_format(format); + + let mut row = prettytable::row!["Name", "Memory", "Avg", "Median", "Min .. Max"]; + if !results[0].tracked_memory { + row.remove_cell(1); + } + let has_output_value = results.iter().any(|r| r.output_value.is_some()); + if has_output_value { + row.add_cell(Cell::new(output_value_column_title)); + } + table.set_titles(row); + for result in results { + let (avg_str, median_str) = + avg_median_str(&result.stats, result.input_size_in_bytes, result.old_stats); + let min_max = min_max_str(&result.stats, result.input_size_in_bytes); + let memory_string = + memory_str(&result.stats, result.old_stats, result.tracked_memory); + let mut row = Row::new(vec![ + Cell::new(&result.bench_id.bench_name), + Cell::new(&memory_string), + Cell::new(&avg_str), + Cell::new(&median_str), + Cell::new(&min_max), + ]); + if has_output_value { + row.add_cell(Cell::new( + result.output_value.as_ref().unwrap_or(&"".to_string()), + )); + } + if !result.tracked_memory { + row.remove_cell(1); + } + table.add_row(row); + } + table.printstd(); } - table.add_row(row); + _ => {} } - table.printstd(); } }