run benchmarks for 500ms, switch reporter to plugin system

README.md

@@ -103,7 +103,6 @@ Disabling the NMI watchdog should help:
 
 ### TODO
 
-- [ ] Move reporter to plugin API
 - [ ] Improve the reporter api. Currently the reporter gets preaggregated data.
 
 #### Maybe Later Features:

src/bench.rs

@@ -178,8 +178,8 @@ impl<'a, I, O> NamedBench<'a, I, O> {
     #[inline]
     /// Each group has its own number of iterations. This is not the final num_iter
     pub fn sample_and_get_iter(&mut self, input: &'a I) -> usize {
-        // We want to run the benchmark for 100ms
-        const TARGET_MS_PER_BENCH: u64 = 100;
+        // We want to run the benchmark for 500ms
+        const TARGET_MS_PER_BENCH: u64 = 500;
         const TARGET_NS_PER_BENCH: u128 = TARGET_MS_PER_BENCH as u128 * 1_000_000;
         {
             // Preliminary test if function is very slow

src/bench_id.rs

@@ -1,52 +1,3 @@
-use std::{
-    ops::Deref,
-    sync::{Arc, Once},
-};
-
-use yansi::Paint;
-
-/// The bench runners name is like a header and should only be printed if there are tests to be
-/// run. Since this information is available at the time of creation, it will be handled when
-/// executing the benches instead.
-#[derive(Clone)]
-pub struct PrintOnce {
-    inner: Arc<PrintOnceInner>,
-}
-
-impl Deref for PrintOnce {
-    type Target = str;
-    fn deref(&self) -> &Self::Target {
-        &self.inner.name
-    }
-}
-struct PrintOnceInner {
-    name: String,
-    print_once: Once,
-}
-
-impl PrintOnce {
-    /// Create a new PrintOnce instance
-    pub fn new(name: String) -> Self {
-        PrintOnce {
-            inner: Arc::new(PrintOnceInner {
-                name,
-                print_once: Once::new(),
-            }),
-        }
-    }
-
-    /// Print the name. This will only print the name once.
-    pub fn print_name(&self) {
-        self.inner.print_once.call_once(|| {
-            println!("{}", self.get_name().black().on_red().invert().bold());
-        });
-    }
-    /// Get the name
-    pub fn get_name(&self) -> &str {
-        &self.inner.name
-    }
-}
-
 /// BenchId is a unique identifier for a benchmark.
 /// It has three components:
 /// - runner_name: The name of the runner that executed the benchmark.

src/bench_input_group.rs

@@ -1,9 +1,9 @@
 use std::{alloc::GlobalAlloc, mem};
 
 use crate::output_value::OutputValue;
+use crate::plugins::EventManager;
 use crate::{
-    bench::NamedBench, bench_id::BenchId, bench_runner::BenchRunner, parse_args, report::Reporter,
-    BenchGroup, Config,
+    bench::NamedBench, bench_id::BenchId, bench_runner::BenchRunner, parse_args, BenchGroup, Config,
 };
 use peakmem_alloc::*;
 
@@ -152,9 +152,10 @@ impl<I: 'static, O: OutputValue + 'static> InputGroup<I, O> {
         &mut self.runner.config
     }
 
-    /// Set the reporter to be used for the benchmarks. See [Reporter] for more information.
-    pub fn set_reporter<R: Reporter + 'static>(&mut self, reporter: R) {
-        self.runner.set_reporter(reporter);
+    /// Returns the event manager, which can be used to add listeners to the benchmarks.
+    /// See [crate::plugins::EventManager] for more information.
+    pub fn get_event_manager(&mut self) -> &mut EventManager {
+        self.runner.get_event_manager()
     }
 }
 

src/bench_runner.rs

@@ -4,16 +4,16 @@ use std::{alloc::GlobalAlloc, cmp::Ordering};
 use crate::output_value::OutputValue;
 use crate::plugins::alloc::AllocPerBench;
 use crate::plugins::{BingganEvents, EventManager};
+use crate::report::PlainReporter;
 use crate::{
     bench::{Bench, InputWithBenchmark, NamedBench},
-    bench_id::{BenchId, PrintOnce},
+    bench_id::BenchId,
     black_box, parse_args,
-    report::{report_group, Reporter},
+    report::report_group,
     BenchGroup, Config,
 };
 use core::mem::size_of;
 use peakmem_alloc::*;
-use yansi::Paint;
 
 /// The main struct to run benchmarks.
 ///
@@ -25,9 +25,8 @@ pub struct BenchRunner {
     input_size_in_bytes: Option<usize>,
 
     /// Name of the test
-    pub(crate) name: Option<PrintOnce>,
+    pub(crate) name: Option<String>,
 
-    reporter: Box<dyn Reporter>,
     listeners: EventManager,
 }
 
@@ -63,14 +62,15 @@ impl BenchRunner {
         use yansi::Condition;
         yansi::whenever(Condition::TTY_AND_COLOR);
 
+        let mut event_manager = EventManager::new();
+        event_manager.add_listener_if_absent(PlainReporter::new());
+
         BenchRunner {
             cache_trasher: CacheTrasher::new(1024 * 1024 * 16),
             config: options,
             input_size_in_bytes: None,
             name: None,
-            //reporter: Box::new(crate::report::TableReporter {}),
-            reporter: Box::new(crate::report::PlainReporter::new()),
-            listeners: EventManager::new(),
+            listeners: event_manager,
         }
     }
 
@@ -84,7 +84,7 @@ impl BenchRunner {
     /// runner.
     /// It is also used to distinguish when writing the results to disk.
     pub fn set_name<S: AsRef<str>>(&mut self, name: S) {
-        self.name = Some(PrintOnce::new(name.as_ref().to_string()));
+        self.name = Some(name.as_ref().to_string());
     }
 
     /// Set the peak mem allocator to be used for the benchmarks.
@@ -94,11 +94,6 @@ impl BenchRunner {
         self.listeners.add_listener_if_absent(alloc);
     }
 
-    /// Set the reporter to be used for the benchmarks. See [Reporter] for more information.
-    pub fn set_reporter<R: Reporter + 'static>(&mut self, reporter: R) {
-        self.reporter = Box::new(reporter);
-    }
-
     /// Enables throughput reporting. The throughput will be valid for all inputs that are
     /// registered afterwards.
     pub fn set_input_size(&mut self, input_size: usize) {
@@ -147,9 +142,6 @@ impl BenchRunner {
         if group.is_empty() {
             return;
         }
-        if let Some(runner_name) = &self.name {
-            runner_name.print_name();
-        }
         #[cfg(target_os = "linux")]
         {
             use crate::plugins::profiler::PerfCounterPerBench;
@@ -164,9 +156,6 @@ impl BenchRunner {
             group_name,
             output_value_column_title,
         });
-        if let Some(name) = &group_name {
-            println!("{}", name.black().on_yellow().invert().bold());
-        }
 
         const MAX_GROUP_SIZE: usize = 5;
         if self.config.verbose && group.len() > MAX_GROUP_SIZE {
@@ -180,7 +169,7 @@ impl BenchRunner {
         // If the group is quite big, we don't want to create too big chunks, which causes
         // slow tests, therefore a chunk is at most 5 elements large.
         for group in group.chunks_mut(MAX_GROUP_SIZE) {
-            Self::detect_and_set_num_iter(group, self.config.verbose);
+            Self::detect_and_set_num_iter(group, self.config.verbose, &mut self.listeners);
 
             if self.config.interleave {
                 Self::run_interleaved(
@@ -198,7 +187,6 @@ impl BenchRunner {
             self.name.as_deref(),
             group_name,
             group,
-            &*self.reporter,
             output_value_column_title,
             &mut self.listeners,
         );
@@ -271,7 +259,11 @@ impl BenchRunner {
     }
 
     /// Detect how often each bench should be run if it is not set manually.
-    fn detect_and_set_num_iter<'b>(benches: &mut [Box<dyn Bench<'b> + 'b>], verbose: bool) {
+    fn detect_and_set_num_iter<'b>(
+        benches: &mut [Box<dyn Bench<'b> + 'b>],
+        verbose: bool,
+        events: &mut EventManager,
+    ) {
         if let Some(num_iter) = env::var("NUM_ITER_BENCH")
             .ok()
             .and_then(|val| val.parse::<usize>().ok())
@@ -305,6 +297,9 @@ impl BenchRunner {
         let max_num_iter = max_num_iter.min(min_num_iter * 10);
         // We round up, so that we may get the same number of iterations between runs
         let max_num_iter = round_up(max_num_iter as u64) as usize;
+        events.emit(BingganEvents::GroupNumIters {
+            num_iter: max_num_iter,
+        });
         if verbose {
             println!("Set common iterations of {} for group", max_num_iter);
         }

src/lib.rs

@@ -33,8 +33,7 @@
 //! See the [plugins] module for more information on how to register custom plugins.
 //!
 //! ## Reporting
-//! See the [report] module for more information on how to customize the output.
-//! Notice: This may be replaced by [plugins] later on.
+//! See the [report] module for more information on how to customize the benchmark result reporting.
 //!
 //! # Perf Integration
 //! Binggan can integrate with perf to report hardware performance counters.

src/plugins/events.rs

@@ -13,6 +13,13 @@ use std::any::Any;
 /// Events that can be emitted by the benchmark runner.
 #[derive(Debug, Clone, Copy)]
 pub enum BingganEvents<'a> {
+    /// The number of iterations for a group has been set.
+    GroupNumIters {
+        /// The number of iterations for each bench in the group. The whole group has the same
+        /// number of iterations to be a fair comparison between the benches in the group.
+        /// The previous event was `GroupStart`.
+        num_iter: usize,
+    },
     /// Profiling of the group started
     GroupStart {
         /// The name of the runner
@@ -79,6 +86,13 @@ impl EventManager {
         }
     }
 
+    /// Removes any listeners with the same name and sets the new listener.
+    pub fn replace_listener<L: EventListener + 'static>(&mut self, listener: L) {
+        self.remove_listener_by_name(listener.name());
+        self.listeners
+            .push((listener.name().to_owned(), Box::new(listener)));
+    }
+
     /// Add a new listener to the event manager if it is not already present by name.
     pub fn add_listener_if_absent<L: EventListener + 'static>(&mut self, listener: L) {
         if self.get_listener(listener.name()).is_some() {

src/plugins/mod.rs

@@ -1,5 +1,13 @@
 //! The plugin system works by registering to events.
 //!
+//! The `BingganEvents` enum contains all the events that can be emitted.
+//! The `EventListener` trait is used to listen to these events.
+//!
+//! The `BenchRunner` has an `EventManager` which can be used to add listeners.
+//! The listeners can be used to track memory consumption, report results, etc.
+//!
+//! `name` is used to identify the listener.
+//!
 //! # Example
 //! ```rust
 //! use binggan::*;