From 13ab097a134102470c84f25f23e4b437f0f593e7 Mon Sep 17 00:00:00 2001 From: YangKian <1207783292@qq.com> Date: Sun, 14 Jan 2024 12:12:32 +0800 Subject: [PATCH 1/4] Feature: add `Raft::data_metrics()` and `Raft::server_metrics()` --- openraft/src/metrics/mod.rs | 4 + openraft/src/metrics/raft_metrics.rs | 152 ++++++++++++++++++ openraft/src/raft/mod.rs | 45 ++++++ openraft/src/raft/raft_inner.rs | 4 + tests/tests/metrics/main.rs | 1 + .../t10_server_metrics_and_data_metrics.rs | 54 +++++++ 6 files changed, 260 insertions(+) create mode 100644 tests/tests/metrics/t10_server_metrics_and_data_metrics.rs diff --git a/openraft/src/metrics/mod.rs b/openraft/src/metrics/mod.rs index f67f3e7a5..15cc2474c 100644 --- a/openraft/src/metrics/mod.rs +++ b/openraft/src/metrics/mod.rs @@ -38,7 +38,11 @@ mod wait_condition; use std::collections::BTreeMap; pub use metric::Metric; +pub use raft_metrics::is_data_metrics_changed; +pub use raft_metrics::is_server_metrics_changed; +pub use raft_metrics::RaftDataMetrics; pub use raft_metrics::RaftMetrics; +pub use raft_metrics::RaftServerMetrics; pub use wait::Wait; pub use wait::WaitError; pub(crate) use wait_condition::Condition; diff --git a/openraft/src/metrics/raft_metrics.rs b/openraft/src/metrics/raft_metrics.rs index e17c52318..03bc416fc 100644 --- a/openraft/src/metrics/raft_metrics.rs +++ b/openraft/src/metrics/raft_metrics.rs @@ -140,3 +140,155 @@ where } } } + +pub fn is_data_metrics_changed(old: &RaftMetrics, new: &RaftMetrics) -> bool +where + NID: NodeId, + N: Node, +{ + new.last_log_index.ne(&old.last_log_index) + || new.last_applied.ne(&old.last_applied) + || new.snapshot.ne(&old.snapshot) + || new.purged.ne(&old.purged) + || new.replication.ne(&old.replication) +} + +pub fn is_server_metrics_changed(old: &RaftMetrics, new: &RaftMetrics) -> bool +where + NID: NodeId, + N: Node, +{ + new.current_term.ne(&old.current_term) + || new.vote.ne(&old.vote) + || new.state.ne(&old.state) + || new.current_leader.ne(&old.current_leader) + || new.membership_config.ne(&old.membership_config) +} + +/// Subset of RaftMetrics, only include data-related metrics +#[derive(Clone, Debug, Default, PartialEq, Eq)] +#[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize), serde(bound = ""))] +pub struct RaftDataMetrics +where NID: NodeId +{ + pub last_log_index: Option, + pub last_applied: Option>, + pub snapshot: Option>, + pub purged: Option>, + pub replication: Option>, +} + +impl fmt::Display for RaftDataMetrics +where NID: NodeId +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "DataMetrics{{")?; + + write!( + f, + "last_log:{}, last_applied:{}, snapshot:{}, purged:{}, replication:{{{}}}", + DisplayOption(&self.last_log_index), + DisplayOption(&self.last_applied), + DisplayOption(&self.snapshot), + DisplayOption(&self.purged), + self.replication + .as_ref() + .map(|x| { x.iter().map(|(k, v)| format!("{}:{}", k, DisplayOption(v))).collect::>().join(",") }) + .unwrap_or_default(), + )?; + + write!(f, "}}")?; + Ok(()) + } +} + +impl MessageSummary> for RaftDataMetrics +where NID: NodeId +{ + fn summary(&self) -> String { + self.to_string() + } +} + +impl From> for RaftDataMetrics +where + NID: NodeId, + N: Node, +{ + fn from(metrics: RaftMetrics) -> Self { + Self { + last_log_index: metrics.last_log_index, + last_applied: metrics.last_applied, + snapshot: metrics.snapshot, + purged: metrics.purged, + replication: metrics.replication, + } + } +} + +/// Subset of RaftMetrics, only include server-related metrics +#[derive(Clone, Debug, Default, PartialEq, Eq)] +#[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize), serde(bound = ""))] +pub struct RaftServerMetrics +where + NID: NodeId, + N: Node, +{ + pub id: NID, + pub current_term: u64, + pub vote: Vote, + pub state: ServerState, + pub current_leader: Option, + pub membership_config: Arc>, +} + +impl fmt::Display for RaftServerMetrics +where + NID: NodeId, + N: Node, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "Metrics{{")?; + + write!( + f, + "id:{}, {:?}, term:{}, vote:{}, leader:{}, membership:{}", + self.id, + self.state, + self.current_term, + self.vote, + DisplayOption(&self.current_leader), + self.membership_config.summary(), + )?; + + write!(f, "}}")?; + Ok(()) + } +} + +impl MessageSummary> for RaftServerMetrics +where + NID: NodeId, + N: Node, +{ + fn summary(&self) -> String { + self.to_string() + } +} + +impl From> for RaftServerMetrics +where + NID: NodeId, + N: Node, +{ + fn from(metrics: RaftMetrics) -> Self { + Self { + id: metrics.id, + current_term: metrics.current_term, + vote: metrics.vote, + state: metrics.state, + current_leader: metrics.current_leader, + membership_config: metrics.membership_config, + } + } +} diff --git a/openraft/src/raft/mod.rs b/openraft/src/raft/mod.rs index 8475eab6d..dc60092b1 100644 --- a/openraft/src/raft/mod.rs +++ b/openraft/src/raft/mod.rs @@ -51,7 +51,11 @@ use crate::error::InitializeError; use crate::error::InstallSnapshotError; use crate::error::RaftError; use crate::membership::IntoNodes; +use crate::metrics::is_data_metrics_changed; +use crate::metrics::is_server_metrics_changed; +use crate::metrics::RaftDataMetrics; use crate::metrics::RaftMetrics; +use crate::metrics::RaftServerMetrics; use crate::metrics::Wait; use crate::metrics::WaitError; use crate::network::RaftNetworkFactory; @@ -174,8 +178,37 @@ where C: RaftTypeConfig let (tx_api, rx_api) = mpsc::unbounded_channel(); let (tx_notify, rx_notify) = mpsc::unbounded_channel(); let (tx_metrics, rx_metrics) = watch::channel(RaftMetrics::new_initial(id)); + let (tx_data_metrics, rx_data_metrics) = watch::channel(RaftDataMetrics::default()); + let (tx_server_metrics, rx_server_metrics) = watch::channel(RaftServerMetrics::default()); let (tx_shutdown, rx_shutdown) = oneshot::channel(); + let mut raft_metrics_rx = rx_metrics.clone(); + + #[allow(clippy::let_underscore_future)] + let _ = C::AsyncRuntime::spawn(async move { + let mut last = RaftMetrics::new_initial(id); + loop { + let latest = raft_metrics_rx.borrow().clone(); + if is_data_metrics_changed(&last, &latest) { + if let Err(err) = tx_data_metrics.send(latest.clone().into()) { + tracing::error!(error=%err, id=display(id), "error reporting data metrics"); + } + } + + if is_server_metrics_changed(&last, &latest) { + if let Err(err) = tx_server_metrics.send(latest.clone().into()) { + tracing::error!(error=%err, id=display(id), "error reporting server metrics"); + } + } + + last = latest; + if let Err(e) = raft_metrics_rx.changed().await { + tracing::info!(error=%e, id=display(id), "metrics sender closed, so close data_metrics sender and server_metrics sender"); + return; + } + } + }); + let tick_handle = Tick::spawn( Duration::from_millis(config.heartbeat_interval * 3 / 2), tx_notify.clone(), @@ -240,6 +273,8 @@ where C: RaftTypeConfig tick_handle, tx_api, rx_metrics, + rx_data_metrics, + rx_server_metrics, tx_shutdown: Mutex::new(Some(tx_shutdown)), core_state: Mutex::new(CoreState::Running(core_handle)), }; @@ -822,6 +857,16 @@ where C: RaftTypeConfig self.inner.rx_metrics.clone() } + /// Get a handle to the data metrics channel. + pub fn data_metrics(&self) -> watch::Receiver> { + self.inner.rx_data_metrics.clone() + } + + /// Get a handle to the server metrics channel. + pub fn server_metrics(&self) -> watch::Receiver> { + self.inner.rx_server_metrics.clone() + } + /// Get a handle to wait for the metrics to satisfy some condition. /// /// If `timeout` is `None`, then it will wait forever(10 years). diff --git a/openraft/src/raft/raft_inner.rs b/openraft/src/raft/raft_inner.rs index 69712ce4e..f8b7c9fcb 100644 --- a/openraft/src/raft/raft_inner.rs +++ b/openraft/src/raft/raft_inner.rs @@ -11,6 +11,8 @@ use crate::core::raft_msg::external_command::ExternalCommand; use crate::core::raft_msg::RaftMsg; use crate::core::TickHandle; use crate::error::Fatal; +use crate::metrics::RaftDataMetrics; +use crate::metrics::RaftServerMetrics; use crate::raft::core_state::CoreState; use crate::AsyncRuntime; use crate::Config; @@ -28,6 +30,8 @@ where C: RaftTypeConfig pub(in crate::raft) tick_handle: TickHandle, pub(in crate::raft) tx_api: mpsc::UnboundedSender>, pub(in crate::raft) rx_metrics: watch::Receiver>, + pub(in crate::raft) rx_data_metrics: watch::Receiver>, + pub(in crate::raft) rx_server_metrics: watch::Receiver>, // TODO(xp): it does not need to be a async mutex. #[allow(clippy::type_complexity)] diff --git a/tests/tests/metrics/main.rs b/tests/tests/metrics/main.rs index 409d07b78..f212f60d9 100644 --- a/tests/tests/metrics/main.rs +++ b/tests/tests/metrics/main.rs @@ -9,6 +9,7 @@ mod fixtures; mod t10_current_leader; mod t10_purged; +mod t10_server_metrics_and_data_metrics; mod t20_metrics_state_machine_consistency; mod t30_leader_metrics; mod t40_metrics_wait; diff --git a/tests/tests/metrics/t10_server_metrics_and_data_metrics.rs b/tests/tests/metrics/t10_server_metrics_and_data_metrics.rs new file mode 100644 index 000000000..99d6816e3 --- /dev/null +++ b/tests/tests/metrics/t10_server_metrics_and_data_metrics.rs @@ -0,0 +1,54 @@ +use std::sync::Arc; + +use anyhow::Result; +use maplit::btreeset; +use openraft::Config; +#[allow(unused_imports)] use pretty_assertions::assert_eq; +#[allow(unused_imports)] use pretty_assertions::assert_ne; + +use crate::fixtures::init_default_ut_tracing; +use crate::fixtures::RaftRouter; + +/// Server metrics and data metrics method should work. +#[async_entry::test(worker_threads = 8, init = "init_default_ut_tracing()", tracing_span = "debug")] +async fn server_metrics_and_data_metrics() -> Result<()> { + // Setup test dependencies. + let config = Arc::new( + Config { + enable_heartbeat: false, + enable_elect: false, + ..Default::default() + } + .validate()?, + ); + let mut router = RaftRouter::new(config.clone()); + + tracing::info!("--- initializing cluster"); + let mut log_index = router.new_cluster(btreeset! {0,1,2}, btreeset! {}).await?; + + let node = router.get_raft_handle(&0)?; + let mut server_metrics = node.server_metrics(); + let data_metrics = node.data_metrics(); + + let current_leader = router.current_leader(0).await; + let leader = server_metrics.borrow_and_update().current_leader; + assert_eq!(leader, current_leader, "current_leader should be {:?}", current_leader); + + // Write some logs. + let n = 10; + tracing::info!(log_index, "--- write {} logs", n); + log_index += router.client_request_many(0, "foo", n).await?; + + let last_log_index = data_metrics.borrow().last_log_index; + assert_eq!( + last_log_index, + Some(log_index), + "last_log_index should be {:?}", + Some(log_index) + ); + assert!( + !server_metrics.borrow().has_changed(), + "server metrics should not update" + ); + Ok(()) +} From e43168eccb8995503ea2212d8ecc266e9637cdb8 Mon Sep 17 00:00:00 2001 From: YangKian <1207783292@qq.com> Date: Sun, 14 Jan 2024 12:34:27 +0800 Subject: [PATCH 2/4] Fixup: change Metrics to ServerMetrics for RaftServerMetrics fmt --- openraft/src/metrics/raft_metrics.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openraft/src/metrics/raft_metrics.rs b/openraft/src/metrics/raft_metrics.rs index 03bc416fc..091214d5a 100644 --- a/openraft/src/metrics/raft_metrics.rs +++ b/openraft/src/metrics/raft_metrics.rs @@ -248,7 +248,7 @@ where N: Node, { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "Metrics{{")?; + write!(f, "ServerMetrics{{")?; write!( f, From 27a5eea212a3771a8451ba76325e8c7b8ab63cde Mon Sep 17 00:00:00 2001 From: YangKian <1207783292@qq.com> Date: Sun, 14 Jan 2024 17:42:53 +0800 Subject: [PATCH 3/4] Refactor: simplify metrics producing process --- openraft/src/core/raft_core.rs | 39 ++++++++++- openraft/src/metrics/mod.rs | 2 - openraft/src/metrics/raft_metrics.rs | 65 +------------------ openraft/src/raft/mod.rs | 31 +-------- .../t10_server_metrics_and_data_metrics.rs | 6 +- 5 files changed, 44 insertions(+), 99 deletions(-) diff --git a/openraft/src/core/raft_core.rs b/openraft/src/core/raft_core.rs index 59c8f1e6d..d5ccb4127 100644 --- a/openraft/src/core/raft_core.rs +++ b/openraft/src/core/raft_core.rs @@ -55,7 +55,7 @@ use crate::error::RPCError; use crate::error::Timeout; use crate::log_id::LogIdOptionExt; use crate::log_id::RaftLogId; -use crate::metrics::RaftMetrics; +use crate::metrics::{RaftDataMetrics, RaftMetrics, RaftServerMetrics}; use crate::metrics::ReplicationMetrics; use crate::network::RPCOption; use crate::network::RPCTypes; @@ -198,6 +198,8 @@ where pub(crate) rx_notify: mpsc::UnboundedReceiver>, pub(crate) tx_metrics: watch::Sender>, + pub(crate) tx_data_metrics: watch::Sender>, + pub(crate) tx_server_metrics: watch::Sender>, pub(crate) command_state: CommandState, @@ -522,6 +524,7 @@ where pub(crate) fn report_metrics(&self, replication: Option>) { let st = &self.engine.state; + let membership_config = st.membership_state.effective().stored_membership().clone(); let m = RaftMetrics { running_state: Ok(()), id: self.id, @@ -537,10 +540,10 @@ where // --- cluster --- state: st.server_state, current_leader: self.current_leader(), - membership_config: st.membership_state.effective().stored_membership().clone(), + membership_config: membership_config.clone(), // --- replication --- - replication, + replication: replication.clone(), }; tracing::debug!("report_metrics: {}", m.summary()); @@ -549,6 +552,36 @@ where if let Err(err) = res { tracing::error!(error=%err, id=display(self.id), "error reporting metrics"); } + + let data_metrics = RaftDataMetrics { + last_log: st.last_log_id().copied(), + last_applied: st.io_applied().copied(), + snapshot: st.io_snapshot_last_log_id().copied(), + purged: st.io_purged().copied(), + replication, + }; + self.tx_data_metrics.send_if_modified(|metrix| { + if data_metrics.ne(metrix) { + *metrix = data_metrics.clone(); + return true; + } + false + }); + + let server_metrics = RaftServerMetrics { + id: self.id, + vote: *st.io_state().vote(), + state: st.server_state, + current_leader: self.current_leader(), + membership_config, + }; + self.tx_server_metrics.send_if_modified(|metrix| { + if server_metrics.ne(metrix) { + *metrix = server_metrics.clone(); + return true; + } + false + }); } /// Handle the admin command `initialize`. diff --git a/openraft/src/metrics/mod.rs b/openraft/src/metrics/mod.rs index 15cc2474c..8d672195a 100644 --- a/openraft/src/metrics/mod.rs +++ b/openraft/src/metrics/mod.rs @@ -38,8 +38,6 @@ mod wait_condition; use std::collections::BTreeMap; pub use metric::Metric; -pub use raft_metrics::is_data_metrics_changed; -pub use raft_metrics::is_server_metrics_changed; pub use raft_metrics::RaftDataMetrics; pub use raft_metrics::RaftMetrics; pub use raft_metrics::RaftServerMetrics; diff --git a/openraft/src/metrics/raft_metrics.rs b/openraft/src/metrics/raft_metrics.rs index 091214d5a..e0bd6362b 100644 --- a/openraft/src/metrics/raft_metrics.rs +++ b/openraft/src/metrics/raft_metrics.rs @@ -141,37 +141,13 @@ where } } -pub fn is_data_metrics_changed(old: &RaftMetrics, new: &RaftMetrics) -> bool -where - NID: NodeId, - N: Node, -{ - new.last_log_index.ne(&old.last_log_index) - || new.last_applied.ne(&old.last_applied) - || new.snapshot.ne(&old.snapshot) - || new.purged.ne(&old.purged) - || new.replication.ne(&old.replication) -} - -pub fn is_server_metrics_changed(old: &RaftMetrics, new: &RaftMetrics) -> bool -where - NID: NodeId, - N: Node, -{ - new.current_term.ne(&old.current_term) - || new.vote.ne(&old.vote) - || new.state.ne(&old.state) - || new.current_leader.ne(&old.current_leader) - || new.membership_config.ne(&old.membership_config) -} - /// Subset of RaftMetrics, only include data-related metrics #[derive(Clone, Debug, Default, PartialEq, Eq)] #[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize), serde(bound = ""))] pub struct RaftDataMetrics where NID: NodeId { - pub last_log_index: Option, + pub last_log: Option>, pub last_applied: Option>, pub snapshot: Option>, pub purged: Option>, @@ -187,7 +163,7 @@ where NID: NodeId write!( f, "last_log:{}, last_applied:{}, snapshot:{}, purged:{}, replication:{{{}}}", - DisplayOption(&self.last_log_index), + DisplayOption(&self.last_log), DisplayOption(&self.last_applied), DisplayOption(&self.snapshot), DisplayOption(&self.purged), @@ -210,22 +186,6 @@ where NID: NodeId } } -impl From> for RaftDataMetrics -where - NID: NodeId, - N: Node, -{ - fn from(metrics: RaftMetrics) -> Self { - Self { - last_log_index: metrics.last_log_index, - last_applied: metrics.last_applied, - snapshot: metrics.snapshot, - purged: metrics.purged, - replication: metrics.replication, - } - } -} - /// Subset of RaftMetrics, only include server-related metrics #[derive(Clone, Debug, Default, PartialEq, Eq)] #[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize), serde(bound = ""))] @@ -235,7 +195,6 @@ where N: Node, { pub id: NID, - pub current_term: u64, pub vote: Vote, pub state: ServerState, pub current_leader: Option, @@ -252,10 +211,9 @@ where write!( f, - "id:{}, {:?}, term:{}, vote:{}, leader:{}, membership:{}", + "id:{}, {:?}, vote:{}, leader:{}, membership:{}", self.id, self.state, - self.current_term, self.vote, DisplayOption(&self.current_leader), self.membership_config.summary(), @@ -275,20 +233,3 @@ where self.to_string() } } - -impl From> for RaftServerMetrics -where - NID: NodeId, - N: Node, -{ - fn from(metrics: RaftMetrics) -> Self { - Self { - id: metrics.id, - current_term: metrics.current_term, - vote: metrics.vote, - state: metrics.state, - current_leader: metrics.current_leader, - membership_config: metrics.membership_config, - } - } -} diff --git a/openraft/src/raft/mod.rs b/openraft/src/raft/mod.rs index dc60092b1..69e0f5c16 100644 --- a/openraft/src/raft/mod.rs +++ b/openraft/src/raft/mod.rs @@ -51,8 +51,6 @@ use crate::error::InitializeError; use crate::error::InstallSnapshotError; use crate::error::RaftError; use crate::membership::IntoNodes; -use crate::metrics::is_data_metrics_changed; -use crate::metrics::is_server_metrics_changed; use crate::metrics::RaftDataMetrics; use crate::metrics::RaftMetrics; use crate::metrics::RaftServerMetrics; @@ -182,33 +180,6 @@ where C: RaftTypeConfig let (tx_server_metrics, rx_server_metrics) = watch::channel(RaftServerMetrics::default()); let (tx_shutdown, rx_shutdown) = oneshot::channel(); - let mut raft_metrics_rx = rx_metrics.clone(); - - #[allow(clippy::let_underscore_future)] - let _ = C::AsyncRuntime::spawn(async move { - let mut last = RaftMetrics::new_initial(id); - loop { - let latest = raft_metrics_rx.borrow().clone(); - if is_data_metrics_changed(&last, &latest) { - if let Err(err) = tx_data_metrics.send(latest.clone().into()) { - tracing::error!(error=%err, id=display(id), "error reporting data metrics"); - } - } - - if is_server_metrics_changed(&last, &latest) { - if let Err(err) = tx_server_metrics.send(latest.clone().into()) { - tracing::error!(error=%err, id=display(id), "error reporting server metrics"); - } - } - - last = latest; - if let Err(e) = raft_metrics_rx.changed().await { - tracing::info!(error=%e, id=display(id), "metrics sender closed, so close data_metrics sender and server_metrics sender"); - return; - } - } - }); - let tick_handle = Tick::spawn( Duration::from_millis(config.heartbeat_interval * 3 / 2), tx_notify.clone(), @@ -257,6 +228,8 @@ where C: RaftTypeConfig rx_notify, tx_metrics, + tx_data_metrics, + tx_server_metrics, command_state: CommandState::default(), span: core_span, diff --git a/tests/tests/metrics/t10_server_metrics_and_data_metrics.rs b/tests/tests/metrics/t10_server_metrics_and_data_metrics.rs index 99d6816e3..4e0f006bf 100644 --- a/tests/tests/metrics/t10_server_metrics_and_data_metrics.rs +++ b/tests/tests/metrics/t10_server_metrics_and_data_metrics.rs @@ -39,12 +39,12 @@ async fn server_metrics_and_data_metrics() -> Result<()> { tracing::info!(log_index, "--- write {} logs", n); log_index += router.client_request_many(0, "foo", n).await?; - let last_log_index = data_metrics.borrow().last_log_index; + let last_log_index = data_metrics.borrow().last_log.unwrap_or_default().index; assert_eq!( last_log_index, - Some(log_index), + log_index, "last_log_index should be {:?}", - Some(log_index) + log_index ); assert!( !server_metrics.borrow().has_changed(), From 0b005bbaf414ef8ab4e98b1cb6482fad4a8672da Mon Sep 17 00:00:00 2001 From: YangKian <1207783292@qq.com> Date: Sun, 14 Jan 2024 17:46:31 +0800 Subject: [PATCH 4/4] Chore: fmt --- openraft/src/core/raft_core.rs | 4 +++- tests/tests/metrics/t10_server_metrics_and_data_metrics.rs | 7 +------ 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/openraft/src/core/raft_core.rs b/openraft/src/core/raft_core.rs index d5ccb4127..5bd10d554 100644 --- a/openraft/src/core/raft_core.rs +++ b/openraft/src/core/raft_core.rs @@ -55,7 +55,9 @@ use crate::error::RPCError; use crate::error::Timeout; use crate::log_id::LogIdOptionExt; use crate::log_id::RaftLogId; -use crate::metrics::{RaftDataMetrics, RaftMetrics, RaftServerMetrics}; +use crate::metrics::RaftDataMetrics; +use crate::metrics::RaftMetrics; +use crate::metrics::RaftServerMetrics; use crate::metrics::ReplicationMetrics; use crate::network::RPCOption; use crate::network::RPCTypes; diff --git a/tests/tests/metrics/t10_server_metrics_and_data_metrics.rs b/tests/tests/metrics/t10_server_metrics_and_data_metrics.rs index 4e0f006bf..c85204e46 100644 --- a/tests/tests/metrics/t10_server_metrics_and_data_metrics.rs +++ b/tests/tests/metrics/t10_server_metrics_and_data_metrics.rs @@ -40,12 +40,7 @@ async fn server_metrics_and_data_metrics() -> Result<()> { log_index += router.client_request_many(0, "foo", n).await?; let last_log_index = data_metrics.borrow().last_log.unwrap_or_default().index; - assert_eq!( - last_log_index, - log_index, - "last_log_index should be {:?}", - log_index - ); + assert_eq!(last_log_index, log_index, "last_log_index should be {:?}", log_index); assert!( !server_metrics.borrow().has_changed(), "server metrics should not update"