Skip to content

Commit

Permalink
Update large tables statistics (#2146)
Browse files Browse the repository at this point in the history
# Description
After #2135 is deployed in
staging, large tables row count metrics are not updated for hours.
Statistics come from `pg_class` table, which is randomly updated by the
DB itself according to internal thresholds. In order to guarantee
statistics update for large tables, a new background task with hourly
rate triggers `ANALYZE {large_table_name}`, which takes around 15s in
staging. That is still significantly faster compared to the initial
`SELECT COUNT(*)` query(150-600s).

# Changes
A new background task running each hour executes `ANALYZE` for each
large table.

## How to test
Logs observation
  • Loading branch information
squadgazzz authored Dec 11, 2023
1 parent ce4b58f commit 9f62d7d
Show file tree
Hide file tree
Showing 2 changed files with 36 additions and 6 deletions.
37 changes: 35 additions & 2 deletions crates/autopilot/src/database.rs
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,9 @@ mod quotes;
pub mod recent_settlements;

use {
sqlx::{PgConnection, PgPool},
sqlx::{Executor, PgConnection, PgPool},
std::time::Duration,
tracing::Instrument,
};

#[derive(Debug, Clone)]
Expand Down Expand Up @@ -50,6 +51,15 @@ impl Postgres {

Ok(())
}

pub async fn update_large_tables_stats(&self) -> sqlx::Result<()> {
for &table in database::LARGE_TABLES {
let mut ex = self.0.acquire().await?;
analyze_table(&mut ex, table).await?;
}

Ok(())
}
}

async fn count_rows_in_table(ex: &mut PgConnection, table: &str) -> sqlx::Result<i64> {
Expand All @@ -62,6 +72,11 @@ async fn estimate_rows_in_table(ex: &mut PgConnection, table: &str) -> sqlx::Res
sqlx::query_scalar(&query).fetch_one(ex).await
}

async fn analyze_table(ex: &mut PgConnection, table: &str) -> sqlx::Result<()> {
let query = format!("ANALYZE {table};");
ex.execute(sqlx::query(&query)).await.map(|_| ())
}

async fn count_unused_app_data(ex: &mut PgConnection) -> sqlx::Result<i64> {
let query = r#"
SELECT
Expand Down Expand Up @@ -99,7 +114,16 @@ impl Metrics {
}
}

pub async fn database_metrics(db: Postgres) -> ! {
pub fn run_database_metrics_work(db: Postgres) {
let span = tracing::info_span!("database_metrics");
// Spawn the task for updating large table statistics
tokio::spawn(update_large_tables_stats(db.clone()).instrument(span.clone()));

// Spawn the task for database metrics
tokio::task::spawn(database_metrics(db).instrument(span));
}

async fn database_metrics(db: Postgres) -> ! {
loop {
if let Err(err) = db.update_database_metrics().await {
tracing::error!(?err, "failed to update table rows metric");
Expand All @@ -108,6 +132,15 @@ pub async fn database_metrics(db: Postgres) -> ! {
}
}

async fn update_large_tables_stats(db: Postgres) -> ! {
loop {
if let Err(err) = db.update_large_tables_stats().await {
tracing::error!(?err, "failed to update large tables stats");
}
tokio::time::sleep(Duration::from_secs(60 * 60)).await;
}
}

#[cfg(test)]
mod tests {
use super::*;
Expand Down
5 changes: 1 addition & 4 deletions crates/autopilot/src/run.rs
Original file line number Diff line number Diff line change
Expand Up @@ -97,10 +97,7 @@ pub async fn run(args: Arguments) {
assert!(args.shadow.is_none(), "cannot run in shadow mode");

let db = Postgres::new(args.db_url.as_str()).await.unwrap();
tokio::task::spawn(
crate::database::database_metrics(db.clone())
.instrument(tracing::info_span!("database_metrics")),
);
crate::database::run_database_metrics_work(db.clone());

let http_factory = HttpClientFactory::new(&args.http_client);
let web3 = shared::ethrpc::web3(
Expand Down

0 comments on commit 9f62d7d

Please sign in to comment.