Merge remote-tracking branch 'origin/main' into eliza/dropshot-1180

Cargo.toml

@@ -346,10 +346,10 @@ cookie = "0.18"
 criterion = { version = "0.5.1", features = [ "async_tokio" ] }
 crossbeam = "0.8"
 crossterm = { version = "0.28.1", features = ["event-stream"] }
-crucible-agent-client = { git = "https://github.com/oxidecomputer/crucible", rev = "2cfc7e0c8572b3bfafbfc838c4e6d658f442d239" }
-crucible-pantry-client = { git = "https://github.com/oxidecomputer/crucible", rev = "2cfc7e0c8572b3bfafbfc838c4e6d658f442d239" }
-crucible-smf = { git = "https://github.com/oxidecomputer/crucible", rev = "2cfc7e0c8572b3bfafbfc838c4e6d658f442d239" }
-crucible-common = { git = "https://github.com/oxidecomputer/crucible", rev = "2cfc7e0c8572b3bfafbfc838c4e6d658f442d239" }
+crucible-agent-client = { git = "https://github.com/oxidecomputer/crucible", rev = "5a41b826171c7d2a8412fa833377ab1df25ee8ec" }
+crucible-pantry-client = { git = "https://github.com/oxidecomputer/crucible", rev = "5a41b826171c7d2a8412fa833377ab1df25ee8ec" }
+crucible-smf = { git = "https://github.com/oxidecomputer/crucible", rev = "5a41b826171c7d2a8412fa833377ab1df25ee8ec" }
+crucible-common = { git = "https://github.com/oxidecomputer/crucible", rev = "5a41b826171c7d2a8412fa833377ab1df25ee8ec" }
 csv = "1.3.0"
 curve25519-dalek = "4"
 datatest-stable = "0.2.9"
@@ -492,7 +492,7 @@ omicron-test-utils = { path = "test-utils" }
 omicron-workspace-hack = "0.1.0"
 omicron-zone-package = "0.11.1"
 oxide-client = { path = "clients/oxide-client" }
-oxide-vpc = { git = "https://github.com/oxidecomputer/opte", rev = "98247c27846133a80fdb8f730f0c57e72d766561", features = [ "api", "std" ] }
+oxide-vpc = { git = "https://github.com/oxidecomputer/opte", rev = "b56afeeb14e0042cbd7bda85b166ed86ee17820e", features = [ "api", "std" ] }
 oxlog = { path = "dev-tools/oxlog" }
 oxnet = { git = "https://github.com/oxidecomputer/oxnet" }
 once_cell = "1.20.2"
@@ -502,7 +502,7 @@ openapiv3 = "2.0.0"
 # must match samael's crate!
 openssl = "0.10"
 openssl-sys = "0.9"
-opte-ioctl = { git = "https://github.com/oxidecomputer/opte", rev = "98247c27846133a80fdb8f730f0c57e72d766561" }
+opte-ioctl = { git = "https://github.com/oxidecomputer/opte", rev = "b56afeeb14e0042cbd7bda85b166ed86ee17820e" }
 oso = "0.27"
 owo-colors = "4.1.0"
 oximeter = { path = "oximeter/oximeter" }
@@ -539,10 +539,10 @@ prettyplease = { version = "0.2.25", features = ["verbatim"] }
 proc-macro2 = "1.0"
 progenitor = "0.8.0"
 progenitor-client = "0.8.0"
-bhyve_api = { git = "https://github.com/oxidecomputer/propolis", rev = "6936f1a949d155da38d3148abd42caef337dea04" }
-propolis_api_types = { git = "https://github.com/oxidecomputer/propolis", rev = "6936f1a949d155da38d3148abd42caef337dea04" }
-propolis-client = { git = "https://github.com/oxidecomputer/propolis", rev = "6936f1a949d155da38d3148abd42caef337dea04" }
-propolis-mock-server = { git = "https://github.com/oxidecomputer/propolis", rev = "6936f1a949d155da38d3148abd42caef337dea04" }
+bhyve_api = { git = "https://github.com/oxidecomputer/propolis", rev = "220a6f367c18f2452dbc4fa9086f3fe73b961739" }
+propolis_api_types = { git = "https://github.com/oxidecomputer/propolis", rev = "220a6f367c18f2452dbc4fa9086f3fe73b961739" }
+propolis-client = { git = "https://github.com/oxidecomputer/propolis", rev = "220a6f367c18f2452dbc4fa9086f3fe73b961739" }
+propolis-mock-server = { git = "https://github.com/oxidecomputer/propolis", rev = "220a6f367c18f2452dbc4fa9086f3fe73b961739" }
 proptest = "1.5.0"
 qorb = "0.2.1"
 quote = "1.0"

dev-tools/clickhouse-cluster-dev/src/main.rs

@@ -22,6 +22,16 @@ use std::time::Duration;
 async fn main() -> Result<()> {
     let request_timeout = Duration::from_secs(15);
     let (logctx, path) = default_clickhouse_log_ctx_and_path();
+
+    if path.exists() {
+        let deployment =
+            default_clickhouse_cluster_test_deployment(path.clone());
+        slog::info!(logctx.log, "Stopping test clickhouse nodes");
+        deployment.teardown()?;
+        slog::info!(logctx.log, "Removing previous temporary test directory");
+        std::fs::remove_dir_all(&path)?;
+    }
+
     std::fs::create_dir(&path)?;
 
     slog::info!(logctx.log, "Setting up a ClickHouse cluster");

dev-tools/omdb/src/bin/omdb/oximeter.rs

@@ -7,11 +7,15 @@
 use crate::helpers::CONNECTION_OPTIONS_HEADING;
 use crate::Omdb;
 use anyhow::Context;
+use chrono::SecondsFormat;
 use clap::Args;
 use clap::Subcommand;
 use futures::TryStreamExt;
 use internal_dns_types::names::ServiceName;
+use oximeter_client::types::FailedCollection;
+use oximeter_client::types::ProducerDetails;
 use oximeter_client::types::ProducerEndpoint;
+use oximeter_client::types::SuccessfulCollection;
 use oximeter_client::Client;
 use slog::Logger;
 use std::net::SocketAddr;
@@ -41,6 +45,11 @@ pub struct OximeterArgs {
 enum OximeterCommands {
     /// List the producers the collector is assigned to poll.
     ListProducers,
+    /// Fetch details about a single assigned producer.
+    ProducerDetails {
+        /// The ID of the producer to fetch.
+        producer_id: Uuid,
+    },
 }
 
 impl OximeterArgs {
@@ -81,9 +90,26 @@ impl OximeterArgs {
             OximeterCommands::ListProducers => {
                 self.list_producers(client).await
             }
+            OximeterCommands::ProducerDetails { producer_id } => {
+                self.producer_details(client, producer_id).await
+            }
         }
     }
 
+    async fn producer_details(
+        &self,
+        client: Client,
+        producer_id: Uuid,
+    ) -> anyhow::Result<()> {
+        let details = client
+            .producer_details(&producer_id)
+            .await
+            .context("failed to fetch producer details")?
+            .into_inner();
+        print_producer_details(details);
+        Ok(())
+    }
+
     async fn list_producers(&self, client: Client) -> anyhow::Result<()> {
         let info = client
             .collector_info()
@@ -120,11 +146,168 @@ struct Producer {
 
 impl From<ProducerEndpoint> for Producer {
     fn from(p: ProducerEndpoint) -> Self {
-        let interval = Duration::new(p.interval.secs, p.interval.nanos);
         Self {
             id: p.id,
             address: p.address.parse().unwrap(),
-            interval: humantime::format_duration(interval).to_string(),
+            interval: duration_to_humantime(&p.interval),
+        }
+    }
+}
+
+fn duration_to_humantime(d: &oximeter_client::types::Duration) -> String {
+    let interval = Duration::new(d.secs, d.nanos);
+    humantime::format_duration(interval).to_string()
+}
+
+const WIDTH: usize = 12;
+
+fn print_producer_details(details: ProducerDetails) {
+    println!();
+    println!("{:>WIDTH$}: {}", "ID", details.id);
+    println!("{:>WIDTH$}: {}", "Address", details.address);
+    println!(
+        "{:>WIDTH$}: {}",
+        "Registered",
+        details.registered.to_rfc3339_opts(SecondsFormat::Millis, true)
+    );
+    println!(
+        "{:>WIDTH$}: {}",
+        "Updated",
+        details.updated.to_rfc3339_opts(SecondsFormat::Millis, true)
+    );
+    println!(
+        "{:>WIDTH$}: {}",
+        "Interval",
+        duration_to_humantime(&details.interval)
+    );
+    println!("{:>WIDTH$}: {}", "Successes", details.n_collections);
+    println!("{:>WIDTH$}: {}", "Failures", details.n_failures);
+    println!();
+    print_last_success(details.last_success.as_ref());
+    println!();
+    print_last_failure(details.last_failure.as_ref());
+}
+
+fn print_last_success(maybe_success: Option<&SuccessfulCollection>) {
+    print!("{:>WIDTH$}: ", "Last success");
+    match maybe_success {
+        None => println!("None"),
+        Some(success) => {
+            println!();
+            println!(
+                "{:>WIDTH$}: {}",
+                "Started at",
+                success.started_at.to_rfc3339_opts(SecondsFormat::Millis, true)
+            );
+            println!(
+                "{:>WIDTH$}: {:?}",
+                "Queued for",
+                Duration::new(
+                    success.time_queued.secs,
+                    success.time_queued.nanos
+                )
+            );
+            println!(
+                "{:>WIDTH$}: {:?}",
+                "Duration",
+                Duration::new(
+                    success.time_collecting.secs,
+                    success.time_collecting.nanos
+                )
+            );
+            println!("{:>WIDTH$}: {}", "Samples", success.n_samples);
         }
     }
 }
+
+fn print_last_failure(maybe_failure: Option<&FailedCollection>) {
+    print!("{:>WIDTH$}: ", "Last failure");
+    match maybe_failure {
+        None => println!("None"),
+        Some(failure) => {
+            println!();
+            println!(
+                "{:>WIDTH$}: {}",
+                "Started at",
+                failure.started_at.to_rfc3339_opts(SecondsFormat::Millis, true)
+            );
+            println!(
+                "{:>WIDTH$}: {:?}",
+                "Queued for",
+                Duration::new(
+                    failure.time_queued.secs,
+                    failure.time_queued.nanos
+                )
+            );
+            println!(
+                "{:>WIDTH$}: {:?}",
+                "Duration",
+                Duration::new(
+                    failure.time_collecting.secs,
+                    failure.time_collecting.nanos
+                )
+            );
+            println!("{:>WIDTH$}: {}", "Reason", failure.reason);
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::print_producer_details;
+    use chrono::Utc;
+    use oximeter_client::types::FailedCollection;
+    use oximeter_client::types::ProducerDetails;
+    use oximeter_client::types::SuccessfulCollection;
+    use std::time::Duration;
+    use uuid::Uuid;
+
+    #[test]
+    fn test_print_producer_details_success_only() {
+        let now = Utc::now();
+        let details = ProducerDetails {
+            id: Uuid::new_v4(),
+            address: "[::1]:12345".parse().unwrap(),
+            interval: Duration::from_secs(10).into(),
+            last_success: Some(SuccessfulCollection {
+                n_samples: 100,
+                started_at: now,
+                time_collecting: Duration::from_millis(100).into(),
+                time_queued: Duration::from_millis(10).into(),
+            }),
+            last_failure: None,
+            n_collections: 1,
+            n_failures: 0,
+            registered: now,
+            updated: now,
+        };
+        print_producer_details(details);
+    }
+
+    #[test]
+    fn test_print_producer_details_with_failure() {
+        let now = Utc::now();
+        let details = ProducerDetails {
+            id: Uuid::new_v4(),
+            interval: Duration::from_secs(10).into(),
+            address: "[::1]:12345".parse().unwrap(),
+            last_success: Some(SuccessfulCollection {
+                n_samples: 100,
+                started_at: now,
+                time_collecting: Duration::from_millis(100).into(),
+                time_queued: Duration::from_millis(10).into(),
+            }),
+            last_failure: Some(FailedCollection {
+                started_at: now,
+                time_collecting: Duration::from_millis(100).into(),
+                time_queued: Duration::from_millis(10).into(),
+                reason: String::from("unreachable"),
+            }),
+            n_collections: 1,
+            n_failures: 1,
+            registered: now,
+            updated: now,
+        };
+        print_producer_details(details);
+    }
+}

dev-tools/omdb/tests/test_all_output.rs

@@ -10,6 +10,7 @@
 use dropshot::Method;
 use expectorate::assert_contents;
 use http::StatusCode;
+use nexus_test_utils::wait_for_producer;
 use nexus_test_utils::{OXIMETER_UUID, PRODUCER_UUID};
 use nexus_test_utils_macros::nexus_test;
 use nexus_types::deployment::Blueprint;
@@ -23,6 +24,7 @@ use std::fmt::Write;
 use std::net::IpAddr;
 use std::path::Path;
 use subprocess::Exec;
+use uuid::Uuid;
 
 /// name of the "omdb" executable
 const CMD_OMDB: &str = env!("CARGO_BIN_EXE_omdb");
@@ -274,6 +276,11 @@ async fn test_omdb_success_cases(cptestctx: &ControlPlaneTestContext) {
     let mut ox_output = String::new();
     let ox = ox_url.clone();
 
+    wait_for_producer(
+        &cptestctx.oximeter,
+        PRODUCER_UUID.parse::<Uuid>().unwrap(),
+    )
+    .await;
     do_run_no_redactions(
         &mut ox_output,
         move |exec| exec.env("OMDB_OXIMETER_URL", &ox),
@@ -423,6 +430,11 @@ async fn test_omdb_env_settings(cptestctx: &ControlPlaneTestContext) {
     // Case 2: is covered by the success tests above.
     let ox_args1 = &["oximeter", "--oximeter-url", &ox_url, "list-producers"];
     let mut ox_output1 = String::new();
+    wait_for_producer(
+        &cptestctx.oximeter,
+        PRODUCER_UUID.parse::<Uuid>().unwrap(),
+    )
+    .await;
     do_run_no_redactions(
         &mut ox_output1,
         move |exec| exec,

dev-tools/omdb/tests/usage_errors.out

@@ -761,8 +761,9 @@ Query oximeter collector state
 Usage: omdb oximeter [OPTIONS] <COMMAND>
 
 Commands:
-  list-producers  List the producers the collector is assigned to poll
-  help            Print this message or the help of the given subcommand(s)
+  list-producers    List the producers the collector is assigned to poll
+  producer-details  Fetch details about a single assigned producer
+  help              Print this message or the help of the given subcommand(s)
 
 Options:
       --log-level <LOG_LEVEL>  log level filter [env: LOG_LEVEL=] [default: warn]

nexus/db-queries/src/db/datastore/region_snapshot_replacement.rs

@@ -11,7 +11,6 @@ use crate::db;
 use crate::db::datastore::SQL_BATCH_SIZE;
 use crate::db::error::public_error_from_diesel;
 use crate::db::error::ErrorHandler;
-use crate::db::lookup::LookupPath;
 use crate::db::model::RegionSnapshot;
 use crate::db::model::RegionSnapshotReplacement;
 use crate::db::model::RegionSnapshotReplacementState;
@@ -64,10 +63,19 @@ impl DataStore {
         opctx: &OpContext,
         request: RegionSnapshotReplacement,
     ) -> Result<(), Error> {
-        let (.., db_snapshot) = LookupPath::new(opctx, &self)
-            .snapshot_id(request.old_snapshot_id)
-            .fetch()
-            .await?;
+        // Note: if `LookupPath` is used here, it will not be able to retrieve
+        // deleted snapshots
+        let db_snapshot = match self
+            .snapshot_get(opctx, request.old_snapshot_id)
+            .await?
+        {
+            Some(db_snapshot) => db_snapshot,
+            None => {
+                return Err(Error::internal_error(
+                    "cannot perform region snapshot replacement without snapshot volume"
+                ));
+            }
+        };
 
         self.insert_region_snapshot_replacement_request_with_volume_id(
             opctx,

nexus/external-api/output/nexus_tags.txt

@@ -30,6 +30,7 @@ probe_create                             POST     /experimental/v1/probes
 probe_delete                             DELETE   /experimental/v1/probes/{probe}
 probe_list                               GET      /experimental/v1/probes
 probe_view                               GET      /experimental/v1/probes/{probe}
+timeseries_query                         POST     /v1/timeseries/query
 
 API operations found with tag "images"
 OPERATION ID                             METHOD   URL PATH