diff --git a/.cargo/config.toml b/.cargo/config.toml
index c5b6fcd9d4..209d15c760 100644
--- a/.cargo/config.toml
+++ b/.cargo/config.toml
@@ -8,6 +8,7 @@
 # CI scripts:
 # - .github/buildomat/build-and-test.sh
 # - .github/buildomat/jobs/clippy.sh
+# - .github/buildomat/jobs/check-features.sh
 # - .github/workflows/rust.yml
 #
 [build]
diff --git a/.github/buildomat/jobs/check-features.sh b/.github/buildomat/jobs/check-features.sh
new file mode 100644
index 0000000000..4ba97ec02f
--- /dev/null
+++ b/.github/buildomat/jobs/check-features.sh
@@ -0,0 +1,34 @@
+#!/bin/bash
+#:
+#: name = "check-features (helios)"
+#: variety = "basic"
+#: target = "helios-2.0"
+#: rust_toolchain = true
+#: output_rules = [
+#:  "/out/*",
+#: ]
+
+# Run the check-features `xtask` on illumos, testing compilation of feature combinations.
+
+set -o errexit
+set -o pipefail
+set -o xtrace
+
+cargo --version
+rustc --version
+
+#
+# Set up our PATH for use with this workspace.
+#
+source ./env.sh
+export PATH="$PATH:$PWD/out/cargo-hack"
+
+banner prerequisites
+ptime -m bash ./tools/install_builder_prerequisites.sh -y
+
+#
+# Check feature combinations with the `cargo xtask check-features` command.
+#
+banner hack-check
+export CARGO_INCREMENTAL=0
+ptime -m timeout 2h cargo xtask check-features --ci
diff --git a/.github/buildomat/jobs/clippy.sh b/.github/buildomat/jobs/clippy.sh
index 71aa04c907..4040691b72 100755
--- a/.github/buildomat/jobs/clippy.sh
+++ b/.github/buildomat/jobs/clippy.sh
@@ -10,7 +10,7 @@
 # (that we want to check) is conditionally-compiled on illumos only.
 #
 # Note that `cargo clippy` includes `cargo check, so this ends up checking all
-# of our code.
+# of our (default) code.
 
 set -o errexit
 set -o pipefail
diff --git a/.github/buildomat/jobs/deploy.sh b/.github/buildomat/jobs/deploy.sh
index e4f59aff5f..8820378e1c 100755
--- a/.github/buildomat/jobs/deploy.sh
+++ b/.github/buildomat/jobs/deploy.sh
@@ -2,7 +2,7 @@
 #:
 #: name = "helios / deploy"
 #: variety = "basic"
-#: target = "lab-2.0-opte-0.32"
+#: target = "lab-2.0-opte-0.33"
 #: output_rules = [
 #:  "%/var/svc/log/oxide-sled-agent:default.log*",
 #:  "%/zone/oxz_*/root/var/svc/log/oxide-*.log*",
diff --git a/.github/workflows/hakari.yml b/.github/workflows/hakari.yml
index a9beb49ed5..3a31a5323d 100644
--- a/.github/workflows/hakari.yml
+++ b/.github/workflows/hakari.yml
@@ -24,7 +24,7 @@ jobs:
         with:
           toolchain: stable
       - name: Install cargo-hakari
-        uses: taiki-e/install-action@0256b3ea9ae3d751755a35cbb0608979a842f1d2 # v2
+        uses: taiki-e/install-action@ea7e5189a7664872699532b4cd92a443f520624e # v2
         with:
           tool: cargo-hakari
       - name: Check workspace-hack Cargo.toml is up-to-date
diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml
index 2ef2783108..94d25e7dfa 100644
--- a/.github/workflows/rust.yml
+++ b/.github/workflows/rust.yml
@@ -53,7 +53,7 @@ jobs:
       run: cargo run --bin omicron-package -- -t default check
 
   # Note that `cargo clippy` includes `cargo check, so this ends up checking all
-  # of our code.
+  # of our (default) code.
   clippy-lint:
     runs-on: ubuntu-22.04
     env:
@@ -82,6 +82,36 @@ jobs:
     - name: Run Clippy Lints
       run: cargo xtask clippy
 
+  check-features:
+    runs-on: ubuntu-22.04
+    env:
+      CARGO_INCREMENTAL: 0
+    steps:
+    # This repo is unstable and unnecessary: https://github.com/microsoft/linux-package-repositories/issues/34
+    - name: Disable packages.microsoft.com repo
+      run: sudo rm -f /etc/apt/sources.list.d/microsoft-prod.list
+    - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+      with:
+        ref: ${{ github.event.pull_request.head.sha }} # see omicron#4461
+    - uses: Swatinem/rust-cache@23bce251a8cd2ffc3c1075eaa2367cf899916d84 # v2.7.3
+      if: ${{ github.ref != 'refs/heads/main' }}
+    - name: Report cargo version
+      run: cargo --version
+    - name: Update PATH
+      run: |
+        set -x
+        export PATH="./out/cargo-hack:$PATH"
+        source "./env.sh"; echo "PATH=$PATH" >> "$GITHUB_ENV"
+    - name: Print PATH
+      run: echo $PATH
+    - name: Print GITHUB_ENV
+      run: cat "$GITHUB_ENV"
+    - name: Install Pre-Requisites
+      run: ./tools/install_builder_prerequisites.sh -y
+    - name: Run Check on Feature Combinations (Feature-Powerset, No-Dev-Deps)
+      timeout-minutes: 120 # 2 hours
+      run: cargo xtask check-features --ci
+
   # This is just a test build of docs.  Publicly available docs are built via
   # the separate "rustdocs" repo.
   build-docs:
diff --git a/Cargo.lock b/Cargo.lock
index 845f64b659..29c3f90c15 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -166,7 +166,7 @@ dependencies = [
  "omicron-workspace-hack",
  "proc-macro2",
  "quote",
- "syn 2.0.68",
+ "syn 2.0.71",
 ]
 
 [[package]]
@@ -273,7 +273,7 @@ checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.68",
+ "syn 2.0.71",
 ]
 
 [[package]]
@@ -295,7 +295,7 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.68",
+ "syn 2.0.71",
 ]
 
 [[package]]
@@ -306,7 +306,7 @@ checksum = "6e0c28dcc82d7c8ead5cb13beb15405b57b8546e93215673ff8ca0349a028107"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.68",
+ "syn 2.0.71",
 ]
 
 [[package]]
@@ -359,7 +359,7 @@ dependencies = [
  "quote",
  "serde",
  "serde_tokenstream",
- "syn 2.0.68",
+ "syn 2.0.71",
 ]
 
 [[package]]
@@ -463,9 +463,9 @@ dependencies = [
 [[package]]
 name = "bhyve_api"
 version = "0.0.0"
-source = "git+https://github.com/oxidecomputer/propolis?rev=5ebf9626e0ad274eb515d206d102cb09d2d51f15#5ebf9626e0ad274eb515d206d102cb09d2d51f15"
+source = "git+https://github.com/oxidecomputer/propolis?rev=66d1ee7d4a5829dbbf02a152091ea051023b5b8b#66d1ee7d4a5829dbbf02a152091ea051023b5b8b"
 dependencies = [
- "bhyve_api_sys 0.0.0 (git+https://github.com/oxidecomputer/propolis?rev=5ebf9626e0ad274eb515d206d102cb09d2d51f15)",
+ "bhyve_api_sys 0.0.0 (git+https://github.com/oxidecomputer/propolis?rev=66d1ee7d4a5829dbbf02a152091ea051023b5b8b)",
  "libc",
  "strum",
 ]
@@ -483,7 +483,7 @@ dependencies = [
 [[package]]
 name = "bhyve_api_sys"
 version = "0.0.0"
-source = "git+https://github.com/oxidecomputer/propolis?rev=5ebf9626e0ad274eb515d206d102cb09d2d51f15#5ebf9626e0ad274eb515d206d102cb09d2d51f15"
+source = "git+https://github.com/oxidecomputer/propolis?rev=66d1ee7d4a5829dbbf02a152091ea051023b5b8b#66d1ee7d4a5829dbbf02a152091ea051023b5b8b"
 dependencies = [
  "libc",
  "strum",
@@ -517,7 +517,7 @@ dependencies = [
  "regex",
  "rustc-hash",
  "shlex",
- "syn 2.0.68",
+ "syn 2.0.71",
  "which",
 ]
 
@@ -550,7 +550,7 @@ checksum = "1657dce144574f921af10a92876a96f0ca05dd830900598d21d91c8e4cf78f74"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.68",
+ "syn 2.0.71",
 ]
 
 [[package]]
@@ -858,7 +858,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4895c018bb228aa6b3ba1a0285543fcb4b704734c3fb1f72afaa75aa769500c1"
 dependencies = [
  "serde",
- "toml 0.8.14",
+ "toml 0.8.15",
 ]
 
 [[package]]
@@ -1018,9 +1018,9 @@ dependencies = [
 
 [[package]]
 name = "clap"
-version = "4.5.4"
+version = "4.5.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "90bc066a67923782aa8515dbaea16946c5bcc5addbd668bb80af688e53e548a0"
+checksum = "64acc1846d54c1fe936a78dc189c34e28d3f5afc348403f28ecf53660b9b8462"
 dependencies = [
  "clap_builder",
  "clap_derive",
@@ -1028,9 +1028,9 @@ dependencies = [
 
 [[package]]
 name = "clap_builder"
-version = "4.5.2"
+version = "4.5.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ae129e2e766ae0ec03484e609954119f123cc1fe650337e155d03b022f24f7b4"
+checksum = "6fb8393d67ba2e7bfaf28a23458e4e2b543cc73a99595511eb207fdb8aede942"
 dependencies = [
  "anstream",
  "anstyle",
@@ -1041,14 +1041,14 @@ dependencies = [
 
 [[package]]
 name = "clap_derive"
-version = "4.5.4"
+version = "4.5.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "528131438037fd55894f62d6e9f068b8f45ac57ffa77517819645d10aed04f64"
+checksum = "2bac35c6dafb060fd4d275d9a4ffae97917c13a6327903a8be2153cd964f7085"
 dependencies = [
  "heck 0.5.0",
  "proc-macro2",
  "quote",
- "syn 2.0.68",
+ "syn 2.0.71",
 ]
 
 [[package]]
@@ -1220,12 +1220,6 @@ version = "0.8.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f"
 
-[[package]]
-name = "corncobs"
-version = "0.1.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f9236877021b66ad90f833d8a73a7acb702b985b64c5986682d9f1f1a184f0fb"
-
 [[package]]
 name = "cpufeatures"
 version = "0.2.12"
@@ -1539,7 +1533,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.68",
+ "syn 2.0.71",
 ]
 
 [[package]]
@@ -1563,7 +1557,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "strsim",
- "syn 2.0.68",
+ "syn 2.0.71",
 ]
 
 [[package]]
@@ -1574,7 +1568,7 @@ checksum = "733cabb43482b1a1b53eee8583c2b9e8684d592215ea83efd305dd31bc2f0178"
 dependencies = [
  "darling_core",
  "quote",
- "syn 2.0.68",
+ "syn 2.0.71",
 ]
 
 [[package]]
@@ -1608,13 +1602,13 @@ dependencies = [
  "quote",
  "serde",
  "serde_tokenstream",
- "syn 2.0.68",
+ "syn 2.0.71",
 ]
 
 [[package]]
 name = "ddm-admin-client"
 version = "0.1.0"
-source = "git+https://github.com/oxidecomputer/maghemite?rev=1b385990e8648b221fd11f018f2a7ec425461c6c#1b385990e8648b221fd11f018f2a7ec425461c6c"
+source = "git+https://github.com/oxidecomputer/maghemite?rev=220dd026e83142b83bd93123f465a64dd4600201#220dd026e83142b83bd93123f465a64dd4600201"
 dependencies = [
  "oxnet",
  "percent-encoding",
@@ -1652,7 +1646,7 @@ dependencies = [
  "proc-macro-error",
  "proc-macro2",
  "quote",
- "syn 2.0.68",
+ "syn 2.0.71",
 ]
 
 [[package]]
@@ -1685,7 +1679,7 @@ checksum = "5fe87ce4529967e0ba1dcf8450bab64d97dfd5010a6256187ffe2e43e6f0e049"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.68",
+ "syn 2.0.71",
 ]
 
 [[package]]
@@ -1706,7 +1700,7 @@ checksum = "62d671cc41a825ebabc75757b62d3d168c577f9149b2d49ece1dad1f72119d25"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.68",
+ "syn 2.0.71",
 ]
 
 [[package]]
@@ -1727,7 +1721,7 @@ dependencies = [
  "darling",
  "proc-macro2",
  "quote",
- "syn 2.0.68",
+ "syn 2.0.71",
 ]
 
 [[package]]
@@ -1737,7 +1731,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "206868b8242f27cecce124c19fd88157fbd0dd334df2587f36417bafbc85097b"
 dependencies = [
  "derive_builder_core",
- "syn 2.0.68",
+ "syn 2.0.71",
 ]
 
 [[package]]
@@ -1750,7 +1744,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "rustc_version 0.4.0",
- "syn 2.0.68",
+ "syn 2.0.71",
 ]
 
 [[package]]
@@ -1814,7 +1808,7 @@ dependencies = [
  "diesel_table_macro_syntax",
  "proc-macro2",
  "quote",
- "syn 2.0.68",
+ "syn 2.0.71",
 ]
 
 [[package]]
@@ -1823,7 +1817,7 @@ version = "0.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "fc5557efc453706fed5e4fa85006fe9817c224c3f480a34c7e5959fd700921c5"
 dependencies = [
- "syn 2.0.68",
+ "syn 2.0.71",
 ]
 
 [[package]]
@@ -1929,6 +1923,7 @@ dependencies = [
  "camino-tempfile",
  "chrono",
  "clap",
+ "dns-server-api",
  "dns-service-client",
  "dropshot",
  "expectorate",
@@ -1950,7 +1945,7 @@ dependencies = [
  "tempfile",
  "thiserror",
  "tokio",
- "toml 0.8.14",
+ "toml 0.8.15",
  "trust-dns-client",
  "trust-dns-proto",
  "trust-dns-resolver",
@@ -1958,6 +1953,17 @@ dependencies = [
  "uuid",
 ]
 
+[[package]]
+name = "dns-server-api"
+version = "0.1.0"
+dependencies = [
+ "chrono",
+ "dropshot",
+ "omicron-workspace-hack",
+ "schemars",
+ "serde",
+]
+
 [[package]]
 name = "dns-service-client"
 version = "0.1.0"
@@ -2021,14 +2027,14 @@ dependencies = [
  "serde",
  "serde_json",
  "slog",
- "toml 0.8.14",
+ "toml 0.8.15",
  "uuid",
 ]
 
 [[package]]
 name = "dropshot"
 version = "0.10.2-dev"
-source = "git+https://github.com/oxidecomputer/dropshot?branch=main#9fef3961c0b89aa8ab8e186dc0c89f8f4f811eea"
+source = "git+https://github.com/oxidecomputer/dropshot?branch=main#7b594d01f47ca783c5d4a25ca2b256602580fe92"
 dependencies = [
  "async-stream",
  "async-trait",
@@ -2064,7 +2070,7 @@ dependencies = [
  "slog-term",
  "tokio",
  "tokio-rustls 0.25.0",
- "toml 0.8.14",
+ "toml 0.8.15",
  "usdt",
  "uuid",
  "version_check",
@@ -2074,14 +2080,14 @@ dependencies = [
 [[package]]
 name = "dropshot_endpoint"
 version = "0.10.2-dev"
-source = "git+https://github.com/oxidecomputer/dropshot?branch=main#9fef3961c0b89aa8ab8e186dc0c89f8f4f811eea"
+source = "git+https://github.com/oxidecomputer/dropshot?branch=main#7b594d01f47ca783c5d4a25ca2b256602580fe92"
 dependencies = [
  "heck 0.5.0",
  "proc-macro2",
  "quote",
  "serde",
  "serde_tokenstream",
- "syn 2.0.68",
+ "syn 2.0.71",
 ]
 
 [[package]]
@@ -2111,19 +2117,10 @@ dependencies = [
  "digest",
  "elliptic-curve",
  "rfc6979",
- "signature 2.2.0",
+ "signature",
  "spki",
 ]
 
-[[package]]
-name = "ed25519"
-version = "1.5.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "91cff35c70bba8a626e3185d8cd48cc11b5437e1a5bcd15b9b5fa3c64b6dfee7"
-dependencies = [
- "signature 1.6.4",
-]
-
 [[package]]
 name = "ed25519"
 version = "2.2.3"
@@ -2131,7 +2128,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "115531babc129696a58c64a4fef0a8bf9e9698629fb97e9e40767d235cfbcd53"
 dependencies = [
  "pkcs8",
- "signature 2.2.0",
+ "signature",
 ]
 
 [[package]]
@@ -2141,7 +2138,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4a3daa8e81a3963a60642bcc1f90a670680bd4a77535faa384e9d1c79d620871"
 dependencies = [
  "curve25519-dalek",
- "ed25519 2.2.3",
+ "ed25519",
  "rand_core 0.6.4",
  "serde",
  "sha2",
@@ -2236,7 +2233,7 @@ dependencies = [
  "serde_json",
  "socket2 0.5.7",
  "tokio",
- "toml 0.8.14",
+ "toml 0.8.15",
  "trust-dns-resolver",
  "uuid",
 ]
@@ -2496,7 +2493,7 @@ checksum = "1a5c6c585bc94aaf2c7b51dd4c2ba22680844aba4c687be581871a6f518c5742"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.68",
+ "syn 2.0.71",
 ]
 
 [[package]]
@@ -2608,7 +2605,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.68",
+ "syn 2.0.71",
 ]
 
 [[package]]
@@ -2703,7 +2700,7 @@ version = "0.1.0"
 source = "git+https://github.com/oxidecomputer/management-gateway-service?rev=c85a4ca043aaa389df12aac5348d8a3feda28762#c85a4ca043aaa389df12aac5348d8a3feda28762"
 dependencies = [
  "bitflags 2.5.0",
- "hubpack 0.1.2",
+ "hubpack",
  "serde",
  "serde_repr",
  "smoltcp 0.9.1",
@@ -2724,14 +2721,14 @@ dependencies = [
  "fxhash",
  "gateway-messages",
  "hex",
- "hubpack 0.1.2",
+ "hubpack",
  "hubtools",
  "lru-cache",
  "nix 0.27.1",
  "once_cell",
  "paste",
  "serde",
- "serde-big-array 0.5.1",
+ "serde-big-array",
  "slog",
  "slog-error-chain",
  "socket2 0.5.7",
@@ -3213,35 +3210,16 @@ dependencies = [
  "tokio",
 ]
 
-[[package]]
-name = "hubpack"
-version = "0.1.0"
-source = "git+https://github.com/cbiffle/hubpack.git?rev=df08cc3a6e1f97381cd0472ae348e310f0119e25#df08cc3a6e1f97381cd0472ae348e310f0119e25"
-dependencies = [
- "hubpack_derive 0.1.0",
- "serde",
-]
-
 [[package]]
 name = "hubpack"
 version = "0.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "61a0b84aeae519f65e0ba3aa998327080993426024edbd5cc38dbaf5ec524303"
 dependencies = [
- "hubpack_derive 0.1.1",
+ "hubpack_derive",
  "serde",
 ]
 
-[[package]]
-name = "hubpack_derive"
-version = "0.1.0"
-source = "git+https://github.com/cbiffle/hubpack.git?rev=df08cc3a6e1f97381cd0472ae348e310f0119e25#df08cc3a6e1f97381cd0472ae348e310f0119e25"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 1.0.109",
-]
-
 [[package]]
 name = "hubpack_derive"
 version = "0.1.1"
@@ -3296,7 +3274,7 @@ dependencies = [
  "httpdate",
  "itoa",
  "pin-project-lite",
- "socket2 0.5.7",
+ "socket2 0.4.10",
  "tokio",
  "tower-service",
  "tracing",
@@ -3480,7 +3458,7 @@ dependencies = [
 [[package]]
 name = "illumos-sys-hdrs"
 version = "0.1.0"
-source = "git+https://github.com/oxidecomputer/opte?rev=915975f6d1729db95619f752148974016912412f#915975f6d1729db95619f752148974016912412f"
+source = "git+https://github.com/oxidecomputer/opte?rev=3dc9a3dd8d3c623f0cf2c659c7119ce0c026a96d#3dc9a3dd8d3c623f0cf2c659c7119ce0c026a96d"
 
 [[package]]
 name = "illumos-utils"
@@ -3488,7 +3466,7 @@ version = "0.1.0"
 dependencies = [
  "anyhow",
  "async-trait",
- "bhyve_api 0.0.0 (git+https://github.com/oxidecomputer/propolis?rev=5ebf9626e0ad274eb515d206d102cb09d2d51f15)",
+ "bhyve_api 0.0.0 (git+https://github.com/oxidecomputer/propolis?rev=66d1ee7d4a5829dbbf02a152091ea051023b5b8b)",
  "byteorder",
  "camino",
  "camino-tempfile",
@@ -3514,7 +3492,7 @@ dependencies = [
  "smf",
  "thiserror",
  "tokio",
- "toml 0.8.14",
+ "toml 0.8.15",
  "uuid",
  "whoami",
  "zone 0.3.0",
@@ -3606,7 +3584,7 @@ dependencies = [
  "hex-literal",
  "http 0.2.12",
  "illumos-utils",
- "installinator-artifact-client",
+ "installinator-client",
  "installinator-common",
  "ipcc",
  "itertools 0.12.1",
@@ -3638,43 +3616,35 @@ dependencies = [
 ]
 
 [[package]]
-name = "installinator-artifact-client"
+name = "installinator-api"
 version = "0.1.0"
 dependencies = [
+ "anyhow",
+ "dropshot",
+ "hyper 0.14.28",
  "installinator-common",
+ "omicron-common",
  "omicron-workspace-hack",
- "progenitor",
- "regress",
- "reqwest",
  "schemars",
  "serde",
- "serde_json",
  "slog",
- "update-engine",
  "uuid",
 ]
 
 [[package]]
-name = "installinator-artifactd"
+name = "installinator-client"
 version = "0.1.0"
 dependencies = [
- "anyhow",
- "async-trait",
- "clap",
- "dropshot",
- "expectorate",
- "hyper 0.14.28",
  "installinator-common",
- "omicron-common",
- "omicron-test-utils",
  "omicron-workspace-hack",
- "openapi-lint",
- "openapiv3",
+ "progenitor",
+ "regress",
+ "reqwest",
  "schemars",
  "serde",
  "serde_json",
  "slog",
- "subprocess",
+ "update-engine",
  "uuid",
 ]
 
@@ -3903,10 +3873,10 @@ dependencies = [
 [[package]]
 name = "kstat-macro"
 version = "0.1.0"
-source = "git+https://github.com/oxidecomputer/opte?rev=915975f6d1729db95619f752148974016912412f#915975f6d1729db95619f752148974016912412f"
+source = "git+https://github.com/oxidecomputer/opte?rev=3dc9a3dd8d3c623f0cf2c659c7119ce0c026a96d#3dc9a3dd8d3c623f0cf2c659c7119ce0c026a96d"
 dependencies = [
  "quote",
- "syn 2.0.68",
+ "syn 2.0.71",
 ]
 
 [[package]]
@@ -4319,7 +4289,7 @@ dependencies = [
 [[package]]
 name = "mg-admin-client"
 version = "0.1.0"
-source = "git+https://github.com/oxidecomputer/maghemite?rev=1b385990e8648b221fd11f018f2a7ec425461c6c#1b385990e8648b221fd11f018f2a7ec425461c6c"
+source = "git+https://github.com/oxidecomputer/maghemite?rev=220dd026e83142b83bd93123f465a64dd4600201#220dd026e83142b83bd93123f465a64dd4600201"
 dependencies = [
  "anyhow",
  "chrono",
@@ -4399,7 +4369,7 @@ dependencies = [
  "cfg-if",
  "proc-macro2",
  "quote",
- "syn 2.0.68",
+ "syn 2.0.71",
 ]
 
 [[package]]
@@ -4572,7 +4542,7 @@ dependencies = [
  "serde_json",
  "serde_with",
  "tokio-postgres",
- "toml 0.8.14",
+ "toml 0.8.15",
  "uuid",
 ]
 
@@ -4776,7 +4746,7 @@ dependencies = [
  "omicron-workspace-hack",
  "proc-macro2",
  "quote",
- "syn 2.0.68",
+ "syn 2.0.71",
 ]
 
 [[package]]
@@ -4909,6 +4879,34 @@ dependencies = [
  "slog-error-chain",
 ]
 
+[[package]]
+name = "nexus-saga-recovery"
+version = "0.1.0"
+dependencies = [
+ "chrono",
+ "futures",
+ "nexus-auth",
+ "nexus-db-model",
+ "nexus-db-queries",
+ "nexus-test-utils",
+ "nexus-test-utils-macros",
+ "nexus-types",
+ "omicron-common",
+ "omicron-rpaths",
+ "omicron-test-utils",
+ "omicron-workspace-hack",
+ "once_cell",
+ "pq-sys",
+ "pretty_assertions",
+ "serde",
+ "serde_json",
+ "slog",
+ "slog-error-chain",
+ "steno",
+ "tokio",
+ "uuid",
+]
+
 [[package]]
 name = "nexus-test-interface"
 version = "0.1.0"
@@ -4974,7 +4972,7 @@ version = "0.1.0"
 dependencies = [
  "omicron-workspace-hack",
  "quote",
- "syn 2.0.68",
+ "syn 2.0.71",
 ]
 
 [[package]]
@@ -5145,7 +5143,7 @@ checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.68",
+ "syn 2.0.71",
 ]
 
 [[package]]
@@ -5337,7 +5335,7 @@ dependencies = [
  "thiserror",
  "tokio",
  "tokio-postgres",
- "toml 0.8.14",
+ "toml 0.8.15",
  "url",
 ]
 
@@ -5385,7 +5383,7 @@ dependencies = [
  "test-strategy",
  "thiserror",
  "tokio",
- "toml 0.8.14",
+ "toml 0.8.15",
  "uuid",
 ]
 
@@ -5410,7 +5408,7 @@ dependencies = [
  "slog",
  "thiserror",
  "tokio",
- "toml 0.8.14",
+ "toml 0.8.15",
  "uuid",
 ]
 
@@ -5445,7 +5443,7 @@ dependencies = [
  "subprocess",
  "tokio",
  "tokio-postgres",
- "toml 0.8.14",
+ "toml 0.8.15",
 ]
 
 [[package]]
@@ -5487,7 +5485,7 @@ dependencies = [
  "tokio",
  "tokio-stream",
  "tokio-tungstenite 0.20.1",
- "toml 0.8.14",
+ "toml 0.8.15",
  "uuid",
 ]
 
@@ -5549,6 +5547,7 @@ dependencies = [
  "nexus-reconfigurator-execution",
  "nexus-reconfigurator-planning",
  "nexus-reconfigurator-preparation",
+ "nexus-saga-recovery",
  "nexus-test-interface",
  "nexus-test-utils",
  "nexus-test-utils-macros",
@@ -5580,7 +5579,7 @@ dependencies = [
  "pq-sys",
  "pretty_assertions",
  "progenitor-client",
- "propolis-client 0.1.0 (git+https://github.com/oxidecomputer/propolis?rev=5ebf9626e0ad274eb515d206d102cb09d2d51f15)",
+ "propolis-client 0.1.0 (git+https://github.com/oxidecomputer/propolis?rev=66d1ee7d4a5829dbbf02a152091ea051023b5b8b)",
  "rand 0.8.5",
  "rcgen",
  "ref-cast",
@@ -5652,6 +5651,7 @@ dependencies = [
  "nexus-db-model",
  "nexus-db-queries",
  "nexus-reconfigurator-preparation",
+ "nexus-saga-recovery",
  "nexus-test-utils",
  "nexus-test-utils-macros",
  "nexus-types",
@@ -5712,7 +5712,7 @@ dependencies = [
  "tar",
  "thiserror",
  "tokio",
- "toml 0.8.14",
+ "toml 0.8.15",
  "walkdir",
 ]
 
@@ -5760,7 +5760,7 @@ dependencies = [
  "slog-term",
  "tar",
  "tokio",
- "toml 0.8.14",
+ "toml 0.8.15",
  "tufaceous-lib",
 ]
 
@@ -5832,7 +5832,7 @@ dependencies = [
  "oximeter-producer",
  "oxnet",
  "pretty_assertions",
- "propolis-client 0.1.0 (git+https://github.com/oxidecomputer/propolis?rev=5ebf9626e0ad274eb515d206d102cb09d2d51f15)",
+ "propolis-client 0.1.0 (git+https://github.com/oxidecomputer/propolis?rev=66d1ee7d4a5829dbbf02a152091ea051023b5b8b)",
  "propolis-mock-server",
  "rand 0.8.5",
  "rcgen",
@@ -5863,7 +5863,7 @@ dependencies = [
  "tokio",
  "tokio-stream",
  "tokio-util",
- "toml 0.8.14",
+ "toml 0.8.15",
  "usdt",
  "uuid",
  "zeroize",
@@ -6008,11 +6008,12 @@ dependencies = [
  "similar",
  "slog",
  "smallvec 1.13.2",
+ "socket2 0.5.7",
  "spin 0.9.8",
  "string_cache",
  "subtle",
  "syn 1.0.109",
- "syn 2.0.68",
+ "syn 2.0.71",
  "time",
  "time-macros",
  "tokio",
@@ -6022,7 +6023,7 @@ dependencies = [
  "toml 0.7.8",
  "toml_datetime",
  "toml_edit 0.19.15",
- "toml_edit 0.22.14",
+ "toml_edit 0.22.16",
  "tracing",
  "trust-dns-proto",
  "unicode-bidi",
@@ -6106,9 +6107,11 @@ dependencies = [
  "atomicwrites",
  "camino",
  "clap",
+ "dns-server-api",
  "dropshot",
  "fs-err",
  "indent_write",
+ "installinator-api",
  "nexus-internal-api",
  "omicron-workspace-hack",
  "openapi-lint",
@@ -6117,6 +6120,7 @@ dependencies = [
  "serde_json",
  "similar",
  "supports-color",
+ "wicketd-api",
 ]
 
 [[package]]
@@ -6153,7 +6157,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.68",
+ "syn 2.0.71",
 ]
 
 [[package]]
@@ -6177,7 +6181,7 @@ dependencies = [
 [[package]]
 name = "opte"
 version = "0.1.0"
-source = "git+https://github.com/oxidecomputer/opte?rev=915975f6d1729db95619f752148974016912412f#915975f6d1729db95619f752148974016912412f"
+source = "git+https://github.com/oxidecomputer/opte?rev=3dc9a3dd8d3c623f0cf2c659c7119ce0c026a96d#3dc9a3dd8d3c623f0cf2c659c7119ce0c026a96d"
 dependencies = [
  "cfg-if",
  "dyn-clone",
@@ -6194,7 +6198,7 @@ dependencies = [
 [[package]]
 name = "opte-api"
 version = "0.1.0"
-source = "git+https://github.com/oxidecomputer/opte?rev=915975f6d1729db95619f752148974016912412f#915975f6d1729db95619f752148974016912412f"
+source = "git+https://github.com/oxidecomputer/opte?rev=3dc9a3dd8d3c623f0cf2c659c7119ce0c026a96d#3dc9a3dd8d3c623f0cf2c659c7119ce0c026a96d"
 dependencies = [
  "illumos-sys-hdrs",
  "ipnetwork",
@@ -6206,7 +6210,7 @@ dependencies = [
 [[package]]
 name = "opte-ioctl"
 version = "0.1.0"
-source = "git+https://github.com/oxidecomputer/opte?rev=915975f6d1729db95619f752148974016912412f#915975f6d1729db95619f752148974016912412f"
+source = "git+https://github.com/oxidecomputer/opte?rev=3dc9a3dd8d3c623f0cf2c659c7119ce0c026a96d#3dc9a3dd8d3c623f0cf2c659c7119ce0c026a96d"
 dependencies = [
  "libc",
  "libnet 0.1.0 (git+https://github.com/oxidecomputer/netadm-sys)",
@@ -6280,7 +6284,7 @@ dependencies = [
 [[package]]
 name = "oxide-vpc"
 version = "0.1.0"
-source = "git+https://github.com/oxidecomputer/opte?rev=915975f6d1729db95619f752148974016912412f#915975f6d1729db95619f752148974016912412f"
+source = "git+https://github.com/oxidecomputer/opte?rev=3dc9a3dd8d3c623f0cf2c659c7119ce0c026a96d#3dc9a3dd8d3c623f0cf2c659c7119ce0c026a96d"
 dependencies = [
  "cfg-if",
  "illumos-sys-hdrs",
@@ -6304,8 +6308,8 @@ dependencies = [
  "oximeter-macro-impl",
  "oximeter-timeseries-macro",
  "prettyplease",
- "syn 2.0.68",
- "toml 0.8.14",
+ "syn 2.0.71",
+ "toml 0.8.15",
  "uuid",
 ]
 
@@ -6360,7 +6364,7 @@ dependencies = [
  "subprocess",
  "thiserror",
  "tokio",
- "toml 0.8.14",
+ "toml 0.8.15",
  "uuid",
 ]
 
@@ -6418,6 +6422,7 @@ dependencies = [
  "approx",
  "bytes",
  "chrono",
+ "criterion",
  "float-ord",
  "heck 0.5.0",
  "num",
@@ -6436,9 +6441,9 @@ dependencies = [
  "serde_json",
  "slog-error-chain",
  "strum",
- "syn 2.0.68",
+ "syn 2.0.71",
  "thiserror",
- "toml 0.8.14",
+ "toml 0.8.15",
  "trybuild",
  "uuid",
 ]
@@ -6475,7 +6480,7 @@ dependencies = [
  "omicron-workspace-hack",
  "proc-macro2",
  "quote",
- "syn 2.0.68",
+ "syn 2.0.71",
 ]
 
 [[package]]
@@ -6511,7 +6516,7 @@ dependencies = [
  "oximeter-impl",
  "proc-macro2",
  "quote",
- "syn 2.0.68",
+ "syn 2.0.71",
 ]
 
 [[package]]
@@ -6667,7 +6672,7 @@ dependencies = [
  "regex",
  "regex-syntax 0.8.3",
  "structmeta 0.3.0",
- "syn 2.0.68",
+ "syn 2.0.71",
 ]
 
 [[package]]
@@ -6835,7 +6840,7 @@ dependencies = [
  "pest_meta",
  "proc-macro2",
  "quote",
- "syn 2.0.68",
+ "syn 2.0.71",
 ]
 
 [[package]]
@@ -6905,7 +6910,7 @@ checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.68",
+ "syn 2.0.71",
 ]
 
 [[package]]
@@ -7169,7 +7174,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "5f12335488a2f3b0a83b14edad48dca9879ce89b2edd10e80237e4e852dd645e"
 dependencies = [
  "proc-macro2",
- "syn 2.0.68",
+ "syn 2.0.71",
 ]
 
 [[package]]
@@ -7265,7 +7270,7 @@ dependencies = [
  "schemars",
  "serde",
  "serde_json",
- "syn 2.0.68",
+ "syn 2.0.71",
  "thiserror",
  "typify",
  "unicode-ident",
@@ -7285,7 +7290,7 @@ dependencies = [
  "serde_json",
  "serde_tokenstream",
  "serde_yaml",
- "syn 2.0.68",
+ "syn 2.0.71",
 ]
 
 [[package]]
@@ -7321,7 +7326,7 @@ dependencies = [
 [[package]]
 name = "propolis-client"
 version = "0.1.0"
-source = "git+https://github.com/oxidecomputer/propolis?rev=5ebf9626e0ad274eb515d206d102cb09d2d51f15#5ebf9626e0ad274eb515d206d102cb09d2d51f15"
+source = "git+https://github.com/oxidecomputer/propolis?rev=66d1ee7d4a5829dbbf02a152091ea051023b5b8b#66d1ee7d4a5829dbbf02a152091ea051023b5b8b"
 dependencies = [
  "async-trait",
  "base64 0.21.7",
@@ -7363,7 +7368,7 @@ dependencies = [
 [[package]]
 name = "propolis-mock-server"
 version = "0.0.0"
-source = "git+https://github.com/oxidecomputer/propolis?rev=5ebf9626e0ad274eb515d206d102cb09d2d51f15#5ebf9626e0ad274eb515d206d102cb09d2d51f15"
+source = "git+https://github.com/oxidecomputer/propolis?rev=66d1ee7d4a5829dbbf02a152091ea051023b5b8b#66d1ee7d4a5829dbbf02a152091ea051023b5b8b"
 dependencies = [
  "anyhow",
  "atty",
@@ -7373,7 +7378,7 @@ dependencies = [
  "futures",
  "hyper 0.14.28",
  "progenitor",
- "propolis_types 0.0.0 (git+https://github.com/oxidecomputer/propolis?rev=5ebf9626e0ad274eb515d206d102cb09d2d51f15)",
+ "propolis_types 0.0.0 (git+https://github.com/oxidecomputer/propolis?rev=66d1ee7d4a5829dbbf02a152091ea051023b5b8b)",
  "rand 0.8.5",
  "reqwest",
  "schemars",
@@ -7405,7 +7410,7 @@ dependencies = [
 [[package]]
 name = "propolis_types"
 version = "0.0.0"
-source = "git+https://github.com/oxidecomputer/propolis?rev=5ebf9626e0ad274eb515d206d102cb09d2d51f15#5ebf9626e0ad274eb515d206d102cb09d2d51f15"
+source = "git+https://github.com/oxidecomputer/propolis?rev=66d1ee7d4a5829dbbf02a152091ea051023b5b8b#66d1ee7d4a5829dbbf02a152091ea051023b5b8b"
 dependencies = [
  "schemars",
  "serde",
@@ -7776,7 +7781,7 @@ checksum = "bcc303e793d3734489387d205e9b186fac9c6cfacedd98cbb2e8a5943595f3e6"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.68",
+ "syn 2.0.71",
 ]
 
 [[package]]
@@ -7999,7 +8004,7 @@ dependencies = [
  "rand_core 0.6.4",
  "serde",
  "sha2",
- "signature 2.2.0",
+ "signature",
  "spki",
  "subtle",
  "zeroize",
@@ -8030,7 +8035,7 @@ dependencies = [
  "regex",
  "relative-path",
  "rustc_version 0.4.0",
- "syn 2.0.68",
+ "syn 2.0.71",
  "unicode-ident",
 ]
 
@@ -8178,7 +8183,7 @@ dependencies = [
  "serde",
  "tempfile",
  "thiserror",
- "toml 0.8.14",
+ "toml 0.8.15",
  "toolchain_find",
 ]
 
@@ -8344,17 +8349,6 @@ version = "1.0.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f"
 
-[[package]]
-name = "salty"
-version = "0.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "77cdd38ed8bfe51e53ee991aae0791b94349d0a05cfdecd283835a8a965d4c37"
-dependencies = [
- "ed25519 1.5.3",
- "subtle",
- "zeroize",
-]
-
 [[package]]
 name = "samael"
 version = "0.0.15"
@@ -8433,7 +8427,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "serde_derive_internals",
- "syn 2.0.68",
+ "syn 2.0.71",
 ]
 
 [[package]]
@@ -8459,7 +8453,7 @@ checksum = "7f81c2fde025af7e69b1d1420531c8a8811ca898919db177141a85313b1cb932"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.68",
+ "syn 2.0.71",
 ]
 
 [[package]]
@@ -8542,15 +8536,6 @@ dependencies = [
  "serde_derive",
 ]
 
-[[package]]
-name = "serde-big-array"
-version = "0.4.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3323f09a748af288c3dc2474ea6803ee81f118321775bffa3ac8f7e65c5e90e7"
-dependencies = [
- "serde",
-]
-
 [[package]]
 name = "serde-big-array"
 version = "0.5.1"
@@ -8588,7 +8573,7 @@ checksum = "e0cd7e117be63d3c3678776753929474f3b04a43a080c744d6b0ae2a8c28e222"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.68",
+ "syn 2.0.71",
 ]
 
 [[package]]
@@ -8599,7 +8584,7 @@ checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.68",
+ "syn 2.0.71",
 ]
 
 [[package]]
@@ -8649,7 +8634,7 @@ checksum = "6c64451ba24fc7a6a2d60fc75dd9c83c90903b19028d4eff35e88fc1e86564e9"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.68",
+ "syn 2.0.71",
 ]
 
 [[package]]
@@ -8670,7 +8655,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "serde",
- "syn 2.0.68",
+ "syn 2.0.71",
 ]
 
 [[package]]
@@ -8712,7 +8697,7 @@ dependencies = [
  "darling",
  "proc-macro2",
  "quote",
- "syn 2.0.68",
+ "syn 2.0.71",
 ]
 
 [[package]]
@@ -8814,12 +8799,6 @@ dependencies = [
  "tokio",
 ]
 
-[[package]]
-name = "signature"
-version = "1.6.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "74233d3b3b2f6d4b006dc19dee745e73e2a6bfb6f93607cd3b02bd5b00797d7c"
-
 [[package]]
 name = "signature"
 version = "2.2.0"
@@ -9060,7 +9039,7 @@ source = "git+https://github.com/oxidecomputer/slog-error-chain?branch=main#15f6
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.68",
+ "syn 2.0.71",
 ]
 
 [[package]]
@@ -9187,7 +9166,7 @@ dependencies = [
  "heck 0.4.1",
  "proc-macro2",
  "quote",
- "syn 2.0.68",
+ "syn 2.0.71",
 ]
 
 [[package]]
@@ -9227,10 +9206,9 @@ dependencies = [
  "serde",
  "slog",
  "slog-dtrace",
- "sprockets-rot",
  "thiserror",
  "tokio",
- "toml 0.8.14",
+ "toml 0.8.15",
 ]
 
 [[package]]
@@ -9258,40 +9236,12 @@ dependencies = [
  "der",
 ]
 
-[[package]]
-name = "sprockets-common"
-version = "0.1.0"
-source = "git+https://github.com/oxidecomputer/sprockets?rev=77df31efa5619d0767ffc837ef7468101608aee9#77df31efa5619d0767ffc837ef7468101608aee9"
-dependencies = [
- "derive_more",
- "hubpack 0.1.0",
- "salty",
- "serde",
- "serde-big-array 0.4.1",
-]
-
-[[package]]
-name = "sprockets-rot"
-version = "0.1.0"
-source = "git+https://github.com/oxidecomputer/sprockets?rev=77df31efa5619d0767ffc837ef7468101608aee9#77df31efa5619d0767ffc837ef7468101608aee9"
-dependencies = [
- "corncobs",
- "derive_more",
- "hubpack 0.1.0",
- "rand 0.8.5",
- "salty",
- "serde",
- "sprockets-common",
- "tinyvec",
-]
-
 [[package]]
 name = "sqlformat"
-version = "0.2.3"
+version = "0.2.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ce81b7bd7c4493975347ef60d8c7e8b742d4694f4c49f93e0a12ea263938176c"
+checksum = "f895e3734318cc55f1fe66258926c9b910c124d47520339efecbb6c59cec7c1f"
 dependencies = [
- "itertools 0.12.1",
  "nom",
  "unicode_categories",
 ]
@@ -9314,7 +9264,7 @@ checksum = "01b2e185515564f15375f593fb966b5718bc624ba77fe49fa4616ad619690554"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.68",
+ "syn 2.0.71",
 ]
 
 [[package]]
@@ -9324,7 +9274,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2ff9eaf853dec4c8802325d8b6d3dffa86cc707fd7a1a4cdbf416e13b061787a"
 dependencies = [
  "quote",
- "syn 2.0.68",
+ "syn 2.0.71",
 ]
 
 [[package]]
@@ -9410,7 +9360,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "structmeta-derive 0.2.0",
- "syn 2.0.68",
+ "syn 2.0.71",
 ]
 
 [[package]]
@@ -9422,7 +9372,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "structmeta-derive 0.3.0",
- "syn 2.0.68",
+ "syn 2.0.71",
 ]
 
 [[package]]
@@ -9433,7 +9383,7 @@ checksum = "a60bcaff7397072dca0017d1db428e30d5002e00b6847703e2e42005c95fbe00"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.68",
+ "syn 2.0.71",
 ]
 
 [[package]]
@@ -9444,14 +9394,14 @@ checksum = "152a0b65a590ff6c3da95cabe2353ee04e6167c896b28e3b14478c2636c922fc"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.68",
+ "syn 2.0.71",
 ]
 
 [[package]]
 name = "strum"
-version = "0.26.2"
+version = "0.26.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5d8cec3501a5194c432b2b7976db6b7d10ec95c253208b45f83f7136aa985e29"
+checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06"
 dependencies = [
  "strum_macros 0.26.4",
 ]
@@ -9479,7 +9429,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "rustversion",
- "syn 2.0.68",
+ "syn 2.0.71",
 ]
 
 [[package]]
@@ -9492,7 +9442,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "rustversion",
- "syn 2.0.68",
+ "syn 2.0.71",
 ]
 
 [[package]]
@@ -9539,9 +9489,9 @@ dependencies = [
 
 [[package]]
 name = "syn"
-version = "2.0.68"
+version = "2.0.71"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "901fa70d88b9d6c98022e23b4136f9f3e54e4662c3bc1bd1d84a42a9a0f0c1e9"
+checksum = "b146dcf730474b4bcd16c311627b31ede9ab149045db4d6088b3becaea046462"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -9622,9 +9572,9 @@ checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369"
 
 [[package]]
 name = "tar"
-version = "0.4.40"
+version = "0.4.41"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b16afcea1f22891c49a00c751c7b63b2233284064f11a200fc624137c51e2ddb"
+checksum = "cb797dad5fb5b76fcf519e702f4a589483b5ef06567f160c392832c1f5e44909"
 dependencies = [
  "filetime",
  "libc",
@@ -9715,7 +9665,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "structmeta 0.2.0",
- "syn 2.0.68",
+ "syn 2.0.71",
 ]
 
 [[package]]
@@ -9731,22 +9681,22 @@ dependencies = [
 
 [[package]]
 name = "thiserror"
-version = "1.0.60"
+version = "1.0.62"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "579e9083ca58dd9dcf91a9923bb9054071b9ebbd800b342194c9feb0ee89fc18"
+checksum = "f2675633b1499176c2dff06b0856a27976a8f9d436737b4cf4f312d4d91d8bbb"
 dependencies = [
  "thiserror-impl",
 ]
 
 [[package]]
 name = "thiserror-impl"
-version = "1.0.60"
+version = "1.0.62"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e2470041c06ec3ac1ab38d0356a6119054dedaea53e12fbefc0de730a1c08524"
+checksum = "d20468752b09f49e909e55a5d338caa8bedf615594e9d80bc4c565d30faf798c"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.68",
+ "syn 2.0.71",
 ]
 
 [[package]]
@@ -9883,7 +9833,7 @@ checksum = "8d9ef545650e79f30233c0003bcc2504d7efac6dad25fca40744de773fe2049c"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.68",
+ "syn 2.0.71",
 ]
 
 [[package]]
@@ -9929,9 +9879,9 @@ dependencies = [
 
 [[package]]
 name = "tokio"
-version = "1.38.0"
+version = "1.38.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ba4f4a02a7a80d6f274636f0aa95c7e383b912d41fe721a31f29e29698585a4a"
+checksum = "eb2caba9f80616f438e09748d5acda951967e1ea58508ef53d9c6402485a46df"
 dependencies = [
  "backtrace",
  "bytes",
@@ -9954,7 +9904,7 @@ checksum = "5f5ae998a069d4b5aba8ee9dad856af7d520c3699e6159b185c2acd48155d39a"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.68",
+ "syn 2.0.71",
 ]
 
 [[package]]
@@ -10085,14 +10035,14 @@ dependencies = [
 
 [[package]]
 name = "toml"
-version = "0.8.14"
+version = "0.8.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6f49eb2ab21d2f26bd6db7bf383edc527a7ebaee412d17af4d40fdccd442f335"
+checksum = "ac2caab0bf757388c6c0ae23b3293fdb463fee59434529014f85e3263b995c28"
 dependencies = [
  "serde",
  "serde_spanned",
  "toml_datetime",
- "toml_edit 0.22.14",
+ "toml_edit 0.22.16",
 ]
 
 [[package]]
@@ -10119,9 +10069,9 @@ dependencies = [
 
 [[package]]
 name = "toml_edit"
-version = "0.22.14"
+version = "0.22.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f21c7aaf97f1bd9ca9d4f9e73b0a6c74bd5afef56f2bc931943a6e1c37e04e38"
+checksum = "278f3d518e152219c994ce877758516bca5e118eaed6996192a774fb9fbf0788"
 dependencies = [
  "indexmap 2.2.6",
  "serde",
@@ -10231,7 +10181,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.68",
+ "syn 2.0.71",
 ]
 
 [[package]]
@@ -10347,7 +10297,7 @@ dependencies = [
  "serde_derive",
  "serde_json",
  "termcolor",
- "toml 0.8.14",
+ "toml 0.8.15",
 ]
 
 [[package]]
@@ -10407,7 +10357,7 @@ dependencies = [
  "slog",
  "tar",
  "tokio",
- "toml 0.8.14",
+ "toml 0.8.15",
  "tough",
  "url",
  "zip",
@@ -10508,7 +10458,7 @@ dependencies = [
  "semver 1.0.23",
  "serde",
  "serde_json",
- "syn 2.0.68",
+ "syn 2.0.71",
  "thiserror",
  "unicode-ident",
 ]
@@ -10525,7 +10475,7 @@ dependencies = [
  "serde",
  "serde_json",
  "serde_tokenstream",
- "syn 2.0.68",
+ "syn 2.0.71",
  "typify-impl",
 ]
 
@@ -10742,7 +10692,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "serde_tokenstream",
- "syn 2.0.68",
+ "syn 2.0.71",
  "usdt-impl",
 ]
 
@@ -10760,7 +10710,7 @@ dependencies = [
  "quote",
  "serde",
  "serde_json",
- "syn 2.0.68",
+ "syn 2.0.71",
  "thiserror",
  "thread-id",
  "version_check",
@@ -10776,7 +10726,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "serde_tokenstream",
- "syn 2.0.68",
+ "syn 2.0.71",
  "usdt-impl",
 ]
 
@@ -10794,9 +10744,9 @@ checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a"
 
 [[package]]
 name = "uuid"
-version = "1.9.1"
+version = "1.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5de17fd2f7da591098415cff336e12965a28061ddace43b59cb3c430179c9439"
+checksum = "81dfa00651efa65069b0b6b651f4aaa31ba9e3c3ce0137aaad053604ee7e0314"
 dependencies = [
  "getrandom 0.2.14",
  "serde",
@@ -10955,7 +10905,7 @@ dependencies = [
  "once_cell",
  "proc-macro2",
  "quote",
- "syn 2.0.68",
+ "syn 2.0.71",
  "wasm-bindgen-shared",
 ]
 
@@ -10989,7 +10939,7 @@ checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.68",
+ "syn 2.0.71",
  "wasm-bindgen-backend",
  "wasm-bindgen-shared",
 ]
@@ -11092,8 +11042,8 @@ dependencies = [
  "textwrap",
  "tokio",
  "tokio-util",
- "toml 0.8.14",
- "toml_edit 0.22.14",
+ "toml 0.8.15",
+ "toml_edit 0.22.16",
  "tui-tree-widget",
  "unicode-width",
  "update-engine",
@@ -11107,6 +11057,8 @@ name = "wicket-common"
 version = "0.1.0"
 dependencies = [
  "anyhow",
+ "dpd-client",
+ "dropshot",
  "gateway-client",
  "maplit",
  "omicron-common",
@@ -11118,8 +11070,10 @@ dependencies = [
  "serde_json",
  "sha2",
  "sled-hardware-types",
+ "slog",
  "thiserror",
- "toml 0.8.14",
+ "tokio",
+ "toml 0.8.15",
  "update-engine",
 ]
 
@@ -11176,8 +11130,8 @@ dependencies = [
  "hyper 0.14.28",
  "illumos-utils",
  "installinator",
- "installinator-artifact-client",
- "installinator-artifactd",
+ "installinator-api",
+ "installinator-client",
  "installinator-common",
  "internal-dns",
  "itertools 0.12.1",
@@ -11207,7 +11161,7 @@ dependencies = [
  "tokio",
  "tokio-stream",
  "tokio-util",
- "toml 0.8.14",
+ "toml 0.8.15",
  "tough",
  "trust-dns-resolver",
  "tufaceous",
@@ -11217,9 +11171,27 @@ dependencies = [
  "uuid",
  "wicket",
  "wicket-common",
+ "wicketd-api",
  "wicketd-client",
 ]
 
+[[package]]
+name = "wicketd-api"
+version = "0.1.0"
+dependencies = [
+ "bootstrap-agent-client",
+ "dropshot",
+ "gateway-client",
+ "omicron-common",
+ "omicron-passwords",
+ "omicron-workspace-hack",
+ "schemars",
+ "serde",
+ "sled-hardware-types",
+ "slog",
+ "wicket-common",
+]
+
 [[package]]
 name = "wicketd-client"
 version = "0.1.0"
@@ -11521,7 +11493,7 @@ dependencies = [
  "tabled",
  "tar",
  "tokio",
- "toml 0.8.14",
+ "toml 0.8.15",
  "usdt",
 ]
 
@@ -11570,7 +11542,7 @@ checksum = "125139de3f6b9d625c39e2efdd73d41bdac468ccd556556440e322be0e1bbd91"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.68",
+ "syn 2.0.71",
 ]
 
 [[package]]
@@ -11581,7 +11553,7 @@ checksum = "15e934569e47891f7d9411f1a451d947a60e000ab3bd24fbb970f000387d1b3b"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.68",
+ "syn 2.0.71",
 ]
 
 [[package]]
@@ -11601,7 +11573,7 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.68",
+ "syn 2.0.71",
 ]
 
 [[package]]
diff --git a/Cargo.toml b/Cargo.toml
index e5783b39eb..dc464e547f 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -9,7 +9,7 @@ members = [
     "clients/dns-service-client",
     "clients/dpd-client",
     "clients/gateway-client",
-    "clients/installinator-artifact-client",
+    "clients/installinator-client",
     "clients/nexus-client",
     "clients/oxide-client",
     "clients/oximeter-client",
@@ -26,12 +26,13 @@ members = [
     "dev-tools/releng",
     "dev-tools/xtask",
     "dns-server",
+    "dns-server-api",
     "end-to-end-tests",
     "gateway-cli",
     "gateway-test-utils",
     "gateway",
     "illumos-utils",
-    "installinator-artifactd",
+    "installinator-api",
     "installinator-common",
     "installinator",
     "internal-dns-cli",
@@ -55,6 +56,7 @@ members = [
     "nexus/reconfigurator/execution",
     "nexus/reconfigurator/planning",
     "nexus/reconfigurator/preparation",
+    "nexus/saga-recovery",
     "nexus/test-interface",
     "nexus/test-utils-macros",
     "nexus/test-utils",
@@ -86,6 +88,7 @@ members = [
     "wicket-dbg",
     "wicket",
     "wicketd",
+    "wicketd-api",
     "workspace-hack",
     "zone-setup",
 ]
@@ -100,7 +103,7 @@ default-members = [
     "clients/dns-service-client",
     "clients/dpd-client",
     "clients/gateway-client",
-    "clients/installinator-artifact-client",
+    "clients/installinator-client",
     "clients/nexus-client",
     "clients/oxide-client",
     "clients/oximeter-client",
@@ -119,13 +122,14 @@ default-members = [
     # hakari to not work as well and build times to be longer.
     # See omicron#4392.
     "dns-server",
+    "dns-server-api",
     # Do not include end-to-end-tests in the list of default members, as its
     # tests only work on a deployed control plane.
     "gateway-cli",
     "gateway-test-utils",
     "gateway",
     "illumos-utils",
-    "installinator-artifactd",
+    "installinator-api",
     "installinator-common",
     "installinator",
     "internal-dns-cli",
@@ -149,6 +153,7 @@ default-members = [
     "nexus/reconfigurator/execution",
     "nexus/reconfigurator/planning",
     "nexus/reconfigurator/preparation",
+    "nexus/saga-recovery",
     "nexus/test-interface",
     "nexus/test-utils-macros",
     "nexus/test-utils",
@@ -180,6 +185,7 @@ default-members = [
     "wicket-dbg",
     "wicket",
     "wicketd",
+    "wicketd-api",
     "workspace-hack",
     "zone-setup",
 ]
@@ -279,6 +285,7 @@ derive-where = "1.2.7"
 diesel = { version = "2.1.6", features = ["postgres", "r2d2", "chrono", "serde_json", "network-address", "uuid"] }
 diesel-dtrace = { git = "https://github.com/oxidecomputer/diesel-dtrace", branch = "main" }
 dns-server = { path = "dns-server" }
+dns-server-api = { path = "dns-server-api" }
 dns-service-client = { path = "clients/dns-service-client" }
 dpd-client = { path = "clients/dpd-client" }
 dropshot = { git = "https://github.com/oxidecomputer/dropshot", branch = "main", features = [ "usdt-probes" ] }
@@ -318,8 +325,8 @@ indent_write = "2.2.0"
 indexmap = "2.2.6"
 indicatif = { version = "0.17.8", features = ["rayon"] }
 installinator = { path = "installinator" }
-installinator-artifactd = { path = "installinator-artifactd" }
-installinator-artifact-client = { path = "clients/installinator-artifact-client" }
+installinator-api = { path = "installinator-api" }
+installinator-client = { path = "clients/installinator-client" }
 installinator-common = { path = "installinator-common" }
 internal-dns = { path = "internal-dns" }
 ipcc = { path = "ipcc" }
@@ -338,8 +345,8 @@ macaddr = { version = "1.0.1", features = ["serde_std"] }
 maplit = "1.0.2"
 mockall = "0.12"
 newtype_derive = "0.1.6"
-mg-admin-client = { git = "https://github.com/oxidecomputer/maghemite", rev = "1b385990e8648b221fd11f018f2a7ec425461c6c" }
-ddm-admin-client = { git = "https://github.com/oxidecomputer/maghemite", rev = "1b385990e8648b221fd11f018f2a7ec425461c6c" }
+mg-admin-client = { git = "https://github.com/oxidecomputer/maghemite", rev = "220dd026e83142b83bd93123f465a64dd4600201" }
+ddm-admin-client = { git = "https://github.com/oxidecomputer/maghemite", rev = "220dd026e83142b83bd93123f465a64dd4600201" }
 multimap = "0.10.0"
 nexus-auth = { path = "nexus/auth" }
 nexus-client = { path = "clients/nexus-client" }
@@ -356,6 +363,7 @@ nexus-networking = { path = "nexus/networking" }
 nexus-reconfigurator-execution = { path = "nexus/reconfigurator/execution" }
 nexus-reconfigurator-planning = { path = "nexus/reconfigurator/planning" }
 nexus-reconfigurator-preparation = { path = "nexus/reconfigurator/preparation" }
+nexus-saga-recovery = { path = "nexus/saga-recovery" }
 omicron-certificates = { path = "certificates" }
 omicron-passwords = { path = "passwords" }
 omicron-workspace-hack = "0.1.0"
@@ -378,14 +386,14 @@ omicron-sled-agent = { path = "sled-agent" }
 omicron-test-utils = { path = "test-utils" }
 omicron-zone-package = "0.11.0"
 oxide-client = { path = "clients/oxide-client" }
-oxide-vpc = { git = "https://github.com/oxidecomputer/opte", rev = "915975f6d1729db95619f752148974016912412f", features = [ "api", "std" ] }
+oxide-vpc = { git = "https://github.com/oxidecomputer/opte", rev = "3dc9a3dd8d3c623f0cf2c659c7119ce0c026a96d", features = [ "api", "std" ] }
 once_cell = "1.19.0"
 openapi-lint = { git = "https://github.com/oxidecomputer/openapi-lint", branch = "main" }
 openapiv3 = "2.0.0"
 # must match samael's crate!
 openssl = "0.10"
 openssl-sys = "0.9"
-opte-ioctl = { git = "https://github.com/oxidecomputer/opte", rev = "915975f6d1729db95619f752148974016912412f" }
+opte-ioctl = { git = "https://github.com/oxidecomputer/opte", rev = "3dc9a3dd8d3c623f0cf2c659c7119ce0c026a96d" }
 oso = "0.27"
 owo-colors = "4.0.0"
 oximeter = { path = "oximeter/oximeter" }
@@ -414,9 +422,9 @@ prettyplease = { version = "0.2.20", features = ["verbatim"] }
 proc-macro2 = "1.0"
 progenitor = { git = "https://github.com/oxidecomputer/progenitor", branch = "main" }
 progenitor-client = { git = "https://github.com/oxidecomputer/progenitor", branch = "main" }
-bhyve_api = { git = "https://github.com/oxidecomputer/propolis", rev = "5ebf9626e0ad274eb515d206d102cb09d2d51f15" }
-propolis-client = { git = "https://github.com/oxidecomputer/propolis", rev = "5ebf9626e0ad274eb515d206d102cb09d2d51f15" }
-propolis-mock-server = { git = "https://github.com/oxidecomputer/propolis", rev = "5ebf9626e0ad274eb515d206d102cb09d2d51f15" }
+bhyve_api = { git = "https://github.com/oxidecomputer/propolis", rev = "66d1ee7d4a5829dbbf02a152091ea051023b5b8b" }
+propolis-client = { git = "https://github.com/oxidecomputer/propolis", rev = "66d1ee7d4a5829dbbf02a152091ea051023b5b8b" }
+propolis-mock-server = { git = "https://github.com/oxidecomputer/propolis", rev = "66d1ee7d4a5829dbbf02a152091ea051023b5b8b" }
 proptest = "1.4.0"
 quote = "1.0"
 rand = "0.8.5"
@@ -477,10 +485,7 @@ slog-term = "2.9.1"
 smf = "0.2"
 socket2 = { version = "0.5", features = ["all"] }
 sp-sim = { path = "sp-sim" }
-sprockets-common = { git = "https://github.com/oxidecomputer/sprockets", rev = "77df31efa5619d0767ffc837ef7468101608aee9" }
-sprockets-host = { git = "https://github.com/oxidecomputer/sprockets", rev = "77df31efa5619d0767ffc837ef7468101608aee9" }
-sprockets-rot = { git = "https://github.com/oxidecomputer/sprockets", rev = "77df31efa5619d0767ffc837ef7468101608aee9" }
-sqlformat = "0.2.3"
+sqlformat = "0.2.4"
 sqlparser = { version = "0.45.0", features = [ "visitor" ] }
 static_assertions = "1.1.0"
 # Please do not change the Steno version to a Git dependency.  It makes it
@@ -502,12 +507,12 @@ textwrap = "0.16.1"
 test-strategy = "0.3.1"
 thiserror = "1.0"
 tofino = { git = "https://github.com/oxidecomputer/tofino", branch = "main" }
-tokio = "1.37.0"
+tokio = "1.38.1"
 tokio-postgres = { version = "0.7", features = [ "with-chrono-0_4", "with-uuid-1" ] }
 tokio-stream = "0.1.15"
 tokio-tungstenite = "0.20"
 tokio-util = { version = "0.7.10", features = ["io", "io-util"] }
-toml = "0.8.12"
+toml = "0.8.15"
 toml_edit = "0.22.12"
 tough = { version = "0.17.1", features = [ "http" ] }
 trust-dns-client = "0.22"
@@ -530,6 +535,7 @@ walkdir = "2.5"
 whoami = "1.5"
 wicket = { path = "wicket" }
 wicket-common = { path = "wicket-common" }
+wicketd-api = { path = "wicketd-api" }
 wicketd-client = { path = "clients/wicketd-client" }
 zeroize = { version = "1.7.0", features = ["zeroize_derive", "std"] }
 zip = { version = "0.6.6", default-features = false, features = ["deflate","bzip2"] }
@@ -677,8 +683,6 @@ opt-level = 3
 opt-level = 3
 [profile.dev.package.rsa]
 opt-level = 3
-[profile.dev.package.salty]
-opt-level = 3
 [profile.dev.package.signature]
 opt-level = 3
 [profile.dev.package.subtle]
diff --git a/README.adoc b/README.adoc
index 1ef4bd8601..4979411d73 100644
--- a/README.adoc
+++ b/README.adoc
@@ -112,6 +112,21 @@ cargo nextest run
 
 We check that certain system library dependencies are not leaked outside of their intended binaries via `cargo xtask verify-libraries` in CI. If you are adding a new dependency on a illumos/helios library it is recommended that you update xref:.cargo/xtask.toml[] with an allow list of where you expect the dependency to show up. For example some libraries such as `libnvme.so.1` are only available in the global zone and therefore will not be present in any other zone. This check is here to help us catch any leakage before we go to deploy on a rack. You can inspect a compiled binary in the target directory for what it requires by using `elfedit` - for example `elfedit -r -e 'dyn:tag NEEDED' /path/to/omicron/target/debug/sled-agent`.
 
+=== Checking feature flag combinations
+
+To ensure that varying combinations of features compile, run `cargo xtask check-features`, which executes the https://github.com/taiki-e/cargo-hack[`cargo hack`] subcommand under the hood.
+
+This `xtask` is run in CI using the `--ci` parameter , which automatically exludes certain `image-*` features that purposefully cause compiler errors if set and uses a pre-built binary.
+
+If `cargo hack` is not already installed in omicron's `out/` directory, a pre-built binary will be installed automatically depending on your operating system and architecture.
+
+To limit the max number of simultaneous feature flags combined for checking, run the `xtask` with the `--depth <NUM>` flag:
+
+[source,text]
+----
+$ cargo xtask check-features --depth 2
+----
+
 === Rust packages in Omicron
 
 NOTE: The term "package" is overloaded: most programming languages and operating systems have their own definitions of a package.  On top of that, Omicron bundles up components into our own kind of "package" that gets delivered via the install and update systems.  These are described in the `package-manifest.toml` file in the root of the repo.  In this section, we're just concerned with Rust packages.
diff --git a/clients/installinator-artifact-client/Cargo.toml b/clients/installinator-client/Cargo.toml
similarity index 92%
rename from clients/installinator-artifact-client/Cargo.toml
rename to clients/installinator-client/Cargo.toml
index f1e896864f..ca2de0476a 100644
--- a/clients/installinator-artifact-client/Cargo.toml
+++ b/clients/installinator-client/Cargo.toml
@@ -1,5 +1,5 @@
 [package]
-name = "installinator-artifact-client"
+name = "installinator-client"
 version = "0.1.0"
 edition = "2021"
 license = "MPL-2.0"
diff --git a/clients/installinator-artifact-client/src/lib.rs b/clients/installinator-client/src/lib.rs
similarity index 91%
rename from clients/installinator-artifact-client/src/lib.rs
rename to clients/installinator-client/src/lib.rs
index 96806c2cab..a39ff3ff80 100644
--- a/clients/installinator-artifact-client/src/lib.rs
+++ b/clients/installinator-client/src/lib.rs
@@ -2,10 +2,10 @@
 // License, v. 2.0. If a copy of the MPL was not distributed with this
 // file, You can obtain one at https://mozilla.org/MPL/2.0/.
 
-//! Interface for making API requests to installinator-artifactd.
+//! Interface for installinator to make API requests.
 
 progenitor::generate_api!(
-    spec = "../../openapi/installinator-artifactd.json",
+    spec = "../../openapi/installinator.json",
     inner_type = slog::Logger,
     pre_hook = (|log: &slog::Logger, request: &reqwest::Request| {
         slog::debug!(log, "client request";
diff --git a/clients/sled-agent-client/src/lib.rs b/clients/sled-agent-client/src/lib.rs
index 42eefaf8b5..8a63cecd4f 100644
--- a/clients/sled-agent-client/src/lib.rs
+++ b/clients/sled-agent-client/src/lib.rs
@@ -593,6 +593,36 @@ impl From<omicron_common::api::internal::shared::NetworkInterfaceKind>
     }
 }
 
+impl From<omicron_common::api::internal::shared::SledIdentifiers>
+    for types::SledIdentifiers
+{
+    fn from(
+        value: omicron_common::api::internal::shared::SledIdentifiers,
+    ) -> Self {
+        Self {
+            model: value.model,
+            rack_id: value.rack_id,
+            revision: value.revision,
+            serial: value.serial,
+            sled_id: value.sled_id,
+        }
+    }
+}
+
+impl From<types::SledIdentifiers>
+    for omicron_common::api::internal::shared::SledIdentifiers
+{
+    fn from(value: types::SledIdentifiers) -> Self {
+        Self {
+            model: value.model,
+            rack_id: value.rack_id,
+            revision: value.revision,
+            serial: value.serial,
+            sled_id: value.sled_id,
+        }
+    }
+}
+
 /// Exposes additional [`Client`] interfaces for use by the test suite. These
 /// are bonus endpoints, not generated in the real client.
 #[async_trait]
diff --git a/clients/wicketd-client/src/lib.rs b/clients/wicketd-client/src/lib.rs
index 6198c6cf9e..bb377de31e 100644
--- a/clients/wicketd-client/src/lib.rs
+++ b/clients/wicketd-client/src/lib.rs
@@ -27,21 +27,11 @@ progenitor::generate_api!(
         RackNetworkConfigV2 = { derives = [PartialEq, Eq, PartialOrd, Ord] },
         RackOperationStatus = { derives = [PartialEq, Eq, PartialOrd, Ord] },
         RackResetId = { derives = [PartialEq, Eq, PartialOrd, Ord] },
-        RackV1Inventory = { derives = [PartialEq, Eq, PartialOrd, Ord]},
         RotImageDetails = { derives = [PartialEq, Eq, PartialOrd, Ord]},
-        RotInventory = { derives = [PartialEq, Eq, PartialOrd, Ord]},
-        RotSlot = { derives = [PartialEq, Eq, PartialOrd, Ord]},
-        RotState = { derives = [PartialEq, Eq, PartialOrd, Ord]},
-        SpComponentCaboose = { derives = [PartialEq, Eq, PartialOrd, Ord] },
-        SpComponentInfo = { derives = [PartialEq, Eq, PartialOrd, Ord]},
-        SpIgnition = { derives = [PartialEq, Eq, PartialOrd, Ord]},
-        SpIgnitionSystemType= { derives = [PartialEq, Eq, PartialOrd, Ord]},
-        SpInventory = { derives = [PartialEq, Eq, PartialOrd, Ord]},
-        SpState = { derives = [PartialEq, Eq, PartialOrd, Ord] },
-        StartUpdateOptions = { derives = [Default]},
         UplinkConfig = { derives = [PartialEq, Eq, PartialOrd, Ord] },
     },
     replace = {
+        AbortUpdateOptions = wicket_common::rack_update::AbortUpdateOptions,
         AllowedSourceIps = omicron_common::api::internal::shared::AllowedSourceIps,
         Baseboard = sled_hardware_types::Baseboard,
         BgpAuthKey = wicket_common::rack_setup::BgpAuthKey,
@@ -52,9 +42,11 @@ progenitor::generate_api!(
         BgpPeerAuthKind = wicket_common::rack_setup::BgpPeerAuthKind,
         BgpPeerConfig = omicron_common::api::internal::shared::BgpPeerConfig,
         BootstrapSledDescription = wicket_common::rack_setup::BootstrapSledDescription,
+        ClearUpdateStateOptions = wicket_common::rack_update::ClearUpdateStateOptions,
         ClearUpdateStateResponse = wicket_common::rack_update::ClearUpdateStateResponse,
         CurrentRssUserConfigInsensitive = wicket_common::rack_setup::CurrentRssUserConfigInsensitive,
         Duration = std::time::Duration,
+        EventReportForUplinkPreflightCheckSpec = wicket_common::preflight_check::EventReport,
         EventReportForWicketdEngineSpec = wicket_common::update_events::EventReport,
         GetBgpAuthKeyInfoResponse = wicket_common::rack_setup::GetBgpAuthKeyInfoResponse,
         ImportExportPolicy = omicron_common::api::internal::shared::ImportExportPolicy,
@@ -67,15 +59,31 @@ progenitor::generate_api!(
         PortSpeed = omicron_common::api::internal::shared::PortSpeed,
         ProgressEventForGenericSpec = update_engine::events::ProgressEvent<update_engine::NestedSpec>,
         ProgressEventForInstallinatorSpec = installinator_common::ProgressEvent,
+        ProgressEventForUplinkPreflightSpec = wicket_common::preflight_check::ProgressEvent,
         ProgressEventForWicketdEngineSpec = wicket_common::update_events::ProgressEvent,
         PutRssUserConfigInsensitive = wicket_common::rack_setup::PutRssUserConfigInsensitive,
+        RackV1Inventory = wicket_common::inventory::RackV1Inventory,
+        RotInventory = wicket_common::inventory::RotInventory,
+        RotSlot = wicket_common::inventory::RotSlot,
+        RotState = wicket_common::inventory::RotState,
         RouteConfig = omicron_common::api::internal::shared::RouteConfig,
-        SpIdentifier = wicket_common::rack_update::SpIdentifier,
-        SpType = wicket_common::rack_update::SpType,
+        SpComponentCaboose = wicket_common::inventory::SpComponentCaboose,
+        SpComponentInfo = wicket_common::inventory::SpComponentInfo,
+        SpIdentifier = wicket_common::inventory::SpIdentifier,
+        SpIgnition = wicket_common::inventory::SpIgnition,
+        SpIgnitionSystemType = wicket_common::inventory::SpIgnitionSystemType,
+        SpInventory = wicket_common::inventory::SpInventory,
+        SpState = wicket_common::inventory::SpState,
+        SpType = wicket_common::inventory::SpType,
+        StartUpdateOptions = wicket_common::rack_update::StartUpdateOptions,
         StepEventForGenericSpec = update_engine::events::StepEvent<update_engine::NestedSpec>,
+        StepEventForUplinkPreflightSpec = wicket_common::preflight_check::StepEvent,
         StepEventForInstallinatorSpec = installinator_common::StepEvent,
         StepEventForWicketdEngineSpec = wicket_common::update_events::StepEvent,
         SwitchLocation = omicron_common::api::internal::shared::SwitchLocation,
+        UpdateSimulatedResult = wicket_common::rack_update::UpdateSimulatedResult,
+        UpdateTestError = wicket_common::rack_update::UpdateTestError,
+        UplinkPreflightStepId = wicket_common::preflight_check::UplinkPreflightStepId,
         UserSpecifiedBgpPeerConfig = wicket_common::rack_setup::UserSpecifiedBgpPeerConfig,
         UserSpecifiedImportExportPolicy = wicket_common::rack_setup::UserSpecifiedImportExportPolicy,
         UserSpecifiedPortConfig = wicket_common::rack_setup::UserSpecifiedPortConfig,
diff --git a/common/src/api/internal/shared.rs b/common/src/api/internal/shared.rs
index 884b4dc165..24bb339112 100644
--- a/common/src/api/internal/shared.rs
+++ b/common/src/api/internal/shared.rs
@@ -702,6 +702,26 @@ pub struct ResolvedVpcRouteSet {
     pub routes: HashSet<ResolvedVpcRoute>,
 }
 
+/// Identifiers for a single sled.
+///
+/// This is intended primarily to be used in timeseries, to identify
+/// sled from which metric data originates.
+#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq)]
+pub struct SledIdentifiers {
+    /// Control plane ID of the rack this sled is a member of
+    pub rack_id: Uuid,
+    /// Control plane ID for the sled itself
+    pub sled_id: Uuid,
+    /// Model name of the sled
+    pub model: String,
+    /// Revision number of the sled
+    pub revision: u32,
+    /// Serial number of the sled
+    //
+    // NOTE: This is only guaranteed to be unique within a model.
+    pub serial: String,
+}
+
 #[cfg(test)]
 mod tests {
     use crate::api::internal::shared::AllowedSourceIps;
diff --git a/dev-tools/omdb/Cargo.toml b/dev-tools/omdb/Cargo.toml
index e5d898509c..0990fdb11c 100644
--- a/dev-tools/omdb/Cargo.toml
+++ b/dev-tools/omdb/Cargo.toml
@@ -33,6 +33,7 @@ nexus-config.workspace = true
 nexus-db-model.workspace = true
 nexus-db-queries.workspace = true
 nexus-reconfigurator-preparation.workspace = true
+nexus-saga-recovery.workspace = true
 nexus-types.workspace = true
 omicron-common.workspace = true
 omicron-uuid-kinds.workspace = true
diff --git a/dev-tools/omdb/src/bin/omdb/db.rs b/dev-tools/omdb/src/bin/omdb/db.rs
index f0f7be0b83..98669ddc06 100644
--- a/dev-tools/omdb/src/bin/omdb/db.rs
+++ b/dev-tools/omdb/src/bin/omdb/db.rs
@@ -62,6 +62,7 @@ use nexus_db_model::IpAttachState;
 use nexus_db_model::IpKind;
 use nexus_db_model::NetworkInterface;
 use nexus_db_model::NetworkInterfaceKind;
+use nexus_db_model::PhysicalDisk;
 use nexus_db_model::Probe;
 use nexus_db_model::Project;
 use nexus_db_model::Region;
@@ -96,7 +97,10 @@ use nexus_types::deployment::Blueprint;
 use nexus_types::deployment::BlueprintZoneDisposition;
 use nexus_types::deployment::BlueprintZoneFilter;
 use nexus_types::deployment::BlueprintZoneType;
+use nexus_types::deployment::DiskFilter;
 use nexus_types::deployment::SledFilter;
+use nexus_types::external_api::views::PhysicalDiskPolicy;
+use nexus_types::external_api::views::PhysicalDiskState;
 use nexus_types::external_api::views::SledPolicy;
 use nexus_types::external_api::views::SledState;
 use nexus_types::identity::Resource;
@@ -281,12 +285,14 @@ pub struct DbFetchOptions {
 enum DbCommands {
     /// Print information about the rack
     Rack(RackArgs),
-    /// Print information about disks
+    /// Print information about virtual disks
     Disks(DiskArgs),
     /// Print information about internal and external DNS
     Dns(DnsArgs),
     /// Print information about collected hardware/software inventory
     Inventory(InventoryArgs),
+    /// Print information about physical disks
+    PhysicalDisks(PhysicalDisksArgs),
     /// Save the current Reconfigurator inputs to a file
     ReconfiguratorSave(ReconfiguratorSaveArgs),
     /// Print information about regions
@@ -407,8 +413,8 @@ enum InventoryCommands {
     Cabooses,
     /// list and show details from particular collections
     Collections(CollectionsArgs),
-    /// show all physical disks every found
-    PhysicalDisks(PhysicalDisksArgs),
+    /// show all physical disks ever found
+    PhysicalDisks(InvPhysicalDisksArgs),
     /// list all root of trust pages ever found
     RotPages,
 }
@@ -437,7 +443,7 @@ struct CollectionsShowArgs {
 }
 
 #[derive(Debug, Args, Clone, Copy)]
-struct PhysicalDisksArgs {
+struct InvPhysicalDisksArgs {
     #[clap(long)]
     collection_id: Option<CollectionUuid>,
 
@@ -445,6 +451,13 @@ struct PhysicalDisksArgs {
     sled_id: Option<SledUuid>,
 }
 
+#[derive(Debug, Args)]
+struct PhysicalDisksArgs {
+    /// Show disks that match the given filter
+    #[clap(short = 'F', long, value_enum)]
+    filter: Option<DiskFilter>,
+}
+
 #[derive(Debug, Args)]
 struct ReconfiguratorSaveArgs {
     /// where to save the output
@@ -611,6 +624,15 @@ impl DbArgs {
                 )
                 .await
             }
+            DbCommands::PhysicalDisks(args) => {
+                cmd_db_physical_disks(
+                    &opctx,
+                    &datastore,
+                    &self.fetch_opts,
+                    args,
+                )
+                .await
+            }
             DbCommands::ReconfiguratorSave(reconfig_save_args) => {
                 cmd_db_reconfigurator_save(
                     &opctx,
@@ -1385,6 +1407,68 @@ async fn cmd_db_disk_physical(
     Ok(())
 }
 
+#[derive(Tabled)]
+#[tabled(rename_all = "SCREAMING_SNAKE_CASE")]
+struct PhysicalDiskRow {
+    id: Uuid,
+    serial: String,
+    vendor: String,
+    model: String,
+    sled_id: Uuid,
+    policy: PhysicalDiskPolicy,
+    state: PhysicalDiskState,
+}
+
+impl From<PhysicalDisk> for PhysicalDiskRow {
+    fn from(d: PhysicalDisk) -> Self {
+        PhysicalDiskRow {
+            id: d.id(),
+            serial: d.serial.clone(),
+            vendor: d.vendor.clone(),
+            model: d.model.clone(),
+            sled_id: d.sled_id,
+            policy: d.disk_policy.into(),
+            state: d.disk_state.into(),
+        }
+    }
+}
+
+/// Run `omdb db physical-disks`.
+async fn cmd_db_physical_disks(
+    opctx: &OpContext,
+    datastore: &DataStore,
+    fetch_opts: &DbFetchOptions,
+    args: &PhysicalDisksArgs,
+) -> Result<(), anyhow::Error> {
+    let limit = fetch_opts.fetch_limit;
+    let filter = match args.filter {
+        Some(filter) => filter,
+        None => {
+            eprintln!(
+                "note: listing all in-service disks \
+                 (use -F to filter, e.g. -F in-service)"
+            );
+            DiskFilter::InService
+        }
+    };
+
+    let sleds = datastore
+        .physical_disk_list(&opctx, &first_page(limit), filter)
+        .await
+        .context("listing physical disks")?;
+    check_limit(&sleds, limit, || String::from("listing physical disks"));
+
+    let rows = sleds.into_iter().map(|s| PhysicalDiskRow::from(s));
+    let table = tabled::Table::new(rows)
+        .with(tabled::settings::Style::empty())
+        .with(tabled::settings::Padding::new(1, 1, 0, 0))
+        .to_string();
+
+    println!("{}", table);
+
+    Ok(())
+}
+
 // SERVICES
 
 // Snapshots
@@ -2795,7 +2879,12 @@ async fn cmd_db_validate_region_snapshots(
         use crucible_agent_client::types::State;
         use crucible_agent_client::Client as CrucibleAgentClient;
 
-        let url = format!("http://{}", dataset.address());
+        let Some(dataset_addr) = dataset.address() else {
+            eprintln!("Dataset {} missing an IP address", dataset.id());
+            continue;
+        };
+
+        let url = format!("http://{}", dataset_addr);
         let client = CrucibleAgentClient::new(&url);
 
         let actual_region_snapshots = client
@@ -2856,7 +2945,7 @@ async fn cmd_db_validate_region_snapshots(
                                     dataset_id: region_snapshot.dataset_id,
                                     region_id: region_snapshot.region_id,
                                     snapshot_id: region_snapshot.snapshot_id,
-                                    dataset_addr: dataset.address(),
+                                    dataset_addr,
                                     error: String::from(
                                         "region snapshot was deleted, please remove its record",
                                     ),
@@ -2871,7 +2960,7 @@ async fn cmd_db_validate_region_snapshots(
                                     dataset_id: region_snapshot.dataset_id,
                                     region_id: region_snapshot.region_id,
                                     snapshot_id: region_snapshot.snapshot_id,
-                                    dataset_addr: dataset.address(),
+                                    dataset_addr,
                                     error: String::from(
                                         "NEXUS BUG: region snapshot was deleted, but the higher level snapshot was not!",
                                     ),
@@ -2900,7 +2989,7 @@ async fn cmd_db_validate_region_snapshots(
                                 dataset_id: region_snapshot.dataset_id,
                                 region_id: region_snapshot.region_id,
                                 snapshot_id: region_snapshot.snapshot_id,
-                                dataset_addr: dataset.address(),
+                                dataset_addr,
                                 error: format!(
                                     "AGENT BUG: region snapshot was deleted but has a running snapshot in state {:?}!",
                                     running_snapshot.state,
@@ -2950,7 +3039,12 @@ async fn cmd_db_validate_region_snapshots(
         use crucible_agent_client::types::State;
         use crucible_agent_client::Client as CrucibleAgentClient;
 
-        let url = format!("http://{}", dataset.address());
+        let Some(dataset_addr) = dataset.address() else {
+            eprintln!("Dataset {} missing an IP address", dataset.id());
+            continue;
+        };
+
+        let url = format!("http://{}", dataset_addr);
         let client = CrucibleAgentClient::new(&url);
 
         let actual_region_snapshots = client
@@ -2968,7 +3062,7 @@ async fn cmd_db_validate_region_snapshots(
                     dataset_id: dataset.id(),
                     region_id: region.id(),
                     snapshot_id,
-                    dataset_addr: dataset.address(),
+                    dataset_addr,
                     error: String::from(
                         "Nexus does not know about this snapshot!",
                     ),
@@ -2993,7 +3087,7 @@ async fn cmd_db_validate_region_snapshots(
                             dataset_id: dataset.id(),
                             region_id: region.id(),
                             snapshot_id,
-                            dataset_addr: dataset.address(),
+                            dataset_addr,
                             error: String::from(
                                 "Nexus does not know about this running snapshot!"
                             ),
@@ -3187,7 +3281,7 @@ async fn cmd_db_inventory_cabooses(
 async fn cmd_db_inventory_physical_disks(
     conn: &DataStoreConnection<'_>,
     limit: NonZeroU32,
-    args: PhysicalDisksArgs,
+    args: InvPhysicalDisksArgs,
 ) -> Result<(), anyhow::Error> {
     #[derive(Tabled)]
     #[tabled(rename_all = "SCREAMING_SNAKE_CASE")]
diff --git a/dev-tools/omdb/src/bin/omdb/nexus.rs b/dev-tools/omdb/src/bin/omdb/nexus.rs
index fb74ddd89b..8649d15aa6 100644
--- a/dev-tools/omdb/src/bin/omdb/nexus.rs
+++ b/dev-tools/omdb/src/bin/omdb/nexus.rs
@@ -24,15 +24,18 @@ use nexus_client::types::BackgroundTask;
 use nexus_client::types::BackgroundTasksActivateRequest;
 use nexus_client::types::CurrentStatus;
 use nexus_client::types::LastResult;
+use nexus_client::types::PhysicalDiskPath;
 use nexus_client::types::SledSelector;
 use nexus_client::types::UninitializedSledId;
 use nexus_db_queries::db::lookup::LookupPath;
+use nexus_saga_recovery::LastPass;
 use nexus_types::deployment::Blueprint;
 use nexus_types::internal_api::background::LookupRegionPortStatus;
 use nexus_types::internal_api::background::RegionReplacementDriverStatus;
 use nexus_types::inventory::BaseboardId;
 use omicron_uuid_kinds::CollectionUuid;
 use omicron_uuid_kinds::GenericUuid;
+use omicron_uuid_kinds::PhysicalDiskUuid;
 use omicron_uuid_kinds::SledUuid;
 use reedline::DefaultPrompt;
 use reedline::DefaultPromptSegment;
@@ -256,6 +259,8 @@ enum SledsCommands {
     Add(SledAddArgs),
     /// Expunge a sled (DANGEROUS)
     Expunge(SledExpungeArgs),
+    /// Expunge a disk (DANGEROUS)
+    ExpungeDisk(DiskExpungeArgs),
 }
 
 #[derive(Debug, Args)]
@@ -277,6 +282,17 @@ struct SledExpungeArgs {
     sled_id: SledUuid,
 }
 
+#[derive(Debug, Args)]
+struct DiskExpungeArgs {
+    // expunge is _extremely_ dangerous, so we also require a database
+    // connection to perform some safety checks
+    #[clap(flatten)]
+    db_url_opts: DbUrlOptions,
+
+    /// Physical disk ID
+    physical_disk_id: PhysicalDiskUuid,
+}
+
 impl NexusArgs {
     /// Run a `omdb nexus` subcommand.
     pub(crate) async fn run_cmd(
@@ -401,6 +417,13 @@ impl NexusArgs {
                 let token = omdb.check_allow_destructive()?;
                 cmd_nexus_sled_expunge(&client, args, omdb, log, token).await
             }
+            NexusCommands::Sleds(SledsArgs {
+                command: SledsCommands::ExpungeDisk(args),
+            }) => {
+                let token = omdb.check_allow_destructive()?;
+                cmd_nexus_sled_expunge_disk(&client, args, omdb, log, token)
+                    .await
+            }
         }
     }
 }
@@ -1083,6 +1106,136 @@ fn print_task_details(bgtask: &BackgroundTask, details: &serde_json::Value) {
                 }
             }
         };
+    } else if name == "saga_recovery" {
+        match serde_json::from_value::<nexus_saga_recovery::Report>(
+            details.clone(),
+        ) {
+            Err(error) => eprintln!(
+                "warning: failed to interpret task details: {:?}: {:?}",
+                error, details
+            ),
+
+            Ok(report) => {
+                println!("    since Nexus started:");
+                println!(
+                    "        sagas recovered:         {:3}",
+                    report.ntotal_recovered
+                );
+                println!(
+                    "        sagas recovery errors:   {:3}",
+                    report.ntotal_failures,
+                );
+                println!(
+                    "        sagas observed started:  {:3}",
+                    report.ntotal_started
+                );
+                println!(
+                    "        sagas inferred finished: {:3}",
+                    report.ntotal_finished
+                );
+                println!(
+                    "        missing from SEC:        {:3}",
+                    report.ntotal_sec_errors_missing,
+                );
+                println!(
+                    "        bad state in SEC:        {:3}",
+                    report.ntotal_sec_errors_bad_state,
+                );
+                match report.last_pass {
+                    LastPass::NeverStarted => {
+                        println!("    never run");
+                    }
+                    LastPass::Failed { message } => {
+                        println!("    last pass FAILED: {}", message);
+                    }
+                    LastPass::Success(success) => {
+                        println!("    last pass:");
+                        println!(
+                            "        found sagas: {:3} \
+                            (in-progress, assigned to this Nexus)",
+                            success.nfound
+                        );
+                        println!(
+                            "        recovered:   {:3} (successfully)",
+                            success.nrecovered
+                        );
+                        println!("        failed:      {:3}", success.nfailed);
+                        println!(
+                            "        skipped:     {:3} (already running)",
+                            success.nskipped
+                        );
+                        println!(
+                            "        removed:     {:3} (newly finished)",
+                            success.nskipped
+                        );
+                    }
+                };
+
+                if report.recent_recoveries.is_empty() {
+                    println!("    no recovered sagas");
+                } else {
+                    println!(
+                        "    recently recovered sagas ({}):",
+                        report.recent_recoveries.len()
+                    );
+
+                    #[derive(Tabled)]
+                    #[tabled(rename_all = "SCREAMING_SNAKE_CASE")]
+                    struct SagaRow {
+                        time: String,
+                        saga_id: String,
+                    }
+                    let table_rows =
+                        report.recent_recoveries.iter().map(|r| SagaRow {
+                            time: r
+                                .time
+                                .to_rfc3339_opts(SecondsFormat::Secs, true),
+                            saga_id: r.saga_id.to_string(),
+                        });
+                    let table = tabled::Table::new(table_rows)
+                        .with(tabled::settings::Style::empty())
+                        .with(tabled::settings::Padding::new(0, 1, 0, 0))
+                        .to_string();
+                    println!(
+                        "{}",
+                        textwrap::indent(&table.to_string(), "        ")
+                    );
+                }
+
+                if report.recent_failures.is_empty() {
+                    println!("    no saga recovery failures");
+                } else {
+                    println!(
+                        "    recent sagas recovery failures ({}):",
+                        report.recent_failures.len()
+                    );
+
+                    #[derive(Tabled)]
+                    #[tabled(rename_all = "SCREAMING_SNAKE_CASE")]
+                    struct SagaRow<'a> {
+                        time: String,
+                        saga_id: String,
+                        message: &'a str,
+                    }
+                    let table_rows =
+                        report.recent_failures.iter().map(|r| SagaRow {
+                            time: r
+                                .time
+                                .to_rfc3339_opts(SecondsFormat::Secs, true),
+                            saga_id: r.saga_id.to_string(),
+                            message: &r.message,
+                        });
+                    let table = tabled::Table::new(table_rows)
+                        .with(tabled::settings::Style::empty())
+                        .with(tabled::settings::Padding::new(0, 1, 0, 0))
+                        .to_string();
+                    println!(
+                        "{}",
+                        textwrap::indent(&table.to_string(), "        ")
+                    );
+                }
+            }
+        }
     } else if name == "lookup_region_port" {
         match serde_json::from_value::<LookupRegionPortStatus>(details.clone())
         {
@@ -1422,7 +1575,7 @@ async fn cmd_nexus_sleds_list_uninitialized(
         cubby: u16,
         serial: String,
         part: String,
-        revision: i64,
+        revision: u32,
     }
     let rows = sleds.into_iter().map(|sled| UninitializedSledRow {
         rack_id: sled.rack_id,
@@ -1458,6 +1611,39 @@ async fn cmd_nexus_sled_add(
     Ok(())
 }
 
+struct ConfirmationPrompt(Reedline);
+
+impl ConfirmationPrompt {
+    fn new() -> Self {
+        Self(Reedline::create())
+    }
+
+    fn read(&mut self, message: &str) -> Result<String, anyhow::Error> {
+        let prompt = DefaultPrompt::new(
+            DefaultPromptSegment::Basic(message.to_string()),
+            DefaultPromptSegment::Empty,
+        );
+        if let Ok(reedline::Signal::Success(input)) = self.0.read_line(&prompt)
+        {
+            Ok(input)
+        } else {
+            bail!("expungement aborted")
+        }
+    }
+
+    fn read_and_validate(
+        &mut self,
+        message: &str,
+        expected: &str,
+    ) -> Result<(), anyhow::Error> {
+        let input = self.read(message)?;
+        if input != expected {
+            bail!("Aborting, input did not match expected value");
+        }
+        Ok(())
+    }
+}
+
 /// Runs `omdb nexus sleds expunge`
 async fn cmd_nexus_sled_expunge(
     client: &nexus_client::Client,
@@ -1487,20 +1673,7 @@ async fn cmd_nexus_sled_expunge(
         .with_context(|| format!("failed to find sled {}", args.sled_id))?;
 
     // Helper to get confirmation messages from the user.
-    let mut line_editor = Reedline::create();
-    let mut read_with_prompt = move |message: &str| {
-        let prompt = DefaultPrompt::new(
-            DefaultPromptSegment::Basic(message.to_string()),
-            DefaultPromptSegment::Empty,
-        );
-        if let Ok(reedline::Signal::Success(input)) =
-            line_editor.read_line(&prompt)
-        {
-            Ok(input)
-        } else {
-            bail!("expungement aborted")
-        }
-    };
+    let mut prompt = ConfirmationPrompt::new();
 
     // Now check whether its sled-agent or SP were found in the most recent
     // inventory collection.
@@ -1530,11 +1703,7 @@ async fn cmd_nexus_sled_expunge(
                      proceed anyway?",
                     args.sled_id, collection.time_done,
                 );
-                let confirm = read_with_prompt("y/N")?;
-                if confirm != "y" {
-                    eprintln!("expungement not confirmed: aborting");
-                    return Ok(());
-                }
+                prompt.read_and_validate("y/N", "y")?;
             }
         }
         None => {
@@ -1552,11 +1721,7 @@ async fn cmd_nexus_sled_expunge(
         args.sled_id,
         sled.serial_number(),
     );
-    let confirm = read_with_prompt("sled serial number")?;
-    if confirm != sled.serial_number() {
-        eprintln!("sled serial number not confirmed: aborting");
-        return Ok(());
-    }
+    prompt.read_and_validate("sled serial number", sled.serial_number())?;
 
     let old_policy = client
         .sled_expunge(&SledSelector { sled: args.sled_id.into_untyped_uuid() })
@@ -1569,3 +1734,118 @@ async fn cmd_nexus_sled_expunge(
     );
     Ok(())
 }
+
+/// Runs `omdb nexus sleds expunge-disk`
+async fn cmd_nexus_sled_expunge_disk(
+    client: &nexus_client::Client,
+    args: &DiskExpungeArgs,
+    omdb: &Omdb,
+    log: &slog::Logger,
+    _destruction_token: DestructiveOperationToken,
+) -> Result<(), anyhow::Error> {
+    use nexus_db_queries::context::OpContext;
+
+    let datastore = args.db_url_opts.connect(omdb, log).await?;
+    let opctx = OpContext::for_tests(log.clone(), datastore.clone());
+    let opctx = &opctx;
+
+    // First, we need to look up the disk so we can lookup identity information.
+    let (_authz_physical_disk, physical_disk) =
+        LookupPath::new(opctx, &datastore)
+            .physical_disk(args.physical_disk_id.into_untyped_uuid())
+            .fetch()
+            .await
+            .with_context(|| {
+                format!(
+                    "failed to find physical disk {}",
+                    args.physical_disk_id
+                )
+            })?;
+
+    // Helper to get confirmation messages from the user.
+    let mut prompt = ConfirmationPrompt::new();
+
+    // Now check whether its sled-agent was found in the most recent
+    // inventory collection.
+    match datastore
+        .inventory_get_latest_collection(opctx)
+        .await
+        .context("loading latest collection")?
+    {
+        Some(collection) => {
+            let disk_identity = omicron_common::disk::DiskIdentity {
+                vendor: physical_disk.vendor.clone(),
+                serial: physical_disk.serial.clone(),
+                model: physical_disk.model.clone(),
+            };
+
+            let mut sleds_containing_disk = vec![];
+
+            for (sled_id, sled_agent) in collection.sled_agents {
+                for sled_disk in sled_agent.disks {
+                    if sled_disk.identity == disk_identity {
+                        sleds_containing_disk.push(sled_id);
+                    }
+                }
+            }
+
+            match sleds_containing_disk.len() {
+                0 => {}
+                1 => {
+                    eprintln!(
+                        "WARNING: physical disk {} is PRESENT in the most \
+                         recent inventory collection (spotted at {}). Although \
+                         expunging a running disk is supported, it is safer \
+                         to expunge a disk from a system where it has been \
+                         removed. Are you sure you want to proceed anyway?",
+                        args.physical_disk_id, collection.time_done,
+                    );
+                    prompt.read_and_validate("y/N", "y")?;
+                }
+                _ => {
+                    // This should be impossible due to a unique database index,
+                    // "vendor_serial_model_unique".
+                    //
+                    // Even if someone tried moving a disk, it would need to be
+                    // decommissioned before being re-commissioned elsewhere.
+                    //
+                    // However, we still print out an error message here in the
+                    // (unlikely) even that it happens anyway.
+                    eprintln!(
+                        "ERROR: physical disk {} is PRESENT MULTIPLE TIMES in \
+                        the most recent inventory collection (spotted at {}).
+                        This should not be possible, and is an indication of a \
+                        database issue.",
+                        args.physical_disk_id, collection.time_done,
+                    );
+                    bail!("Physical Disk appeared on multiple sleds");
+                }
+            }
+        }
+        None => {
+            eprintln!(
+                "ERROR: cannot verify the physical disk inventory status \
+                 because there are no inventory collections present. Please \
+                 ensure that inventory may be collected."
+            );
+            bail!("No inventory");
+        }
+    }
+
+    eprintln!(
+        "WARNING: This operation will PERMANENTLY and IRRECOVABLY mark physical disk \
+        {} ({}) expunged. To proceed, type the physical disk's serial number.",
+        args.physical_disk_id,
+        physical_disk.serial,
+    );
+    prompt.read_and_validate("disk serial number", &physical_disk.serial)?;
+
+    client
+        .physical_disk_expunge(&PhysicalDiskPath {
+            disk_id: args.physical_disk_id.into_untyped_uuid(),
+        })
+        .await
+        .context("expunging disk")?;
+    eprintln!("expunged disk {}", args.physical_disk_id);
+    Ok(())
+}
diff --git a/dev-tools/omdb/tests/env.out b/dev-tools/omdb/tests/env.out
index 66a48ab394..75acc5c584 100644
--- a/dev-tools/omdb/tests/env.out
+++ b/dev-tools/omdb/tests/env.out
@@ -118,6 +118,10 @@ task: "region_replacement_driver"
     drive region replacements forward to completion
 
 
+task: "saga_recovery"
+    recovers sagas assigned to this Nexus
+
+
 task: "service_firewall_rule_propagation"
     propagates VPC firewall rules for Omicron services with external network
     connectivity
@@ -254,6 +258,10 @@ task: "region_replacement_driver"
     drive region replacements forward to completion
 
 
+task: "saga_recovery"
+    recovers sagas assigned to this Nexus
+
+
 task: "service_firewall_rule_propagation"
     propagates VPC firewall rules for Omicron services with external network
     connectivity
@@ -377,6 +385,10 @@ task: "region_replacement_driver"
     drive region replacements forward to completion
 
 
+task: "saga_recovery"
+    recovers sagas assigned to this Nexus
+
+
 task: "service_firewall_rule_propagation"
     propagates VPC firewall rules for Omicron services with external network
     connectivity
diff --git a/dev-tools/omdb/tests/successes.out b/dev-tools/omdb/tests/successes.out
index a65098d7aa..982a9d8403 100644
--- a/dev-tools/omdb/tests/successes.out
+++ b/dev-tools/omdb/tests/successes.out
@@ -319,6 +319,10 @@ task: "region_replacement_driver"
     drive region replacements forward to completion
 
 
+task: "saga_recovery"
+    recovers sagas assigned to this Nexus
+
+
 task: "service_firewall_rule_propagation"
     propagates VPC firewall rules for Omicron services with external network
     connectivity
@@ -534,6 +538,27 @@ task: "region_replacement_driver"
     number of region replacement finish sagas started ok: 0
     number of errors: 0
 
+task: "saga_recovery"
+  configured period: every 10m
+  currently executing: no
+  last completed activation: <REDACTED ITERATIONS>, triggered by a periodic timer firing
+    started at <REDACTED     TIMESTAMP> (<REDACTED DURATION>s ago) and ran for <REDACTED DURATION>ms
+    since Nexus started:
+        sagas recovered:           0
+        sagas recovery errors:     0
+        sagas observed started:    0
+        sagas inferred finished:   0
+        missing from SEC:          0
+        bad state in SEC:          0
+    last pass:
+        found sagas:   0 (in-progress, assigned to this Nexus)
+        recovered:     0 (successfully)
+        failed:        0
+        skipped:       0 (already running)
+        removed:       0 (newly finished)
+    no recovered sagas
+    no saga recovery failures
+
 task: "service_firewall_rule_propagation"
   configured period: every 5m
   currently executing: no
diff --git a/dev-tools/omdb/tests/usage_errors.out b/dev-tools/omdb/tests/usage_errors.out
index 8762907e81..3d6f2af112 100644
--- a/dev-tools/omdb/tests/usage_errors.out
+++ b/dev-tools/omdb/tests/usage_errors.out
@@ -105,9 +105,10 @@ Usage: omdb db [OPTIONS] <COMMAND>
 
 Commands:
   rack                 Print information about the rack
-  disks                Print information about disks
+  disks                Print information about virtual disks
   dns                  Print information about internal and external DNS
   inventory            Print information about collected hardware/software inventory
+  physical-disks       Print information about physical disks
   reconfigurator-save  Save the current Reconfigurator inputs to a file
   region               Print information about regions
   region-replacement   Query for information about region replacements, optionally manually
@@ -146,9 +147,10 @@ Usage: omdb db [OPTIONS] <COMMAND>
 
 Commands:
   rack                 Print information about the rack
-  disks                Print information about disks
+  disks                Print information about virtual disks
   dns                  Print information about internal and external DNS
   inventory            Print information about collected hardware/software inventory
+  physical-disks       Print information about physical disks
   reconfigurator-save  Save the current Reconfigurator inputs to a file
   region               Print information about regions
   region-replacement   Query for information about region replacements, optionally manually
@@ -185,7 +187,7 @@ termination: Exited(2)
 stdout:
 ---------------------------------------------
 stderr:
-Print information about disks
+Print information about virtual disks
 
 Usage: omdb db disks [OPTIONS] <COMMAND>
 
@@ -526,6 +528,7 @@ Commands:
   list-uninitialized  List all uninitialized sleds
   add                 Add an uninitialized sled
   expunge             Expunge a sled (DANGEROUS)
+  expunge-disk        Expunge a disk (DANGEROUS)
   help                Print this message or the help of the given subcommand(s)
 
 Options:
diff --git a/dev-tools/openapi-manager/Cargo.toml b/dev-tools/openapi-manager/Cargo.toml
index b50aeec69f..aa0cfacfd5 100644
--- a/dev-tools/openapi-manager/Cargo.toml
+++ b/dev-tools/openapi-manager/Cargo.toml
@@ -12,9 +12,11 @@ anyhow.workspace = true
 atomicwrites.workspace = true
 camino.workspace = true
 clap.workspace = true
+dns-server-api.workspace = true
 dropshot.workspace = true
 fs-err.workspace = true
 indent_write.workspace = true
+installinator-api.workspace = true
 nexus-internal-api.workspace = true
 omicron-workspace-hack.workspace = true
 openapiv3.workspace = true
@@ -23,3 +25,4 @@ owo-colors.workspace = true
 serde_json.workspace = true
 similar.workspace = true
 supports-color.workspace = true
+wicketd-api.workspace = true
diff --git a/dev-tools/openapi-manager/src/spec.rs b/dev-tools/openapi-manager/src/spec.rs
index 37330d6922..83f0f4dd57 100644
--- a/dev-tools/openapi-manager/src/spec.rs
+++ b/dev-tools/openapi-manager/src/spec.rs
@@ -15,13 +15,43 @@ use openapiv3::OpenAPI;
 pub fn all_apis() -> Vec<ApiSpec> {
     vec![
         ApiSpec {
-            title: "Nexus internal API".to_string(),
-            version: "0.0.1".to_string(),
-            description: "Nexus internal API".to_string(),
+            title: "Internal DNS",
+            version: "0.0.1",
+            description: "API for the internal DNS server",
+            boundary: ApiBoundary::Internal,
+            api_description:
+                dns_server_api::dns_server_api::stub_api_description,
+            filename: "dns-server.json",
+            extra_validation: None,
+        },
+        ApiSpec {
+            title: "Installinator API",
+            version: "0.0.1",
+            description: "API for installinator to fetch artifacts \
+                and report progress",
+            boundary: ApiBoundary::Internal,
+            api_description:
+                installinator_api::installinator_api::stub_api_description,
+            filename: "installinator.json",
+            extra_validation: None,
+        },
+        ApiSpec {
+            title: "Nexus internal API",
+            version: "0.0.1",
+            description: "Nexus internal API",
             boundary: ApiBoundary::Internal,
             api_description:
                 nexus_internal_api::nexus_internal_api_mod::stub_api_description,
-            filename: "nexus-internal.json".to_string(),
+            filename: "nexus-internal.json",
+            extra_validation: None,
+        },
+        ApiSpec {
+            title: "Oxide Technician Port Control Service",
+            version: "0.0.1",
+            description: "API for use by the technician port TUI: wicket",
+            boundary: ApiBoundary::Internal,
+            api_description: wicketd_api::wicketd_api_mod::stub_api_description,
+            filename: "wicketd.json",
             extra_validation: None,
         },
         // Add your APIs here! Please keep this list sorted by filename.
@@ -30,13 +60,13 @@ pub fn all_apis() -> Vec<ApiSpec> {
 
 pub struct ApiSpec {
     /// The title.
-    pub title: String,
+    pub title: &'static str,
 
     /// The version.
-    pub version: String,
+    pub version: &'static str,
 
     /// The description string.
-    pub description: String,
+    pub description: &'static str,
 
     /// Whether this API is internal or external.
     pub boundary: ApiBoundary,
@@ -47,7 +77,7 @@ pub struct ApiSpec {
         fn() -> Result<ApiDescription<StubContext>, ApiDescriptionBuildErrors>,
 
     /// The JSON filename to write the API description to.
-    pub filename: String,
+    pub filename: &'static str,
 
     /// Extra validation to perform on the OpenAPI spec, if any.
     pub extra_validation: Option<fn(&OpenAPI) -> anyhow::Result<()>>,
diff --git a/dev-tools/releng/src/main.rs b/dev-tools/releng/src/main.rs
index 1bd3b69ac9..ee649e79b2 100644
--- a/dev-tools/releng/src/main.rs
+++ b/dev-tools/releng/src/main.rs
@@ -41,7 +41,7 @@ use crate::job::Jobs;
 /// to as "v8", "version 8", or "release 8" to customers). The use of semantic
 /// versioning is mostly to hedge for perhaps wanting something more granular in
 /// the future.
-const BASE_VERSION: Version = Version::new(9, 0, 0);
+const BASE_VERSION: Version = Version::new(10, 0, 0);
 
 const RETRY_ATTEMPTS: usize = 3;
 
diff --git a/dev-tools/xtask/src/check_features.rs b/dev-tools/xtask/src/check_features.rs
new file mode 100644
index 0000000000..a9dbc2bff7
--- /dev/null
+++ b/dev-tools/xtask/src/check_features.rs
@@ -0,0 +1,212 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+//! Subcommand: cargo xtask check-features
+
+use anyhow::{bail, Result};
+use camino::Utf8PathBuf;
+use clap::Parser;
+use std::{collections::HashSet, process::Command};
+
+const SUPPORTED_ARCHITECTURES: [&str; 1] = ["x86_64"];
+const CI_EXCLUDED_FEATURES: [&str; 2] = ["image-trampoline", "image-standard"];
+
+#[derive(Parser)]
+pub struct Args {
+    /// Run in CI mode, with a default set of features excluded.
+    #[clap(long, default_value_t = false)]
+    ci: bool,
+    /// Features to exclude from the check.
+    #[clap(long, value_name = "FEATURES")]
+    exclude_features: Option<Vec<String>>,
+    /// Depth of the feature powerset to check.
+    #[clap(long, value_name = "NUM")]
+    depth: Option<usize>,
+    /// Error format passed to `cargo hack check`.
+    #[clap(long, value_name = "FMT")]
+    message_format: Option<String>,
+    /// Version of `cargo-hack` to install. By default, we download a pre-built
+    /// version.
+    #[clap(long, value_name = "VERSION")]
+    install_version: Option<String>,
+}
+
+/// Run `cargo hack check`.
+pub fn run_cmd(args: Args) -> Result<()> {
+    // We cannot specify both `--ci` and `--install-version`, as the former
+    // implies we are using a pre-built version.
+    if args.ci && args.install_version.is_some() {
+        bail!("cannot specify --ci and --install-version together");
+    }
+
+    let cargo =
+        std::env::var("CARGO").unwrap_or_else(|_| String::from("cargo"));
+
+    let mut command = Command::new(&cargo);
+
+    // Add the `hack check` subcommand.
+    command.args(&["hack", "check"]);
+
+    if args.ci {
+        install_prebuilt_cargo_hack(&cargo)?;
+
+        let ex = if let Some(mut features) = args.exclude_features {
+            // Extend the list of features to exclude with the CI defaults.
+            features.extend(
+                CI_EXCLUDED_FEATURES.into_iter().map(|s| s.to_string()),
+            );
+
+            // Remove duplicates.
+            let excludes = features.into_iter().collect::<HashSet<_>>();
+
+            excludes.into_iter().collect::<Vec<_>>().join(",")
+        } else {
+            CI_EXCLUDED_FEATURES.join(",")
+        };
+
+        // Add the `--exclude-features` flag if we are running in CI mode.
+        command.args(["--exclude-features", &ex]);
+    } else {
+        install_cargo_hack(&cargo, args.install_version)?;
+        // Add "only" the `--exclude-features` flag if it was provided.
+        if let Some(features) = args.exclude_features {
+            command.args(["--exclude-features", &features.join(",")]);
+        }
+    }
+
+    if let Some(depth) = args.depth {
+        command.args(&["--depth", &depth.to_string()]);
+    }
+
+    // Pass along the `--message-format` flag if it was provided.
+    if let Some(fmt) = args.message_format {
+        command.args(["--message-format", &fmt]);
+    }
+
+    command
+        // Make sure we check everything.
+        .arg("--workspace")
+        // We want to check the binaries.
+        .arg("--bins")
+        // We want to check the feature powerset.
+        .arg("--feature-powerset")
+        // We will not check the dev-dependencies, which should covered by tests.
+        .arg("--no-dev-deps");
+
+    exec(command)
+}
+
+/// The supported operating systems.
+enum Os {
+    Illumos,
+    Linux,
+    Mac,
+}
+
+/// Get the current OS.
+fn os_name() -> Result<Os> {
+    let os = match std::env::consts::OS {
+        "linux" => Os::Linux,
+        "macos" => Os::Mac,
+        "solaris" | "illumos" => Os::Illumos,
+        other => bail!("OS not supported: {other}"),
+    };
+    Ok(os)
+}
+
+/// This is a workaround for the lack of a CARGO_WORKSPACE_DIR environment
+/// variable, as suggested in <https://github.com/rust-lang/cargo/issues/3946#issuecomment-1433384192>.
+/// A better workaround might be to set this in the `[env]` section of
+/// `.cargo/config.toml`.
+fn project_root() -> Utf8PathBuf {
+    Utf8PathBuf::from(&concat!(env!("CARGO_MANIFEST_DIR"), "/.."))
+}
+
+/// Get the path to the `out` directory from the project root/workspace
+/// directory.
+fn out_dir() -> Utf8PathBuf {
+    project_root().join("out/cargo-hack")
+}
+
+/// Install `cargo-hack` if the `install-version` was specified; otherwise,
+/// download a pre-built version if it's not already in our `out` directory.
+fn install_cargo_hack(cargo: &str, version: Option<String>) -> Result<()> {
+    if let Some(version) = version {
+        let mut command = Command::new(cargo);
+
+        eprintln!(
+            "installing cargo-hack at version {} to {}",
+            version,
+            env!("CARGO_HOME")
+        );
+        command.args(&["install", "cargo-hack", "--version", &version]);
+        exec(command)
+    } else if !out_dir().exists() {
+        install_prebuilt_cargo_hack(cargo)
+    } else {
+        let out_dir = out_dir();
+        eprintln!("cargo-hack found in {}", out_dir);
+        Ok(())
+    }
+}
+
+/// Download a pre-built version of `cargo-hack` to the `out` directory via the
+/// download `xtask`.
+fn install_prebuilt_cargo_hack(cargo: &str) -> Result<()> {
+    let mut command = Command::new(cargo);
+
+    let out_dir = out_dir();
+    eprintln!(
+        "cargo-hack not found in {}, downloading a pre-built version",
+        out_dir
+    );
+
+    let os = os_name()?;
+    match os {
+        Os::Illumos | Os::Linux | Os::Mac
+            if SUPPORTED_ARCHITECTURES.contains(&std::env::consts::ARCH) =>
+        {
+            // Download the pre-built version of `cargo-hack` via our
+            // download `xtask`.
+            command.args(&["xtask", "download", "cargo-hack"]);
+        }
+        _ => {
+            bail!(
+                "cargo-hack is not pre-built for this os {} / arch {}",
+                std::env::consts::OS,
+                std::env::consts::ARCH
+            );
+        }
+    }
+
+    exec(command)
+}
+
+/// Execute the command and check the exit status.
+fn exec(mut command: Command) -> Result<()> {
+    let cargo =
+        std::env::var("CARGO").unwrap_or_else(|_| String::from("cargo"));
+
+    eprintln!(
+        "running: {:?} {}",
+        &cargo,
+        command
+            .get_args()
+            .map(|arg| format!("{:?}", arg.to_str().unwrap()))
+            .collect::<Vec<_>>()
+            .join(" ")
+    );
+
+    let exit_status = command
+        .spawn()
+        .expect("failed to spawn child process")
+        .wait()
+        .expect("failed to wait for child process");
+
+    if !exit_status.success() {
+        bail!("cargo-hack install failed: {}", exit_status);
+    }
+
+    Ok(())
+}
diff --git a/dev-tools/xtask/src/download.rs b/dev-tools/xtask/src/download.rs
index 2790a638a7..b5910e3915 100644
--- a/dev-tools/xtask/src/download.rs
+++ b/dev-tools/xtask/src/download.rs
@@ -17,6 +17,7 @@ use std::io::Write;
 use std::os::unix::fs::PermissionsExt;
 use std::sync::OnceLock;
 use std::time::Duration;
+use strum::Display;
 use strum::EnumIter;
 use strum::IntoEnumIterator;
 use tar::Archive;
@@ -25,6 +26,9 @@ use tokio::process::Command;
 
 const BUILDOMAT_URL: &'static str =
     "https://buildomat.eng.oxide.computer/public/file";
+const CARGO_HACK_URL: &'static str =
+    "https://github.com/taiki-e/cargo-hack/releases/download";
+
 const RETRY_ATTEMPTS: usize = 3;
 
 /// What is being downloaded?
@@ -44,6 +48,9 @@ enum Target {
     /// Download all targets
     All,
 
+    /// `cargo hack` binary
+    CargoHack,
+
     /// Clickhouse binary
     Clickhouse,
 
@@ -124,6 +131,7 @@ pub async fn run_cmd(args: DownloadArgs) -> Result<()> {
                     Target::All => {
                         bail!("We should have already filtered this 'All' target out?");
                     }
+                    Target::CargoHack => downloader.download_cargo_hack().await,
                     Target::Clickhouse => downloader.download_clickhouse().await,
                     Target::Cockroach => downloader.download_cockroach().await,
                     Target::Console => downloader.download_console().await,
@@ -151,12 +159,19 @@ pub async fn run_cmd(args: DownloadArgs) -> Result<()> {
     Ok(())
 }
 
+#[derive(Display)]
 enum Os {
     Illumos,
     Linux,
     Mac,
 }
 
+#[derive(Display)]
+enum Arch {
+    X86_64,
+    Aarch64,
+}
+
 impl Os {
     fn env_name(&self) -> &'static str {
         match self {
@@ -177,6 +192,15 @@ fn os_name() -> Result<Os> {
     Ok(os)
 }
 
+fn arch() -> Result<Arch> {
+    let arch = match std::env::consts::ARCH {
+        "x86_64" => Arch::X86_64,
+        "aarch64" => Arch::Aarch64,
+        other => bail!("Architecture not supported: {other}"),
+    };
+    Ok(arch)
+}
+
 struct Downloader<'a> {
     log: Logger,
 
@@ -218,7 +242,7 @@ async fn get_values_from_file<const N: usize>(
 
     let content = tokio::fs::read_to_string(&path)
         .await
-        .context("Failed to read {path}")?;
+        .with_context(|| format!("Failed to read {path}"))?;
     for line in content.lines() {
         let line = line.trim();
         let Some((key, value)) = line.split_once('=') else {
@@ -432,6 +456,59 @@ async fn download_file_and_verify(
 }
 
 impl<'a> Downloader<'a> {
+    async fn download_cargo_hack(&self) -> Result<()> {
+        let os = os_name()?;
+        let arch = arch()?;
+
+        let download_dir = self.output_dir.join("downloads");
+        let destination_dir = self.output_dir.join("cargo-hack");
+
+        let checksums_path = self.versions_dir.join("cargo_hack_checksum");
+        let [checksum] = get_values_from_file(
+            [&format!("CIDL_SHA256_{}", os.env_name())],
+            &checksums_path,
+        )
+        .await?;
+
+        let versions_path = self.versions_dir.join("cargo_hack_version");
+        let version = tokio::fs::read_to_string(&versions_path)
+            .await
+            .context("Failed to read version from {versions_path}")?;
+        let version = version.trim();
+
+        let (platform, supported_arch) = match (os, arch) {
+            (Os::Illumos, Arch::X86_64) => ("unknown-illumos", "x86_64"),
+            (Os::Linux, Arch::X86_64) => ("unknown-linux-gnu", "x86_64"),
+            (Os::Linux, Arch::Aarch64) => ("unknown-linux-gnu", "aarch64"),
+            (Os::Mac, Arch::X86_64) => ("apple-darwin", "x86_64"),
+            (Os::Mac, Arch::Aarch64) => ("apple-darwin", "aarch64"),
+            (os, arch) => bail!("Unsupported OS/arch: {os}/{arch}"),
+        };
+
+        let tarball_filename =
+            format!("cargo-hack-{supported_arch}-{platform}.tar.gz");
+        let tarball_url =
+            format!("{CARGO_HACK_URL}/v{version}/{tarball_filename}");
+
+        let tarball_path = download_dir.join(&tarball_filename);
+
+        tokio::fs::create_dir_all(&download_dir).await?;
+        tokio::fs::create_dir_all(&destination_dir).await?;
+
+        download_file_and_verify(
+            &self.log,
+            &tarball_path,
+            &tarball_url,
+            ChecksumAlgorithm::Sha2,
+            &checksum,
+        )
+        .await?;
+
+        unpack_tarball(&self.log, &tarball_path, &destination_dir).await?;
+
+        Ok(())
+    }
+
     async fn download_clickhouse(&self) -> Result<()> {
         let os = os_name()?;
 
diff --git a/dev-tools/xtask/src/main.rs b/dev-tools/xtask/src/main.rs
index d0a61272a9..0ea2332c31 100644
--- a/dev-tools/xtask/src/main.rs
+++ b/dev-tools/xtask/src/main.rs
@@ -10,6 +10,7 @@ use anyhow::{Context, Result};
 use cargo_metadata::Metadata;
 use clap::{Parser, Subcommand};
 
+mod check_features;
 mod check_workspace_deps;
 mod clippy;
 mod download;
@@ -38,6 +39,8 @@ enum Cmds {
     /// Run Argon2 hash with specific parameters (quick performance check)
     Argon2(external::External),
 
+    /// Check that all features are flagged correctly
+    CheckFeatures(check_features::Args),
     /// Check that dependencies are not duplicated in any packages in the
     /// workspace
     CheckWorkspaceDeps,
@@ -91,6 +94,7 @@ async fn main() -> Result<()> {
             external.cargo_args(["--release"]).exec_example("argon2")
         }
         Cmds::Clippy(args) => clippy::run_cmd(args),
+        Cmds::CheckFeatures(args) => check_features::run_cmd(args),
         Cmds::CheckWorkspaceDeps => check_workspace_deps::run_cmd(),
         Cmds::Download(args) => download::run_cmd(args).await,
         Cmds::Openapi(external) => external.exec_bin("openapi-manager"),
diff --git a/dns-server-api/Cargo.toml b/dns-server-api/Cargo.toml
new file mode 100644
index 0000000000..c87af14e0d
--- /dev/null
+++ b/dns-server-api/Cargo.toml
@@ -0,0 +1,15 @@
+[package]
+name = "dns-server-api"
+version = "0.1.0"
+edition = "2021"
+license = "MPL-2.0"
+
+[lints]
+workspace = true
+
+[dependencies]
+chrono.workspace = true
+dropshot.workspace = true
+omicron-workspace-hack.workspace = true
+schemars.workspace = true
+serde.workspace = true
diff --git a/dns-server-api/src/lib.rs b/dns-server-api/src/lib.rs
new file mode 100644
index 0000000000..2c59caf0c5
--- /dev/null
+++ b/dns-server-api/src/lib.rs
@@ -0,0 +1,160 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+//! Dropshot API for configuring DNS namespace.
+//!
+//! ## Shape of the API
+//!
+//! The DNS configuration API has just two endpoints: PUT and GET of the entire
+//! DNS configuration.  This is pretty anti-REST.  But it's important to think
+//! about how this server fits into the rest of the system.  When changes are
+//! made to DNS data, they're grouped together and assigned a monotonically
+//! increasing generation number.  The DNS data is first stored into CockroachDB
+//! and then propagated from a distributed fleet of Nexus instances to a
+//! distributed fleet of these DNS servers.  If we accepted individual updates to
+//! DNS names, then propagating a particular change would be non-atomic, and
+//! Nexus would have to do a lot more work to ensure (1) that all changes were
+//! propagated (even if it crashes) and (2) that they were propagated in the
+//! correct order (even if two Nexus instances concurrently propagate separate
+//! changes).
+//!
+//! This DNS server supports hosting multiple zones.  We could imagine supporting
+//! separate endpoints to update the DNS data for a particular zone.  That feels
+//! nicer (although it's not clear what it would buy us).  But as with updates to
+//! multiple names, Nexus's job is potentially much easier if the entire state
+//! for all zones is updated at once.  (Otherwise, imagine how Nexus would
+//! implement _renaming_ one zone to another without loss of service.  With
+//! a combined endpoint and generation number for all zones, all that's necessary
+//! is to configure a new zone with all the same names, and then remove the old
+//! zone later in another update.  That can be managed by the same mechanism in
+//! Nexus that manages regular name updates.  On the other hand, if there were
+//! separate endpoints with separate generation numbers, then Nexus has more to
+//! keep track of in order to do the rename safely.)
+//!
+//! See RFD 367 for more on DNS propagation.
+//!
+//! ## ETags and Conditional Requests
+//!
+//! It's idiomatic in HTTP use ETags and conditional requests to provide
+//! synchronization.  We could define an ETag to be just the current generation
+//! number of the server and honor standard `if-match` headers to fail requests
+//! where the generation number doesn't match what the client expects.  This
+//! would be fine, but it's rather annoying:
+//!
+//! 1. When the client wants to propagate generation X, the client would have
+//!    make an extra request just to fetch the current ETag, just so it can put
+//!    it into the conditional request.
+//!
+//! 2. If some other client changes the configuration in the meantime, the
+//!    conditional request would fail and the client would have to take another
+//!    lap (fetching the current config and potentially making another
+//!    conditional PUT).
+//!
+//! 3. This approach would make synchronization opt-in.  If a client (or just
+//!    one errant code path) neglected to set the if-match header, we could do
+//!    the wrong thing and cause the system to come to rest with the wrong DNS
+//!    data.
+//!
+//! Since the semantics here are so simple (we only ever want to move the
+//! generation number forward), we don't bother with ETags or conditional
+//! requests.  Instead we have the server implement the behavior we want, which
+//! is that when a request comes in to update DNS data to generation X, the
+//! server replies with one of:
+//!
+//! (1) the update has been applied and the server is now running generation X
+//!     (client treats this as success)
+//!
+//! (2) the update was not applied because the server is already at generation X
+//!     (client treats this as success)
+//!
+//! (3) the update was not applied because the server is already at a newer
+//!     generation
+//!     (client probably starts the whole propagation process over because its
+//!     current view of the world is out of date)
+//!
+//! This way, the DNS data can never move backwards and the client only ever has
+//! to make one request.
+//!
+//! ## Concurrent updates
+//!
+//! Given that we've got just one API to update the all DNS zones, and given
+//! that might therefore take a minute for a large zone, and also that there may
+//! be multiple Nexus instances trying to do it at the same time, we need to
+//! think a bit about what should happen if two Nexus do try to do it at the same
+//! time.  Spoiler: we immediately fail any request to update the DNS data if
+//! there's already an update in progress.
+//!
+//! What else could we do?  We could queue the incoming request behind the
+//! in-progress one.  How large do we allow that queue to grow?  At some point
+//! we'll need to stop queueing them.  So why bother at all?
+
+use std::{
+    collections::HashMap,
+    net::{Ipv4Addr, Ipv6Addr},
+};
+
+use dropshot::{HttpError, HttpResponseOk, RequestContext};
+use schemars::JsonSchema;
+use serde::{Deserialize, Serialize};
+
+#[dropshot::api_description]
+pub trait DnsServerApi {
+    type Context;
+
+    #[endpoint(
+        method = GET,
+        path = "/config",
+    )]
+    async fn dns_config_get(
+        rqctx: RequestContext<Self::Context>,
+    ) -> Result<HttpResponseOk<DnsConfig>, HttpError>;
+
+    #[endpoint(
+        method = PUT,
+        path = "/config",
+    )]
+    async fn dns_config_put(
+        rqctx: RequestContext<Self::Context>,
+        rq: dropshot::TypedBody<DnsConfigParams>,
+    ) -> Result<dropshot::HttpResponseUpdatedNoContent, dropshot::HttpError>;
+}
+
+#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema)]
+pub struct DnsConfigParams {
+    pub generation: u64,
+    pub time_created: chrono::DateTime<chrono::Utc>,
+    pub zones: Vec<DnsConfigZone>,
+}
+
+#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema)]
+pub struct DnsConfig {
+    pub generation: u64,
+    pub time_created: chrono::DateTime<chrono::Utc>,
+    pub time_applied: chrono::DateTime<chrono::Utc>,
+    pub zones: Vec<DnsConfigZone>,
+}
+
+#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema)]
+pub struct DnsConfigZone {
+    pub zone_name: String,
+    pub records: HashMap<String, Vec<DnsRecord>>,
+}
+
+#[allow(clippy::upper_case_acronyms)]
+#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema, PartialEq)]
+#[serde(tag = "type", content = "data")]
+pub enum DnsRecord {
+    A(Ipv4Addr),
+    AAAA(Ipv6Addr),
+    SRV(SRV),
+}
+
+#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema, PartialEq)]
+#[serde(rename = "Srv")]
+pub struct SRV {
+    pub prio: u16,
+    pub weight: u16,
+    pub port: u16,
+    pub target: String,
+}
diff --git a/dns-server/Cargo.toml b/dns-server/Cargo.toml
index 237d2a2fbb..d11dabaf85 100644
--- a/dns-server/Cargo.toml
+++ b/dns-server/Cargo.toml
@@ -12,6 +12,7 @@ anyhow.workspace = true
 camino.workspace = true
 chrono.workspace = true
 clap.workspace = true
+dns-server-api.workspace = true
 dns-service-client.workspace = true
 dropshot.workspace = true
 http.workspace = true
diff --git a/dns-server/src/bin/apigen.rs b/dns-server/src/bin/apigen.rs
deleted file mode 100644
index e130ee0211..0000000000
--- a/dns-server/src/bin/apigen.rs
+++ /dev/null
@@ -1,29 +0,0 @@
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this
-// file, You can obtain one at https://mozilla.org/MPL/2.0/.
-
-//! Generate the OpenAPI spec for the DNS server
-
-use anyhow::{bail, Result};
-use dns_server::http_server::api;
-use std::fs::File;
-use std::io;
-
-fn usage(args: &[String]) -> String {
-    format!("{} [output path]", args[0])
-}
-
-fn main() -> Result<()> {
-    let args: Vec<String> = std::env::args().collect();
-
-    let mut out = match args.len() {
-        1 => Box::new(io::stdout()) as Box<dyn io::Write>,
-        2 => Box::new(File::create(args[1].clone())?) as Box<dyn io::Write>,
-        _ => bail!(usage(&args)),
-    };
-
-    let api = api();
-    let openapi = api.openapi("Internal DNS", "v0.1.0");
-    openapi.write(&mut out)?;
-    Ok(())
-}
diff --git a/dns-server/src/dns_server.rs b/dns-server/src/dns_server.rs
index 01a8430b62..5c761f2aa3 100644
--- a/dns-server/src/dns_server.rs
+++ b/dns-server/src/dns_server.rs
@@ -7,12 +7,12 @@
 //! The facilities here handle binding a UDP socket, receiving DNS messages on
 //! that socket, and replying to them.
 
-use crate::dns_types::DnsRecord;
 use crate::storage;
 use crate::storage::QueryError;
 use crate::storage::Store;
 use anyhow::anyhow;
 use anyhow::Context;
+use dns_server_api::DnsRecord;
 use pretty_hex::*;
 use serde::Deserialize;
 use slog::{debug, error, info, o, trace, Logger};
@@ -234,12 +234,7 @@ fn dns_record_to_record(
             Ok(aaaa)
         }
 
-        DnsRecord::SRV(crate::dns_types::SRV {
-            prio,
-            weight,
-            port,
-            target,
-        }) => {
+        DnsRecord::SRV(dns_server_api::SRV { prio, weight, port, target }) => {
             let tgt = Name::from_str(&target).map_err(|error| {
                 RequestError::ServFail(anyhow!(
                     "serialization failed due to bad SRV target {:?}: {:#}",
diff --git a/dns-server/src/dns_types.rs b/dns-server/src/dns_types.rs
deleted file mode 100644
index 941124feb6..0000000000
--- a/dns-server/src/dns_types.rs
+++ /dev/null
@@ -1,50 +0,0 @@
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this
-// file, You can obtain one at https://mozilla.org/MPL/2.0/.
-
-//! types describing DNS records and configuration
-
-use schemars::JsonSchema;
-use serde::{Deserialize, Serialize};
-use std::collections::HashMap;
-use std::net::Ipv4Addr;
-use std::net::Ipv6Addr;
-
-#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema)]
-pub struct DnsConfigParams {
-    pub generation: u64,
-    pub time_created: chrono::DateTime<chrono::Utc>,
-    pub zones: Vec<DnsConfigZone>,
-}
-
-#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema)]
-pub struct DnsConfig {
-    pub generation: u64,
-    pub time_created: chrono::DateTime<chrono::Utc>,
-    pub time_applied: chrono::DateTime<chrono::Utc>,
-    pub zones: Vec<DnsConfigZone>,
-}
-
-#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema)]
-pub struct DnsConfigZone {
-    pub zone_name: String,
-    pub records: HashMap<String, Vec<DnsRecord>>,
-}
-
-#[allow(clippy::upper_case_acronyms)]
-#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema, PartialEq)]
-#[serde(tag = "type", content = "data")]
-pub enum DnsRecord {
-    A(Ipv4Addr),
-    AAAA(Ipv6Addr),
-    SRV(SRV),
-}
-
-#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema, PartialEq)]
-#[serde(rename = "Srv")]
-pub struct SRV {
-    pub prio: u16,
-    pub weight: u16,
-    pub port: u16,
-    pub target: String,
-}
diff --git a/dns-server/src/http_server.rs b/dns-server/src/http_server.rs
index e50346d828..84ffbc90e9 100644
--- a/dns-server/src/http_server.rs
+++ b/dns-server/src/http_server.rs
@@ -4,102 +4,12 @@
 
 //! Dropshot server for configuring DNS namespace
 
-// Shape of the API
-// ------------------------------
-//
-// The DNS configuration API has just two endpoints: PUT and GET of the entire
-// DNS configuration.  This is pretty anti-REST.  But it's important to think
-// about how this server fits into the rest of the system.  When changes are
-// made to DNS data, they're grouped together and assigned a monotonically
-// increasing generation number.  The DNS data is first stored into CockroachDB
-// and then propagated from a distributed fleet of Nexus instances to a
-// distributed fleet of these DNS servers.  If we accepted individual updates to
-// DNS names, then propagating a particular change would be non-atomic, and
-// Nexus would have to do a lot more work to ensure (1) that all changes were
-// propagated (even if it crashes) and (2) that they were propagated in the
-// correct order (even if two Nexus instances concurrently propagate separate
-// changes).
-//
-// This DNS server supports hosting multiple zones.  We could imagine supporting
-// separate endpoints to update the DNS data for a particular zone.  That feels
-// nicer (although it's not clear what it would buy us).  But as with updates to
-// multiple names, Nexus's job is potentially much easier if the entire state
-// for all zones is updated at once.  (Otherwise, imagine how Nexus would
-// implement _renaming_ one zone to another without loss of service.  With
-// a combined endpoint and generation number for all zones, all that's necessary
-// is to configure a new zone with all the same names, and then remove the old
-// zone later in another update.  That can be managed by the same mechanism in
-// Nexus that manages regular name updates.  On the other hand, if there were
-// separate endpoints with separate generation numbers, then Nexus has more to
-// keep track of in order to do the rename safely.)
-//
-// See RFD 367 for more on DNS propagation.
-//
-//
-// ETags and Conditional Requests
-// ------------------------------
-//
-// It's idiomatic in HTTP use ETags and conditional requests to provide
-// synchronization.  We could define an ETag to be just the current generation
-// number of the server and honor standard `if-match` headers to fail requests
-// where the generation number doesn't match what the client expects.  This
-// would be fine, but it's rather annoying:
-//
-// (1) When the client wants to propagate generation X, the client would have
-//     make an extra request just to fetch the current ETag, just so it can put
-//     it into the conditional request.
-//
-// (2) If some other client changes the configuration in the meantime, the
-//     conditional request would fail and the client would  have to take another
-//     lap (fetching the current config and potentially making another
-//     conditional PUT).
-//
-// (3) This approach would make synchronization opt-in.  If a client (or just
-//     one errant code path) neglected to set the if-match header, we could do
-//     the wrong thing and cause the system to come to rest with the wrong DNS
-//     data.
-//
-// Since the semantics here are so simple (we only ever want to move the
-// generation number forward), we don't bother with ETags or conditional
-// requests.  Instead we have the server implement the behavior we want, which
-// is that when a request comes in to update DNS data to generation X, the
-// server replies with one of:
-//
-// (1) the update has been applied and the server is now running generation X
-//     (client treats this as success)
-//
-// (2) the update was not applied because the server is already at generation X
-//     (client treats this as success)
-//
-// (3) the update was not applied because the server is already at a newer
-//     generation
-//     (client probably starts the whole propagation process over because its
-//     current view of the world is out of date)
-//
-// This way, the DNS data can never move backwards and the client only ever has
-// to make one request.
-//
-//
-// Concurrent updates
-// ------------------
-//
-// Given that we've got just one API to update the all DNS zones, and given
-// that might therefore take a minute for a large zone, and also that there may
-// be multiple Nexus instances trying to do it at the same time, we need to
-// think a bit about what should happen if two Nexus do try to do it at the same
-// time.  Spoiler: we immediately fail any request to update the DNS data if
-// there's already an update in progress.
-//
-// What else could we do?  We could queue the incoming request behind the
-// in-progress one.  How large do we allow that queue to grow?  At some point
-// we'll need to stop queueing them.  So why bother at all?
-
-use crate::dns_types::{DnsConfig, DnsConfigParams};
 use crate::storage::{self, UpdateError};
+use dns_server_api::{DnsConfig, DnsConfigParams, DnsServerApi};
 use dns_service_client::{
     ERROR_CODE_BAD_UPDATE_GENERATION, ERROR_CODE_UPDATE_IN_PROGRESS,
 };
-use dropshot::{endpoint, RequestContext};
+use dropshot::RequestContext;
 
 pub struct Context {
     store: storage::Store,
@@ -112,41 +22,40 @@ impl Context {
 }
 
 pub fn api() -> dropshot::ApiDescription<Context> {
-    let mut api = dropshot::ApiDescription::new();
-
-    api.register(dns_config_get).expect("register dns_config_get");
-    api.register(dns_config_put).expect("register dns_config_update");
-    api
+    dns_server_api::dns_server_api::api_description::<DnsServerApiImpl>()
+        .expect("registered DNS server entrypoints")
 }
 
-#[endpoint(
-    method = GET,
-    path = "/config",
-)]
-async fn dns_config_get(
-    rqctx: RequestContext<Context>,
-) -> Result<dropshot::HttpResponseOk<DnsConfig>, dropshot::HttpError> {
-    let apictx = rqctx.context();
-    let config = apictx.store.dns_config().await.map_err(|e| {
-        dropshot::HttpError::for_internal_error(format!(
-            "internal error: {:?}",
-            e
-        ))
-    })?;
-    Ok(dropshot::HttpResponseOk(config))
-}
+enum DnsServerApiImpl {}
+
+impl DnsServerApi for DnsServerApiImpl {
+    type Context = Context;
 
-#[endpoint(
-    method = PUT,
-    path = "/config",
-)]
-async fn dns_config_put(
-    rqctx: RequestContext<Context>,
-    rq: dropshot::TypedBody<DnsConfigParams>,
-) -> Result<dropshot::HttpResponseUpdatedNoContent, dropshot::HttpError> {
-    let apictx = rqctx.context();
-    apictx.store.dns_config_update(&rq.into_inner(), &rqctx.request_id).await?;
-    Ok(dropshot::HttpResponseUpdatedNoContent())
+    async fn dns_config_get(
+        rqctx: RequestContext<Context>,
+    ) -> Result<dropshot::HttpResponseOk<DnsConfig>, dropshot::HttpError> {
+        let apictx = rqctx.context();
+        let config = apictx.store.dns_config().await.map_err(|e| {
+            dropshot::HttpError::for_internal_error(format!(
+                "internal error: {:?}",
+                e
+            ))
+        })?;
+        Ok(dropshot::HttpResponseOk(config))
+    }
+
+    async fn dns_config_put(
+        rqctx: RequestContext<Context>,
+        rq: dropshot::TypedBody<DnsConfigParams>,
+    ) -> Result<dropshot::HttpResponseUpdatedNoContent, dropshot::HttpError>
+    {
+        let apictx = rqctx.context();
+        apictx
+            .store
+            .dns_config_update(&rq.into_inner(), &rqctx.request_id)
+            .await?;
+        Ok(dropshot::HttpResponseUpdatedNoContent())
+    }
 }
 
 impl From<UpdateError> for dropshot::HttpError {
diff --git a/dns-server/src/lib.rs b/dns-server/src/lib.rs
index ea8625a667..424159e41d 100644
--- a/dns-server/src/lib.rs
+++ b/dns-server/src/lib.rs
@@ -43,7 +43,6 @@
 //!    the persistent DNS data
 
 pub mod dns_server;
-pub mod dns_types;
 pub mod http_server;
 pub mod storage;
 
@@ -139,6 +138,7 @@ impl TransientServer {
                 bind_address: "[::1]:0".parse().unwrap(),
                 request_body_max_bytes: 4 * 1024 * 1024,
                 default_handler_task_mode: dropshot::HandlerTaskMode::Detached,
+                log_headers: vec![],
             },
         )
         .await?;
diff --git a/dns-server/src/storage.rs b/dns-server/src/storage.rs
index 21fb9ebdc6..85b2e79b8b 100644
--- a/dns-server/src/storage.rs
+++ b/dns-server/src/storage.rs
@@ -92,9 +92,9 @@
 // backwards-compatible way (but obviously one wouldn't get the scaling benefits
 // while continuing to use the old API).
 
-use crate::dns_types::{DnsConfig, DnsConfigParams, DnsConfigZone, DnsRecord};
 use anyhow::{anyhow, Context};
 use camino::Utf8PathBuf;
+use dns_server_api::{DnsConfig, DnsConfigParams, DnsConfigZone, DnsRecord};
 use serde::{Deserialize, Serialize};
 use sled::transaction::ConflictableTransactionError;
 use slog::{debug, error, info, o, warn};
@@ -777,13 +777,13 @@ impl<'a, 'b> Drop for UpdateGuard<'a, 'b> {
 #[cfg(test)]
 mod test {
     use super::{Config, Store, UpdateError};
-    use crate::dns_types::DnsConfigParams;
-    use crate::dns_types::DnsConfigZone;
-    use crate::dns_types::DnsRecord;
     use crate::storage::QueryError;
     use anyhow::Context;
     use camino::Utf8PathBuf;
     use camino_tempfile::Utf8TempDir;
+    use dns_server_api::DnsConfigParams;
+    use dns_server_api::DnsConfigZone;
+    use dns_server_api::DnsRecord;
     use omicron_test_utils::dev::test_setup_log;
     use std::collections::BTreeSet;
     use std::collections::HashMap;
diff --git a/dns-server/tests/basic_test.rs b/dns-server/tests/basic_test.rs
index 19666e82c1..b3b7f37378 100644
--- a/dns-server/tests/basic_test.rs
+++ b/dns-server/tests/basic_test.rs
@@ -419,6 +419,7 @@ fn test_config(
         bind_address: "[::1]:0".to_string().parse().unwrap(),
         request_body_max_bytes: 1024,
         default_handler_task_mode: HandlerTaskMode::Detached,
+        log_headers: vec![],
     };
 
     Ok((tmp_dir, config_storage, config_dropshot, logctx))
diff --git a/dns-server/tests/openapi_test.rs b/dns-server/tests/openapi_test.rs
deleted file mode 100644
index 490680eda4..0000000000
--- a/dns-server/tests/openapi_test.rs
+++ /dev/null
@@ -1,27 +0,0 @@
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this
-// file, You can obtain one at https://mozilla.org/MPL/2.0/.
-
-use expectorate::assert_contents;
-use omicron_test_utils::dev::test_cmds::assert_exit_code;
-use omicron_test_utils::dev::test_cmds::path_to_executable;
-use omicron_test_utils::dev::test_cmds::run_command;
-use omicron_test_utils::dev::test_cmds::EXIT_SUCCESS;
-use openapiv3::OpenAPI;
-use subprocess::Exec;
-
-const CMD_API_GEN: &str = env!("CARGO_BIN_EXE_apigen");
-
-#[test]
-fn test_dns_server_openapi() {
-    let exec = Exec::cmd(path_to_executable(CMD_API_GEN));
-    let (exit_status, stdout, stderr) = run_command(exec);
-    assert_exit_code(exit_status, EXIT_SUCCESS, &stderr);
-
-    let spec: OpenAPI =
-        serde_json::from_str(&stdout).expect("stdout was not valid OpenAPI");
-    let errors = openapi_lint::validate(&spec);
-    assert!(errors.is_empty(), "{}", errors.join("\n\n"));
-
-    assert_contents("../openapi/dns-server.json", &stdout);
-}
diff --git a/docs/how-to-run-simulated.adoc b/docs/how-to-run-simulated.adoc
index de19b70f04..86f7a0915b 100644
--- a/docs/how-to-run-simulated.adoc
+++ b/docs/how-to-run-simulated.adoc
@@ -94,6 +94,10 @@ omicron-dev: external DNS:          [::1]:54342
 
 === Running the pieces by hand
 
+There are many reasons it's useful to run the pieces of the stack by hand, especially during development and debugging: to test stopping and starting a component while the rest of the stack remains online; to run one component in a custom environment; to use a custom binary; to use a custom config file; to run under the debugger or with extra tracing enabled; etc.
+
+CAUTION: This process does not currently work.  See https://github.com/oxidecomputer/omicron/issues/4421[omicron#4421] for details.  The pieces here may still be useful for reference.
+
 . Start CockroachDB using `omicron-dev db-run`:
 +
 [source,text]
@@ -181,6 +185,8 @@ omicron-dev: using /tmp/.tmpFH6v8h and /tmp/.tmpkUjDji for ClickHouse data stora
 $ cargo run --bin=nexus -- nexus/examples/config.toml
 ----
 Nexus can also serve the web console. Instructions for downloading (or building) the console's static assets and pointing Nexus to them are https://github.com/oxidecomputer/console/blob/main/docs/serve-from-nexus.md[here]. Without console assets, Nexus will still start and run normally as an API. A few link:./nexus/src/external_api/console_api.rs[console-specific routes] will 404.
++
+CAUTION: This step does not currently work.  See https://github.com/oxidecomputer/omicron/issues/4421[omicron#4421] for details.
 
 . `dns-server` is run similar to Nexus, except that the bind addresses are specified on the command line:
 +
@@ -207,9 +213,98 @@ Dec 02 18:00:01.093 DEBG registered endpoint, path: /producers, method: POST, lo
 ...
 ----
 
+=== Using both `omicron-dev run-all` and running Nexus manually
+
+While it's often useful to run _some_ part of the stack by hand (see above), if you only want to run your own Nexus, one option is to run `omicron-dev run-all` first to get a whole simulated stack up, then run a second Nexus by hand with a custom config file.
+
+To do this, first run `omicron-dev run-all`:
+
+[source,text]
+----
+$ cargo run --bin=omicron-dev -- run-all
+    Finished `dev` profile [unoptimized + debuginfo] target(s) in 0.95s
+     Running `target/debug/omicron-dev run-all`
+omicron-dev: setting up all services ... 
+log file: /dangerzone/omicron_tmp/omicron-dev-omicron-dev.29765.0.log
+note: configured to log to "/dangerzone/omicron_tmp/omicron-dev-omicron-dev.29765.0.log"
+DB URL: postgresql://root@[::1]:43256/omicron?sslmode=disable
+DB address: [::1]:43256
+log file: /dangerzone/omicron_tmp/omicron-dev-omicron-dev.29765.2.log
+note: configured to log to "/dangerzone/omicron_tmp/omicron-dev-omicron-dev.29765.2.log"
+log file: /dangerzone/omicron_tmp/omicron-dev-omicron-dev.29765.3.log
+note: configured to log to "/dangerzone/omicron_tmp/omicron-dev-omicron-dev.29765.3.log"
+omicron-dev: services are running.
+omicron-dev: nexus external API:    127.0.0.1:12220
+omicron-dev: nexus internal API:    [::1]:12221
+omicron-dev: cockroachdb pid:       29769
+omicron-dev: cockroachdb URL:       postgresql://root@[::1]:43256/omicron?sslmode=disable
+omicron-dev: cockroachdb directory: /dangerzone/omicron_tmp/.tmpikyLO8
+omicron-dev: internal DNS HTTP:     http://[::1]:39841
+omicron-dev: internal DNS:          [::1]:54025
+omicron-dev: external DNS name:     oxide-dev.test
+omicron-dev: external DNS HTTP:     http://[::1]:63482
+omicron-dev: external DNS:          [::1]:45276
+omicron-dev:   e.g. `dig @::1 -p 45276 test-suite-silo.sys.oxide-dev.test`
+omicron-dev: management gateway:    http://[::1]:49188 (switch0)
+omicron-dev: management gateway:    http://[::1]:39352 (switch1)
+omicron-dev: silo name:             test-suite-silo
+omicron-dev: privileged user name:  test-privileged
+----
+
+You'll need to note:
+
+* the TCP ports for the two management gateways (`49188` and `39352` here for switch0 and switch1, respectively)
+* the TCP port for internal DNS  (`54025` here)
+* the TCP port in the CockroachDB URL (`43256` here)
+
+Next, you'll need to customize the Nexus configuration file.  Start with nexus/examples/config-second.toml (_not_ nexus/examples/config.toml, which uses various values that conflict with what `omicron-dev run-all` uses).  You should only need to modify the block at the **bottom** of the file:
+
+[source,toml]
+----
+################################################################################
+# INSTRUCTIONS: To run Nexus against an existing stack started with            #
+# `omicron-dev run-all`, you should only have to modify values in this 	       #
+# section. 								       #
+# 									       #
+# Modify the port numbers below based on the output of `omicron-dev run-all`   #
+################################################################################
+
+[mgd]
+# Look for "management gateway:    http://[::1]:49188 (switch0)"
+# The "http://" does not go in this string -- just the socket address.
+switch0.address = "[::1]:49188"
+
+# Look for "management gateway:    http://[::1]:39352 (switch1)"
+# The "http://" does not go in this string -- just the socket address.
+switch1.address = "[::1]:39352"
+
+[deployment.internal_dns]
+# Look for "internal DNS:          [::1]:54025"
+# and adjust the port number below.
+address = "[::1]:54025"
+# You should not need to change this.
+type = "from_address"
+
+[deployment.database]
+# Look for "cockroachdb URL:       postgresql://root@[::1]:43256/omicron?sslmode=disable"
+# and adjust the port number below.
+url = "postgresql://root@[::1]:43256/omicron?sslmode=disable"
+# You should not need to change this.
+type = "from_url"
+################################################################################
+----
+
+So it's:
+
+* Copy the example config file: `cp nexus/examples/config-second.toml config-second.toml`
+* Edit as described above: `vim config-second.toml`
+* Start Nexus like above, but with this config file: `cargo run --bin=nexus -- config-second.toml`
+
+=== Using the stack
+
 Once everything is up and running, you can use the system in a few ways:
 
-* Use the browser-based console.  The Nexus log output will show what IP address and port it's listening on.  This is also configured in the config file.  If you're using the defaults, you can reach the console at `http://127.0.0.1:12220/projects`.  Depending on the environment where you're running this, you may need an ssh tunnel or the like to reach this from your browser.
+* Use the browser-based console.  The Nexus log output will show what IP address and port it's listening on.  This is also configured in the config file.  If you're using the defaults with `omicron-dev run-all`, you can reach the console at `http://127.0.0.1:12220/projects`.  If you ran a second Nexus using the `config-second.toml` config file, it will be on port `12222` instead (because that config file specifies port 12222).  Depending on the environment where you're running this, you may need an ssh tunnel or the like to reach this from your browser.
 * Use the xref:cli.adoc[`oxide` CLI].
 
 == Running with TLS
diff --git a/flake.nix b/flake.nix
index 831a0aaea2..4fee8d3b0a 100644
--- a/flake.nix
+++ b/flake.nix
@@ -9,7 +9,7 @@
     };
   };
 
-  outputs = { self, nixpkgs, rust-overlay, ... }:
+  outputs = { nixpkgs, rust-overlay, ... }:
     let
       overlays = [ (import rust-overlay) ];
       pkgs = import nixpkgs {
@@ -381,11 +381,16 @@
           };
         };
 
-      devShells.x86_64-linux.default =
-        pkgs.mkShell.override
+      devShells.x86_64-linux.default = with pkgs;
+        mkShell.override
           {
-            # use Clang as the C compiler for all C libraries
-            stdenv = pkgs.clangStdenv;
+            stdenv =
+              # use Mold as the linker rather than ld, for faster builds. Mold
+              # to require substantially less memory to link Nexus and its
+              # avoiding swapping on memory-constrained dev systems.
+              stdenvAdapters.useMoldLinker
+                # use Clang as the C compiler for all C libraries.
+                clangStdenv;
           }
           {
             inherit buildInputs;
@@ -398,10 +403,10 @@
             ];
 
             name = "omicron";
-            DEP_PQ_LIBDIRS = "${pkgs.postgresql.lib}/lib";
-            LIBCLANG_PATH = "${pkgs.libclang.lib}/lib";
-            OPENSSL_DIR = "${pkgs.openssl.dev}";
-            OPENSSL_LIB_DIR = "${pkgs.openssl.out}/lib";
+            DEP_PQ_LIBDIRS = "${postgresql.lib}/lib";
+            LIBCLANG_PATH = "${libclang.lib}/lib";
+            OPENSSL_DIR = "${openssl.dev}";
+            OPENSSL_LIB_DIR = "${openssl.out}/lib";
 
             MG_OPENAPI_PATH = mgOpenAPI;
             DDM_OPENAPI_PATH = ddmOpenAPI;
diff --git a/gateway/src/lib.rs b/gateway/src/lib.rs
index 1354f30a0a..be8c84d7db 100644
--- a/gateway/src/lib.rs
+++ b/gateway/src/lib.rs
@@ -98,6 +98,7 @@ fn start_dropshot_server(
         bind_address: SocketAddr::V6(addr),
         request_body_max_bytes,
         default_handler_task_mode: HandlerTaskMode::Detached,
+        log_headers: vec![],
     };
     let http_server_starter = dropshot::HttpServerStarter::new(
         &dropshot,
diff --git a/illumos-utils/src/opte/port.rs b/illumos-utils/src/opte/port.rs
index a692a02304..4cfe351776 100644
--- a/illumos-utils/src/opte/port.rs
+++ b/illumos-utils/src/opte/port.rs
@@ -30,17 +30,6 @@ pub struct PortData {
     pub(crate) subnet: IpNet,
     /// Information about the virtual gateway, aka OPTE
     pub(crate) gateway: Gateway,
-    /// Name of the VNIC the OPTE port is bound to.
-    // TODO-remove(#2932): Remove this once we can put Viona directly on top of an
-    // OPTE port device.
-    //
-    // NOTE: This is intentionally not an actual `Vnic` object. We'd like to
-    // delete the VNIC manually in `PortInner::drop`, because we _can't_ delete
-    // the xde device if we fail to delete the VNIC. See
-    // https://github.com/oxidecomputer/opte/issues/178 for more details. This
-    // can be changed back to a real VNIC when that is resolved, and the Drop
-    // impl below can simplify to just call `drop(self.vnic)`.
-    pub(crate) vnic: String,
 }
 
 #[derive(Debug)]
@@ -57,18 +46,6 @@ impl core::ops::Deref for PortInner {
 #[cfg(target_os = "illumos")]
 impl Drop for PortInner {
     fn drop(&mut self) {
-        if let Err(e) = crate::dladm::Dladm::delete_vnic(&self.vnic) {
-            eprintln!(
-                "WARNING: Failed to delete OPTE port overlay VNIC \
-                while dropping port. The VNIC will not be cleaned up \
-                properly, and the xde device itself will not be deleted. \
-                Both the VNIC and the xde device must be deleted out \
-                of band, and it will not be possible to recreate the xde \
-                device until then. Error: {:?}",
-                e
-            );
-            return;
-        }
         let err = match opte_ioctl::OpteHdl::open(opte_ioctl::OpteHdl::XDE_CTL)
         {
             Ok(hdl) => {
@@ -81,9 +58,8 @@ impl Drop for PortInner {
             Err(e) => e,
         };
         eprintln!(
-            "WARNING: OPTE port overlay VNIC deleted, but failed \
-            to delete the xde device. It must be deleted out \
-            of band, and it will not be possible to recreate the xde \
+            "WARNING: Failed to delete the xde device. It must be deleted
+            out of band, and it will not be possible to recreate the xde \
             device until then. Error: {:?}",
             err,
         );
@@ -130,10 +106,6 @@ impl Port {
         &self.inner.subnet
     }
 
-    pub fn vnic_name(&self) -> &str {
-        &self.inner.vnic
-    }
-
     pub fn slot(&self) -> u8 {
         self.inner.slot
     }
diff --git a/illumos-utils/src/opte/port_manager.rs b/illumos-utils/src/opte/port_manager.rs
index 984e3c55fa..b6d28d1b06 100644
--- a/illumos-utils/src/opte/port_manager.rs
+++ b/illumos-utils/src/opte/port_manager.rs
@@ -304,57 +304,6 @@ impl PortManager {
             rules,
         })?;
 
-        // TODO-remove(#2932): Create a VNIC on top of this device, to hook Viona into.
-        //
-        // Viona is the illumos MAC provider that implements the VIRTIO
-        // specification. It sits on top of a MAC provider, which is responsible
-        // for delivering frames to the underlying data link. The guest includes
-        // a driver that handles the virtio-net specification on their side,
-        // which talks to Viona.
-        //
-        // In theory, Viona works with any MAC provider. However, there are
-        // implicit assumptions, in both Viona _and_ MAC, that require Viona to
-        // be built on top of a VNIC specifically. There is probably a good deal
-        // of work required to relax that assumption, so in the meantime, we
-        // create a superfluous VNIC on the OPTE device, solely so Viona can use
-        // it.
-        let vnic = {
-            let vnic_name = format!("v{}", port_name);
-            #[cfg(target_os = "illumos")]
-            if let Err(e) = crate::dladm::Dladm::create_vnic(
-                &crate::dladm::PhysicalLink(port_name.clone()),
-                &vnic_name,
-                Some(nic.mac),
-                None,
-                1500,
-            ) {
-                slog::warn!(
-                    self.inner.log,
-                    "Failed to create overlay VNIC for xde device";
-                    "port_name" => &port_name,
-                    "err" => ?e
-                );
-                if let Err(e) = hdl.delete_xde(&port_name) {
-                    slog::warn!(
-                        self.inner.log,
-                        "Failed to clean up xde device after failure to create overlay VNIC";
-                        "err" => ?e
-                    );
-                }
-                return Err(e.into());
-            }
-            debug!(
-                self.inner.log,
-                "Created overlay VNIC for xde device";
-                "port_name" => &port_name,
-                "vnic_name" => &vnic_name,
-            );
-
-            // NOTE: We intentionally use a string rather than the Vnic type
-            // here. See the notes on the `opte::PortInner::vnic` field.
-            vnic_name
-        };
-
         let (port, ticket) = {
             let mut ports = self.inner.ports.lock().unwrap();
             let ticket = PortTicket::new(nic.id, nic.kind, self.inner.clone());
@@ -366,7 +315,6 @@ impl PortManager {
                 vni,
                 subnet: nic.subnet,
                 gateway,
-                vnic,
             });
             let old = ports.insert((nic.id, nic.kind), port.clone());
             assert!(
diff --git a/illumos-utils/src/running_zone.rs b/illumos-utils/src/running_zone.rs
index c4e68e0c50..ea24a6f502 100644
--- a/illumos-utils/src/running_zone.rs
+++ b/illumos-utils/src/running_zone.rs
@@ -10,6 +10,7 @@ use crate::link::{Link, VnicAllocator};
 use crate::opte::{Port, PortTicket};
 use crate::svc::wait_for_service;
 use crate::zone::{AddressRequest, IPADM, ZONE_PREFIX};
+use crate::zpool::{PathInPool, ZpoolName};
 use camino::{Utf8Path, Utf8PathBuf};
 use camino_tempfile::Utf8TempDir;
 use ipnetwork::IpNetwork;
@@ -101,60 +102,6 @@ pub enum EnsureAddressError {
     OpteGatewayConfig(#[from] RunCommandError),
 }
 
-/// Errors returned from [`RunningZone::get`].
-#[derive(thiserror::Error, Debug)]
-pub enum GetZoneError {
-    #[error("While looking up zones with prefix '{prefix}', could not get zones: {err}")]
-    GetZones {
-        prefix: String,
-        #[source]
-        err: crate::zone::AdmError,
-    },
-
-    #[error("Invalid Utf8 path: {0}")]
-    FromPathBuf(#[from] camino::FromPathBufError),
-
-    #[error("Zone with prefix '{prefix}' not found")]
-    NotFound { prefix: String },
-
-    #[error("Cannot get zone '{name}': it is in the {state:?} state instead of running")]
-    NotRunning { name: String, state: zone::State },
-
-    #[error(
-        "Cannot get zone '{name}': Failed to acquire control interface {err}"
-    )]
-    ControlInterface {
-        name: String,
-        #[source]
-        err: crate::zone::GetControlInterfaceError,
-    },
-
-    #[error("Cannot get zone '{name}': Failed to create addrobj: {err}")]
-    AddrObject {
-        name: String,
-        #[source]
-        err: crate::addrobj::ParseError,
-    },
-
-    #[error(
-        "Cannot get zone '{name}': Failed to ensure address exists: {err}"
-    )]
-    EnsureAddress {
-        name: String,
-        #[source]
-        err: crate::zone::EnsureAddressError,
-    },
-
-    #[error(
-        "Cannot get zone '{name}': Incorrect bootstrap interface access {err}"
-    )]
-    BootstrapInterface {
-        name: String,
-        #[source]
-        err: crate::zone::GetBootstrapInterfaceError,
-    },
-}
-
 #[cfg(target_os = "illumos")]
 static REAPER_THREAD: OnceLock<thread::JoinHandle<()>> = OnceLock::new();
 
@@ -407,6 +354,11 @@ impl RunningZone {
         self.inner.root()
     }
 
+    /// Returns the zpool on which the filesystem path has been placed.
+    pub fn root_zpool(&self) -> Option<&ZpoolName> {
+        self.inner.zonepath.pool.as_ref()
+    }
+
     pub fn control_interface(&self) -> AddrObject {
         AddrObject::new(self.inner.get_control_vnic_name(), "omicron6").unwrap()
     }
@@ -659,15 +611,13 @@ impl RunningZone {
                 port_idx,
             }
         })?;
-        // TODO-remove(#2932): Switch to using port directly once vnic is no longer needed.
-        let addrobj =
-            AddrObject::new(port.vnic_name(), name).map_err(|err| {
-                EnsureAddressError::AddrObject {
-                    request: AddressRequest::Dhcp,
-                    zone: self.inner.name.clone(),
-                    err,
-                }
-            })?;
+        let addrobj = AddrObject::new(port.name(), name).map_err(|err| {
+            EnsureAddressError::AddrObject {
+                request: AddressRequest::Dhcp,
+                zone: self.inner.name.clone(),
+                err,
+            }
+        })?;
         let zone = Some(self.inner.name.as_ref());
         if let IpAddr::V4(gateway) = port.gateway().ip() {
             let addr =
@@ -686,7 +636,7 @@ impl RunningZone {
                 &private_ip.to_string(),
                 "-interface",
                 "-ifp",
-                port.vnic_name(),
+                port.name(),
             ])?;
             self.run_cmd(&[
                 "/usr/sbin/route",
@@ -800,95 +750,6 @@ impl RunningZone {
         Ok(())
     }
 
-    /// Looks up a running zone based on the `zone_prefix`, if one already exists.
-    ///
-    /// - If the zone was found, is running, and has a network interface, it is
-    /// returned.
-    /// - If the zone was not found `Error::NotFound` is returned.
-    /// - If the zone was found, but not running, `Error::NotRunning` is
-    /// returned.
-    /// - Other errors may be returned attempting to look up and accessing an
-    /// address on the zone.
-    pub async fn get(
-        log: &Logger,
-        vnic_allocator: &VnicAllocator<Etherstub>,
-        zone_prefix: &str,
-        addrtype: AddressRequest,
-    ) -> Result<Self, GetZoneError> {
-        let zone_info = Zones::get()
-            .await
-            .map_err(|err| GetZoneError::GetZones {
-                prefix: zone_prefix.to_string(),
-                err,
-            })?
-            .into_iter()
-            .find(|zone_info| zone_info.name().starts_with(&zone_prefix))
-            .ok_or_else(|| GetZoneError::NotFound {
-                prefix: zone_prefix.to_string(),
-            })?;
-
-        if zone_info.state() != zone::State::Running {
-            return Err(GetZoneError::NotRunning {
-                name: zone_info.name().to_string(),
-                state: zone_info.state(),
-            });
-        }
-
-        let zone_name = zone_info.name();
-        let vnic_name =
-            Zones::get_control_interface(zone_name).map_err(|err| {
-                GetZoneError::ControlInterface {
-                    name: zone_name.to_string(),
-                    err,
-                }
-            })?;
-        let addrobj = AddrObject::new_control(&vnic_name).map_err(|err| {
-            GetZoneError::AddrObject { name: zone_name.to_string(), err }
-        })?;
-        Zones::ensure_address(Some(zone_name), &addrobj, addrtype).map_err(
-            |err| GetZoneError::EnsureAddress {
-                name: zone_name.to_string(),
-                err,
-            },
-        )?;
-
-        let control_vnic = vnic_allocator
-            .wrap_existing(vnic_name)
-            .expect("Failed to wrap valid control VNIC");
-
-        // The bootstrap address for a running zone never changes,
-        // so there's no need to call `Zones::ensure_address`.
-        // Currently, only the switch zone has a bootstrap interface.
-        let bootstrap_vnic = Zones::get_bootstrap_interface(zone_name)
-            .map_err(|err| GetZoneError::BootstrapInterface {
-                name: zone_name.to_string(),
-                err,
-            })?
-            .map(|name| {
-                vnic_allocator
-                    .wrap_existing(name)
-                    .expect("Failed to wrap valid bootstrap VNIC")
-            });
-
-        Ok(Self {
-            id: zone_info.id().map(|x| {
-                x.try_into().expect("zoneid_t is expected to be an i32")
-            }),
-            inner: InstalledZone {
-                log: log.new(o!("zone" => zone_name.to_string())),
-                zonepath: zone_info.path().to_path_buf().try_into()?,
-                name: zone_name.to_string(),
-                control_vnic,
-                // TODO(https://github.com/oxidecomputer/omicron/issues/725)
-                //
-                // Re-initialize guest_vnic state by inspecting the zone.
-                opte_ports: vec![],
-                links: vec![],
-                bootstrap_vnic,
-            },
-        })
-    }
-
     /// Return references to the OPTE ports for this zone.
     pub fn opte_ports(&self) -> impl Iterator<Item = &Port> {
         self.inner.opte_ports()
@@ -1084,7 +945,7 @@ pub struct InstalledZone {
     log: Logger,
 
     // Filesystem path of the zone
-    zonepath: Utf8PathBuf,
+    zonepath: PathInPool,
 
     // Name of the Zone.
     name: String,
@@ -1134,7 +995,7 @@ impl InstalledZone {
 
     /// Returns the filesystem path to the zonepath
     pub fn zonepath(&self) -> &Utf8Path {
-        &self.zonepath
+        &self.zonepath.path
     }
 
     pub fn site_profile_xml_path(&self) -> Utf8PathBuf {
@@ -1150,7 +1011,7 @@ impl InstalledZone {
 
     /// Returns the filesystem path to the zone's root in the GZ.
     pub fn root(&self) -> Utf8PathBuf {
-        self.zonepath.join(Self::ROOT_FS_PATH)
+        self.zonepath.path.join(Self::ROOT_FS_PATH)
     }
 
     /// Return a reference to the links for this zone.
@@ -1206,7 +1067,7 @@ pub struct ZoneBuilder<'a> {
     /// Allocates the NIC used for control plane communication.
     underlay_vnic_allocator: Option<&'a VnicAllocator<Etherstub>>,
     /// Filesystem path at which the installed zone will reside.
-    zone_root_path: Option<&'a Utf8Path>,
+    zone_root_path: Option<PathInPool>,
     /// The directories that will be searched for the image tarball for the
     /// provided zone type ([`Self::with_zone_type`]).
     zone_image_paths: Option<&'a [Utf8PathBuf]>,
@@ -1259,7 +1120,7 @@ impl<'a> ZoneBuilder<'a> {
     }
 
     /// Filesystem path at which the installed zone will reside.
-    pub fn with_zone_root_path(mut self, root_path: &'a Utf8Path) -> Self {
+    pub fn with_zone_root_path(mut self, root_path: PathInPool) -> Self {
         self.zone_root_path = Some(root_path);
         self
     }
@@ -1353,8 +1214,11 @@ impl<'a> ZoneBuilder<'a> {
                 self.zone_type?,
                 self.unique_name,
             );
-            let zonepath = temp_dir
-                .join(self.zone_root_path?.strip_prefix("/").unwrap())
+            let mut zonepath = self.zone_root_path?;
+            zonepath.path = temp_dir
+                .join(
+                    zonepath.path.strip_prefix("/").unwrap()
+                )
                 .join(&full_zone_name);
             let iz = InstalledZone {
                 log: self.log?,
@@ -1384,7 +1248,7 @@ impl<'a> ZoneBuilder<'a> {
         let Self {
             log: Some(log),
             underlay_vnic_allocator: Some(underlay_vnic_allocator),
-            zone_root_path: Some(zone_root_path),
+            zone_root_path: Some(mut zone_root_path),
             zone_image_paths: Some(zone_image_paths),
             zone_type: Some(zone_type),
             unique_name,
@@ -1435,7 +1299,7 @@ impl<'a> ZoneBuilder<'a> {
 
         let mut net_device_names: Vec<String> = opte_ports
             .iter()
-            .map(|(port, _)| port.vnic_name().to_string())
+            .map(|(port, _)| port.name().to_string())
             .chain(std::iter::once(control_vnic.name().to_string()))
             .chain(bootstrap_vnic.as_ref().map(|vnic| vnic.name().to_string()))
             .chain(links.iter().map(|nic| nic.name().to_string()))
@@ -1448,6 +1312,7 @@ impl<'a> ZoneBuilder<'a> {
         net_device_names.sort();
         net_device_names.dedup();
 
+        zone_root_path.path = zone_root_path.path.join(&full_zone_name);
         Zones::install_omicron_zone(
             &log,
             &zone_root_path,
@@ -1468,7 +1333,7 @@ impl<'a> ZoneBuilder<'a> {
 
         Ok(InstalledZone {
             log: log.new(o!("zone" => full_zone_name.clone())),
-            zonepath: zone_root_path.join(&full_zone_name),
+            zonepath: zone_root_path,
             name: full_zone_name,
             control_vnic,
             bootstrap_vnic,
diff --git a/illumos-utils/src/zone.rs b/illumos-utils/src/zone.rs
index deda449a3e..47cc84dce6 100644
--- a/illumos-utils/src/zone.rs
+++ b/illumos-utils/src/zone.rs
@@ -14,6 +14,7 @@ use std::net::{IpAddr, Ipv6Addr};
 
 use crate::addrobj::AddrObject;
 use crate::dladm::{EtherstubVnic, VNIC_PREFIX_BOOTSTRAP, VNIC_PREFIX_CONTROL};
+use crate::zpool::PathInPool;
 use crate::{execute, PFEXEC};
 use omicron_common::address::SLED_PREFIX;
 
@@ -282,7 +283,7 @@ impl Zones {
     #[allow(clippy::too_many_arguments)]
     pub async fn install_omicron_zone(
         log: &Logger,
-        zone_root_path: &Utf8Path,
+        zone_root_path: &PathInPool,
         zone_name: &str,
         zone_image: &Utf8Path,
         datasets: &[zone::Dataset],
@@ -319,10 +320,9 @@ impl Zones {
             true,
             zone::CreationOptions::Blank,
         );
-        let path = zone_root_path.join(zone_name);
         cfg.get_global()
             .set_brand("omicron1")
-            .set_path(&path)
+            .set_path(&zone_root_path.path)
             .set_autoboot(false)
             .set_ip_type(zone::IpType::Exclusive);
         if !limit_priv.is_empty() {
diff --git a/illumos-utils/src/zpool.rs b/illumos-utils/src/zpool.rs
index fa93760f99..5dabbdecc7 100644
--- a/illumos-utils/src/zpool.rs
+++ b/illumos-utils/src/zpool.rs
@@ -5,7 +5,7 @@
 //! Utilities for managing Zpools.
 
 use crate::{execute, ExecutionError, PFEXEC};
-use camino::Utf8Path;
+use camino::{Utf8Path, Utf8PathBuf};
 use std::str::FromStr;
 
 pub use omicron_common::zpool_name::ZpoolName;
@@ -181,6 +181,19 @@ impl FromStr for ZpoolInfo {
 /// Wraps commands for interacting with ZFS pools.
 pub struct Zpool {}
 
+/// A path which exists within a pool.
+///
+/// By storing these types together, it's possible to answer
+/// whether or not a path exists on a particular device.
+// Technically we could re-derive the pool name from the path,
+// but that involves some string parsing, and honestly I'd just
+// Rather Not.
+#[derive(Debug, Clone, Eq, PartialEq)]
+pub struct PathInPool {
+    pub pool: Option<ZpoolName>,
+    pub path: Utf8PathBuf,
+}
+
 #[cfg_attr(any(test, feature = "testing"), mockall::automock, allow(dead_code))]
 impl Zpool {
     pub fn create(
diff --git a/installinator-artifactd/Cargo.toml b/installinator-api/Cargo.toml
similarity index 55%
rename from installinator-artifactd/Cargo.toml
rename to installinator-api/Cargo.toml
index 236ea7a51c..52db4362c6 100644
--- a/installinator-artifactd/Cargo.toml
+++ b/installinator-api/Cargo.toml
@@ -1,5 +1,5 @@
 [package]
-name = "installinator-artifactd"
+name = "installinator-api"
 version = "0.1.0"
 edition = "2021"
 license = "MPL-2.0"
@@ -9,24 +9,12 @@ workspace = true
 
 [dependencies]
 anyhow.workspace = true
-async-trait.workspace = true
-clap.workspace = true
 dropshot.workspace = true
 hyper.workspace = true
+installinator-common.workspace = true
+omicron-common.workspace = true
+omicron-workspace-hack.workspace = true
 schemars.workspace = true
 serde.workspace = true
-serde_json.workspace = true
 slog.workspace = true
 uuid.workspace = true
-
-installinator-common.workspace = true
-omicron-common.workspace = true
-omicron-workspace-hack.workspace = true
-
-[dev-dependencies]
-expectorate.workspace = true
-omicron-test-utils.workspace = true
-openapiv3.workspace = true
-openapi-lint.workspace = true
-serde_json.workspace = true
-subprocess.workspace = true
diff --git a/installinator-api/src/lib.rs b/installinator-api/src/lib.rs
new file mode 100644
index 0000000000..3ff9acffd2
--- /dev/null
+++ b/installinator-api/src/lib.rs
@@ -0,0 +1,169 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+//! The REST API that installinator is a client of.
+//!
+//! Note that most of our APIs are named by their server. This one is instead
+//! named by the client, since it is expected that multiple services will
+//! implement it.
+
+use anyhow::{anyhow, Result};
+use dropshot::{
+    ConfigDropshot, FreeformBody, HandlerTaskMode, HttpError,
+    HttpResponseHeaders, HttpResponseOk, HttpResponseUpdatedNoContent,
+    HttpServerStarter, Path, RequestContext, TypedBody,
+};
+use hyper::{header, Body, StatusCode};
+use installinator_common::EventReport;
+use omicron_common::update::ArtifactHashId;
+use schemars::JsonSchema;
+use serde::Deserialize;
+use uuid::Uuid;
+
+#[derive(Debug, Deserialize, JsonSchema)]
+pub struct ReportQuery {
+    /// A unique identifier for the update.
+    pub update_id: Uuid,
+}
+
+#[dropshot::api_description]
+pub trait InstallinatorApi {
+    type Context;
+
+    /// Fetch an artifact by hash.
+    #[endpoint {
+        method = GET,
+        path = "/artifacts/by-hash/{kind}/{hash}",
+    }]
+    async fn get_artifact_by_hash(
+        rqctx: RequestContext<Self::Context>,
+        path: Path<ArtifactHashId>,
+    ) -> Result<HttpResponseHeaders<HttpResponseOk<FreeformBody>>, HttpError>;
+
+    /// Report progress and completion to the server.
+    ///
+    /// This method requires an `update_id` path parameter. This update ID is
+    /// matched against the server currently performing an update. If the
+    /// server is unaware of the update ID, it will return an HTTP 422
+    /// Unprocessable Entity code.
+    #[endpoint {
+        method = POST,
+        path = "/report-progress/{update_id}",
+    }]
+    async fn report_progress(
+        rqctx: RequestContext<Self::Context>,
+        path: Path<ReportQuery>,
+        report: TypedBody<EventReport>,
+    ) -> Result<HttpResponseUpdatedNoContent, HttpError>;
+}
+
+/// Add a content length header to a response.
+///
+/// Intended to be called by `get_artifact_by_hash` implementations.
+pub fn body_to_artifact_response(
+    size: u64,
+    body: Body,
+) -> HttpResponseHeaders<HttpResponseOk<FreeformBody>> {
+    let mut response =
+        HttpResponseHeaders::new_unnamed(HttpResponseOk(body.into()));
+    let headers = response.headers_mut();
+    headers.append(header::CONTENT_LENGTH, size.into());
+    response
+}
+
+/// The result of processing an installinator event report.
+#[derive(Copy, Clone, Debug, Eq, PartialEq, PartialOrd, Ord, Hash)]
+#[must_use]
+pub enum EventReportStatus {
+    /// This report was processed by the server.
+    Processed,
+
+    /// The update ID was not recognized by the server.
+    UnrecognizedUpdateId,
+
+    /// The progress receiver is closed.
+    ReceiverClosed,
+}
+
+impl EventReportStatus {
+    /// Convert this status to an HTTP result.
+    ///
+    /// Intended to be called by `report_progress` implementations.
+    pub fn to_http_result(
+        self,
+        update_id: Uuid,
+    ) -> Result<HttpResponseUpdatedNoContent, HttpError> {
+        match self {
+            EventReportStatus::Processed => Ok(HttpResponseUpdatedNoContent()),
+            EventReportStatus::UnrecognizedUpdateId => {
+                Err(HttpError::for_client_error(
+                    None,
+                    StatusCode::UNPROCESSABLE_ENTITY,
+                    format!(
+                        "update ID {update_id} unrecognized by this server"
+                    ),
+                ))
+            }
+            EventReportStatus::ReceiverClosed => {
+                Err(HttpError::for_client_error(
+                    None,
+                    StatusCode::GONE,
+                    format!("update ID {update_id}: receiver closed"),
+                ))
+            }
+        }
+    }
+}
+
+/// Creates a default `ConfigDropshot` for the installinator API.
+pub fn default_config(bind_address: std::net::SocketAddr) -> ConfigDropshot {
+    ConfigDropshot {
+        bind_address,
+        // Even though the installinator sets an upper bound on the number of
+        // items in a progress report, they can get pretty large if they
+        // haven't gone through for a bit. Ensure that hitting the max request
+        // size won't cause a failure by setting a generous upper bound for the
+        // request size.
+        //
+        // TODO: replace with an endpoint-specific option once
+        // https://github.com/oxidecomputer/dropshot/pull/618 lands and is
+        // available in omicron.
+        request_body_max_bytes: 4 * 1024 * 1024,
+        default_handler_task_mode: HandlerTaskMode::Detached,
+        log_headers: vec![],
+    }
+}
+
+/// Make an `HttpServerStarter` for the installinator API with default settings.
+pub fn make_server_starter<T: InstallinatorApi>(
+    context: T::Context,
+    bind_address: std::net::SocketAddr,
+    log: &slog::Logger,
+) -> Result<HttpServerStarter<T::Context>> {
+    let dropshot_config = dropshot::ConfigDropshot {
+        bind_address,
+        // Even though the installinator sets an upper bound on the number
+        // of items in a progress report, they can get pretty large if they
+        // haven't gone through for a bit. Ensure that hitting the max
+        // request size won't cause a failure by setting a generous upper
+        // bound for the request size.
+        //
+        // TODO: replace with an endpoint-specific option once
+        // https://github.com/oxidecomputer/dropshot/pull/618 lands and is
+        // available in omicron.
+        request_body_max_bytes: 4 * 1024 * 1024,
+        default_handler_task_mode: HandlerTaskMode::Detached,
+        log_headers: vec![],
+    };
+
+    let api = crate::installinator_api::api_description::<T>()?;
+    let server =
+        dropshot::HttpServerStarter::new(&dropshot_config, api, context, &log)
+            .map_err(|error| {
+                anyhow!(error)
+                    .context("failed to create installinator artifact server")
+            })?;
+
+    Ok(server)
+}
diff --git a/installinator-artifactd/src/bin/installinator-artifactd.rs b/installinator-artifactd/src/bin/installinator-artifactd.rs
deleted file mode 100644
index abe63bbe31..0000000000
--- a/installinator-artifactd/src/bin/installinator-artifactd.rs
+++ /dev/null
@@ -1,38 +0,0 @@
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this
-// file, You can obtain one at https://mozilla.org/MPL/2.0/.
-
-//! Executable that generates OpenAPI definitions for the installinator artifact server.
-
-use anyhow::Result;
-use clap::Parser;
-use omicron_common::cmd::CmdError;
-
-#[derive(Debug, Parser)]
-#[clap(name = "installinator-artifactd")]
-enum Args {
-    /// Print the external OpenAPI Spec document and exit
-    Openapi,
-    // NOTE: this server is not intended to be run as a standalone service. Instead, it should be
-    // embedded as part of other servers (e.g. wicketd).
-}
-
-fn main() {
-    if let Err(cmd_error) = do_run() {
-        omicron_common::cmd::fatal(cmd_error);
-    }
-}
-
-fn do_run() -> Result<(), CmdError> {
-    let args = Args::parse();
-
-    match args {
-        Args::Openapi => {
-            installinator_artifactd::run_openapi().map_err(|error| {
-                CmdError::Failure(
-                    error.context("failed to generate OpenAPI spec"),
-                )
-            })
-        }
-    }
-}
diff --git a/installinator-artifactd/src/http_entrypoints.rs b/installinator-artifactd/src/http_entrypoints.rs
deleted file mode 100644
index 13163e007b..0000000000
--- a/installinator-artifactd/src/http_entrypoints.rs
+++ /dev/null
@@ -1,115 +0,0 @@
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this
-// file, You can obtain one at https://mozilla.org/MPL/2.0/.
-
-// Copyright 2022 Oxide Computer Company
-
-use dropshot::{
-    endpoint, ApiDescription, ApiDescriptionRegisterError, FreeformBody,
-    HttpError, HttpResponseHeaders, HttpResponseOk,
-    HttpResponseUpdatedNoContent, Path, RequestContext, TypedBody,
-};
-use hyper::{header, Body, StatusCode};
-use installinator_common::EventReport;
-use omicron_common::update::ArtifactHashId;
-use schemars::JsonSchema;
-use serde::Deserialize;
-use uuid::Uuid;
-
-use crate::{context::ServerContext, EventReportStatus};
-
-type ArtifactServerApiDesc = ApiDescription<ServerContext>;
-
-/// Return a description of the artifact server api for use in generating an OpenAPI spec
-pub fn api() -> ArtifactServerApiDesc {
-    fn register_endpoints(
-        api: &mut ArtifactServerApiDesc,
-    ) -> Result<(), ApiDescriptionRegisterError> {
-        api.register(get_artifact_by_hash)?;
-        api.register(report_progress)?;
-        Ok(())
-    }
-
-    let mut api = ArtifactServerApiDesc::new();
-    if let Err(err) = register_endpoints(&mut api) {
-        panic!("failed to register entrypoints: {}", err);
-    }
-    api
-}
-
-/// Fetch an artifact by hash.
-#[endpoint {
-    method = GET,
-    path = "/artifacts/by-hash/{kind}/{hash}",
-}]
-async fn get_artifact_by_hash(
-    rqctx: RequestContext<ServerContext>,
-    path: Path<ArtifactHashId>,
-) -> Result<HttpResponseHeaders<HttpResponseOk<FreeformBody>>, HttpError> {
-    match rqctx
-        .context()
-        .artifact_store
-        .get_artifact_by_hash(&path.into_inner())
-        .await
-    {
-        Some((size, body)) => Ok(body_to_artifact_response(size, body)),
-        None => {
-            Err(HttpError::for_not_found(None, "Artifact not found".into()))
-        }
-    }
-}
-
-#[derive(Debug, Deserialize, JsonSchema)]
-pub(crate) struct ReportQuery {
-    /// A unique identifier for the update.
-    pub(crate) update_id: Uuid,
-}
-
-/// Report progress and completion to the server.
-///
-/// This method requires an `update_id` path parameter. This update ID is
-/// matched against the server currently performing an update. If the server
-/// is unaware of the update ID, it will return an HTTP 422 Unprocessable Entity
-/// code.
-#[endpoint {
-    method = POST,
-    path = "/report-progress/{update_id}",
-}]
-async fn report_progress(
-    rqctx: RequestContext<ServerContext>,
-    path: Path<ReportQuery>,
-    report: TypedBody<EventReport>,
-) -> Result<HttpResponseUpdatedNoContent, HttpError> {
-    let update_id = path.into_inner().update_id;
-    match rqctx
-        .context()
-        .artifact_store
-        .report_progress(update_id, report.into_inner())
-        .await?
-    {
-        EventReportStatus::Processed => Ok(HttpResponseUpdatedNoContent()),
-        EventReportStatus::UnrecognizedUpdateId => {
-            Err(HttpError::for_client_error(
-                None,
-                StatusCode::UNPROCESSABLE_ENTITY,
-                format!("update ID {update_id} unrecognized by this server"),
-            ))
-        }
-        EventReportStatus::ReceiverClosed => Err(HttpError::for_client_error(
-            None,
-            StatusCode::GONE,
-            format!("update ID {update_id}: receiver closed"),
-        )),
-    }
-}
-
-fn body_to_artifact_response(
-    size: u64,
-    body: Body,
-) -> HttpResponseHeaders<HttpResponseOk<FreeformBody>> {
-    let mut response =
-        HttpResponseHeaders::new_unnamed(HttpResponseOk(body.into()));
-    let headers = response.headers_mut();
-    headers.append(header::CONTENT_LENGTH, size.into());
-    response
-}
diff --git a/installinator-artifactd/src/lib.rs b/installinator-artifactd/src/lib.rs
deleted file mode 100644
index c54ed78a97..0000000000
--- a/installinator-artifactd/src/lib.rs
+++ /dev/null
@@ -1,29 +0,0 @@
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this
-// file, You can obtain one at https://mozilla.org/MPL/2.0/.
-
-// Copyright 2023 Oxide Computer Company
-
-mod context;
-mod http_entrypoints;
-mod server;
-mod store;
-
-pub use context::ServerContext;
-pub use server::ArtifactServer;
-pub use store::{ArtifactGetter, EventReportStatus};
-
-use anyhow::Result;
-
-/// Run the OpenAPI generator for the API; which emits the OpenAPI spec
-/// to stdout.
-pub fn run_openapi() -> Result<()> {
-    http_entrypoints::api()
-        .openapi("Oxide Installinator Artifact Server", "0.0.1")
-        .description("API for use by the installinator to retrieve artifacts")
-        .contact_url("https://oxide.computer")
-        .contact_email("api@oxide.computer")
-        .write(&mut std::io::stdout())?;
-
-    Ok(())
-}
diff --git a/installinator-artifactd/src/server.rs b/installinator-artifactd/src/server.rs
deleted file mode 100644
index 88b622b756..0000000000
--- a/installinator-artifactd/src/server.rs
+++ /dev/null
@@ -1,74 +0,0 @@
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this
-// file, You can obtain one at https://mozilla.org/MPL/2.0/.
-
-// Copyright 2023 Oxide Computer Company
-
-//! The installinator artifact server.
-
-use std::net::SocketAddrV6;
-
-use anyhow::{anyhow, Result};
-use dropshot::{HandlerTaskMode, HttpServer};
-
-use crate::{
-    context::ServerContext,
-    store::{ArtifactGetter, ArtifactStore},
-};
-
-/// The installinator artifact server.
-#[derive(Debug)]
-pub struct ArtifactServer {
-    address: SocketAddrV6,
-    log: slog::Logger,
-    store: ArtifactStore,
-}
-
-impl ArtifactServer {
-    /// Creates a new artifact server with the given address.
-    pub fn new<Getter: ArtifactGetter>(
-        getter: Getter,
-        address: SocketAddrV6,
-        log: &slog::Logger,
-    ) -> Self {
-        let log = log.new(slog::o!("component" => "installinator artifactd"));
-        let store = ArtifactStore::new(getter, &log);
-        Self { address, log, store }
-    }
-
-    /// Starts the artifact server.
-    ///
-    /// This returns an `HttpServer`, which can be awaited to completion.
-    pub fn start(self) -> Result<HttpServer<ServerContext>> {
-        let context = ServerContext { artifact_store: self.store };
-
-        let dropshot_config = dropshot::ConfigDropshot {
-            bind_address: std::net::SocketAddr::V6(self.address),
-            // Even though the installinator sets an upper bound on the number
-            // of items in a progress report, they can get pretty large if they
-            // haven't gone through for a bit. Ensure that hitting the max
-            // request size won't cause a failure by setting a generous upper
-            // bound for the request size.
-            //
-            // TODO: replace with an endpoint-specific option once
-            // https://github.com/oxidecomputer/dropshot/pull/618 lands and is
-            // available in omicron.
-            request_body_max_bytes: 4 * 1024 * 1024,
-            default_handler_task_mode: HandlerTaskMode::Detached,
-        };
-
-        let api = crate::http_entrypoints::api();
-        let server = dropshot::HttpServerStarter::new(
-            &dropshot_config,
-            api,
-            context,
-            &self.log,
-        )
-        .map_err(|error| {
-            anyhow!(error)
-                .context("failed to create installinator artifact server")
-        })?;
-
-        Ok(server.start())
-    }
-}
diff --git a/installinator-artifactd/src/store.rs b/installinator-artifactd/src/store.rs
deleted file mode 100644
index 12e2880893..0000000000
--- a/installinator-artifactd/src/store.rs
+++ /dev/null
@@ -1,79 +0,0 @@
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this
-// file, You can obtain one at https://mozilla.org/MPL/2.0/.
-
-// Copyright 2023 Oxide Computer Company
-
-use std::fmt;
-
-use async_trait::async_trait;
-use dropshot::HttpError;
-use hyper::Body;
-use installinator_common::EventReport;
-use omicron_common::update::ArtifactHashId;
-use slog::Logger;
-use uuid::Uuid;
-
-/// Represents a way to fetch artifacts.
-#[async_trait]
-pub trait ArtifactGetter: fmt::Debug + Send + Sync + 'static {
-    /// Gets an artifact by hash, returning it as a [`Body`].
-    async fn get_by_hash(&self, id: &ArtifactHashId) -> Option<(u64, Body)>;
-
-    /// Reports update progress events from the installinator.
-    async fn report_progress(
-        &self,
-        update_id: Uuid,
-        report: EventReport,
-    ) -> Result<EventReportStatus, HttpError>;
-}
-
-/// The status returned by [`ArtifactGetter::report_progress`].
-#[derive(Copy, Clone, Debug, Eq, PartialEq, PartialOrd, Ord, Hash)]
-#[must_use]
-pub enum EventReportStatus {
-    /// This report was processed by the server.
-    Processed,
-
-    /// The update ID was not recognized by the server.
-    UnrecognizedUpdateId,
-
-    /// The progress receiver is closed.
-    ReceiverClosed,
-}
-
-/// The artifact store -- a simple wrapper around a dynamic [`ArtifactGetter`] that does some basic
-/// logging.
-#[derive(Debug)]
-pub(crate) struct ArtifactStore {
-    log: Logger,
-    getter: Box<dyn ArtifactGetter>,
-    // TODO: implement this
-}
-
-impl ArtifactStore {
-    pub(crate) fn new<Getter: ArtifactGetter>(
-        getter: Getter,
-        log: &Logger,
-    ) -> Self {
-        let log = log.new(slog::o!("component" => "artifact store"));
-        Self { log, getter: Box::new(getter) }
-    }
-
-    pub(crate) async fn get_artifact_by_hash(
-        &self,
-        id: &ArtifactHashId,
-    ) -> Option<(u64, Body)> {
-        slog::debug!(self.log, "Artifact requested by hash: {:?}", id);
-        self.getter.get_by_hash(id).await
-    }
-
-    pub(crate) async fn report_progress(
-        &self,
-        update_id: Uuid,
-        report: EventReport,
-    ) -> Result<EventReportStatus, HttpError> {
-        slog::debug!(self.log, "Report for {update_id}: {report:?}");
-        self.getter.report_progress(update_id, report).await
-    }
-}
diff --git a/installinator-artifactd/tests/integration_tests/mod.rs b/installinator-artifactd/tests/integration_tests/mod.rs
deleted file mode 100644
index ebb67c3880..0000000000
--- a/installinator-artifactd/tests/integration_tests/mod.rs
+++ /dev/null
@@ -1,5 +0,0 @@
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this
-// file, You can obtain one at https://mozilla.org/MPL/2.0/.
-
-mod openapi;
diff --git a/installinator-artifactd/tests/integration_tests/openapi.rs b/installinator-artifactd/tests/integration_tests/openapi.rs
deleted file mode 100644
index 09441731d0..0000000000
--- a/installinator-artifactd/tests/integration_tests/openapi.rs
+++ /dev/null
@@ -1,39 +0,0 @@
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this
-// file, You can obtain one at https://mozilla.org/MPL/2.0/.
-
-use std::path::PathBuf;
-
-use expectorate::assert_contents;
-use omicron_test_utils::dev::test_cmds::{
-    assert_exit_code, path_to_executable, run_command, EXIT_SUCCESS,
-};
-use openapiv3::OpenAPI;
-use subprocess::Exec;
-
-// name of executable
-const CMD_SERVER: &str = env!("CARGO_BIN_EXE_installinator-artifactd");
-
-fn path_to_server() -> PathBuf {
-    path_to_executable(CMD_SERVER)
-}
-
-#[test]
-fn test_server_openapi() {
-    let exec = Exec::cmd(path_to_server()).arg("openapi");
-    let (exit_status, stdout_text, stderr_text) = run_command(exec);
-    assert_exit_code(exit_status, EXIT_SUCCESS, &stderr_text);
-    assert_contents("tests/output/cmd-server-openapi-stderr", &stderr_text);
-
-    let spec: OpenAPI = serde_json::from_str(&stdout_text)
-        .expect("stdout was not valid OpenAPI");
-
-    // Check for lint errors.
-    let errors = openapi_lint::validate(&spec);
-    assert!(errors.is_empty(), "{}", errors.join("\n\n"));
-
-    // Confirm that the output hasn't changed. It's expected that we'll change
-    // this file as the API evolves, but pay attention to the diffs to ensure
-    // that the changes match your expectations.
-    assert_contents("../openapi/installinator-artifactd.json", &stdout_text);
-}
diff --git a/installinator-artifactd/tests/mod.rs b/installinator-artifactd/tests/mod.rs
deleted file mode 100644
index 66fee5d99c..0000000000
--- a/installinator-artifactd/tests/mod.rs
+++ /dev/null
@@ -1,17 +0,0 @@
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this
-// file, You can obtain one at https://mozilla.org/MPL/2.0/.
-
-//! Integration tests for the installinator artifact server.
-//!
-//! Why use this weird layer of indirection, you might ask?  Cargo chooses to
-//! compile *each file* within the "tests/" subdirectory as a separate crate.
-//! This means that doing "file-granularity" conditional compilation is
-//! difficult, since a file like "test_for_illumos_only.rs" would get compiled
-//! and tested regardless of the contents of "mod.rs".
-//!
-//! However, by lumping all tests into a submodule, all integration tests are
-//! joined into a single crate, which itself can filter individual files
-//! by (for example) choice of target OS.
-
-mod integration_tests;
diff --git a/installinator-artifactd/tests/output/cmd-server-openapi-stderr b/installinator-artifactd/tests/output/cmd-server-openapi-stderr
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/installinator/Cargo.toml b/installinator/Cargo.toml
index c21c3f2ee2..00dfb6440b 100644
--- a/installinator/Cargo.toml
+++ b/installinator/Cargo.toml
@@ -20,7 +20,7 @@ futures.workspace = true
 hex.workspace = true
 http.workspace = true
 illumos-utils.workspace = true
-installinator-artifact-client.workspace = true
+installinator-client.workspace = true
 installinator-common.workspace = true
 ipcc.workspace = true
 itertools.workspace = true
diff --git a/installinator/src/artifact.rs b/installinator/src/artifact.rs
index 734759a2c2..12e85e0938 100644
--- a/installinator/src/artifact.rs
+++ b/installinator/src/artifact.rs
@@ -7,7 +7,7 @@ use std::net::SocketAddr;
 use anyhow::{Context, Result};
 use clap::Args;
 use futures::StreamExt;
-use installinator_artifact_client::ClientError;
+use installinator_client::ClientError;
 use installinator_common::EventReport;
 use ipcc::{InstallinatorImageId, Ipcc};
 use omicron_common::update::{ArtifactHash, ArtifactHashId};
@@ -63,7 +63,7 @@ impl ArtifactIdOpts {
 #[derive(Debug)]
 pub(crate) struct ArtifactClient {
     log: slog::Logger,
-    client: installinator_artifact_client::Client,
+    client: installinator_client::Client,
 }
 
 impl ArtifactClient {
@@ -81,8 +81,7 @@ impl ArtifactClient {
         let log = log.new(
             slog::o!("component" => "ArtifactClient", "peer" => addr.to_string()),
         );
-        let client =
-            installinator_artifact_client::Client::new(&endpoint, log.clone());
+        let client = installinator_client::Client::new(&endpoint, log.clone());
         Self { log, client }
     }
 
diff --git a/installinator/src/errors.rs b/installinator/src/errors.rs
index 1349cf7d89..577d0d6f4d 100644
--- a/installinator/src/errors.rs
+++ b/installinator/src/errors.rs
@@ -4,7 +4,7 @@
 
 use std::{net::SocketAddr, time::Duration};
 
-use installinator_artifact_client::ClientError;
+use installinator_client::ClientError;
 use thiserror::Error;
 
 #[derive(Debug, Error)]
diff --git a/installinator/src/mock_peers.rs b/installinator/src/mock_peers.rs
index 434276649f..ccb35a2f06 100644
--- a/installinator/src/mock_peers.rs
+++ b/installinator/src/mock_peers.rs
@@ -16,7 +16,7 @@ use std::{
 use anyhow::{bail, Result};
 use async_trait::async_trait;
 use bytes::Bytes;
-use installinator_artifact_client::{ClientError, ResponseValue};
+use installinator_client::{ClientError, ResponseValue};
 use installinator_common::EventReport;
 use omicron_common::update::ArtifactHashId;
 use proptest::prelude::*;
@@ -342,7 +342,7 @@ impl MockPeer {
                 tokio::time::sleep(after).await;
                 _ = sender
                     .send(Err(ClientError::ErrorResponse(ResponseValue::new(
-                        installinator_artifact_client::types::Error {
+                        installinator_client::types::Error {
                             error_code: None,
                             message: format!("not-found error after {after:?}"),
                             request_id: "mock-request-id".to_owned(),
@@ -356,7 +356,7 @@ impl MockPeer {
                 tokio::time::sleep(after).await;
                 _ = sender
                     .send(Err(ClientError::ErrorResponse(ResponseValue::new(
-                        installinator_artifact_client::types::Error {
+                        installinator_client::types::Error {
                             error_code: None,
                             message: format!("forbidden error after {after:?}"),
                             request_id: "mock-request-id".to_owned(),
@@ -526,7 +526,7 @@ impl PeersImpl for MockReportPeers {
             Ok(())
         } else if peer == Self::invalid_peer() {
             Err(ClientError::ErrorResponse(ResponseValue::new(
-                installinator_artifact_client::types::Error {
+                installinator_client::types::Error {
                     error_code: None,
                     message: "invalid peer => HTTP 422".to_owned(),
                     request_id: "mock-request-id".to_owned(),
diff --git a/installinator/src/peers.rs b/installinator/src/peers.rs
index 644507da4b..3d2e05077d 100644
--- a/installinator/src/peers.rs
+++ b/installinator/src/peers.rs
@@ -16,7 +16,7 @@ use buf_list::BufList;
 use bytes::Bytes;
 use display_error_chain::DisplayErrorChain;
 use futures::{Stream, StreamExt};
-use installinator_artifact_client::ClientError;
+use installinator_client::ClientError;
 use installinator_common::{
     EventReport, InstallinatorProgressMetadata, StepContext, StepProgress,
 };
diff --git a/internal-dns/src/resolver.rs b/internal-dns/src/resolver.rs
index cf5def01c5..fdd5dce428 100644
--- a/internal-dns/src/resolver.rs
+++ b/internal-dns/src/resolver.rs
@@ -434,6 +434,7 @@ mod test {
                     bind_address: "[::1]:0".parse().unwrap(),
                     request_body_max_bytes: 8 * 1024,
                     default_handler_task_mode: HandlerTaskMode::Detached,
+                    log_headers: vec![],
                 },
             )
             .await
diff --git a/nexus-config/src/nexus_config.rs b/nexus-config/src/nexus_config.rs
index 5ca1d2d6ed..3bc3a36126 100644
--- a/nexus-config/src/nexus_config.rs
+++ b/nexus-config/src/nexus_config.rs
@@ -383,6 +383,8 @@ pub struct BackgroundTaskConfig {
     pub v2p_mapping_propagation: V2PMappingPropagationConfig,
     /// configuration for abandoned VMM reaper task
     pub abandoned_vmm_reaper: AbandonedVmmReaperConfig,
+    /// configuration for saga recovery task
+    pub saga_recovery: SagaRecoveryConfig,
     /// configuration for lookup region port task
     pub lookup_region_port: LookupRegionPortConfig,
 }
@@ -566,6 +568,14 @@ pub struct AbandonedVmmReaperConfig {
     pub period_secs: Duration,
 }
 
+#[serde_as]
+#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
+pub struct SagaRecoveryConfig {
+    /// period (in seconds) for periodic activations of this background task
+    #[serde_as(as = "DurationSeconds<u64>")]
+    pub period_secs: Duration,
+}
+
 #[serde_as]
 #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
 pub struct RegionReplacementDriverConfig {
@@ -824,6 +834,7 @@ mod test {
             service_firewall_propagation.period_secs = 300
             v2p_mapping_propagation.period_secs = 30
             abandoned_vmm_reaper.period_secs = 60
+            saga_recovery.period_secs = 60
             lookup_region_port.period_secs = 60
             [default_region_allocation_strategy]
             type = "random"
@@ -972,6 +983,9 @@ mod test {
                         abandoned_vmm_reaper: AbandonedVmmReaperConfig {
                             period_secs: Duration::from_secs(60),
                         },
+                        saga_recovery: SagaRecoveryConfig {
+                            period_secs: Duration::from_secs(60),
+                        },
                         lookup_region_port: LookupRegionPortConfig {
                             period_secs: Duration::from_secs(60),
                         },
@@ -1047,6 +1061,7 @@ mod test {
             service_firewall_propagation.period_secs = 300
             v2p_mapping_propagation.period_secs = 30
             abandoned_vmm_reaper.period_secs = 60
+            saga_recovery.period_secs = 60
             lookup_region_port.period_secs = 60
             [default_region_allocation_strategy]
             type = "random"
@@ -1174,6 +1189,12 @@ mod test {
         let example_config = NexusConfig::from_file(config_path)
             .expect("example config file is not valid");
 
+        // The second example config file should be valid.
+        let config_path = "../nexus/examples/config-second.toml";
+        println!("checking {:?}", config_path);
+        let _ = NexusConfig::from_file(config_path)
+            .expect("second example config file is not valid");
+
         // The config file used for the tests should also be valid.  The tests
         // won't clear the runway anyway if this file isn't valid.  But it's
         // helpful to verify this here explicitly as well.
diff --git a/nexus/Cargo.toml b/nexus/Cargo.toml
index 359ea616d4..8d256aad5a 100644
--- a/nexus/Cargo.toml
+++ b/nexus/Cargo.toml
@@ -47,6 +47,7 @@ nexus-client.workspace = true
 nexus-config.workspace = true
 nexus-internal-api.workspace = true
 nexus-networking.workspace = true
+nexus-saga-recovery.workspace = true
 nexus-test-interface.workspace = true
 num-integer.workspace = true
 once_cell.workspace = true
diff --git a/nexus/auth/src/context.rs b/nexus/auth/src/context.rs
index 0aac0900c5..161ce6493b 100644
--- a/nexus/auth/src/context.rs
+++ b/nexus/auth/src/context.rs
@@ -236,6 +236,25 @@ impl OpContext {
         }
     }
 
+    /// Creates a new `OpContext` just like the given one, but with a different
+    /// identity.
+    ///
+    /// This is only intended for tests.
+    pub fn child_with_authn(&self, authn: authn::Context) -> OpContext {
+        let created_instant = Instant::now();
+        let created_walltime = SystemTime::now();
+
+        OpContext {
+            log: self.log.clone(),
+            authn: Arc::new(authn),
+            authz: self.authz.clone(),
+            created_instant,
+            created_walltime,
+            metadata: self.metadata.clone(),
+            kind: self.kind,
+        }
+    }
+
     /// Check whether the actor performing this request is authorized for
     /// `action` on `resource`.
     pub async fn authorize<Resource>(
diff --git a/nexus/db-model/src/dataset.rs b/nexus/db-model/src/dataset.rs
index 65c0070509..a9dee990b9 100644
--- a/nexus/db-model/src/dataset.rs
+++ b/nexus/db-model/src/dataset.rs
@@ -36,8 +36,8 @@ pub struct Dataset {
 
     pub pool_id: Uuid,
 
-    ip: ipv6::Ipv6Addr,
-    port: SqlU16,
+    ip: Option<ipv6::Ipv6Addr>,
+    port: Option<SqlU16>,
 
     pub kind: DatasetKind,
     pub size_used: Option<i64>,
@@ -47,7 +47,7 @@ impl Dataset {
     pub fn new(
         id: Uuid,
         pool_id: Uuid,
-        addr: SocketAddrV6,
+        addr: Option<SocketAddrV6>,
         kind: DatasetKind,
     ) -> Self {
         let size_used = match kind {
@@ -59,19 +59,19 @@ impl Dataset {
             time_deleted: None,
             rcgen: Generation::new(),
             pool_id,
-            ip: addr.ip().into(),
-            port: addr.port().into(),
+            ip: addr.map(|addr| addr.ip().into()),
+            port: addr.map(|addr| addr.port().into()),
             kind,
             size_used,
         }
     }
 
-    pub fn address(&self) -> SocketAddrV6 {
-        self.address_with_port(self.port.into())
+    pub fn address(&self) -> Option<SocketAddrV6> {
+        self.address_with_port(self.port?.into())
     }
 
-    pub fn address_with_port(&self, port: u16) -> SocketAddrV6 {
-        SocketAddrV6::new(Ipv6Addr::from(self.ip), port, 0, 0)
+    pub fn address_with_port(&self, port: u16) -> Option<SocketAddrV6> {
+        Some(SocketAddrV6::new(Ipv6Addr::from(self.ip?), port, 0, 0))
     }
 }
 
diff --git a/nexus/db-model/src/lib.rs b/nexus/db-model/src/lib.rs
index 30dc82965d..f28f886f6c 100644
--- a/nexus/db-model/src/lib.rs
+++ b/nexus/db-model/src/lib.rs
@@ -176,6 +176,7 @@ pub use region_replacement_step::*;
 pub use region_snapshot::*;
 pub use role_assignment::*;
 pub use role_builtin::*;
+pub use saga_types::*;
 pub use schema_versions::*;
 pub use semver_version::*;
 pub use service_kind::*;
diff --git a/nexus/db-model/src/physical_disk.rs b/nexus/db-model/src/physical_disk.rs
index c6ef97ee1f..d4a1dcd33c 100644
--- a/nexus/db-model/src/physical_disk.rs
+++ b/nexus/db-model/src/physical_disk.rs
@@ -85,3 +85,75 @@ impl DatastoreCollectionConfig<super::Zpool> for PhysicalDisk {
     type CollectionTimeDeletedColumn = physical_disk::dsl::time_deleted;
     type CollectionIdColumn = zpool::dsl::sled_id;
 }
+
+mod diesel_util {
+    use diesel::{
+        helper_types::{And, EqAny},
+        prelude::*,
+        query_dsl::methods::FilterDsl,
+    };
+    use nexus_types::{
+        deployment::DiskFilter,
+        external_api::views::{PhysicalDiskPolicy, PhysicalDiskState},
+    };
+
+    /// An extension trait to apply a [`DiskFilter`] to a Diesel expression.
+    ///
+    /// This is applicable to any Diesel expression which includes the `physical_disk`
+    /// table.
+    ///
+    /// This needs to live here, rather than in `nexus-db-queries`, because it
+    /// names the `DbPhysicalDiskPolicy` type which is private to this crate.
+    pub trait ApplyPhysicalDiskFilterExt {
+        type Output;
+
+        /// Applies a [`DiskFilter`] to a Diesel expression.
+        fn physical_disk_filter(self, filter: DiskFilter) -> Self::Output;
+    }
+
+    impl<E> ApplyPhysicalDiskFilterExt for E
+    where
+        E: FilterDsl<PhysicalDiskFilterQuery>,
+    {
+        type Output = E::Output;
+
+        fn physical_disk_filter(self, filter: DiskFilter) -> Self::Output {
+            use crate::schema::physical_disk::dsl as physical_disk_dsl;
+
+            // These are only boxed for ease of reference above.
+            let all_matching_policies: BoxedIterator<
+                crate::PhysicalDiskPolicy,
+            > = Box::new(
+                PhysicalDiskPolicy::all_matching(filter).map(Into::into),
+            );
+            let all_matching_states: BoxedIterator<crate::PhysicalDiskState> =
+                Box::new(
+                    PhysicalDiskState::all_matching(filter).map(Into::into),
+                );
+
+            FilterDsl::filter(
+                self,
+                physical_disk_dsl::disk_policy
+                    .eq_any(all_matching_policies)
+                    .and(
+                        physical_disk_dsl::disk_state
+                            .eq_any(all_matching_states),
+                    ),
+            )
+        }
+    }
+
+    type BoxedIterator<T> = Box<dyn Iterator<Item = T>>;
+    type PhysicalDiskFilterQuery = And<
+        EqAny<
+            crate::schema::physical_disk::disk_policy,
+            BoxedIterator<crate::PhysicalDiskPolicy>,
+        >,
+        EqAny<
+            crate::schema::physical_disk::disk_state,
+            BoxedIterator<crate::PhysicalDiskState>,
+        >,
+    >;
+}
+
+pub use diesel_util::ApplyPhysicalDiskFilterExt;
diff --git a/nexus/db-model/src/schema.rs b/nexus/db-model/src/schema.rs
index 89ae6c18c5..dc57de9263 100644
--- a/nexus/db-model/src/schema.rs
+++ b/nexus/db-model/src/schema.rs
@@ -1022,8 +1022,8 @@ table! {
 
         pool_id -> Uuid,
 
-        ip -> Inet,
-        port -> Int4,
+        ip -> Nullable<Inet>,
+        port -> Nullable<Int4>,
 
         kind -> crate::DatasetKindEnum,
         size_used -> Nullable<Int8>,
diff --git a/nexus/db-model/src/schema_versions.rs b/nexus/db-model/src/schema_versions.rs
index 3e740590c5..cc34a3581c 100644
--- a/nexus/db-model/src/schema_versions.rs
+++ b/nexus/db-model/src/schema_versions.rs
@@ -17,7 +17,7 @@ use std::collections::BTreeMap;
 ///
 /// This must be updated when you change the database schema.  Refer to
 /// schema/crdb/README.adoc in the root of this repository for details.
-pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(82, 0, 0);
+pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(83, 0, 0);
 
 /// List of all past database schema versions, in *reverse* order
 ///
@@ -29,6 +29,7 @@ static KNOWN_VERSIONS: Lazy<Vec<KnownVersion>> = Lazy::new(|| {
         // |  leaving the first copy as an example for the next person.
         // v
         // KnownVersion::new(next_int, "unique-dirname-with-the-sql-files"),
+        KnownVersion::new(83, "dataset-address-optional"),
         KnownVersion::new(82, "region-port"),
         KnownVersion::new(81, "add-nullable-filesystem-pool"),
         KnownVersion::new(80, "add-instance-id-to-migrations"),
diff --git a/nexus/db-model/src/sled.rs b/nexus/db-model/src/sled.rs
index c177650991..b02a082f07 100644
--- a/nexus/db-model/src/sled.rs
+++ b/nexus/db-model/src/sled.rs
@@ -26,7 +26,7 @@ use uuid::Uuid;
 pub struct SledBaseboard {
     pub serial_number: String,
     pub part_number: String,
-    pub revision: i64,
+    pub revision: u32,
 }
 
 /// Hardware information about the sled.
@@ -53,7 +53,7 @@ pub struct Sled {
     is_scrimlet: bool,
     serial_number: String,
     part_number: String,
-    revision: i64,
+    revision: SqlU32,
 
     pub usable_hardware_threads: SqlU32,
     pub usable_physical_ram: ByteCount,
@@ -128,7 +128,7 @@ impl From<Sled> for views::Sled {
             baseboard: shared::Baseboard {
                 serial: sled.serial_number,
                 part: sled.part_number,
-                revision: sled.revision,
+                revision: *sled.revision,
             },
             policy: sled.policy.into(),
             state: sled.state.into(),
@@ -155,7 +155,7 @@ impl From<Sled> for params::SledAgentInfo {
             baseboard: Baseboard {
                 serial: sled.serial_number.clone(),
                 part: sled.part_number.clone(),
-                revision: sled.revision,
+                revision: *sled.revision,
             },
             usable_hardware_threads: sled.usable_hardware_threads.into(),
             usable_physical_ram: sled.usable_physical_ram.into(),
@@ -192,7 +192,7 @@ pub struct SledUpdate {
     is_scrimlet: bool,
     serial_number: String,
     part_number: String,
-    revision: i64,
+    revision: SqlU32,
 
     pub usable_hardware_threads: SqlU32,
     pub usable_physical_ram: ByteCount,
@@ -221,7 +221,7 @@ impl SledUpdate {
             is_scrimlet: hardware.is_scrimlet,
             serial_number: baseboard.serial_number,
             part_number: baseboard.part_number,
-            revision: baseboard.revision,
+            revision: SqlU32(baseboard.revision),
             usable_hardware_threads: SqlU32::new(
                 hardware.usable_hardware_threads,
             ),
diff --git a/nexus/db-model/src/switch.rs b/nexus/db-model/src/switch.rs
index 159888d91e..d5c523e9c4 100644
--- a/nexus/db-model/src/switch.rs
+++ b/nexus/db-model/src/switch.rs
@@ -1,5 +1,9 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
 use super::Generation;
-use crate::schema::switch;
+use crate::{schema::switch, SqlU32};
 use chrono::{DateTime, Utc};
 use db_macros::Asset;
 use nexus_types::{external_api::shared, external_api::views, identity::Asset};
@@ -12,7 +16,7 @@ use uuid::Uuid;
 pub struct SwitchBaseboard {
     pub serial_number: String,
     pub part_number: String,
-    pub revision: i64,
+    pub revision: u32,
 }
 
 /// Database representation of a Switch.
@@ -28,7 +32,7 @@ pub struct Switch {
 
     serial_number: String,
     part_number: String,
-    revision: i64,
+    revision: SqlU32,
 }
 
 impl Switch {
@@ -37,7 +41,7 @@ impl Switch {
         id: Uuid,
         serial_number: String,
         part_number: String,
-        revision: i64,
+        revision: u32,
         rack_id: Uuid,
     ) -> Self {
         Self {
@@ -47,7 +51,7 @@ impl Switch {
             rack_id,
             serial_number,
             part_number,
-            revision,
+            revision: SqlU32(revision),
         }
     }
 }
@@ -60,7 +64,7 @@ impl From<Switch> for views::Switch {
             baseboard: shared::Baseboard {
                 serial: switch.serial_number,
                 part: switch.part_number,
-                revision: switch.revision,
+                revision: *switch.revision,
             },
         }
     }
diff --git a/nexus/db-queries/src/db/datastore/dataset.rs b/nexus/db-queries/src/db/datastore/dataset.rs
index 3f1df24e45..a08e346fe8 100644
--- a/nexus/db-queries/src/db/datastore/dataset.rs
+++ b/nexus/db-queries/src/db/datastore/dataset.rs
@@ -290,7 +290,7 @@ mod test {
             .dataset_insert_if_not_exists(Dataset::new(
                 Uuid::new_v4(),
                 zpool_id,
-                "[::1]:0".parse().unwrap(),
+                Some("[::1]:0".parse().unwrap()),
                 DatasetKind::Crucible,
             ))
             .await
@@ -323,7 +323,7 @@ mod test {
             .dataset_insert_if_not_exists(Dataset::new(
                 dataset1.id(),
                 zpool_id,
-                "[::1]:12345".parse().unwrap(),
+                Some("[::1]:12345".parse().unwrap()),
                 DatasetKind::Cockroach,
             ))
             .await
@@ -339,7 +339,7 @@ mod test {
             .dataset_upsert(Dataset::new(
                 Uuid::new_v4(),
                 zpool_id,
-                "[::1]:0".parse().unwrap(),
+                Some("[::1]:0".parse().unwrap()),
                 DatasetKind::Cockroach,
             ))
             .await
@@ -371,7 +371,7 @@ mod test {
             .dataset_insert_if_not_exists(Dataset::new(
                 dataset1.id(),
                 zpool_id,
-                "[::1]:12345".parse().unwrap(),
+                Some("[::1]:12345".parse().unwrap()),
                 DatasetKind::Cockroach,
             ))
             .await
diff --git a/nexus/db-queries/src/db/datastore/mod.rs b/nexus/db-queries/src/db/datastore/mod.rs
index 461e71d88a..2540790477 100644
--- a/nexus/db-queries/src/db/datastore/mod.rs
+++ b/nexus/db-queries/src/db/datastore/mod.rs
@@ -394,9 +394,9 @@ mod test {
         BlockSize, ConsoleSession, Dataset, DatasetKind, ExternalIp,
         PhysicalDisk, PhysicalDiskKind, PhysicalDiskPolicy, PhysicalDiskState,
         Project, Rack, Region, SiloUser, SledBaseboard, SledSystemHardware,
-        SledUpdate, SshKey, VpcSubnet, Zpool,
+        SledUpdate, SshKey, Zpool,
     };
-    use crate::db::queries::vpc_subnet::FilterConflictingVpcSubnetRangesQuery;
+    use crate::db::queries::vpc_subnet::InsertVpcSubnetQuery;
     use chrono::{Duration, Utc};
     use futures::stream;
     use futures::StreamExt;
@@ -892,7 +892,8 @@ mod test {
                 .collect()
                 .await;
 
-            let bogus_addr = SocketAddrV6::new(Ipv6Addr::LOCALHOST, 8080, 0, 0);
+            let bogus_addr =
+                Some(SocketAddrV6::new(Ipv6Addr::LOCALHOST, 8080, 0, 0));
 
             let datasets = stream::iter(zpools)
                 .map(|zpool| {
@@ -1266,7 +1267,8 @@ mod test {
             .collect()
             .await;
 
-        let bogus_addr = SocketAddrV6::new(Ipv6Addr::LOCALHOST, 8080, 0, 0);
+        let bogus_addr =
+            Some(SocketAddrV6::new(Ipv6Addr::LOCALHOST, 8080, 0, 0));
 
         // 1 dataset per zpool
         stream::iter(zpool_ids.clone())
@@ -1365,7 +1367,8 @@ mod test {
                 .collect()
                 .await;
 
-        let bogus_addr = SocketAddrV6::new(Ipv6Addr::LOCALHOST, 8080, 0, 0);
+        let bogus_addr =
+            Some(SocketAddrV6::new(Ipv6Addr::LOCALHOST, 8080, 0, 0));
 
         // 1 dataset per zpool
         stream::iter(zpool_ids)
@@ -1444,7 +1447,8 @@ mod test {
                 physical_disk_id,
             )
             .await;
-            let bogus_addr = SocketAddrV6::new(Ipv6Addr::LOCALHOST, 8080, 0, 0);
+            let bogus_addr =
+                Some(SocketAddrV6::new(Ipv6Addr::LOCALHOST, 8080, 0, 0));
             let dataset = Dataset::new(
                 Uuid::new_v4(),
                 zpool_id,
@@ -1599,11 +1603,7 @@ mod test {
             "172.30.0.0/22".parse().unwrap(),
             "fd00::/64".parse().unwrap(),
         );
-        let values = FilterConflictingVpcSubnetRangesQuery::new(subnet);
-        let query =
-            diesel::insert_into(db::schema::vpc_subnet::dsl::vpc_subnet)
-                .values(values)
-                .returning(VpcSubnet::as_returning());
+        let query = InsertVpcSubnetQuery::new(subnet);
         println!("{}", diesel::debug_query(&query));
         let explanation = query.explain_async(&conn).await.unwrap();
         assert!(
diff --git a/nexus/db-queries/src/db/datastore/physical_disk.rs b/nexus/db-queries/src/db/datastore/physical_disk.rs
index e51d59075e..11e056d19b 100644
--- a/nexus/db-queries/src/db/datastore/physical_disk.rs
+++ b/nexus/db-queries/src/db/datastore/physical_disk.rs
@@ -26,7 +26,8 @@ use crate::transaction_retry::OptionalError;
 use async_bb8_diesel::AsyncRunQueryDsl;
 use chrono::Utc;
 use diesel::prelude::*;
-use nexus_types::deployment::SledFilter;
+use nexus_db_model::ApplyPhysicalDiskFilterExt;
+use nexus_types::deployment::{DiskFilter, SledFilter};
 use omicron_common::api::external::CreateResult;
 use omicron_common::api::external::DataPageParams;
 use omicron_common::api::external::DeleteResult;
@@ -247,11 +248,13 @@ impl DataStore {
         &self,
         opctx: &OpContext,
         pagparams: &DataPageParams<'_, Uuid>,
+        disk_filter: DiskFilter,
     ) -> ListResultVec<PhysicalDisk> {
         opctx.authorize(authz::Action::Read, &authz::FLEET).await?;
         use db::schema::physical_disk::dsl;
         paginated(dsl::physical_disk, dsl::id, pagparams)
             .filter(dsl::time_deleted.is_null())
+            .physical_disk_filter(disk_filter)
             .select(PhysicalDisk::as_select())
             .load_async(&*self.pool_connection_authorized(opctx).await?)
             .await
diff --git a/nexus/db-queries/src/db/datastore/region.rs b/nexus/db-queries/src/db/datastore/region.rs
index 6832665944..3b1c20c1df 100644
--- a/nexus/db-queries/src/db/datastore/region.rs
+++ b/nexus/db-queries/src/db/datastore/region.rs
@@ -496,7 +496,13 @@ impl DataStore {
 
         let dataset = self.dataset_get(region.dataset_id()).await?;
 
-        Ok(Some(SocketAddrV6::new(*dataset.address().ip(), port, 0, 0)))
+        let Some(address) = dataset.address() else {
+            return Err(Error::internal_error(
+                "Dataset for Crucible region does know IP address",
+            ));
+        };
+
+        Ok(Some(SocketAddrV6::new(*address.ip(), port, 0, 0)))
     }
 
     pub async fn regions_missing_ports(
diff --git a/nexus/db-queries/src/db/datastore/saga.rs b/nexus/db-queries/src/db/datastore/saga.rs
index c42d14d0d7..939929e665 100644
--- a/nexus/db-queries/src/db/datastore/saga.rs
+++ b/nexus/db-queries/src/db/datastore/saga.rs
@@ -5,14 +5,19 @@
 //! [`DataStore`] methods on [`db::saga_types::Saga`]s.
 
 use super::DataStore;
+use super::SQL_BATCH_SIZE;
 use crate::db;
 use crate::db::error::public_error_from_diesel;
 use crate::db::error::ErrorHandler;
 use crate::db::model::Generation;
+use crate::db::pagination::paginated;
+use crate::db::pagination::paginated_multicolumn;
+use crate::db::pagination::Paginator;
 use crate::db::update_and_check::UpdateAndCheck;
 use crate::db::update_and_check::UpdateStatus;
 use async_bb8_diesel::AsyncRunQueryDsl;
 use diesel::prelude::*;
+use nexus_auth::context::OpContext;
 use omicron_common::api::external::Error;
 use omicron_common::api::external::LookupType;
 use omicron_common::api::external::ResourceType;
@@ -42,6 +47,15 @@ impl DataStore {
         // owning this saga.
         diesel::insert_into(dsl::saga_node_event)
             .values(event.clone())
+            // (saga_id, node_id, event_type) is the primary key, and this is
+            // expected to be idempotent.
+            //
+            // Consider the situation where a saga event gets recorded and
+            // committed, but there's a network reset which makes the client
+            // (us) believe that the event wasn't recorded. If we retry the
+            // event, we want to not fail with a conflict.
+            .on_conflict((dsl::saga_id, dsl::node_id, dsl::event_type))
+            .do_nothing()
             .execute_async(&*self.pool_connection_unauthorized().await?)
             .await
             .map_err(|e| {
@@ -53,6 +67,28 @@ impl DataStore {
         Ok(())
     }
 
+    /// Update the state of a saga in the database.
+    ///
+    /// This function is meant to be called in a loop, so that in the event of
+    /// network flakiness, the operation is retried until successful.
+    ///
+    /// ## About conflicts
+    ///
+    /// Currently, if the value of `saga_state` in the database is the same as
+    /// the value we're trying to set it to, the update will be a no-op. That
+    /// is okay, because at any time only one SEC will update the saga. (For
+    /// now, we're implementing saga adoption only in cases where the original
+    /// SEC/Nexus has been expunged.)
+    ///
+    /// However, in the future, it may be possible for multiple SECs to try and
+    /// update the same saga, and overwrite each other's state. For example,
+    /// one SEC might try and update the state to Running while the other one
+    /// updates it to Done. That case would have to be carefully considered and
+    /// tested here, probably using the (currently unused)
+    /// `current_adopt_generation` field to enable optimistic concurrency.
+    ///
+    /// To reiterate, we are *not* considering the case where several SECs try
+    /// to update the same saga. That will be a future enhancement.
     pub async fn saga_update_state(
         &self,
         saga_id: steno::SagaId,
@@ -99,4 +135,378 @@ impl DataStore {
             )),
         }
     }
+
+    /// Returns a list of unfinished sagas assigned to SEC `sec_id`, making as
+    /// many queries as needed (in batches) to get them all
+    pub async fn saga_list_recovery_candidates_batched(
+        &self,
+        opctx: &OpContext,
+        sec_id: db::saga_types::SecId,
+    ) -> Result<Vec<db::saga_types::Saga>, Error> {
+        let mut sagas = vec![];
+        let mut paginator = Paginator::new(SQL_BATCH_SIZE);
+        let conn = self.pool_connection_authorized(opctx).await?;
+        while let Some(p) = paginator.next() {
+            use db::schema::saga::dsl;
+
+            let mut batch =
+                paginated(dsl::saga, dsl::id, &p.current_pagparams())
+                    .filter(dsl::saga_state.ne(
+                        db::saga_types::SagaCachedState(
+                            steno::SagaCachedState::Done,
+                        ),
+                    ))
+                    .filter(dsl::current_sec.eq(sec_id))
+                    .select(db::saga_types::Saga::as_select())
+                    .load_async(&*conn)
+                    .await
+                    .map_err(|e| {
+                        public_error_from_diesel(e, ErrorHandler::Server)
+                    })?;
+
+            paginator = p.found_batch(&batch, &|row| row.id);
+            sagas.append(&mut batch);
+        }
+        Ok(sagas)
+    }
+
+    /// Returns a list of all saga log entries for the given saga, making as
+    /// many queries as needed (in batches) to get them all
+    pub async fn saga_fetch_log_batched(
+        &self,
+        opctx: &OpContext,
+        saga_id: db::saga_types::SagaId,
+    ) -> Result<Vec<steno::SagaNodeEvent>, Error> {
+        let mut events = vec![];
+        let mut paginator = Paginator::new(SQL_BATCH_SIZE);
+        let conn = self.pool_connection_authorized(opctx).await?;
+        while let Some(p) = paginator.next() {
+            use db::schema::saga_node_event::dsl;
+            let batch = paginated_multicolumn(
+                dsl::saga_node_event,
+                (dsl::node_id, dsl::event_type),
+                &p.current_pagparams(),
+            )
+            .filter(dsl::saga_id.eq(saga_id))
+            .select(db::saga_types::SagaNodeEvent::as_select())
+            .load_async(&*conn)
+            .await
+            .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?;
+
+            paginator = p.found_batch(&batch, &|row| {
+                (row.node_id, row.event_type.clone())
+            });
+
+            let mut batch = batch
+                .into_iter()
+                .map(|event| steno::SagaNodeEvent::try_from(event))
+                .collect::<Result<Vec<_>, Error>>()?;
+
+            events.append(&mut batch);
+        }
+
+        Ok(events)
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use crate::db::datastore::test_utils::datastore_test;
+    use nexus_db_model::{SagaNodeEvent, SecId};
+    use nexus_test_utils::db::test_setup_database;
+    use omicron_test_utils::dev;
+    use rand::seq::SliceRandom;
+    use uuid::Uuid;
+
+    // Tests pagination in listing sagas that are candidates for recovery
+    #[tokio::test]
+    async fn test_list_candidate_sagas() {
+        // Test setup
+        let logctx = dev::test_setup_log("test_list_candidate_sagas");
+        let mut db = test_setup_database(&logctx.log).await;
+        let (opctx, datastore) = datastore_test(&logctx, &db).await;
+        let sec_id = db::SecId(uuid::Uuid::new_v4());
+        let mut inserted_sagas = (0..SQL_BATCH_SIZE.get() * 2)
+            .map(|_| SagaTestContext::new(sec_id).new_running_db_saga())
+            .collect::<Vec<_>>();
+
+        // Shuffle these sagas into a random order to check that the pagination
+        // order is working as intended on the read path, which we'll do later
+        // in this test.
+        inserted_sagas.shuffle(&mut rand::thread_rng());
+
+        // Insert the batches of unfinished sagas into the database
+        let conn = datastore
+            .pool_connection_unauthorized()
+            .await
+            .expect("Failed to access db connection");
+        diesel::insert_into(db::schema::saga::dsl::saga)
+            .values(inserted_sagas.clone())
+            .execute_async(&*conn)
+            .await
+            .expect("Failed to insert test setup data");
+
+        // List them, expect to see them all in order by ID.
+        let mut observed_sagas = datastore
+            .saga_list_recovery_candidates_batched(&opctx, sec_id)
+            .await
+            .expect("Failed to list unfinished sagas");
+        inserted_sagas.sort_by_key(|a| a.id);
+
+        // Timestamps can change slightly when we insert them.
+        //
+        // Sanitize them to make input/output equality checks easier.
+        let sanitize_timestamps = |sagas: &mut Vec<db::saga_types::Saga>| {
+            for saga in sagas {
+                saga.time_created = chrono::DateTime::UNIX_EPOCH;
+                saga.adopt_time = chrono::DateTime::UNIX_EPOCH;
+            }
+        };
+        sanitize_timestamps(&mut observed_sagas);
+        sanitize_timestamps(&mut inserted_sagas);
+
+        assert_eq!(
+            inserted_sagas, observed_sagas,
+            "Observed sagas did not match inserted sagas"
+        );
+
+        // Test cleanup
+        db.cleanup().await.unwrap();
+        logctx.cleanup_successful();
+    }
+
+    // Tests pagination in loading a saga log
+    #[tokio::test]
+    async fn test_list_unfinished_nodes() {
+        // Test setup
+        let logctx = dev::test_setup_log("test_list_unfinished_nodes");
+        let mut db = test_setup_database(&logctx.log).await;
+        let (opctx, datastore) = datastore_test(&logctx, &db).await;
+        let node_cx = SagaTestContext::new(SecId(Uuid::new_v4()));
+
+        // Create a couple batches of saga events
+        let mut inserted_nodes = (0..SQL_BATCH_SIZE.get() * 2)
+            .flat_map(|i| {
+                // This isn't an exhaustive list of event types, but gives us a
+                // few options to pick from. Since this is a pagination key,
+                // it's important to include a variety here.
+                use steno::SagaNodeEventType::*;
+                [
+                    node_cx.new_db_event(i, Started),
+                    node_cx.new_db_event(i, UndoStarted),
+                    node_cx.new_db_event(i, UndoFinished),
+                ]
+            })
+            .collect::<Vec<_>>();
+
+        // Shuffle these nodes into a random order to check that the pagination
+        // order is working as intended on the read path, which we'll do later
+        // in this test.
+        inserted_nodes.shuffle(&mut rand::thread_rng());
+
+        // Insert them into the database
+        let conn = datastore
+            .pool_connection_unauthorized()
+            .await
+            .expect("Failed to access db connection");
+        diesel::insert_into(db::schema::saga_node_event::dsl::saga_node_event)
+            .values(inserted_nodes.clone())
+            .execute_async(&*conn)
+            .await
+            .expect("Failed to insert test setup data");
+
+        // List them, expect to see them all in order by ID.
+        let observed_nodes = datastore
+            .saga_fetch_log_batched(
+                &opctx,
+                nexus_db_model::saga_types::SagaId::from(node_cx.saga_id),
+            )
+            .await
+            .expect("Failed to list nodes of unfinished saga");
+        inserted_nodes.sort_by_key(|a| (a.node_id, a.event_type.clone()));
+
+        let inserted_nodes = inserted_nodes
+            .into_iter()
+            .map(|node| steno::SagaNodeEvent::try_from(node))
+            .collect::<Result<Vec<_>, _>>()
+            .expect("Couldn't convert DB nodes to steno nodes");
+
+        // The steno::SagaNodeEvent type doesn't implement PartialEq, so we need
+        // to do this a little manually.
+        assert_eq!(inserted_nodes.len(), observed_nodes.len());
+        for (inserted, observed) in
+            inserted_nodes.iter().zip(observed_nodes.iter())
+        {
+            assert_eq!(inserted.saga_id, observed.saga_id);
+            assert_eq!(inserted.node_id, observed.node_id);
+            assert_eq!(
+                inserted.event_type.label(),
+                observed.event_type.label()
+            );
+        }
+
+        // Test cleanup
+        db.cleanup().await.unwrap();
+        logctx.cleanup_successful();
+    }
+
+    // Tests the special case of listing an empty saga log
+    #[tokio::test]
+    async fn test_list_no_unfinished_nodes() {
+        // Test setup
+        let logctx = dev::test_setup_log("test_list_no_unfinished_nodes");
+        let mut db = test_setup_database(&logctx.log).await;
+        let (opctx, datastore) = datastore_test(&logctx, &db).await;
+        let saga_id = steno::SagaId(Uuid::new_v4());
+
+        // Test that this returns "no nodes" rather than throwing some "not
+        // found" error.
+        let observed_nodes = datastore
+            .saga_fetch_log_batched(
+                &opctx,
+                nexus_db_model::saga_types::SagaId::from(saga_id),
+            )
+            .await
+            .expect("Failed to list nodes of unfinished saga");
+        assert_eq!(observed_nodes.len(), 0);
+
+        // Test cleanup
+        db.cleanup().await.unwrap();
+        logctx.cleanup_successful();
+    }
+
+    #[tokio::test]
+    async fn test_create_event_idempotent() {
+        // Test setup
+        let logctx = dev::test_setup_log("test_create_event_idempotent");
+        let mut db = test_setup_database(&logctx.log).await;
+        let (_, datastore) = datastore_test(&logctx, &db).await;
+        let node_cx = SagaTestContext::new(SecId(Uuid::new_v4()));
+
+        // Generate a bunch of events.
+        let inserted_nodes = (0..2)
+            .flat_map(|i| {
+                use steno::SagaNodeEventType::*;
+                [
+                    node_cx.new_db_event(i, Started),
+                    node_cx.new_db_event(i, UndoStarted),
+                    node_cx.new_db_event(i, UndoFinished),
+                ]
+            })
+            .collect::<Vec<_>>();
+
+        // Insert the events into the database.
+        for node in &inserted_nodes {
+            datastore
+                .saga_create_event(node)
+                .await
+                .expect("inserting first node events");
+        }
+
+        // Insert the events again into the database and ensure that we don't
+        // get a conflict.
+        for node in &inserted_nodes {
+            datastore
+                .saga_create_event(node)
+                .await
+                .expect("inserting duplicate node events");
+        }
+
+        // Test cleanup
+        db.cleanup().await.unwrap();
+        logctx.cleanup_successful();
+    }
+
+    #[tokio::test]
+    async fn test_update_state_idempotent() {
+        // Test setup
+        let logctx = dev::test_setup_log("test_create_event_idempotent");
+        let mut db = test_setup_database(&logctx.log).await;
+        let (_, datastore) = datastore_test(&logctx, &db).await;
+        let node_cx = SagaTestContext::new(SecId(Uuid::new_v4()));
+
+        // Create a saga in the running state.
+        let params = node_cx.new_running_db_saga();
+        datastore
+            .saga_create(&params)
+            .await
+            .expect("creating saga in Running state");
+
+        // Attempt to update its state to Running, which is a no-op -- this
+        // should be idempotent, so expect success.
+        datastore
+            .saga_update_state(
+                node_cx.saga_id,
+                steno::SagaCachedState::Running,
+                node_cx.sec_id,
+                db::model::Generation::new(),
+            )
+            .await
+            .expect("updating state to Running again");
+
+        // Update the state to Done.
+        datastore
+            .saga_update_state(
+                node_cx.saga_id,
+                steno::SagaCachedState::Done,
+                node_cx.sec_id,
+                db::model::Generation::new(),
+            )
+            .await
+            .expect("updating state to Done");
+
+        // Attempt to update its state to Done again, which is a no-op -- this
+        // should be idempotent, so expect success.
+        datastore
+            .saga_update_state(
+                node_cx.saga_id,
+                steno::SagaCachedState::Done,
+                node_cx.sec_id,
+                db::model::Generation::new(),
+            )
+            .await
+            .expect("updating state to Done again");
+
+        // Test cleanup
+        db.cleanup().await.unwrap();
+        logctx.cleanup_successful();
+    }
+
+    /// Helpers to create sagas.
+    struct SagaTestContext {
+        saga_id: steno::SagaId,
+        sec_id: SecId,
+    }
+
+    impl SagaTestContext {
+        fn new(sec_id: SecId) -> Self {
+            Self { saga_id: steno::SagaId(Uuid::new_v4()), sec_id }
+        }
+
+        fn new_running_db_saga(&self) -> db::model::saga_types::Saga {
+            let params = steno::SagaCreateParams {
+                id: self.saga_id,
+                name: steno::SagaName::new("test saga"),
+                dag: serde_json::value::Value::Null,
+                state: steno::SagaCachedState::Running,
+            };
+
+            db::model::saga_types::Saga::new(self.sec_id, params)
+        }
+
+        fn new_db_event(
+            &self,
+            node_id: u32,
+            event_type: steno::SagaNodeEventType,
+        ) -> SagaNodeEvent {
+            let event = steno::SagaNodeEvent {
+                saga_id: self.saga_id,
+                node_id: steno::SagaNodeId::from(node_id),
+                event_type,
+            };
+
+            SagaNodeEvent::new(event, self.sec_id)
+        }
+    }
 }
diff --git a/nexus/db-queries/src/db/datastore/volume.rs b/nexus/db-queries/src/db/datastore/volume.rs
index 84f8e211a8..b13006aa95 100644
--- a/nexus/db-queries/src/db/datastore/volume.rs
+++ b/nexus/db-queries/src/db/datastore/volume.rs
@@ -1164,12 +1164,14 @@ impl DataStore {
 
         let mut targets: Vec<SocketAddrV6> = vec![];
 
-        find_matching_rw_regions_in_volume(
-            &vcr,
-            dataset.address().ip(),
-            &mut targets,
-        )
-        .map_err(|e| Error::internal_error(&e.to_string()))?;
+        let Some(address) = dataset.address() else {
+            return Err(Error::internal_error(
+                "Crucible Dataset missing IP address",
+            ));
+        };
+
+        find_matching_rw_regions_in_volume(&vcr, address.ip(), &mut targets)
+            .map_err(|e| Error::internal_error(&e.to_string()))?;
 
         Ok(targets)
     }
diff --git a/nexus/db-queries/src/db/datastore/vpc.rs b/nexus/db-queries/src/db/datastore/vpc.rs
index fdb9c82fb5..615ecdac93 100644
--- a/nexus/db-queries/src/db/datastore/vpc.rs
+++ b/nexus/db-queries/src/db/datastore/vpc.rs
@@ -40,8 +40,8 @@ use crate::db::pagination::paginated;
 use crate::db::pagination::Paginator;
 use crate::db::queries::vpc::InsertVpcQuery;
 use crate::db::queries::vpc::VniSearchIter;
-use crate::db::queries::vpc_subnet::FilterConflictingVpcSubnetRangesQuery;
-use crate::db::queries::vpc_subnet::SubnetError;
+use crate::db::queries::vpc_subnet::InsertVpcSubnetError;
+use crate::db::queries::vpc_subnet::InsertVpcSubnetQuery;
 use crate::transaction_retry::OptionalError;
 use async_bb8_diesel::AsyncRunQueryDsl;
 use chrono::Utc;
@@ -288,7 +288,7 @@ impl DataStore {
             self.vpc_create_subnet(opctx, &authz_vpc, vpc_subnet.clone())
                 .await
                 .map(|_| ())
-                .map_err(SubnetError::into_external)
+                .map_err(InsertVpcSubnetError::into_external)
                 .or_else(|e| match e {
                     Error::ObjectAlreadyExists { .. } => Ok(()),
                     _ => Err(e),
@@ -809,17 +809,17 @@ impl DataStore {
         opctx: &OpContext,
         authz_vpc: &authz::Vpc,
         subnet: VpcSubnet,
-    ) -> Result<(authz::VpcSubnet, VpcSubnet), SubnetError> {
+    ) -> Result<(authz::VpcSubnet, VpcSubnet), InsertVpcSubnetError> {
         opctx
             .authorize(authz::Action::CreateChild, authz_vpc)
             .await
-            .map_err(SubnetError::External)?;
+            .map_err(InsertVpcSubnetError::External)?;
         assert_eq!(authz_vpc.id(), subnet.vpc_id);
 
         let db_subnet = self.vpc_create_subnet_raw(subnet).await?;
         self.vpc_system_router_ensure_subnet_routes(opctx, authz_vpc.id())
             .await
-            .map_err(SubnetError::External)?;
+            .map_err(InsertVpcSubnetError::External)?;
         Ok((
             authz::VpcSubnet::new(
                 authz_vpc.clone(),
@@ -833,20 +833,16 @@ impl DataStore {
     pub(crate) async fn vpc_create_subnet_raw(
         &self,
         subnet: VpcSubnet,
-    ) -> Result<VpcSubnet, SubnetError> {
-        use db::schema::vpc_subnet::dsl;
-        let values = FilterConflictingVpcSubnetRangesQuery::new(subnet.clone());
+    ) -> Result<VpcSubnet, InsertVpcSubnetError> {
         let conn = self
             .pool_connection_unauthorized()
             .await
-            .map_err(SubnetError::External)?;
-
-        diesel::insert_into(dsl::vpc_subnet)
-            .values(values)
-            .returning(VpcSubnet::as_returning())
+            .map_err(InsertVpcSubnetError::External)?;
+        let query = InsertVpcSubnetQuery::new(subnet.clone());
+        query
             .get_result_async(&*conn)
             .await
-            .map_err(|e| SubnetError::from_diesel(e, &subnet))
+            .map_err(|e| InsertVpcSubnetError::from_diesel(e, &subnet))
     }
 
     pub async fn vpc_delete_subnet(
diff --git a/nexus/db-queries/src/db/mod.rs b/nexus/db-queries/src/db/mod.rs
index c8c8860901..fc44a2f27b 100644
--- a/nexus/db-queries/src/db/mod.rs
+++ b/nexus/db-queries/src/db/mod.rs
@@ -27,7 +27,6 @@ mod pool_connection;
 // sagas.
 pub mod queries;
 mod raw_query_builder;
-mod saga_recovery;
 mod sec_store;
 pub(crate) mod true_or_cast_error;
 mod update_and_check;
@@ -37,8 +36,7 @@ mod update_and_check;
 // full table scans the same way pooled connections do.
 pub use pool_connection::DISALLOW_FULL_TABLE_SCAN_SQL;
 
-#[cfg(test)]
-mod test_utils;
+pub mod test_utils;
 
 pub use nexus_db_fixed_data as fixed_data;
 pub use nexus_db_model as model;
@@ -50,7 +48,6 @@ pub use config::Config;
 pub use datastore::DataStore;
 pub use on_conflict_ext::IncompleteOnConflictExt;
 pub use pool::{DbConnection, Pool};
-pub use saga_recovery::{recover, CompletionTask, RecoveryTask};
 pub use saga_types::SecId;
 pub use sec_store::CockroachDbSecStore;
 
diff --git a/nexus/db-queries/src/db/queries/vpc_subnet.rs b/nexus/db-queries/src/db/queries/vpc_subnet.rs
index 72f2771a1e..8cbf4495ca 100644
--- a/nexus/db-queries/src/db/queries/vpc_subnet.rs
+++ b/nexus/db-queries/src/db/queries/vpc_subnet.rs
@@ -7,407 +7,322 @@
 use crate::db;
 use crate::db::identity::Resource;
 use crate::db::model::VpcSubnet;
-use chrono::{DateTime, Utc};
+use crate::db::schema::vpc_subnet::dsl;
+use crate::db::DbConnection;
 use diesel::pg::Pg;
 use diesel::prelude::*;
 use diesel::query_builder::*;
 use diesel::result::Error as DieselError;
 use diesel::sql_types;
+use ipnetwork::IpNetwork;
 use omicron_common::api::external;
 use ref_cast::RefCast;
 use uuid::Uuid;
 
-/// Errors related to allocating VPC Subnets.
-#[derive(Debug, PartialEq)]
-pub enum SubnetError {
-    /// An IPv4 or IPv6 subnet overlaps with an existing VPC Subnet
-    OverlappingIpRange(ipnetwork::IpNetwork),
-    /// An other error
-    External(external::Error),
-}
-
-impl SubnetError {
-    /// Construct a `SubnetError` from a Diesel error, catching the desired
-    /// cases and building useful errors.
-    pub fn from_diesel(e: DieselError, subnet: &VpcSubnet) -> Self {
-        use crate::db::error;
-        use diesel::result::DatabaseErrorKind;
-        const IPV4_OVERLAP_ERROR_MESSAGE: &str =
-            r#"null value in column "ipv4_block" violates not-null constraint"#;
-        const IPV6_OVERLAP_ERROR_MESSAGE: &str =
-            r#"null value in column "ipv6_block" violates not-null constraint"#;
-        const NAME_CONFLICT_CONSTRAINT: &str = "vpc_subnet_vpc_id_name_key";
-        match e {
-            // Attempt to insert overlapping IPv4 subnet
-            DieselError::DatabaseError(
-                DatabaseErrorKind::NotNullViolation,
-                ref info,
-            ) if info.message() == IPV4_OVERLAP_ERROR_MESSAGE => {
-                SubnetError::OverlappingIpRange(ipnetwork::IpNetwork::V4(
-                    subnet.ipv4_block.0.into(),
-                ))
-            }
-
-            // Attempt to insert overlapping IPv6 subnet
-            DieselError::DatabaseError(
-                DatabaseErrorKind::NotNullViolation,
-                ref info,
-            ) if info.message() == IPV6_OVERLAP_ERROR_MESSAGE => {
-                SubnetError::OverlappingIpRange(ipnetwork::IpNetwork::V6(
-                    subnet.ipv6_block.0.into(),
-                ))
-            }
-
-            // Conflicting name for the subnet within a VPC
-            DieselError::DatabaseError(
-                DatabaseErrorKind::UniqueViolation,
-                ref info,
-            ) if info.constraint_name() == Some(NAME_CONFLICT_CONSTRAINT) => {
-                SubnetError::External(error::public_error_from_diesel(
-                    e,
-                    error::ErrorHandler::Conflict(
-                        external::ResourceType::VpcSubnet,
-                        subnet.identity().name.as_str(),
-                    ),
-                ))
-            }
-
-            // Any other error at all is a bug
-            _ => SubnetError::External(error::public_error_from_diesel(
-                e,
-                error::ErrorHandler::Server,
-            )),
-        }
-    }
-
-    /// Convert into a public error
-    pub fn into_external(self) -> external::Error {
-        match self {
-            SubnetError::OverlappingIpRange(ip) => {
-                external::Error::invalid_request(
-                    format!("IP address range '{}' conflicts with an existing subnet", ip).as_str()
-                )
-            },
-            SubnetError::External(e) => e,
-        }
-    }
-}
-
-/// Generate a subquery that selects any overlapping address ranges of the same
-/// type as the input IP subnet.
+/// Query used to insert VPC Subnets.
 ///
-/// This generates a query that, in full, looks like:
+/// This query is used to idempotently insert a VPC Subnet. The query also looks
+/// for any other subnets in the same VPC whose IP address blocks overlap. All
+/// Subnets are required to have non-overlapping IP blocks.
 ///
-/// ```sql
-/// SELECT
-///     <ip>
-/// FROM
-///     vpc_subnet
-/// WHERE
-///     vpc_id = <vpc_id> AND
-///     time_deleted IS NULL AND
-///     inet_contains_or_equals(ipv*_block, <ip>)
-/// LIMIT 1
-/// ```
-///
-/// The input may be either an IPv4 or IPv6 subnet, and the corresponding column
-/// is compared against. Note that the exact input IP range is returned on
-/// purpose.
-fn push_select_overlapping_ip_range<'a>(
-    mut out: AstPass<'_, 'a, Pg>,
-    vpc_id: &'a Uuid,
-    ip: &'a ipnetwork::IpNetwork,
-) -> diesel::QueryResult<()> {
-    use crate::db::schema::vpc_subnet::dsl;
-    out.push_sql("SELECT ");
-    out.push_bind_param::<sql_types::Inet, ipnetwork::IpNetwork>(ip)?;
-    out.push_sql(" FROM ");
-    VPC_SUBNET_FROM_CLAUSE.walk_ast(out.reborrow())?;
-    out.push_sql(" WHERE ");
-    out.push_identifier(dsl::vpc_id::NAME)?;
-    out.push_sql(" = ");
-    out.push_bind_param::<sql_types::Uuid, Uuid>(vpc_id)?;
-    out.push_sql(" AND ");
-    out.push_identifier(dsl::time_deleted::NAME)?;
-    out.push_sql(" IS NULL AND inet_contains_or_equals(");
-    if ip.is_ipv4() {
-        out.push_identifier(dsl::ipv4_block::NAME)?;
-    } else {
-        out.push_identifier(dsl::ipv6_block::NAME)?;
-    }
-    out.push_sql(", ");
-    out.push_bind_param::<sql_types::Inet, ipnetwork::IpNetwork>(ip)?;
-    out.push_sql(")");
-    Ok(())
-}
-
-/// Generate a subquery that returns NULL if there is an overlapping IP address
-/// range of any type.
+/// Note that this query is idempotent. If a record with the provided primary
+/// key already exists, that record is returned exactly from the DB, without any
+/// other modification or alteration. If callers care, they can inspect the
+/// record to make sure it's what they expected, though that's usually a fraught
+/// endeavor.
 ///
-/// This specifically generates a query that looks like:
+/// Here is the entire query:
 ///
 /// ```sql
-/// SELECT NULLIF(
-///     <ip>,
-///     push_select_overlapping_ip_range(<vpc_id>, <ip>)
-/// )
-/// ```
-///
-/// The `NULLIF` function returns NULL if those two expressions are equal, and
-/// the first expression otherwise. That is, this returns NULL if there exists
-/// an overlapping IP range already in the VPC Subnet table, and the requested
-/// IP range if not.
-fn push_null_if_overlapping_ip_range<'a>(
-    mut out: AstPass<'_, 'a, Pg>,
-    vpc_id: &'a Uuid,
-    ip: &'a ipnetwork::IpNetwork,
-) -> diesel::QueryResult<()> {
-    out.push_sql("SELECT NULLIF(");
-    out.push_bind_param::<sql_types::Inet, ipnetwork::IpNetwork>(ip)?;
-    out.push_sql(", (");
-    push_select_overlapping_ip_range(out.reborrow(), vpc_id, ip)?;
-    out.push_sql("))");
-    Ok(())
-}
-
-/// Generate a CTE that can be used to insert a VPC Subnet, only if the IP
-/// address ranges of that subnet don't overlap with existing Subnets in the
-/// same VPC.
-///
-/// In particular, this generates a CTE like so:
-///
-/// ```sql
-/// WITH candidate(
-///     id,
-///     name,
-///     description,
-///     time_created,
-///     time_modified,
-///     time_deleted,
-///     vpc_id,
-///     rcgen
-/// ) AS (VALUES (
-///     <id>,
-///     <name>,
-///     <description>,
-///     <time_created>,
-///     <time_modified>,
-///     NULL::TIMESTAMPTZ,
-///     <vpc_id>,
-///     0
-/// )),
-/// candidate_ipv4(ipv4_block) AS (
-///     SELECT(
-///         NULLIF(
-///             <ipv4_block>,
-///             (
-///                 SELECT
-///                     ipv4_block
-///                 FROM
-///                     vpc_subnet
-///                 WHERE
-///                     vpc_id = <vpc_id> AND
-///                     time_deleted IS NULL AND
-///                     inet_contains_or_equals(<ipv4_block>, ipv4_block)
-///                 LIMIT 1
+/// WITH
+/// -- This CTE generates a casting error if any live records, other than _this_
+/// -- record, have overlapping IP blocks of either family.
+/// overlap AS MATERIALIZED (
+///     SELECT
+///         -- NOTE: This cast always fails, we just use _how_ it fails to
+///         -- learn which IP block overlaps. The filter `id != <id>` below
+///         -- means we're explicitly ignoring an existing, identical record.
+///         -- So this cast is only run if there is another record in the same
+///         -- VPC with an overlapping subnet, which is exactly the error case
+///         -- we're trying to cacth.
+///         CAST(
+///             IF(
+///                 inet_contains_or_equals(ipv4_block, <ipv4_block>),
+///                'ipv4',
+///                'ipv6'
 ///             )
-///        )
-///   )
-/// ),
-/// candidate_ipv6(ipv6_block) AS (
-///     <same as above, for ipv6>
+///             AS BOOL
+///         )
+///     FROM
+///         vpc_subnet
+///     WHERE
+///         vpc_id = <vpc_id> AND
+///         time_deleted IS NULL AND
+///         id != <id> AND
+///         (
+///             inet_contains_or_equals(ipv4_block, <ipv4_block>) OR
+///             inet_contains_or_equals(ipv6_block, <ipv6_block>)
+///         )
+/// )
+/// INSERT INTO
+///     vpc_subnet
+/// VALUES (
+///     <input data>
 /// )
-/// SELECT *
-/// FROM candidate, candidate_ipv4, candidate_ipv6
+/// ON CONFLICT (id)
+/// -- We use this "no-op" update to allow us to return the actual row from the
+/// -- DB, either the existing or inserted one.
+/// DO UPDATE SET id = id
+/// RETURNING *;
 /// ```
-pub struct FilterConflictingVpcSubnetRangesQuery {
-    // TODO: update with random one if the insertion fails.
+#[derive(Clone, Debug)]
+pub struct InsertVpcSubnetQuery {
+    /// The subnet to insert
     subnet: VpcSubnet,
-
-    // The following fields are derived from the previous field. This begs the
-    // question: "Why bother storing them at all?"
-    //
-    // Diesel's [`diesel::query_builder::ast_pass::AstPass:push_bind_param`] method
-    // requires that the provided value now live as long as the entire AstPass
-    // type. By storing these values in the struct, they'll live at least as
-    // long as the entire call to [`QueryFragment<Pg>::walk_ast`].
-    ipv4_block: ipnetwork::IpNetwork,
-    ipv6_block: ipnetwork::IpNetwork,
+    /// Owned values of the IP blocks to check, for inserting in internal pieces
+    /// of the query.
+    ipv4_block: IpNetwork,
+    ipv6_block: IpNetwork,
 }
 
-impl FilterConflictingVpcSubnetRangesQuery {
+impl InsertVpcSubnetQuery {
+    /// Construct a new query to insert the provided subnet.
     pub fn new(subnet: VpcSubnet) -> Self {
-        let ipv4_block =
-            ipnetwork::Ipv4Network::from(subnet.ipv4_block.0).into();
-        let ipv6_block =
-            ipnetwork::Ipv6Network::from(subnet.ipv6_block.0).into();
+        let ipv4_block = IpNetwork::V4(subnet.ipv4_block.0.into());
+        let ipv6_block = IpNetwork::V6(subnet.ipv6_block.0.into());
         Self { subnet, ipv4_block, ipv6_block }
     }
 }
 
-impl QueryId for FilterConflictingVpcSubnetRangesQuery {
+impl QueryId for InsertVpcSubnetQuery {
     type QueryId = ();
     const HAS_STATIC_QUERY_ID: bool = false;
 }
 
-impl QueryFragment<Pg> for FilterConflictingVpcSubnetRangesQuery {
+impl QueryFragment<Pg> for InsertVpcSubnetQuery {
     fn walk_ast<'a>(
         &'a self,
         mut out: AstPass<'_, 'a, Pg>,
     ) -> diesel::QueryResult<()> {
-        use db::schema::vpc_subnet::dsl;
-
-        // Create the base `candidate` from values provided that need no
-        // verificiation.
-        out.push_sql("SELECT * FROM (WITH candidate(");
-        out.push_identifier(dsl::id::NAME)?;
-        out.push_sql(", ");
-        out.push_identifier(dsl::name::NAME)?;
-        out.push_sql(", ");
-        out.push_identifier(dsl::description::NAME)?;
-        out.push_sql(", ");
-        out.push_identifier(dsl::time_created::NAME)?;
-        out.push_sql(", ");
-        out.push_identifier(dsl::time_modified::NAME)?;
+        out.push_sql("WITH overlap AS MATERIALIZED (SELECT CAST(IF(inet_contains_or_equals(");
+        out.push_identifier(dsl::ipv4_block::NAME)?;
         out.push_sql(", ");
-        out.push_identifier(dsl::time_deleted::NAME)?;
+        out.push_bind_param::<sql_types::Inet, _>(&self.ipv4_block)?;
+        out.push_sql("), ");
+        out.push_bind_param::<sql_types::Text, _>(
+            InsertVpcSubnetError::OVERLAPPING_IPV4_BLOCK_SENTINEL,
+        )?;
         out.push_sql(", ");
+        out.push_bind_param::<sql_types::Text, _>(
+            InsertVpcSubnetError::OVERLAPPING_IPV6_BLOCK_SENTINEL,
+        )?;
+        out.push_sql(") AS BOOL) FROM ");
+        VPC_SUBNET_FROM_CLAUSE.walk_ast(out.reborrow())?;
+        out.push_sql(" WHERE ");
         out.push_identifier(dsl::vpc_id::NAME)?;
-        out.push_sql(",");
-        out.push_identifier(dsl::rcgen::NAME)?;
-        out.push_sql(") AS (VALUES (");
+        out.push_sql(" = ");
+        out.push_bind_param::<sql_types::Uuid, Uuid>(&self.subnet.vpc_id)?;
+        out.push_sql(" AND ");
+        out.push_identifier(dsl::time_deleted::NAME)?;
+        out.push_sql(" IS NULL AND ");
+        out.push_identifier(dsl::id::NAME)?;
+        out.push_sql(" != ");
         out.push_bind_param::<sql_types::Uuid, Uuid>(&self.subnet.identity.id)?;
+        out.push_sql(" AND (inet_contains_or_equals(");
+        out.push_identifier(dsl::ipv4_block::NAME)?;
         out.push_sql(", ");
-        out.push_bind_param::<sql_types::Text, db::model::Name>(
-            db::model::Name::ref_cast(self.subnet.name()),
-        )?;
+        out.push_bind_param::<sql_types::Inet, IpNetwork>(&self.ipv4_block)?;
+        out.push_sql(") OR inet_contains_or_equals(");
+        out.push_identifier(dsl::ipv6_block::NAME)?;
         out.push_sql(", ");
-        out.push_bind_param::<sql_types::Text, String>(
+        out.push_bind_param::<sql_types::Inet, IpNetwork>(&self.ipv6_block)?;
+
+        out.push_sql("))) INSERT INTO ");
+        VPC_SUBNET_FROM_CLAUSE.walk_ast(out.reborrow())?;
+        out.push_sql("VALUES (");
+        out.push_bind_param::<sql_types::Uuid, _>(&self.subnet.identity.id)?;
+        out.push_sql(", ");
+        out.push_bind_param::<sql_types::Text, _>(db::model::Name::ref_cast(
+            self.subnet.name(),
+        ))?;
+        out.push_sql(", ");
+        out.push_bind_param::<sql_types::Text, _>(
             &self.subnet.identity.description,
         )?;
         out.push_sql(", ");
-        out.push_bind_param::<sql_types::Timestamptz, DateTime<Utc>>(
+        out.push_bind_param::<sql_types::Timestamptz, _>(
             &self.subnet.identity.time_created,
         )?;
         out.push_sql(", ");
-        out.push_bind_param::<sql_types::Timestamptz, DateTime<Utc>>(
+        out.push_bind_param::<sql_types::Timestamptz, _>(
             &self.subnet.identity.time_modified,
         )?;
         out.push_sql(", ");
-        out.push_sql("NULL::TIMESTAMPTZ, ");
-        out.push_bind_param::<sql_types::Uuid, Uuid>(&self.subnet.vpc_id)?;
-        out.push_sql(", 0)), ");
-
-        // Push the candidate IPv4 and IPv6 selection subqueries, which return
-        // NULL if the corresponding address range overlaps.
-        out.push_sql("candidate_ipv4(");
-        out.push_identifier(dsl::ipv4_block::NAME)?;
-        out.push_sql(") AS (");
-        push_null_if_overlapping_ip_range(
-            out.reborrow(),
-            &self.subnet.vpc_id,
-            &self.ipv4_block,
+        out.push_bind_param::<sql_types::Nullable<sql_types::Timestamptz>, _>(
+            &self.subnet.identity.time_deleted,
         )?;
-
-        out.push_sql("), candidate_ipv6(");
-        out.push_identifier(dsl::ipv6_block::NAME)?;
-        out.push_sql(") AS (");
-        push_null_if_overlapping_ip_range(
-            out.reborrow(),
-            &self.subnet.vpc_id,
-            &self.ipv6_block,
+        out.push_sql(", ");
+        out.push_bind_param::<sql_types::Uuid, _>(&self.subnet.vpc_id)?;
+        out.push_sql(", ");
+        out.push_bind_param::<sql_types::Int8, _>(&self.subnet.rcgen)?;
+        out.push_sql(", ");
+        out.push_bind_param::<sql_types::Inet, _>(&self.ipv4_block)?;
+        out.push_sql(", ");
+        out.push_bind_param::<sql_types::Inet, _>(&self.ipv6_block)?;
+        out.push_sql(", ");
+        out.push_bind_param::<sql_types::Nullable<sql_types::Uuid>, _>(
+            &self.subnet.custom_router_id,
         )?;
-        out.push_sql(") ");
+        out.push_sql(") ON CONFLICT (");
+        out.push_identifier(dsl::id::NAME)?;
+        out.push_sql(") DO UPDATE SET ");
+        out.push_identifier(dsl::id::NAME)?;
+        out.push_sql(" = ");
+        out.push_bind_param::<sql_types::Uuid, _>(&self.subnet.identity.id)?;
+        out.push_sql(" RETURNING *");
 
-        // Select the entire set of candidate columns.
-        out.push_sql(
-            "SELECT * FROM candidate, candidate_ipv4, candidate_ipv6)",
-        );
         Ok(())
     }
 }
 
-impl Insertable<db::schema::vpc_subnet::table>
-    for FilterConflictingVpcSubnetRangesQuery
-{
-    type Values = FilterConflictingVpcSubnetRangesQueryValues;
+type FromClause<T> =
+    diesel::internal::table_macro::StaticQueryFragmentInstance<T>;
+type VpcSubnetFromClause = FromClause<db::schema::vpc_subnet::table>;
+const VPC_SUBNET_FROM_CLAUSE: VpcSubnetFromClause = VpcSubnetFromClause::new();
 
-    fn values(self) -> Self::Values {
-        FilterConflictingVpcSubnetRangesQueryValues(self)
-    }
+impl RunQueryDsl<DbConnection> for InsertVpcSubnetQuery {}
+impl Query for InsertVpcSubnetQuery {
+    type SqlType = <<VpcSubnet as diesel::Selectable<Pg>>::SelectExpression as diesel::Expression>::SqlType;
 }
 
-/// Used to allow inserting the result of the
-/// `FilterConflictingVpcSubnetRangesQuery`, as in
-/// `diesel::insert_into(foo).values(_). Should not be used directly.
-pub struct FilterConflictingVpcSubnetRangesQueryValues(
-    pub FilterConflictingVpcSubnetRangesQuery,
-);
-
-impl QueryId for FilterConflictingVpcSubnetRangesQueryValues {
-    type QueryId = ();
-    const HAS_STATIC_QUERY_ID: bool = false;
+/// Errors related to inserting VPC Subnets.
+#[derive(Debug, PartialEq)]
+pub enum InsertVpcSubnetError {
+    /// The IPv4 or IPv6 subnet overlaps with an existing VPC Subnet
+    OverlappingIpRange(oxnet::IpNet),
+    /// Any other error
+    External(external::Error),
 }
 
-impl diesel::insertable::CanInsertInSingleQuery<Pg>
-    for FilterConflictingVpcSubnetRangesQueryValues
-{
-    fn rows_to_insert(&self) -> Option<usize> {
-        Some(1)
+impl InsertVpcSubnetError {
+    const OVERLAPPING_IPV4_BLOCK_SENTINEL: &'static str = "ipv4";
+    const OVERLAPPING_IPV4_BLOCK_ERROR_MESSAGE: &'static str =
+        r#"could not parse "ipv4" as type bool: invalid bool value"#;
+    const OVERLAPPING_IPV6_BLOCK_SENTINEL: &'static str = "ipv6";
+    const OVERLAPPING_IPV6_BLOCK_ERROR_MESSAGE: &'static str =
+        r#"could not parse "ipv6" as type bool: invalid bool value"#;
+    const NAME_CONFLICT_CONSTRAINT: &'static str = "vpc_subnet_vpc_id_name_key";
+
+    /// Construct an `InsertError` from a Diesel error, catching the desired
+    /// cases and building useful errors.
+    pub fn from_diesel(e: DieselError, subnet: &VpcSubnet) -> Self {
+        use crate::db::error;
+        use diesel::result::DatabaseErrorKind;
+        match e {
+            // Attempt to insert an overlapping IPv4 subnet
+            DieselError::DatabaseError(
+                DatabaseErrorKind::Unknown,
+                ref info,
+            ) if info.message()
+                == Self::OVERLAPPING_IPV4_BLOCK_ERROR_MESSAGE =>
+            {
+                InsertVpcSubnetError::OverlappingIpRange(
+                    subnet.ipv4_block.0.into(),
+                )
+            }
+
+            // Attempt to insert an overlapping IPv6 subnet
+            DieselError::DatabaseError(
+                DatabaseErrorKind::Unknown,
+                ref info,
+            ) if info.message()
+                == Self::OVERLAPPING_IPV6_BLOCK_ERROR_MESSAGE =>
+            {
+                InsertVpcSubnetError::OverlappingIpRange(
+                    subnet.ipv6_block.0.into(),
+                )
+            }
+
+            // Conflicting name for the subnet within a VPC
+            DieselError::DatabaseError(
+                DatabaseErrorKind::UniqueViolation,
+                ref info,
+            ) if info.constraint_name()
+                == Some(Self::NAME_CONFLICT_CONSTRAINT) =>
+            {
+                InsertVpcSubnetError::External(error::public_error_from_diesel(
+                    e,
+                    error::ErrorHandler::Conflict(
+                        external::ResourceType::VpcSubnet,
+                        subnet.identity().name.as_str(),
+                    ),
+                ))
+            }
+
+            // Any other error at all is a bug
+            _ => InsertVpcSubnetError::External(
+                error::public_error_from_diesel(e, error::ErrorHandler::Server),
+            ),
+        }
     }
-}
 
-impl QueryFragment<Pg> for FilterConflictingVpcSubnetRangesQueryValues {
-    fn walk_ast<'a>(
-        &'a self,
-        mut out: AstPass<'_, 'a, Pg>,
-    ) -> diesel::QueryResult<()> {
-        use db::schema::vpc_subnet::dsl;
-        out.push_sql("(");
-        out.push_identifier(dsl::id::NAME)?;
-        out.push_sql(", ");
-        out.push_identifier(dsl::name::NAME)?;
-        out.push_sql(", ");
-        out.push_identifier(dsl::description::NAME)?;
-        out.push_sql(", ");
-        out.push_identifier(dsl::time_created::NAME)?;
-        out.push_sql(", ");
-        out.push_identifier(dsl::time_modified::NAME)?;
-        out.push_sql(", ");
-        out.push_identifier(dsl::time_deleted::NAME)?;
-        out.push_sql(", ");
-        out.push_identifier(dsl::vpc_id::NAME)?;
-        out.push_sql(", ");
-        out.push_identifier(dsl::rcgen::NAME)?;
-        out.push_sql(", ");
-        out.push_identifier(dsl::ipv4_block::NAME)?;
-        out.push_sql(", ");
-        out.push_identifier(dsl::ipv6_block::NAME)?;
-        out.push_sql(") ");
-        self.0.walk_ast(out)
+    /// Convert into a public error
+    pub fn into_external(self) -> external::Error {
+        match self {
+            InsertVpcSubnetError::OverlappingIpRange(ip) => {
+                external::Error::invalid_request(
+                    format!(
+                        "IP address range '{}' \
+                        conflicts with an existing subnet",
+                        ip,
+                    )
+                    .as_str(),
+                )
+            }
+            InsertVpcSubnetError::External(e) => e,
+        }
     }
 }
 
-type FromClause<T> =
-    diesel::internal::table_macro::StaticQueryFragmentInstance<T>;
-type VpcSubnetFromClause = FromClause<db::schema::vpc_subnet::table>;
-const VPC_SUBNET_FROM_CLAUSE: VpcSubnetFromClause = VpcSubnetFromClause::new();
-
 #[cfg(test)]
 mod test {
-    use super::SubnetError;
+    use super::InsertVpcSubnetError;
+    use super::InsertVpcSubnetQuery;
+    use crate::db::explain::ExplainableAsync as _;
     use crate::db::model::VpcSubnet;
-    use ipnetwork::IpNetwork;
     use nexus_test_utils::db::test_setup_database;
     use omicron_common::api::external::IdentityMetadataCreateParams;
     use omicron_common::api::external::Name;
     use omicron_test_utils::dev;
     use std::convert::TryInto;
     use std::sync::Arc;
-    use uuid::Uuid;
 
     #[tokio::test]
-    async fn test_filter_conflicting_vpc_subnet_ranges_query() {
+    async fn explain_insert_query() {
+        let ipv4_block = "172.30.0.0/24".parse().unwrap();
+        let ipv6_block = "fd12:3456:7890::/64".parse().unwrap();
+        let name = "a-name".to_string().try_into().unwrap();
+        let description = "some description".to_string();
+        let identity = IdentityMetadataCreateParams { name, description };
+        let vpc_id = "d402369d-c9ec-c5ad-9138-9fbee732d53e".parse().unwrap();
+        let subnet_id = "093ad2db-769b-e3c2-bc1c-b46e84ce5532".parse().unwrap();
+        let row =
+            VpcSubnet::new(subnet_id, vpc_id, identity, ipv4_block, ipv6_block);
+        let query = InsertVpcSubnetQuery::new(row);
+        let logctx = dev::test_setup_log("explain_insert_query");
+        let log = logctx.log.new(o!());
+        let mut db = test_setup_database(&log).await;
+        let cfg = crate::db::Config { url: db.pg_config().clone() };
+        let pool = Arc::new(crate::db::Pool::new(&logctx.log, &cfg));
+        let conn = pool.pool().get().await.unwrap();
+        let explain = query.explain_async(&conn).await.unwrap();
+        println!("{explain}");
+        db.cleanup().await.unwrap();
+        logctx.cleanup_successful();
+    }
+
+    #[tokio::test]
+    async fn test_insert_vpc_subnet_query() {
         let make_id =
             |name: &Name, description: &str| IdentityMetadataCreateParams {
                 name: name.clone(),
@@ -427,12 +342,13 @@ mod test {
         let subnet_id = "093ad2db-769b-e3c2-bc1c-b46e84ce5532".parse().unwrap();
         let other_subnet_id =
             "695debcc-e197-447d-ffb2-976150a7b7cf".parse().unwrap();
+        let other_other_subnet_id =
+            "ddbdc2b7-d22f-40d9-98df-fef5da151e0d".parse().unwrap();
         let row =
             VpcSubnet::new(subnet_id, vpc_id, identity, ipv4_block, ipv6_block);
 
         // Setup the test database
-        let logctx =
-            dev::test_setup_log("test_filter_conflicting_vpc_subnet_ranges");
+        let logctx = dev::test_setup_log("test_insert_vpc_subnet_query");
         let log = logctx.log.new(o!());
         let mut db = test_setup_database(&log).await;
         let cfg = crate::db::Config { url: db.pg_config().clone() };
@@ -445,7 +361,10 @@ mod test {
 
         // We should be able to insert anything into an empty table.
         assert!(
-            matches!(db_datastore.vpc_create_subnet_raw(row).await, Ok(_)),
+            matches!(
+                db_datastore.vpc_create_subnet_raw(row.clone()).await,
+                Ok(_)
+            ),
             "Should be able to insert VPC subnet into empty table"
         );
 
@@ -460,10 +379,13 @@ mod test {
         );
         assert!(
             matches!(
-                db_datastore.vpc_create_subnet_raw(new_row).await,
-                Err(SubnetError::OverlappingIpRange(IpNetwork::V4(_)))
+                db_datastore.vpc_create_subnet_raw(new_row.clone()).await,
+                Err(InsertVpcSubnetError::OverlappingIpRange { .. }),
             ),
-            "Should not be able to insert new VPC subnet with the same IPv4 and IPv6 ranges"
+            "Should not be able to insert new VPC subnet with the \
+            same IPv4 and IPv6 ranges,\n\
+            first row: {row:?}\n\
+            new row: {new_row:?}",
         );
 
         // We should be able to insert data with the same ranges, if we change
@@ -483,7 +405,7 @@ mod test {
         // We shouldn't be able to insert a subnet if we change only the
         // IPv4 or IPv6 block. They must _both_ be non-overlapping.
         let new_row = VpcSubnet::new(
-            other_subnet_id,
+            other_other_subnet_id,
             vpc_id,
             make_id(&other_name, &description),
             other_ipv4_block,
@@ -495,11 +417,11 @@ mod test {
             .expect_err("Should not be able to insert VPC Subnet with overlapping IPv6 range");
         assert_eq!(
             err,
-            SubnetError::OverlappingIpRange(ipnetwork::IpNetwork::from(oxnet::IpNet::from(ipv6_block))),
-            "SubnetError variant should include the exact IP range that overlaps"
+            InsertVpcSubnetError::OverlappingIpRange(ipv6_block.into()),
+            "InsertError variant should indicate an IP block overlaps"
         );
         let new_row = VpcSubnet::new(
-            other_subnet_id,
+            other_other_subnet_id,
             vpc_id,
             make_id(&other_name, &description),
             ipv4_block,
@@ -511,14 +433,14 @@ mod test {
             .expect_err("Should not be able to insert VPC Subnet with overlapping IPv4 range");
         assert_eq!(
             err,
-            SubnetError::OverlappingIpRange(ipnetwork::IpNetwork::from(oxnet::IpNet::from(ipv4_block))),
-            "SubnetError variant should include the exact IP range that overlaps"
+            InsertVpcSubnetError::OverlappingIpRange(ipv4_block.into()),
+            "InsertError variant should indicate an IP block overlaps"
         );
 
         // We should get an _external error_ if the IP address ranges are OK,
         // but the name conflicts.
         let new_row = VpcSubnet::new(
-            other_subnet_id,
+            other_other_subnet_id,
             vpc_id,
             make_id(&name, &description),
             other_ipv4_block,
@@ -527,7 +449,7 @@ mod test {
         assert!(
             matches!(
                 db_datastore.vpc_create_subnet_raw(new_row).await,
-                Err(SubnetError::External(_))
+                Err(InsertVpcSubnetError::External(_))
             ),
             "Should get an error inserting a VPC Subnet with unique IP ranges, but the same name"
         );
@@ -535,7 +457,7 @@ mod test {
         // We should be able to insert the row if _both ranges_ are different,
         // and the name is unique as well.
         let new_row = VpcSubnet::new(
-            Uuid::new_v4(),
+            other_other_subnet_id,
             vpc_id,
             make_id(&other_name, &description),
             other_ipv4_block,
@@ -549,4 +471,104 @@ mod test {
         db.cleanup().await.unwrap();
         logctx.cleanup_successful();
     }
+
+    // Helper to verify equality of rows, handling timestamp precision.
+    fn assert_rows_eq(left: &VpcSubnet, right: &VpcSubnet) {
+        assert_eq!(
+            left.identity.id, right.identity.id,
+            "VPC Subnet rows should be equal"
+        );
+        assert_eq!(
+            left.identity.name, right.identity.name,
+            "VPC Subnet rows should be equal"
+        );
+        assert_eq!(
+            left.identity.description, right.identity.description,
+            "VPC Subnet rows should be equal"
+        );
+        // Timestamps in CRDB have microsecond precision, so ensure we're
+        // within 1000 nanos.
+        assert!(
+            (left.identity.time_modified - right.identity.time_modified)
+                .num_nanoseconds()
+                .unwrap()
+                < 1_000,
+            "VPC Subnet rows should be equal",
+        );
+        assert!(
+            (left.identity.time_created - right.identity.time_created)
+                .num_nanoseconds()
+                .unwrap()
+                < 1_000,
+            "VPC Subnet rows should be equal",
+        );
+        assert_eq!(
+            left.identity.time_deleted, right.identity.time_deleted,
+            "VPC Subnet rows should be equal",
+        );
+        assert_eq!(
+            left.vpc_id, right.vpc_id,
+            "VPC Subnet rows should be equal"
+        );
+        assert_eq!(left.rcgen, right.rcgen, "VPC Subnet rows should be equal");
+        assert_eq!(
+            left.ipv4_block, right.ipv4_block,
+            "VPC Subnet rows should be equal"
+        );
+        assert_eq!(
+            left.ipv6_block, right.ipv6_block,
+            "VPC Subnet rows should be equal"
+        );
+        assert_eq!(
+            left.custom_router_id, right.custom_router_id,
+            "VPC Subnet rows should be equal"
+        );
+    }
+
+    // Regression test for https://github.com/oxidecomputer/omicron/issues/6069.
+    #[tokio::test]
+    async fn test_insert_vpc_subnet_query_is_idempotent() {
+        let ipv4_block = "172.30.0.0/24".parse().unwrap();
+        let ipv6_block = "fd12:3456:7890::/64".parse().unwrap();
+        let name = "a-name".to_string().try_into().unwrap();
+        let description = "some description".to_string();
+        let identity = IdentityMetadataCreateParams { name, description };
+        let vpc_id = "d402369d-c9ec-c5ad-9138-9fbee732d53e".parse().unwrap();
+        let subnet_id = "093ad2db-769b-e3c2-bc1c-b46e84ce5532".parse().unwrap();
+        let row =
+            VpcSubnet::new(subnet_id, vpc_id, identity, ipv4_block, ipv6_block);
+
+        // Setup the test database
+        let logctx =
+            dev::test_setup_log("test_insert_vpc_subnet_query_is_idempotent");
+        let log = logctx.log.new(o!());
+        let mut db = test_setup_database(&log).await;
+        let cfg = crate::db::Config { url: db.pg_config().clone() };
+        let pool = Arc::new(crate::db::Pool::new(&logctx.log, &cfg));
+        let db_datastore = Arc::new(
+            crate::db::DataStore::new(&log, Arc::clone(&pool), None)
+                .await
+                .unwrap(),
+        );
+
+        // We should be able to insert anything into an empty table.
+        let inserted = db_datastore
+            .vpc_create_subnet_raw(row.clone())
+            .await
+            .expect("Should be able to insert VPC subnet into empty table");
+        assert_rows_eq(&inserted, &row);
+
+        // We should be able to insert the exact same row again. The IP ranges
+        // overlap, but the ID is also identical, which should not be an error.
+        // This is important for saga idempotency.
+        let inserted = db_datastore
+            .vpc_create_subnet_raw(row.clone())
+            .await
+            .expect(
+            "Must be able to insert the exact same VPC subnet more than once",
+        );
+        assert_rows_eq(&inserted, &row);
+        db.cleanup().await.unwrap();
+        logctx.cleanup_successful();
+    }
 }
diff --git a/nexus/db-queries/src/db/saga_recovery.rs b/nexus/db-queries/src/db/saga_recovery.rs
deleted file mode 100644
index e85011f60f..0000000000
--- a/nexus/db-queries/src/db/saga_recovery.rs
+++ /dev/null
@@ -1,805 +0,0 @@
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this
-// file, You can obtain one at https://mozilla.org/MPL/2.0/.
-
-//! Handles recovery of sagas
-
-use crate::context::OpContext;
-use crate::db;
-use crate::db::datastore::SQL_BATCH_SIZE;
-use crate::db::error::public_error_from_diesel;
-use crate::db::error::ErrorHandler;
-use crate::db::pagination::{paginated, paginated_multicolumn, Paginator};
-use async_bb8_diesel::AsyncRunQueryDsl;
-use diesel::prelude::*;
-use diesel::ExpressionMethods;
-use diesel::SelectableHelper;
-use futures::{future::BoxFuture, TryFutureExt};
-use omicron_common::api::external::Error;
-use omicron_common::api::external::LookupType;
-use omicron_common::api::external::ResourceType;
-use omicron_common::backoff::retry_notify;
-use omicron_common::backoff::retry_policy_internal_service;
-use omicron_common::backoff::BackoffError;
-use std::future::Future;
-use std::pin::Pin;
-use std::sync::Arc;
-use std::task::{Context, Poll};
-
-/// Result type of a [`RecoveryTask`].
-pub type RecoveryResult = Result<CompletionTask, Error>;
-
-/// A future which completes once sagas have been loaded and resumed.
-/// Note that this does not necessarily mean the sagas have completed
-/// execution.
-///
-/// Returns a Result of either:
-/// - A [`CompletionTask`] to track the completion of the resumed sagas, or
-/// - An [`Error`] encountered when attempting to load and resume sagas.
-pub struct RecoveryTask(BoxFuture<'static, RecoveryResult>);
-
-impl Future for RecoveryTask {
-    type Output = RecoveryResult;
-    fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
-        Pin::new(&mut self.get_mut().0).poll(cx)
-    }
-}
-
-/// Result type from a [`CompletionTask`].
-pub type CompletionResult = Result<(), Error>;
-
-/// A future which completes once loaded and resumed sagas have also completed.
-pub struct CompletionTask(BoxFuture<'static, CompletionResult>);
-
-impl Future for CompletionTask {
-    type Output = CompletionResult;
-    fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
-        Pin::new(&mut self.get_mut().0).poll(cx)
-    }
-}
-
-/// Starts an asynchronous task to recover sagas (as after a crash or restart)
-///
-/// More specifically, this task queries the database to list all uncompleted
-/// sagas that are assigned to SEC `sec_id` and for each one:
-///
-/// * loads the saga DAG and log from `datastore`
-/// * uses [`steno::SecClient::saga_resume`] to prepare to resume execution of
-///   the saga using the persistent saga log
-/// * resumes execution of each saga
-///
-/// The returned [`RecoveryTask`] completes once all sagas have been loaded
-/// and resumed, and itself returns a [`CompletionTask`] which completes
-/// when those resumed sagas have finished.
-pub fn recover<T>(
-    opctx: OpContext,
-    sec_id: db::SecId,
-    uctx: Arc<T::ExecContextType>,
-    datastore: Arc<db::DataStore>,
-    sec_client: Arc<steno::SecClient>,
-    registry: Arc<steno::ActionRegistry<T>>,
-) -> RecoveryTask
-where
-    T: steno::SagaType,
-{
-    let join_handle = tokio::spawn(async move {
-        info!(&opctx.log, "start saga recovery");
-
-        // We perform the initial list of sagas using a standard retry policy.
-        // We treat all errors as transient because there's nothing we can do
-        // about any of them except try forever.  As a result, we never expect
-        // an error from the overall operation.
-        // TODO-monitoring we definitely want a way to raise a big red flag if
-        // saga recovery is not completing.
-        // TODO-robustness It would be better to retry the individual database
-        // operations within this operation than retrying the overall operation.
-        // As this is written today, if the listing requires a bunch of pages
-        // and the operation fails partway through, we'll re-fetch all the pages
-        // we successfully fetched before.  If the database is overloaded and
-        // only N% of requests are completing, the probability of this operation
-        // succeeding decreases considerably as the number of separate queries
-        // (pages) goes up.  We'd be much more likely to finish the overall
-        // operation if we didn't throw away the results we did get each time.
-        let found_sagas = retry_notify(
-            retry_policy_internal_service(),
-            || async {
-                list_unfinished_sagas(&opctx, &datastore, &sec_id)
-                    .await
-                    .map_err(BackoffError::transient)
-            },
-            |error, duration| {
-                warn!(
-                    &opctx.log,
-                    "failed to list sagas (will retry after {:?}): {:#}",
-                    duration,
-                    error
-                )
-            },
-        )
-        .await
-        .unwrap();
-
-        info!(&opctx.log, "listed sagas ({} total)", found_sagas.len());
-
-        let recovery_futures = found_sagas.into_iter().map(|saga| async {
-            // TODO-robustness We should put this into a retry loop.  We may
-            // also want to take any failed sagas and put them at the end of the
-            // queue.  It shouldn't really matter, in that the transient
-            // failures here are likely to affect recovery of all sagas.
-            // However, it's conceivable we misclassify a permanent failure as a
-            // transient failure, or that a transient failure is more likely to
-            // affect some sagas than others (e.g, data on a different node, or
-            // it has a larger log that requires more queries).  To avoid one
-            // bad saga ruining the rest, we should try to recover the rest
-            // before we go back to one that's failed.
-            // TODO-debugging want visibility into "abandoned" sagas
-            let saga_id: steno::SagaId = saga.id.into();
-            recover_saga(
-                &opctx,
-                Arc::clone(&uctx),
-                &datastore,
-                &sec_client,
-                Arc::clone(&registry),
-                saga,
-            )
-            .map_err(|error| {
-                warn!(
-                    &opctx.log,
-                    "failed to recover saga {}: {:#}", saga_id, error
-                );
-                error
-            })
-            .await
-        });
-
-        let mut completion_futures = Vec::with_capacity(recovery_futures.len());
-        // Loads and resumes all sagas in serial.
-        for recovery_future in recovery_futures {
-            let saga_complete_future = recovery_future.await?;
-            completion_futures.push(saga_complete_future);
-        }
-        // Returns a future that awaits the completion of all resumed sagas.
-        Ok(CompletionTask(Box::pin(async move {
-            futures::future::try_join_all(completion_futures).await?;
-            Ok(())
-        })))
-    });
-
-    RecoveryTask(Box::pin(async move {
-        // Unwraps join-related errors.
-        join_handle.await.unwrap()
-    }))
-}
-
-/// Queries the database to return a list of uncompleted sagas assigned to SEC
-/// `sec_id`
-// For now, we do the simplest thing: we fetch all the sagas that the
-// caller's going to need before returning any of them.  This is easier to
-// implement than, say, using a channel or some other stream.  In principle
-// we're giving up some opportunity for parallelism.  The caller could be
-// going off and fetching the saga log for the first sagas that we find
-// while we're still listing later sagas.  Doing that properly would require
-// concurrency limits to prevent overload or starvation of other database
-// consumers.
-async fn list_unfinished_sagas(
-    opctx: &OpContext,
-    datastore: &db::DataStore,
-    sec_id: &db::SecId,
-) -> Result<Vec<db::saga_types::Saga>, Error> {
-    trace!(&opctx.log, "listing sagas");
-
-    // Read all sagas in batches.
-    //
-    // Although we could read them all into memory simultaneously, this
-    // risks blocking the DB for an unreasonable amount of time. Instead,
-    // we paginate to avoid cutting off availability to the DB.
-    let mut sagas = vec![];
-    let mut paginator = Paginator::new(SQL_BATCH_SIZE);
-    let conn = datastore.pool_connection_authorized(opctx).await?;
-    while let Some(p) = paginator.next() {
-        use db::schema::saga::dsl;
-
-        let mut batch = paginated(dsl::saga, dsl::id, &p.current_pagparams())
-            .filter(dsl::saga_state.ne(db::saga_types::SagaCachedState(
-                steno::SagaCachedState::Done,
-            )))
-            .filter(dsl::current_sec.eq(*sec_id))
-            .select(db::saga_types::Saga::as_select())
-            .load_async(&*conn)
-            .await
-            .map_err(|e| {
-                public_error_from_diesel(
-                    e,
-                    ErrorHandler::NotFoundByLookup(
-                        ResourceType::SagaDbg,
-                        LookupType::ById(sec_id.0),
-                    ),
-                )
-            })?;
-
-        paginator = p.found_batch(&batch, &|row| row.id);
-        sagas.append(&mut batch);
-    }
-    Ok(sagas)
-}
-
-/// Recovers an individual saga
-///
-/// This function loads the saga log and uses `sec_client` to resume execution.
-///
-/// This function returns a future that completes when the resumed saga
-/// has completed. The saga executor will attempt to execute the saga
-/// regardless of this future - it is for notification purposes only,
-/// and does not need to be polled.
-async fn recover_saga<'a, T>(
-    opctx: &'a OpContext,
-    uctx: Arc<T::ExecContextType>,
-    datastore: &'a db::DataStore,
-    sec_client: &'a steno::SecClient,
-    registry: Arc<steno::ActionRegistry<T>>,
-    saga: db::saga_types::Saga,
-) -> Result<
-    impl core::future::Future<Output = Result<(), Error>> + 'static,
-    Error,
->
-where
-    T: steno::SagaType,
-{
-    let saga_id: steno::SagaId = saga.id.into();
-    let saga_name = saga.name.clone();
-    trace!(opctx.log, "recovering saga: start";
-        "saga_id" => saga_id.to_string(),
-        "saga_name" => saga_name.clone(),
-    );
-
-    let log_events = load_saga_log(&opctx, datastore, &saga).await?;
-    trace!(
-        opctx.log,
-        "recovering saga: loaded log";
-        "saga_id" => ?saga_id,
-        "saga_name" => saga_name.clone()
-    );
-    let saga_completion = sec_client
-        .saga_resume(
-            saga_id,
-            Arc::clone(&uctx),
-            saga.saga_dag,
-            registry,
-            log_events,
-        )
-        .await
-        .map_err(|error| {
-            // TODO-robustness We want to differentiate between retryable and
-            // not here
-            Error::internal_error(&format!(
-                "failed to resume saga: {:#}",
-                error
-            ))
-        })?;
-    sec_client.saga_start(saga_id).await.map_err(|error| {
-        Error::internal_error(&format!("failed to start saga: {:#}", error))
-    })?;
-
-    Ok(async {
-        saga_completion.await.kind.map_err(|e| {
-            Error::internal_error(&format!("Saga failure: {:?}", e))
-        })?;
-        Ok(())
-    })
-}
-
-/// Queries the database to load the full log for the specified saga
-async fn load_saga_log(
-    opctx: &OpContext,
-    datastore: &db::DataStore,
-    saga: &db::saga_types::Saga,
-) -> Result<Vec<steno::SagaNodeEvent>, Error> {
-    // Read all events in batches.
-    //
-    // Although we could read them all into memory simultaneously, this
-    // risks blocking the DB for an unreasonable amount of time. Instead,
-    // we paginate to avoid cutting off availability.
-    let mut events = vec![];
-    let mut paginator = Paginator::new(SQL_BATCH_SIZE);
-    let conn = datastore.pool_connection_authorized(opctx).await?;
-    while let Some(p) = paginator.next() {
-        use db::schema::saga_node_event::dsl;
-        let batch = paginated_multicolumn(
-            dsl::saga_node_event,
-            (dsl::node_id, dsl::event_type),
-            &p.current_pagparams(),
-        )
-        .filter(dsl::saga_id.eq(saga.id))
-        .select(db::saga_types::SagaNodeEvent::as_select())
-        .load_async(&*conn)
-        .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))
-        .await?;
-        paginator =
-            p.found_batch(&batch, &|row| (row.node_id, row.event_type.clone()));
-
-        let mut batch = batch
-            .into_iter()
-            .map(|event| steno::SagaNodeEvent::try_from(event))
-            .collect::<Result<Vec<_>, Error>>()?;
-
-        events.append(&mut batch);
-    }
-    Ok(events)
-}
-
-#[cfg(test)]
-mod test {
-    use super::*;
-    use crate::context::OpContext;
-    use crate::db::test_utils::UnpluggableCockroachDbSecStore;
-    use nexus_test_utils::db::test_setup_database;
-    use omicron_test_utils::dev;
-    use once_cell::sync::Lazy;
-    use pretty_assertions::assert_eq;
-    use rand::seq::SliceRandom;
-    use std::sync::atomic::{AtomicBool, AtomicU32, Ordering};
-    use steno::{
-        new_action_noop_undo, Action, ActionContext, ActionError,
-        ActionRegistry, DagBuilder, Node, SagaDag, SagaId, SagaName, SagaType,
-        SecClient,
-    };
-    use uuid::Uuid;
-
-    // Returns a cockroach DB, as well as a "datastore" interface (which is the
-    // one more frequently used by Nexus).
-    //
-    // The caller is responsible for calling "cleanup().await" on the returned
-    // CockroachInstance - we would normally wrap this in a drop method, but it
-    // is async.
-    async fn new_db(
-        log: &slog::Logger,
-    ) -> (dev::db::CockroachInstance, Arc<db::DataStore>) {
-        let db = test_setup_database(&log).await;
-        let cfg = crate::db::Config { url: db.pg_config().clone() };
-        let pool = Arc::new(db::Pool::new(log, &cfg));
-        let db_datastore = Arc::new(
-            db::DataStore::new(&log, Arc::clone(&pool), None).await.unwrap(),
-        );
-        (db, db_datastore)
-    }
-
-    // The following is our "saga-under-test". It's a simple two-node operation
-    // that tracks how many times it has been called, and provides a mechanism
-    // for detaching storage, to simulate power failure (and meaningfully
-    // recover).
-
-    #[derive(Debug)]
-    struct TestContext {
-        log: slog::Logger,
-
-        // Storage, and instructions on whether or not to detach it
-        // when executing the first saga action.
-        storage: Arc<UnpluggableCockroachDbSecStore>,
-        do_unplug: AtomicBool,
-
-        // Tracks of how many times each node has been reached.
-        n1_count: AtomicU32,
-        n2_count: AtomicU32,
-    }
-
-    impl TestContext {
-        fn new(
-            log: &slog::Logger,
-            storage: Arc<UnpluggableCockroachDbSecStore>,
-        ) -> Self {
-            TestContext {
-                log: log.clone(),
-                storage,
-                do_unplug: AtomicBool::new(false),
-
-                // Counters of how many times the nodes have been invoked.
-                n1_count: AtomicU32::new(0),
-                n2_count: AtomicU32::new(0),
-            }
-        }
-    }
-
-    #[derive(Debug)]
-    struct TestOp;
-    impl SagaType for TestOp {
-        type ExecContextType = TestContext;
-    }
-
-    static ACTION_N1: Lazy<Arc<dyn Action<TestOp>>> =
-        Lazy::new(|| new_action_noop_undo("n1_action", node_one));
-    static ACTION_N2: Lazy<Arc<dyn Action<TestOp>>> =
-        Lazy::new(|| new_action_noop_undo("n2_action", node_two));
-
-    fn registry_create() -> Arc<ActionRegistry<TestOp>> {
-        let mut registry = ActionRegistry::new();
-        registry.register(Arc::clone(&ACTION_N1));
-        registry.register(Arc::clone(&ACTION_N2));
-        Arc::new(registry)
-    }
-
-    fn saga_object_create() -> Arc<SagaDag> {
-        let mut builder = DagBuilder::new(SagaName::new("test-saga"));
-        builder.append(Node::action("n1_out", "NodeOne", ACTION_N1.as_ref()));
-        builder.append(Node::action("n2_out", "NodeTwo", ACTION_N2.as_ref()));
-        let dag = builder.build().unwrap();
-        Arc::new(SagaDag::new(dag, serde_json::Value::Null))
-    }
-
-    async fn node_one(ctx: ActionContext<TestOp>) -> Result<i32, ActionError> {
-        let uctx = ctx.user_data();
-        uctx.n1_count.fetch_add(1, Ordering::SeqCst);
-        info!(&uctx.log, "ACTION: node_one");
-        // If "do_unplug" is true, we detach storage.
-        //
-        // This prevents the SEC from successfully recording that
-        // this node completed, and acts like a crash.
-        if uctx.do_unplug.load(Ordering::SeqCst) {
-            info!(&uctx.log, "Unplugged storage");
-            uctx.storage.set_unplug(true);
-        }
-        Ok(1)
-    }
-
-    async fn node_two(ctx: ActionContext<TestOp>) -> Result<i32, ActionError> {
-        let uctx = ctx.user_data();
-        uctx.n2_count.fetch_add(1, Ordering::SeqCst);
-        info!(&uctx.log, "ACTION: node_two");
-        Ok(2)
-    }
-
-    // Helper function for setting up storage, SEC, and a test context object.
-    fn create_storage_sec_and_context(
-        log: &slog::Logger,
-        db_datastore: Arc<db::DataStore>,
-        sec_id: db::SecId,
-    ) -> (Arc<UnpluggableCockroachDbSecStore>, SecClient, Arc<TestContext>)
-    {
-        let storage = Arc::new(UnpluggableCockroachDbSecStore::new(
-            sec_id,
-            db_datastore,
-            log.new(o!("component" => "SecStore")),
-        ));
-        let sec_client =
-            steno::sec(log.new(o!("component" => "SEC")), storage.clone());
-        let uctx = Arc::new(TestContext::new(&log, storage.clone()));
-        (storage, sec_client, uctx)
-    }
-
-    #[tokio::test]
-    async fn test_failure_during_saga_can_be_recovered() {
-        // Test setup
-        let logctx =
-            dev::test_setup_log("test_failure_during_saga_can_be_recovered");
-        let log = logctx.log.new(o!());
-        let (mut db, db_datastore) = new_db(&log).await;
-        let sec_id = db::SecId(uuid::Uuid::new_v4());
-        let (storage, sec_client, uctx) =
-            create_storage_sec_and_context(&log, db_datastore.clone(), sec_id);
-        let sec_log = log.new(o!("component" => "SEC"));
-        let opctx = OpContext::for_tests(
-            log,
-            Arc::clone(&db_datastore) as Arc<dyn nexus_auth::storage::Storage>,
-        );
-
-        // Create and start a saga.
-        //
-        // Because "do_unplug" is set to true, we should detach storage within
-        // the first node operation.
-        //
-        // We expect the saga will complete successfully, because the
-        // storage subsystem returns "OK" rather than an error.
-        uctx.do_unplug.store(true, Ordering::SeqCst);
-        let saga_id = SagaId(Uuid::new_v4());
-        let future = sec_client
-            .saga_create(
-                saga_id,
-                uctx.clone(),
-                saga_object_create(),
-                registry_create(),
-            )
-            .await
-            .unwrap();
-        sec_client.saga_start(saga_id).await.unwrap();
-        let result = future.await;
-        let output = result.kind.unwrap();
-        assert_eq!(output.lookup_node_output::<i32>("n1_out").unwrap(), 1);
-        assert_eq!(output.lookup_node_output::<i32>("n2_out").unwrap(), 2);
-        assert_eq!(uctx.n1_count.load(Ordering::SeqCst), 1);
-        assert_eq!(uctx.n2_count.load(Ordering::SeqCst), 1);
-
-        // Now we "reboot", by terminating the SEC and creating a new one
-        // using the same storage system.
-        //
-        // We update uctx to prevent the storage system from detaching again.
-        sec_client.shutdown().await;
-        let sec_client = steno::sec(sec_log, storage.clone());
-        uctx.storage.set_unplug(false);
-        uctx.do_unplug.store(false, Ordering::SeqCst);
-
-        // Recover the saga, observing that it re-runs operations and completes.
-        let sec_client = Arc::new(sec_client);
-        recover(
-            opctx,
-            sec_id,
-            uctx.clone(),
-            db_datastore,
-            sec_client.clone(),
-            registry_create(),
-        )
-        .await // Await the loading and resuming of the sagas
-        .unwrap()
-        .await // Awaits the completion of the resumed sagas
-        .unwrap();
-        assert_eq!(uctx.n1_count.load(Ordering::SeqCst), 2);
-        assert_eq!(uctx.n2_count.load(Ordering::SeqCst), 2);
-
-        // Test cleanup
-        let sec_client = Arc::try_unwrap(sec_client).unwrap();
-        sec_client.shutdown().await;
-        db.cleanup().await.unwrap();
-        logctx.cleanup_successful();
-    }
-
-    #[tokio::test]
-    async fn test_successful_saga_does_not_replay_during_recovery() {
-        // Test setup
-        let logctx = dev::test_setup_log(
-            "test_successful_saga_does_not_replay_during_recovery",
-        );
-        let log = logctx.log.new(o!());
-        let (mut db, db_datastore) = new_db(&log).await;
-        let sec_id = db::SecId(uuid::Uuid::new_v4());
-        let (storage, sec_client, uctx) =
-            create_storage_sec_and_context(&log, db_datastore.clone(), sec_id);
-        let sec_log = log.new(o!("component" => "SEC"));
-        let opctx = OpContext::for_tests(
-            log,
-            Arc::clone(&db_datastore) as Arc<dyn nexus_auth::storage::Storage>,
-        );
-
-        // Create and start a saga, which we expect to complete successfully.
-        let saga_id = SagaId(Uuid::new_v4());
-        let future = sec_client
-            .saga_create(
-                saga_id,
-                uctx.clone(),
-                saga_object_create(),
-                registry_create(),
-            )
-            .await
-            .unwrap();
-        sec_client.saga_start(saga_id).await.unwrap();
-        let result = future.await;
-        let output = result.kind.unwrap();
-        assert_eq!(output.lookup_node_output::<i32>("n1_out").unwrap(), 1);
-        assert_eq!(output.lookup_node_output::<i32>("n2_out").unwrap(), 2);
-        assert_eq!(uctx.n1_count.load(Ordering::SeqCst), 1);
-        assert_eq!(uctx.n2_count.load(Ordering::SeqCst), 1);
-
-        // Now we "reboot", by terminating the SEC and creating a new one
-        // using the same storage system.
-        sec_client.shutdown().await;
-        let sec_client = steno::sec(sec_log, storage.clone());
-
-        // Recover the saga, observing that it does not replay the nodes.
-        let sec_client = Arc::new(sec_client);
-        recover(
-            opctx,
-            sec_id,
-            uctx.clone(),
-            db_datastore,
-            sec_client.clone(),
-            registry_create(),
-        )
-        .await
-        .unwrap()
-        .await
-        .unwrap();
-        assert_eq!(uctx.n1_count.load(Ordering::SeqCst), 1);
-        assert_eq!(uctx.n2_count.load(Ordering::SeqCst), 1);
-
-        // Test cleanup
-        let sec_client = Arc::try_unwrap(sec_client).unwrap();
-        sec_client.shutdown().await;
-        db.cleanup().await.unwrap();
-        logctx.cleanup_successful();
-    }
-
-    #[tokio::test]
-    async fn test_list_unfinished_sagas() {
-        // Test setup
-        let logctx = dev::test_setup_log("test_list_unfinished_sagas");
-        let log = logctx.log.new(o!());
-        let (mut db, db_datastore) = new_db(&log).await;
-        let sec_id = db::SecId(uuid::Uuid::new_v4());
-        let opctx = OpContext::for_tests(
-            log,
-            Arc::clone(&db_datastore) as Arc<dyn nexus_auth::storage::Storage>,
-        );
-
-        // Create a couple batches of sagas.
-        let new_running_db_saga = || {
-            let params = steno::SagaCreateParams {
-                id: steno::SagaId(Uuid::new_v4()),
-                name: steno::SagaName::new("test saga"),
-                dag: serde_json::value::Value::Null,
-                state: steno::SagaCachedState::Running,
-            };
-
-            db::model::saga_types::Saga::new(sec_id, params)
-        };
-        let mut inserted_sagas = (0..SQL_BATCH_SIZE.get() * 2)
-            .map(|_| new_running_db_saga())
-            .collect::<Vec<_>>();
-
-        // Shuffle these sagas into a random order to check that the pagination
-        // order is working as intended on the read path, which we'll do later
-        // in this test.
-        inserted_sagas.shuffle(&mut rand::thread_rng());
-
-        // Insert the batches of unfinished sagas into the database
-        let conn = db_datastore
-            .pool_connection_unauthorized()
-            .await
-            .expect("Failed to access db connection");
-        diesel::insert_into(db::schema::saga::dsl::saga)
-            .values(inserted_sagas.clone())
-            .execute_async(&*conn)
-            .await
-            .expect("Failed to insert test setup data");
-
-        // List them, expect to see them all in order by ID.
-        let mut observed_sagas =
-            list_unfinished_sagas(&opctx, &db_datastore, &sec_id)
-                .await
-                .expect("Failed to list unfinished sagas");
-        inserted_sagas.sort_by_key(|a| a.id);
-
-        // Timestamps can change slightly when we insert them.
-        //
-        // Sanitize them to make input/output equality checks easier.
-        let sanitize_timestamps = |sagas: &mut Vec<db::saga_types::Saga>| {
-            for saga in sagas {
-                saga.time_created = chrono::DateTime::UNIX_EPOCH;
-                saga.adopt_time = chrono::DateTime::UNIX_EPOCH;
-            }
-        };
-        sanitize_timestamps(&mut observed_sagas);
-        sanitize_timestamps(&mut inserted_sagas);
-
-        assert_eq!(
-            inserted_sagas, observed_sagas,
-            "Observed sagas did not match inserted sagas"
-        );
-
-        // Test cleanup
-        db.cleanup().await.unwrap();
-        logctx.cleanup_successful();
-    }
-
-    #[tokio::test]
-    async fn test_list_unfinished_nodes() {
-        // Test setup
-        let logctx = dev::test_setup_log("test_list_unfinished_nodes");
-        let log = logctx.log.new(o!());
-        let (mut db, db_datastore) = new_db(&log).await;
-        let sec_id = db::SecId(uuid::Uuid::new_v4());
-        let opctx = OpContext::for_tests(
-            log,
-            Arc::clone(&db_datastore) as Arc<dyn nexus_auth::storage::Storage>,
-        );
-        let saga_id = steno::SagaId(Uuid::new_v4());
-
-        // Create a couple batches of saga events
-        let new_db_saga_nodes =
-            |node_id: u32, event_type: steno::SagaNodeEventType| {
-                let event = steno::SagaNodeEvent {
-                    saga_id,
-                    node_id: steno::SagaNodeId::from(node_id),
-                    event_type,
-                };
-
-                db::model::saga_types::SagaNodeEvent::new(event, sec_id)
-            };
-        let mut inserted_nodes = (0..SQL_BATCH_SIZE.get() * 2)
-            .flat_map(|i| {
-                // This isn't an exhaustive list of event types, but gives us a few
-                // options to pick from. Since this is a pagination key, it's
-                // important to include a variety here.
-                use steno::SagaNodeEventType::*;
-                [
-                    new_db_saga_nodes(i, Started),
-                    new_db_saga_nodes(i, UndoStarted),
-                    new_db_saga_nodes(i, UndoFinished),
-                ]
-            })
-            .collect::<Vec<_>>();
-
-        // Shuffle these nodes into a random order to check that the pagination
-        // order is working as intended on the read path, which we'll do later
-        // in this test.
-        inserted_nodes.shuffle(&mut rand::thread_rng());
-
-        // Insert them into the database
-        let conn = db_datastore
-            .pool_connection_unauthorized()
-            .await
-            .expect("Failed to access db connection");
-        diesel::insert_into(db::schema::saga_node_event::dsl::saga_node_event)
-            .values(inserted_nodes.clone())
-            .execute_async(&*conn)
-            .await
-            .expect("Failed to insert test setup data");
-
-        // List them, expect to see them all in order by ID.
-        //
-        // Note that we need to make up a saga to see this, but the
-        // part of it that actually matters is the ID.
-        let params = steno::SagaCreateParams {
-            id: saga_id,
-            name: steno::SagaName::new("test saga"),
-            dag: serde_json::value::Value::Null,
-            state: steno::SagaCachedState::Running,
-        };
-        let saga = db::model::saga_types::Saga::new(sec_id, params);
-        let observed_nodes = load_saga_log(&opctx, &db_datastore, &saga)
-            .await
-            .expect("Failed to list unfinished nodes");
-        inserted_nodes.sort_by_key(|a| (a.node_id, a.event_type.clone()));
-
-        let inserted_nodes = inserted_nodes
-            .into_iter()
-            .map(|node| steno::SagaNodeEvent::try_from(node))
-            .collect::<Result<Vec<_>, _>>()
-            .expect("Couldn't convert DB nodes to steno nodes");
-
-        // The steno::SagaNodeEvent type doesn't implement PartialEq, so we need to do this
-        // a little manually.
-        assert_eq!(inserted_nodes.len(), observed_nodes.len());
-        for i in 0..inserted_nodes.len() {
-            assert_eq!(inserted_nodes[i].saga_id, observed_nodes[i].saga_id);
-            assert_eq!(inserted_nodes[i].node_id, observed_nodes[i].node_id);
-            assert_eq!(
-                inserted_nodes[i].event_type.label(),
-                observed_nodes[i].event_type.label()
-            );
-        }
-
-        // Test cleanup
-        db.cleanup().await.unwrap();
-        logctx.cleanup_successful();
-    }
-
-    #[tokio::test]
-    async fn test_list_no_unfinished_nodes() {
-        // Test setup
-        let logctx = dev::test_setup_log("test_list_no_unfinished_nodes");
-        let log = logctx.log.new(o!());
-        let (mut db, db_datastore) = new_db(&log).await;
-        let sec_id = db::SecId(uuid::Uuid::new_v4());
-        let opctx = OpContext::for_tests(
-            log,
-            Arc::clone(&db_datastore) as Arc<dyn nexus_auth::storage::Storage>,
-        );
-        let saga_id = steno::SagaId(Uuid::new_v4());
-
-        let params = steno::SagaCreateParams {
-            id: saga_id,
-            name: steno::SagaName::new("test saga"),
-            dag: serde_json::value::Value::Null,
-            state: steno::SagaCachedState::Running,
-        };
-        let saga = db::model::saga_types::Saga::new(sec_id, params);
-
-        // Test that this returns "no nodes" rather than throwing some "not
-        // found" error.
-        let observed_nodes = load_saga_log(&opctx, &db_datastore, &saga)
-            .await
-            .expect("Failed to list unfinished nodes");
-        assert_eq!(observed_nodes.len(), 0);
-
-        // Test cleanup
-        db.cleanup().await.unwrap();
-        logctx.cleanup_successful();
-    }
-}
diff --git a/nexus/db-queries/src/db/sec_store.rs b/nexus/db-queries/src/db/sec_store.rs
index 72de02ff54..0dcc3aa717 100644
--- a/nexus/db-queries/src/db/sec_store.rs
+++ b/nexus/db-queries/src/db/sec_store.rs
@@ -8,7 +8,8 @@ use crate::db::{self, model::Generation};
 use anyhow::Context;
 use async_trait::async_trait;
 use dropshot::HttpError;
-use futures::TryFutureExt;
+use futures::{Future, TryFutureExt};
+use omicron_common::api::external;
 use omicron_common::backoff;
 use slog::Logger;
 use std::fmt;
@@ -66,78 +67,114 @@ impl steno::SecStore for CockroachDbSecStore {
         debug!(&log, "recording saga event");
         let our_event = db::saga_types::SagaNodeEvent::new(event, self.sec_id);
 
-        backoff::retry_notify_ext(
-            // This is an internal service query to CockroachDB.
-            backoff::retry_policy_internal_service(),
+        // Add retries for this operation. saga_create_event is internally
+        // idempotent, so we can retry indefinitely until the event has been
+        // durably recorded.
+        backoff_saga_operation(
+            &log,
             || {
-                // In general, there are some kinds of database errors that are
-                // temporary/server errors (e.g. network failures), and some
-                // that are permanent/client errors (e.g. conflict during
-                // insertion). The permanent ones would require operator
-                // intervention to fix.
-                //
-                // However, there is no way to bubble up errors here, and for
-                // good reason: it is inherent to the nature of sagas that
-                // progress is durably recorded. So within *this* code there is
-                // no option but to retry forever. (Below, however, we do mark
-                // errors that likely require operator intervention.)
-                //
-                // At a higher level, callers should plan for the fact that
-                // record_event (and, so, saga execution) could potentially loop
-                // indefinitely while the datastore (or other dependent
-                // services) are down.
                 self.datastore
                     .saga_create_event(&our_event)
                     .map_err(backoff::BackoffError::transient)
             },
-            move |error, call_count, total_duration| {
-                let http_error = HttpError::from(error.clone());
-                if http_error.status_code.is_client_error() {
-                    error!(
-                        &log,
-                        "client error while recording saga event (likely \
-                         requires operator intervention), retrying anyway";
-                        "error" => &error,
-                        "call_count" => call_count,
-                        "total_duration" => ?total_duration,
-                    );
-                } else if total_duration > Duration::from_secs(20) {
-                    warn!(
-                        &log,
-                        "server error while recording saga event, retrying";
-                        "error" => &error,
-                        "call_count" => call_count,
-                        "total_duration" => ?total_duration,
-                    );
-                } else {
-                    info!(
-                        &log,
-                        "server error while recording saga event, retrying";
-                        "error" => &error,
-                        "call_count" => call_count,
-                        "total_duration" => ?total_duration,
-                    );
-                }
-            },
+            "recording saga event",
         )
         .await
-        .expect("the above backoff retries forever")
     }
 
     async fn saga_update(&self, id: SagaId, update: steno::SagaCachedState) {
         // TODO-robustness We should track the current generation of the saga
         // and use it.  We'll know this either from when it was created or when
         // it was recovered.
-        info!(&self.log, "updating state";
+
+        let log = self.log.new(o!(
             "saga_id" => id.to_string(),
-            "new_state" => update.to_string()
-        );
+            "new_state" => update.to_string(),
+        ));
 
-        // TODO-robustness This should be wrapped with a retry loop rather than
-        // unwrapping the result.  See omicron#2416.
-        self.datastore
-            .saga_update_state(id, update, self.sec_id, Generation::new())
-            .await
-            .unwrap();
+        info!(&log, "updating state");
+
+        // Add retries for this operation. saga_update_state is internally
+        // idempotent, so we can retry indefinitely until the event has been
+        // durably recorded. (But see the note in saga_update_state about how
+        // idempotence is enough for now, but may not be in the future.)
+        backoff_saga_operation(
+            &log,
+            || {
+                self.datastore
+                    .saga_update_state(
+                        id,
+                        update,
+                        self.sec_id,
+                        Generation::new(),
+                    )
+                    .map_err(backoff::BackoffError::transient)
+            },
+            "updating saga state",
+        )
+        .await
     }
 }
+
+/// Implements backoff retry logic for saga operations.
+///
+/// In general, there are some kinds of database errors that are
+/// temporary/server errors (e.g. network failures), and some that are
+/// permanent/client errors (e.g. conflict during insertion). The permanent
+/// ones would require operator intervention to fix.
+///
+/// However, there is no way to bubble up errors from the SEC store, and for
+/// good reason: it is inherent to the nature of sagas that progress is durably
+/// recorded. So inside this code there is no option but to retry forever.
+/// (Below, however, we do mark errors that likely require operator
+/// intervention.)
+///
+/// At a higher level, callers should plan for the fact saga execution could
+/// potentially loop indefinitely while the datastore (or other dependent
+/// services) are down.
+async fn backoff_saga_operation<F, Fut>(log: &Logger, op: F, description: &str)
+where
+    F: Fn() -> Fut,
+    Fut: Future<Output = Result<(), backoff::BackoffError<external::Error>>>,
+{
+    backoff::retry_notify_ext(
+        // This is an internal service query to CockroachDB.
+        backoff::retry_policy_internal_service(),
+        op,
+        move |error, call_count, total_duration| {
+            let http_error = HttpError::from(error.clone());
+            if http_error.status_code.is_client_error() {
+                error!(
+                    &log,
+                    "client error while {description} (likely \
+                     requires operator intervention), retrying anyway";
+                    "error" => &error,
+                    "call_count" => call_count,
+                    "total_duration" => ?total_duration,
+                );
+            } else if total_duration > WARN_DURATION {
+                warn!(
+                    &log,
+                    "server error while {description}, retrying";
+                    "error" => &error,
+                    "call_count" => call_count,
+                    "total_duration" => ?total_duration,
+                );
+            } else {
+                info!(
+                    &log,
+                    "server error while {description}, retrying";
+                    "error" => &error,
+                    "call_count" => call_count,
+                    "total_duration" => ?total_duration,
+                );
+            }
+        },
+    )
+    .await
+    .expect("the above backoff retries forever")
+}
+
+/// Threshold at which logs about server errors during retries switch from INFO
+/// to WARN.
+const WARN_DURATION: Duration = Duration::from_secs(20);
diff --git a/nexus/examples/config-second.toml b/nexus/examples/config-second.toml
new file mode 100644
index 0000000000..ef67749a4b
--- /dev/null
+++ b/nexus/examples/config-second.toml
@@ -0,0 +1,181 @@
+#
+# Example configuration file for running a second Nexus instance locally
+# alongside the stack started by `omicron-dev run-all`.  See the
+# how-to-run-simulated instructions for details.
+#
+
+################################################################################
+# INSTRUCTIONS: To run Nexus against an existing stack started with 	       #
+# `omicron-dev run-all`, see the very bottom of this file.  		       #
+################################################################################
+
+[console]
+# Directory for static assets. Absolute path or relative to CWD.
+static_dir = "out/console-assets"
+session_idle_timeout_minutes = 480 # 8 hours
+session_absolute_timeout_minutes = 1440 # 24 hours
+
+# List of authentication schemes to support.
+[authn]
+schemes_external = ["session_cookie", "access_token"]
+
+[log]
+# Show log messages of this level and more severe
+level = "info"
+
+# Example output to a terminal (with colors)
+mode = "stderr-terminal"
+
+# Example output to a file, appending if it already exists.
+#mode = "file"
+#path = "logs/server.log"
+#if_exists = "append"
+
+# Configuration for interacting with the timeseries database
+[timeseries_db]
+address = "[::1]:8123"
+
+
+
+[deployment]
+# Identifier for this instance of Nexus
+id = "a4ef738a-1fb0-47b1-9da2-4919c7ec7c7f"
+rack_id = "c19a698f-c6f9-4a17-ae30-20d711b8f7dc"
+# Since we expect to be the second instance of Nexus running on this system,
+# pick any available port.
+techport_external_server_port = 0
+
+# Nexus may need to resolve external hosts (e.g. to grab IdP metadata).
+# These are the DNS servers it should use.
+external_dns_servers = ["1.1.1.1", "9.9.9.9"]
+
+[deployment.dropshot_external]
+# IP Address and TCP port on which to listen for the external API
+# This config file uses 12222 to avoid colliding with the usual 12220 that's
+# used by `omicron-dev run-all`
+bind_address = "127.0.0.1:12222"
+# Allow large request bodies to support uploading TUF archives. The number here
+# is picked based on the typical size for tuf-mupdate.zip as of 2024-01
+# (~1.5GiB) and multiplying it by 2.
+#
+# This should be brought back down to a more reasonable value once per-endpoint
+# request body limits are implemented.
+request_body_max_bytes = 3221225472
+# To have Nexus's external HTTP endpoint use TLS, uncomment the line below.  You
+# will also need to provide an initial TLS certificate during rack
+# initialization.  If you're using this config file, you're probably running a
+# simulated system.  In that case, the initial certificate is provided to the
+# simulated sled agent (acting as RSS) via command-line arguments.
+#tls = true
+
+[deployment.dropshot_internal]
+# IP Address and TCP port on which to listen for the internal API
+# This config file uses 12223 to avoid colliding with the usual 12221 that's
+# used by `omicron-dev run-all`
+bind_address = "[::1]:12223"
+request_body_max_bytes = 1048576
+
+#[deployment.internal_dns]
+## These values are overridden at the bottom of this file.
+#type = "from_address"
+#address = "[::1]:3535"
+
+#[deployment.database]
+## These values are overridden at the bottom of this file.
+#type = "from_url"
+#url = "postgresql://root@[::1]:32221/omicron?sslmode=disable"
+
+# Tunable configuration parameters, for testing or experimentation
+[tunables]
+
+# The maximum allowed prefix (thus smallest size) for a VPC Subnet's
+# IPv4 subnetwork. This size allows for ~60 hosts.
+max_vpc_ipv4_subnet_prefix = 26
+
+# Configuration for interacting with the dataplane daemon
+[dendrite.switch0]
+address = "[::1]:12224"
+
+[background_tasks]
+dns_internal.period_secs_config = 60
+dns_internal.period_secs_servers = 60
+dns_internal.period_secs_propagation = 60
+dns_internal.max_concurrent_server_updates = 5
+dns_external.period_secs_config = 60
+dns_external.period_secs_servers = 60
+dns_external.period_secs_propagation = 60
+dns_external.max_concurrent_server_updates = 5
+metrics_producer_gc.period_secs = 60
+# How frequently we check the list of stored TLS certificates.  This is
+# approximately an upper bound on how soon after updating the list of
+# certificates it will take _other_ Nexus instances to notice and stop serving
+# them (on a sunny day).
+external_endpoints.period_secs = 60
+nat_cleanup.period_secs = 30
+bfd_manager.period_secs = 30
+# How frequently to collect hardware/software inventory from the whole system
+# (even if we don't have reason to believe anything has changed).
+inventory.period_secs = 600
+# Maximum number of past collections to keep in the database
+inventory.nkeep = 5
+# Disable inventory collection altogether (for emergencies)
+inventory.disable = false
+phantom_disks.period_secs = 30
+physical_disk_adoption.period_secs = 30
+blueprints.period_secs_load = 10
+blueprints.period_secs_execute = 60
+blueprints.period_secs_collect_crdb_node_ids = 180
+sync_service_zone_nat.period_secs = 30
+switch_port_settings_manager.period_secs = 30
+region_replacement.period_secs = 30
+region_replacement_driver.period_secs = 10
+# How frequently to query the status of active instances.
+instance_watcher.period_secs = 30
+service_firewall_propagation.period_secs = 300
+v2p_mapping_propagation.period_secs = 30
+abandoned_vmm_reaper.period_secs = 60
+saga_recovery.period_secs = 600
+lookup_region_port.period_secs = 60
+
+[default_region_allocation_strategy]
+# allocate region on 3 random distinct zpools, on 3 random distinct sleds.
+type = "random_with_distinct_sleds"
+
+# the same as random_with_distinct_sleds, but without requiring distinct sleds
+# type = "random"
+
+# setting `seed` to a fixed value will make dataset selection ordering use the
+# same shuffling order for every region allocation.
+# seed = 0
+
+################################################################################
+# INSTRUCTIONS: To run Nexus against an existing stack started with            #
+# `omicron-dev run-all`, you should only have to modify values in this 	       #
+# section. 								       #
+# 									       #
+# Modify the port numbers below based on the output of `omicron-dev run-all`   #
+################################################################################
+
+[mgd]
+# Look for "management gateway:    http://[::1]:49188 (switch0)"
+# The "http://" does not go in this string -- just the socket address.
+switch0.address = "[::1]:49188"
+
+# Look for "management gateway:    http://[::1]:39352 (switch1)"
+# The "http://" does not go in this string -- just the socket address.
+switch1.address = "[::1]:39352"
+
+[deployment.internal_dns]
+# Look for "internal DNS:          [::1]:54025"
+# and adjust the port number below.
+address = "[::1]:54025"
+# You should not need to change this.
+type = "from_address"
+
+[deployment.database]
+# Look for "cockroachdb URL:       postgresql://root@[::1]:43256/omicron?sslmode=disable"
+# and adjust the port number below.
+url = "postgresql://root@[::1]:43256/omicron?sslmode=disable"
+# You should not need to change this.
+type = "from_url"
+################################################################################
diff --git a/nexus/examples/config.toml b/nexus/examples/config.toml
index 8c1ab5ca5f..6ec80359ab 100644
--- a/nexus/examples/config.toml
+++ b/nexus/examples/config.toml
@@ -120,6 +120,7 @@ instance_watcher.period_secs = 30
 service_firewall_propagation.period_secs = 300
 v2p_mapping_propagation.period_secs = 30
 abandoned_vmm_reaper.period_secs = 60
+saga_recovery.period_secs = 600
 lookup_region_port.period_secs = 60
 
 [default_region_allocation_strategy]
diff --git a/nexus/internal-api/src/lib.rs b/nexus/internal-api/src/lib.rs
index b2d68036bb..b6de85486a 100644
--- a/nexus/internal-api/src/lib.rs
+++ b/nexus/internal-api/src/lib.rs
@@ -14,7 +14,7 @@ use nexus_types::{
         Blueprint, BlueprintMetadata, BlueprintTarget, BlueprintTargetSet,
     },
     external_api::{
-        params::{SledSelector, UninitializedSledId},
+        params::{PhysicalDiskPath, SledSelector, UninitializedSledId},
         shared::{ProbeInfo, UninitializedSled},
         views::SledPolicy,
     },
@@ -472,6 +472,21 @@ pub trait NexusInternalApi {
         sled: TypedBody<SledSelector>,
     ) -> Result<HttpResponseOk<SledPolicy>, HttpError>;
 
+    /// Mark a physical disk as expunged
+    ///
+    /// This is an irreversible process! It should only be called after
+    /// sufficient warning to the operator.
+    ///
+    /// This is idempotent.
+    #[endpoint {
+        method = POST,
+        path = "/physical-disk/expunge",
+    }]
+    async fn physical_disk_expunge(
+        rqctx: RequestContext<Self::Context>,
+        disk: TypedBody<PhysicalDiskPath>,
+    ) -> Result<HttpResponseUpdatedNoContent, HttpError>;
+
     /// Get all the probes associated with a given sled.
     #[endpoint {
         method = GET,
diff --git a/nexus/reconfigurator/execution/src/datasets.rs b/nexus/reconfigurator/execution/src/datasets.rs
index 51ac45c9df..139c94c53f 100644
--- a/nexus/reconfigurator/execution/src/datasets.rs
+++ b/nexus/reconfigurator/execution/src/datasets.rs
@@ -66,7 +66,7 @@ pub(crate) async fn ensure_dataset_records_exist(
         let dataset = Dataset::new(
             id.into_untyped_uuid(),
             pool_id.into_untyped_uuid(),
-            address,
+            Some(address),
             kind.into(),
         );
         let maybe_inserted = datastore
diff --git a/nexus/reconfigurator/planning/src/system.rs b/nexus/reconfigurator/planning/src/system.rs
index 0499e0ef5b..5f00ea8172 100644
--- a/nexus/reconfigurator/planning/src/system.rs
+++ b/nexus/reconfigurator/planning/src/system.rs
@@ -521,7 +521,7 @@ impl Sled {
                     sled_agent_client::types::Baseboard::Gimlet {
                         identifier: serial.clone(),
                         model: model.clone(),
-                        revision: i64::from(revision),
+                        revision,
                     }
                 }
                 SledHardware::Pc => sled_agent_client::types::Baseboard::Pc {
@@ -591,7 +591,7 @@ impl Sled {
             .map(|sledhw| sled_agent_client::types::Baseboard::Gimlet {
                 identifier: sledhw.baseboard_id.serial_number.clone(),
                 model: sledhw.baseboard_id.part_number.clone(),
-                revision: i64::from(sledhw.sp.baseboard_revision),
+                revision: sledhw.sp.baseboard_revision,
             })
             .unwrap_or(sled_agent_client::types::Baseboard::Unknown);
 
diff --git a/nexus/saga-recovery/Cargo.toml b/nexus/saga-recovery/Cargo.toml
new file mode 100644
index 0000000000..4356cc9789
--- /dev/null
+++ b/nexus/saga-recovery/Cargo.toml
@@ -0,0 +1,41 @@
+[package]
+name = "nexus-saga-recovery"
+version = "0.1.0"
+edition = "2021"
+
+[lints]
+workspace = true
+
+[build-dependencies]
+omicron-rpaths.workspace = true
+
+[dependencies]
+chrono.workspace = true
+futures.workspace = true
+nexus-db-queries.workspace = true
+nexus-db-model.workspace = true
+omicron-common.workspace = true
+# See omicron-rpaths for more about the "pq-sys" dependency.
+pq-sys = "*"
+serde.workspace = true
+serde_json.workspace = true
+slog.workspace = true
+slog-error-chain.workspace = true
+steno.workspace = true
+tokio.workspace = true
+
+omicron-workspace-hack.workspace = true
+
+[dev-dependencies]
+nexus-auth.workspace = true
+nexus-db-queries.workspace = true
+nexus-test-utils.workspace = true
+nexus-test-utils-macros.workspace = true
+nexus-types.workspace = true
+omicron-common.workspace = true
+omicron-test-utils.workspace = true
+once_cell.workspace = true
+pretty_assertions.workspace = true
+steno.workspace = true
+tokio.workspace = true
+uuid.workspace = true
diff --git a/installinator-artifactd/src/context.rs b/nexus/saga-recovery/build.rs
similarity index 50%
rename from installinator-artifactd/src/context.rs
rename to nexus/saga-recovery/build.rs
index beea2593aa..1ba9acd41c 100644
--- a/installinator-artifactd/src/context.rs
+++ b/nexus/saga-recovery/build.rs
@@ -2,12 +2,9 @@
 // License, v. 2.0. If a copy of the MPL was not distributed with this
 // file, You can obtain one at https://mozilla.org/MPL/2.0/.
 
-// Copyright 2023 Oxide Computer Company
-
-//! User provided dropshot server context
-
-use crate::store::ArtifactStore;
-
-pub struct ServerContext {
-    pub(crate) artifact_store: ArtifactStore,
+// See omicron-rpaths for documentation.
+// NOTE: This file MUST be kept in sync with the other build.rs files in this
+// repository.
+fn main() {
+    omicron_rpaths::configure_default_omicron_rpaths();
 }
diff --git a/nexus/saga-recovery/src/lib.rs b/nexus/saga-recovery/src/lib.rs
new file mode 100644
index 0000000000..a83fc28774
--- /dev/null
+++ b/nexus/saga-recovery/src/lib.rs
@@ -0,0 +1,682 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+//! # Saga recovery bookkeeping
+//!
+//! If you're reading this, you first want to read the big block comment in the
+//! saga recovery background task.  It explains important background about what
+//! we're trying to do here.  The rest of this comment assumes you've read all
+//! that.
+//!
+//! ## Saga recovery passes
+//!
+//! For the reasons mentioned in that comment, saga recovery is done in
+//! **passes**.  The state that's kept between passes (when the task is "at
+//! rest") is called the [`RestState`].
+//!
+//! Each saga recovery pass looks like this:
+//!
+//! 1. Start with initial [`RestState`] and [`Report`].
+//! 2. List in-progress sagas from the database.
+//! 3. Collect list of sagas that have been started by the rest of Nexus.
+//! 4. Use [`Plan::new()`] to construct a plan.  The plan examines all the sagas
+//!    reported by the database as well as all the sagas we knew about before
+//!    and determines for each one exactly what's needed.  Each saga falls into
+//!    one of a few buckets:
+//!    * It's clearly not running, but should be, so it needs to be recovered.
+//!    * It's clearly running, so it does not need to be recovered.
+//!    * It has finished running and we can stop keeping track of it.
+//!    * It _may_ be running but we cannot tell because of the intrinsic race
+//!      between steps 2 and 3 above.  We'll keep track of these and resolve the
+//!      ambiguity on the next pass.
+//! 5. Carry out recovery for whatever sagas need to be recovered.  Use
+//!    [`ExecutionBuilder::new()`] to construct a description of what happened.
+//! 6. Update the [`RestState`] and [`Report`] to reflect what happened in this
+//!    pass.
+//!
+//! This process can be repeated forever, as often as wanted, but should not be
+//! run concurrently.
+//!
+//! ## Saga recovery task vs. this crate
+//!
+//! This process is driven by the caller (the saga recovery background task),
+//! with helpers provided by this crate:
+//!
+//! ```text
+//!          Saga recovery task        |      nexus-saga-recovery crate
+//!     ------------------------------------------------------------------------
+//!                                    |
+//!      1. initial `RestState` and   ---> provides `RestState`, `Report`
+//!         `Report`                   |
+//!                                    |
+//!      2. list in-progress sagas     |
+//!                                    |
+//!      3. collect list of sagas  ------> use
+//!         started by Nexus           |   `RestState::update_started_sagas()`
+//!                                    |
+//!      4. make a plan  ----------------> use `Plan::new()`
+//!                                    |   This is where all the decisions
+//!                                    |   about saga recovery get made.
+//!                                    |
+//!      5. follow the plan -------------> use `Plan::sagas_needing_recovery()`
+//!                                    |
+//!         fetch details from db      |
+//!         load sagas into Steno      |
+//!                                    |   use `ExecutionBuilder::new()` to
+//!                                    |   report what's going on
+//!                                    |
+//!      6. update `RestState` and ----->  use `RestState::update_after_pass()`
+//!         `Report`                   |   and `Report::update_after_pass()`
+//! ```
+//!
+//! We do it this way to separate all the tricky planning logic from the
+//! mechanics of loading saga state from the database and handing it over to
+//! Steno (which is simple by comparison).  This crate handles the planning and
+//! reporting.  The saga recovery task handles the database/Steno stuff.  This
+//! is an example of the ["plan-execute" pattern][1] and it makes it much easier
+//! for us to exercise all the different cases in automated tests.  It also
+//! makes it easy to keep status objects for runtime observability and
+//! debugging.  These get exposed to `omdb` and should also be visible in core
+//! files.
+//!
+//! [1]: https://mmapped.blog/posts/29-plan-execute
+
+mod recovery;
+mod status;
+
+pub use recovery::Execution;
+pub use recovery::ExecutionBuilder;
+pub use recovery::Plan;
+pub use recovery::RestState;
+pub use status::DebuggingHistory;
+pub use status::LastPass;
+pub use status::LastPassSuccess;
+pub use status::RecoveryFailure;
+pub use status::RecoverySuccess;
+pub use status::Report;
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use omicron_common::api::external::Error;
+    use omicron_test_utils::dev::test_setup_log;
+    use slog::o;
+    use std::collections::BTreeMap;
+    use std::collections::BTreeSet;
+    use steno::SagaId;
+    use tokio::sync::mpsc;
+    use uuid::Uuid;
+
+    const FAKE_SEC_ID: &str = "03082281-fb2e-4bfd-bce3-997c89a0db2d";
+    pub fn make_fake_saga(saga_id: SagaId) -> nexus_db_model::Saga {
+        let sec_id =
+            nexus_db_model::SecId::from(FAKE_SEC_ID.parse::<Uuid>().unwrap());
+        nexus_db_model::Saga::new(
+            sec_id,
+            steno::SagaCreateParams {
+                id: saga_id,
+                name: steno::SagaName::new("dummy"),
+                state: steno::SagaCachedState::Running,
+                dag: serde_json::Value::Null,
+            },
+        )
+    }
+
+    pub fn make_saga_ids(count: usize) -> Vec<SagaId> {
+        let mut rv = Vec::with_capacity(count);
+        for _ in 0..count {
+            rv.push(SagaId(Uuid::new_v4()));
+        }
+        // Essentially by coincidence, the values we're checking against
+        // are going to be sorted.  Sort this here for convenience.
+        rv.sort();
+        rv
+    }
+
+    /// Simple simulator for saga recovery state
+    ///
+    /// This type exposes functions to simulate things that would happen in
+    /// Nexus (e.g., saga started, saga finished, etc.).  It keeps track of
+    /// what little simulated database state and in-memory state is required to
+    /// exercise all the bookkeeping in this crate.
+    struct Simulator {
+        log: slog::Logger,
+        rest_state: recovery::RestState,
+        started_sagas: Vec<SagaId>,
+        db_list: BTreeMap<SagaId, nexus_db_model::Saga>,
+        snapshot_db_list: Option<BTreeMap<SagaId, nexus_db_model::Saga>>,
+        injected_recovery_errors: BTreeSet<SagaId>,
+    }
+
+    impl Simulator {
+        pub fn new(log: slog::Logger) -> Simulator {
+            Simulator {
+                log,
+                rest_state: recovery::RestState::new(),
+                started_sagas: Vec::new(),
+                db_list: BTreeMap::new(),
+                snapshot_db_list: None,
+                injected_recovery_errors: BTreeSet::new(),
+            }
+        }
+
+        /// Pretend that a particular saga was running in a previous Nexus
+        /// lifetime (and so needs to be recovered).
+        pub fn sim_previously_running_saga(&mut self) -> SagaId {
+            let saga_id = SagaId(Uuid::new_v4());
+            println!("sim: recording previously-running saga {saga_id}");
+            self.db_list.insert(saga_id, make_fake_saga(saga_id));
+            saga_id
+        }
+
+        /// Pretend that Nexus started a new saga (e.g., in response to an API
+        /// request)
+        pub fn sim_normal_saga_start(&mut self) -> SagaId {
+            let saga_id = SagaId(Uuid::new_v4());
+            println!("sim: starting saga {saga_id}");
+            self.db_list.insert(saga_id, make_fake_saga(saga_id));
+            self.started_sagas.push(saga_id);
+            saga_id
+        }
+
+        /// Pretend that Nexus finished running the given saga
+        pub fn sim_normal_saga_done(&mut self, saga_id: SagaId) {
+            println!("sim: finishing saga {saga_id}");
+            assert!(
+                self.db_list.remove(&saga_id).is_some(),
+                "simulated saga finished, but it wasn't running"
+            );
+        }
+
+        /// Configure simulation so that recovery for the specified saga will
+        /// succeed or fail, depending on `fail`.  This will affect all recovery
+        /// passes until the function is called again with a different value.
+        ///
+        /// If this function is not called for a saga, the default behavior is
+        /// that recovery succeeds.
+        pub fn sim_config_recovery_result(
+            &mut self,
+            saga_id: SagaId,
+            fail: bool,
+        ) {
+            println!(
+                "sim: configuring saga {saga_id} recovery to {}",
+                if fail { "fail" } else { "succeed" }
+            );
+            if fail {
+                self.injected_recovery_errors.insert(saga_id);
+            } else {
+                self.injected_recovery_errors.remove(&saga_id);
+            }
+        }
+
+        /// Snapshot the simulated database state and use that state for the
+        /// next recovery pass.
+        ///
+        /// As an example, this can be used to exercise both sides of the race
+        /// between Nexus starting a saga and listing in-progress sagas.  If you
+        /// want to test "listing in-progress" happens first, use this function
+        /// to snapshot the database state, then start a saga, and then do a
+        /// recovery pass.  That recovery pass will act on the snapshotted
+        /// database state.
+        ///
+        /// After the next recovery pass, the snapshotted state will be removed.
+        /// The _next_ recovery pass will use the latest database state unless
+        /// this function is called again.
+        pub fn snapshot_db(&mut self) {
+            println!("sim: snapshotting database");
+            assert!(
+                self.snapshot_db_list.is_none(),
+                "multiple snapshots created between recovery passes"
+            );
+            self.snapshot_db_list = Some(self.db_list.clone());
+        }
+
+        /// Simulate a saga recovery pass
+        pub fn sim_recovery_pass(
+            &mut self,
+        ) -> (recovery::Plan, recovery::Execution, status::LastPassSuccess, usize)
+        {
+            let log = &self.log;
+
+            println!("sim: starting recovery pass");
+
+            // Simulate processing messages that the `new_sagas_started` sagas
+            // just started.
+            let nstarted = self.started_sagas.len();
+            let (tx, mut rx) = mpsc::unbounded_channel();
+            for saga_id in self.started_sagas.drain(..) {
+                tx.send(saga_id).unwrap();
+            }
+            self.rest_state.update_started_sagas(log, &mut rx);
+
+            // Start the recovery pass by planning what to do.
+            let db_sagas = self
+                .snapshot_db_list
+                .take()
+                .unwrap_or_else(|| self.db_list.clone());
+            let plan = recovery::Plan::new(log, &self.rest_state, db_sagas);
+
+            // Simulate execution using the callback to determine whether
+            // recovery for each saga succeeds or not.
+            //
+            // There are a lot of ways we could interleave execution here.  But
+            // in practice, the implementation we care about does these all
+            // serially.  So that's what we test here.
+            let mut execution_builder = recovery::ExecutionBuilder::new();
+            let mut nok = 0;
+            let mut nerrors = 0;
+            for (saga_id, saga) in plan.sagas_needing_recovery() {
+                let saga_log = log.new(o!(
+                    "saga_name" => saga.name.clone(),
+                    "saga_id" => saga_id.to_string(),
+                ));
+
+                execution_builder.saga_recovery_start(*saga_id, saga_log);
+                if self.injected_recovery_errors.contains(saga_id) {
+                    nerrors += 1;
+                    execution_builder.saga_recovery_failure(
+                        *saga_id,
+                        &Error::internal_error("test error"),
+                    );
+                } else {
+                    nok += 1;
+                    execution_builder.saga_recovery_success(*saga_id);
+                }
+            }
+
+            let execution = execution_builder.build();
+            let last_pass = status::LastPassSuccess::new(&plan, &execution);
+            assert_eq!(last_pass.nrecovered, nok);
+            assert_eq!(last_pass.nfailed, nerrors);
+
+            self.rest_state.update_after_pass(&plan, &execution);
+
+            println!("sim: recovery pass result: {:?}", last_pass);
+
+            // We can't tell from the information we have how many were skipped,
+            // removed, or ambiguous.  The caller verifies that.
+            (plan, execution, last_pass, nstarted)
+        }
+    }
+
+    // End-to-end test of the saga recovery bookkeeping, which is basically
+    // everything *except* loading the sagas from the database and restoring
+    // them in Steno.  See the block comment above -- that stuff lives outside
+    // of this crate.
+    //
+    // Tests the following structures used together:
+    //
+    // - RestState
+    // - Plan
+    // - Execution
+    // - Report
+    //
+    // These are hard to test in isolation since they're intended to be used
+    // together in a loop (and so don't export public interfaces for mucking
+    // with internal).
+    #[tokio::test]
+    async fn test_basic() {
+        let logctx = test_setup_log("saga_recovery_basic");
+        let log = &logctx.log;
+
+        // Start with a blank slate.
+        let mut sim = Simulator::new(log.clone());
+        let initial_rest_state = sim.rest_state.clone();
+        let mut report = status::Report::new();
+
+        //
+        // Now, go through a no-op recovery.
+        //
+        let (plan, execution, last_pass_result, nstarted) =
+            sim.sim_recovery_pass();
+        assert_eq!(last_pass_result.nfound, 0);
+        assert_eq!(last_pass_result.nskipped, 0);
+        assert_eq!(last_pass_result.nremoved, 0);
+        assert_eq!(sim.rest_state, initial_rest_state);
+        report.update_after_pass(&plan, execution, nstarted);
+        assert_eq!(report.ntotal_recovered, 0);
+        assert_eq!(report.ntotal_failures, 0);
+        assert_eq!(report.ntotal_started, 0);
+        assert_eq!(report.ntotal_finished, 0);
+
+        //
+        // Now, go through a somewhat general case of recovery.
+        //
+        // First, add a couple of sagas that just showed up in the database.
+        // This covers the case of sagas that were either from a previous Nexus
+        // lifetime or re-assigned from some other Nexus that has been expunged.
+        // We create two so we can exercise success and failure cases for
+        // recovery.
+        //
+        println!("test: general recovery case");
+        let saga_recover_ok = sim.sim_previously_running_saga();
+        let saga_recover_fail = sim.sim_previously_running_saga();
+        sim.sim_config_recovery_result(saga_recover_fail, true);
+
+        // Simulate Nexus starting a couple of sagas in the usual way.  This one
+        // will appear in the database as well as in our set of sagas started.
+        let saga_started_normally_1 = sim.sim_normal_saga_start();
+        let saga_started_normally_2 = sim.sim_normal_saga_start();
+
+        // Start a saga and then finish it immediately.  This is a tricky case
+        // because the recovery pass will see that it started, but not see in
+        // the database, and it won't be able to tell if it finished or just
+        // started.
+        let saga_started_and_finished = sim.sim_normal_saga_start();
+        sim.sim_normal_saga_done(saga_started_and_finished);
+
+        // Take a snapshot.  Subsequent changes will not affect the database
+        // state that's used for the next recovery pass.  We'll use this to
+        // simulate Nexus having started a saga immediately after the database
+        // listing that's used for a recovery pass.
+        sim.snapshot_db();
+        let saga_started_after_listing = sim.sim_normal_saga_start();
+
+        // We're finally ready to carry out a simulation pass and verify what
+        // happened with each of these sagas.
+        let (plan, execution, last_pass_success, nstarted) =
+            sim.sim_recovery_pass();
+        // In the end, there should have been four sagas found in the database:
+        // all of the above except for the one that finished.
+        assert_eq!(4, last_pass_success.nfound);
+        // Two of these needed to be recovered (because they had been previously
+        // running).  One succeeded.
+        assert_eq!(1, last_pass_success.nrecovered);
+        assert_eq!(1, execution.succeeded.len());
+        assert_eq!(saga_recover_ok, execution.succeeded[0].saga_id);
+
+        assert_eq!(1, last_pass_success.nfailed);
+        assert_eq!(1, execution.failed.len());
+        assert_eq!(saga_recover_fail, execution.failed[0].saga_id);
+        // Two sagas should have been found in the database that corresponded to
+        // sagas that had been started normally and did not need to be
+        // recovered.  They would have been skipped.
+        assert_eq!(2, last_pass_success.nskipped);
+        assert_eq!(2, plan.nskipped());
+        // No sagas were removed yet -- we can't do that with only one pass.
+        assert_eq!(0, last_pass_success.nremoved);
+        assert_eq!(0, plan.ninferred_done());
+        // From what the pass could tell, two sagas might be done: the one that
+        // actually finished and the one that started after the database
+        // listing.
+        let mut maybe_done = plan.sagas_maybe_done().collect::<Vec<_>>();
+        maybe_done.sort();
+        let mut expected_maybe_done =
+            vec![saga_started_and_finished, saga_started_after_listing];
+        expected_maybe_done.sort();
+        assert_eq!(maybe_done, expected_maybe_done);
+        report.update_after_pass(&plan, execution, nstarted);
+        assert_eq!(report.ntotal_recovered, 1);
+        assert_eq!(report.ntotal_failures, 1);
+        assert_eq!(report.ntotal_started, 4);
+        assert_eq!(report.ntotal_finished, 0);
+
+        //
+        // Change nothing and run another pass.
+        // This pass allows the system to determine that some sagas are now
+        // done.
+        //
+        println!("test: recovery pass after no changes (1)");
+        let (plan, execution, last_pass_success, nstarted) =
+            sim.sim_recovery_pass();
+        // There's now five sagas in-progress in the database: the same four as
+        // above, plus the one that was started after the snapshot.
+        assert_eq!(5, last_pass_success.nfound);
+        // One of these needs to be recovered because it failed last time.  It
+        // fails again this time.
+        assert_eq!(0, last_pass_success.nrecovered);
+        assert_eq!(0, execution.succeeded.len());
+        assert_eq!(1, last_pass_success.nfailed);
+        assert_eq!(1, execution.failed.len());
+        assert_eq!(saga_recover_fail, execution.failed[0].saga_id);
+        // This time, four sagas should have been found in the database that
+        // correspond to sagas that were started normally and did not need to be
+        // recovered: the two from last time, plus the one that was recovered,
+        // plus the one that was started after the previous snapshot.  These
+        // would have been skipped.
+        assert_eq!(4, last_pass_success.nskipped);
+        assert_eq!(4, plan.nskipped());
+        // This time, the saga that was actually finished should have been
+        // removed.  We could tell this time.
+        assert_eq!(1, last_pass_success.nremoved);
+        assert_eq!(
+            vec![saga_started_and_finished],
+            plan.sagas_inferred_done().collect::<Vec<_>>()
+        );
+        // This time, there are no sagas that might be done.  The one we thought
+        // might have been done last time is now clearly running because it
+        // appears in this database listing.
+        assert_eq!(0, plan.sagas_maybe_done().count());
+        report.update_after_pass(&plan, execution, nstarted);
+        assert_eq!(report.ntotal_recovered, 1);
+        assert_eq!(report.ntotal_failures, 2);
+        assert_eq!(report.ntotal_started, 4);
+        assert_eq!(report.ntotal_finished, 1);
+
+        //
+        // Again, change nothing and run another pass.  This should be a steady
+        // state: if we keep running passes from here, nothing should change.
+        //
+        println!("test: recovery pass after no changes (2)");
+        let (plan, execution, last_pass_success, nstarted) =
+            sim.sim_recovery_pass();
+        // Same as above.
+        assert_eq!(5, last_pass_success.nfound);
+        assert_eq!(0, last_pass_success.nrecovered);
+        assert_eq!(0, execution.succeeded.len());
+        assert_eq!(1, last_pass_success.nfailed);
+        assert_eq!(1, execution.failed.len());
+        assert_eq!(saga_recover_fail, execution.failed[0].saga_id);
+        assert_eq!(4, last_pass_success.nskipped);
+        assert_eq!(4, plan.nskipped());
+        assert_eq!(0, plan.sagas_maybe_done().count());
+        // Here's the only thing that differs from last time.  We removed a saga
+        // before, so this time there's nothing to remove.
+        // removed.  We could tell this time.
+        assert_eq!(0, last_pass_success.nremoved);
+        assert_eq!(0, plan.sagas_inferred_done().count());
+        report.update_after_pass(&plan, execution, nstarted);
+        assert_eq!(report.ntotal_recovered, 1);
+        assert_eq!(report.ntotal_failures, 3);
+        assert_eq!(report.ntotal_started, 4);
+        assert_eq!(report.ntotal_finished, 1);
+
+        //
+        // Once more and make sure nothing changes.
+        //
+        println!("test: recovery pass after no changes (3)");
+        let previous_rest_state = sim.rest_state.clone();
+        let previous_last_pass_success = last_pass_success.clone();
+        let (plan, execution, last_pass_success, nstarted) =
+            sim.sim_recovery_pass();
+        assert_eq!(previous_rest_state, sim.rest_state);
+        assert_eq!(previous_last_pass_success, last_pass_success);
+        report.update_after_pass(&plan, execution, nstarted);
+        assert_eq!(report.ntotal_recovered, 1);
+        assert_eq!(report.ntotal_failures, 4);
+        assert_eq!(report.ntotal_started, 4);
+        assert_eq!(report.ntotal_finished, 1);
+
+        //
+        // This time, fix that saga whose recovery has been failing.
+        //
+        println!("test: recovery pass after removing injected error");
+        sim.sim_config_recovery_result(saga_recover_fail, false);
+        let (plan, execution, last_pass_success, nstarted) =
+            sim.sim_recovery_pass();
+        // Same as above.
+        assert_eq!(5, last_pass_success.nfound);
+        assert_eq!(4, last_pass_success.nskipped);
+        assert_eq!(4, plan.nskipped());
+        assert_eq!(0, last_pass_success.nremoved);
+        assert_eq!(0, plan.sagas_inferred_done().count());
+        assert_eq!(0, plan.sagas_maybe_done().count());
+        // Here's what's different from before.
+        assert_eq!(1, last_pass_success.nrecovered);
+        assert_eq!(1, execution.succeeded.len());
+        assert_eq!(saga_recover_fail, execution.succeeded[0].saga_id);
+        assert_eq!(0, last_pass_success.nfailed);
+        assert_eq!(0, execution.failed.len());
+        report.update_after_pass(&plan, execution, nstarted);
+        assert_eq!(report.ntotal_recovered, 2);
+        assert_eq!(report.ntotal_failures, 4);
+        assert_eq!(report.ntotal_started, 4);
+        assert_eq!(report.ntotal_finished, 1);
+
+        //
+        // After the next pass, we should have one more saga that seems to be
+        // running.
+        //
+        println!("test: recovery pass after no changes (4)");
+        let (plan, execution, last_pass_success, nstarted) =
+            sim.sim_recovery_pass();
+        // Same as above.
+        assert_eq!(5, last_pass_success.nfound);
+        assert_eq!(0, last_pass_success.nremoved);
+        assert_eq!(0, plan.sagas_inferred_done().count());
+        assert_eq!(0, plan.sagas_maybe_done().count());
+        assert_eq!(0, last_pass_success.nfailed);
+        assert_eq!(0, execution.failed.len());
+        // Here's what's different from before.
+        assert_eq!(5, last_pass_success.nskipped);
+        assert_eq!(5, plan.nskipped());
+        assert_eq!(0, last_pass_success.nrecovered);
+        assert_eq!(0, execution.succeeded.len());
+        report.update_after_pass(&plan, execution, nstarted);
+        assert_eq!(report.ntotal_recovered, 2);
+        assert_eq!(report.ntotal_failures, 4);
+        assert_eq!(report.ntotal_started, 4);
+        assert_eq!(report.ntotal_finished, 1);
+
+        //
+        // With another pass, nothing should differ.
+        //
+        println!("test: recovery pass after no changes (5)");
+        let previous_rest_state = sim.rest_state.clone();
+        let previous_last_pass_success = last_pass_success.clone();
+        let (plan, execution, last_pass_success, nstarted) =
+            sim.sim_recovery_pass();
+        assert_eq!(previous_rest_state, sim.rest_state);
+        assert_eq!(previous_last_pass_success, last_pass_success);
+        report.update_after_pass(&plan, execution, nstarted);
+        assert_eq!(report.ntotal_recovered, 2);
+        assert_eq!(report.ntotal_failures, 4);
+        assert_eq!(report.ntotal_started, 4);
+        assert_eq!(report.ntotal_finished, 1);
+
+        //
+        // Now let's complete a couple of different sagas.
+        // It'll take two passes for the system to be sure they're done.
+        //
+        println!("test: recovery pass after completing some sagas");
+        sim.sim_normal_saga_done(saga_started_normally_1);
+        sim.sim_normal_saga_done(saga_started_after_listing);
+        sim.sim_normal_saga_done(saga_recover_fail);
+        let (plan, execution, last_pass_success, nstarted) =
+            sim.sim_recovery_pass();
+        assert_eq!(2, last_pass_success.nfound);
+        assert_eq!(0, last_pass_success.nremoved);
+        assert_eq!(0, plan.sagas_inferred_done().count());
+        assert_eq!(3, plan.sagas_maybe_done().count());
+        assert_eq!(0, last_pass_success.nfailed);
+        assert_eq!(0, execution.failed.len());
+        assert_eq!(2, last_pass_success.nskipped);
+        assert_eq!(2, plan.nskipped());
+        assert_eq!(0, last_pass_success.nrecovered);
+        assert_eq!(0, execution.succeeded.len());
+        report.update_after_pass(&plan, execution, nstarted);
+        assert_eq!(report.ntotal_recovered, 2);
+        assert_eq!(report.ntotal_failures, 4);
+        assert_eq!(report.ntotal_started, 4);
+        assert_eq!(report.ntotal_finished, 1);
+
+        //
+        // With another pass, we can remove those three that finished.
+        //
+        println!("test: recovery pass after no changes (6)");
+        let (plan, execution, last_pass_success, nstarted) =
+            sim.sim_recovery_pass();
+        assert_eq!(2, last_pass_success.nfound);
+        assert_eq!(3, last_pass_success.nremoved);
+        assert_eq!(3, plan.sagas_inferred_done().count());
+        assert_eq!(0, plan.sagas_maybe_done().count());
+        assert_eq!(0, last_pass_success.nfailed);
+        assert_eq!(0, execution.failed.len());
+        assert_eq!(2, last_pass_success.nskipped);
+        assert_eq!(2, plan.nskipped());
+        assert_eq!(0, last_pass_success.nrecovered);
+        assert_eq!(0, execution.succeeded.len());
+        report.update_after_pass(&plan, execution, nstarted);
+        assert_eq!(report.ntotal_recovered, 2);
+        assert_eq!(report.ntotal_failures, 4);
+        assert_eq!(report.ntotal_started, 4);
+        assert_eq!(report.ntotal_finished, 4);
+
+        //
+        // Finish the last two sagas.
+        //
+        println!("test: recovery pass after completing remaining sagas");
+        sim.sim_normal_saga_done(saga_started_normally_2);
+        sim.sim_normal_saga_done(saga_recover_ok);
+        let (plan, execution, last_pass_success, nstarted) =
+            sim.sim_recovery_pass();
+        assert_eq!(0, last_pass_success.nfound);
+        assert_eq!(0, last_pass_success.nremoved);
+        assert_eq!(0, plan.sagas_inferred_done().count());
+        assert_eq!(2, plan.sagas_maybe_done().count());
+        assert_eq!(0, last_pass_success.nfailed);
+        assert_eq!(0, execution.failed.len());
+        assert_eq!(0, last_pass_success.nskipped);
+        assert_eq!(0, plan.nskipped());
+        assert_eq!(0, last_pass_success.nrecovered);
+        assert_eq!(0, execution.succeeded.len());
+        report.update_after_pass(&plan, execution, nstarted);
+        assert_eq!(report.ntotal_recovered, 2);
+        assert_eq!(report.ntotal_failures, 4);
+        assert_eq!(report.ntotal_started, 4);
+        assert_eq!(report.ntotal_finished, 4);
+
+        //
+        // With another pass, remove those last two.
+        //
+        println!("test: recovery pass after no changes (7)");
+        let (plan, execution, last_pass_success, nstarted) =
+            sim.sim_recovery_pass();
+        assert_eq!(0, last_pass_success.nfound);
+        assert_eq!(2, last_pass_success.nremoved);
+        assert_eq!(2, plan.sagas_inferred_done().count());
+        assert_eq!(0, plan.sagas_maybe_done().count());
+        assert_eq!(0, last_pass_success.nfailed);
+        assert_eq!(0, execution.failed.len());
+        assert_eq!(0, last_pass_success.nskipped);
+        assert_eq!(0, plan.nskipped());
+        assert_eq!(0, last_pass_success.nrecovered);
+        assert_eq!(0, execution.succeeded.len());
+        report.update_after_pass(&plan, execution, nstarted);
+        assert_eq!(report.ntotal_recovered, 2);
+        assert_eq!(report.ntotal_failures, 4);
+        assert_eq!(report.ntotal_started, 4);
+        assert_eq!(report.ntotal_finished, 6);
+
+        // At this point, the rest state should match our existing rest state.
+        // This is an extra check to make sure we're not leaking memory related
+        // to old sagas.
+        assert_eq!(sim.rest_state, initial_rest_state);
+
+        //
+        // At this point, we've exercised:
+        // - recovering a saga that we didn't start
+        //   (basic "recovery" path after a crash, plus re-assignment path)
+        // - retrying a saga whose recovery failed (multiple times)
+        // - *not* trying to recover:
+        //   - a newly-started saga
+        //   - a saga that was recovered before
+        // - not hanging on forever to sagas that have finished
+        // - the edge case built into our implementation where we learned that a
+        //   saga was started before it appeared in the database
+        //
+        logctx.cleanup_successful();
+    }
+}
diff --git a/nexus/saga-recovery/src/recovery.rs b/nexus/saga-recovery/src/recovery.rs
new file mode 100644
index 0000000000..0b13e68a49
--- /dev/null
+++ b/nexus/saga-recovery/src/recovery.rs
@@ -0,0 +1,705 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+//! Guts of the saga recovery bookkeeping
+
+use super::status::RecoveryFailure;
+use super::status::RecoverySuccess;
+use chrono::{DateTime, Utc};
+use omicron_common::api::external::Error;
+use slog::{debug, error, info, warn};
+use slog_error_chain::InlineErrorChain;
+use std::collections::BTreeMap;
+use std::collections::BTreeSet;
+use steno::SagaId;
+use tokio::sync::mpsc;
+
+/// Describes state related to saga recovery that needs to be maintained across
+/// multiple passes
+#[derive(Debug, Clone, Eq, PartialEq)]
+pub struct RestState {
+    /// set of sagas that we believe may be running
+    ///
+    /// See the big block comment in the saga recovery background task for more
+    /// on how this works and why.
+    sagas_started: BTreeMap<SagaId, SagaStartInfo>,
+    remove_next: BTreeSet<SagaId>,
+}
+
+/// Describes how we learned that a particular saga might be running
+///
+/// This is only intended for debugging.
+#[derive(Debug, Clone, Eq, PartialEq)]
+#[allow(dead_code)]
+struct SagaStartInfo {
+    time_observed: DateTime<Utc>,
+    source: SagaStartSource,
+}
+
+#[derive(Debug, Clone, Copy, Eq, PartialEq)]
+enum SagaStartSource {
+    StartChannel,
+    Recovered,
+}
+
+impl RestState {
+    /// Returns initial at-rest state related to saga recovery
+    pub fn new() -> RestState {
+        RestState {
+            sagas_started: BTreeMap::new(),
+            remove_next: BTreeSet::new(),
+        }
+    }
+
+    /// Read messages from the channel (signaling sagas that have started
+    /// running) and update the set of sagas that we believe may be running.
+    ///
+    /// See the big block comment in the saga recovery background task for more
+    /// on how this works and why.
+    pub fn update_started_sagas(
+        &mut self,
+        log: &slog::Logger,
+        sagas_started_rx: &mut mpsc::UnboundedReceiver<SagaId>,
+    ) -> usize {
+        let (new_sagas, disconnected) = read_all_from_channel(sagas_started_rx);
+        if disconnected {
+            warn!(
+                log,
+                "sagas_started_rx disconnected (is Nexus shutting down?)"
+            );
+        }
+
+        let rv = new_sagas.len();
+        let time_observed = Utc::now();
+        for saga_id in new_sagas {
+            info!(log, "observed saga start"; "saga_id" => %saga_id);
+            assert!(self
+                .sagas_started
+                .insert(
+                    saga_id,
+                    SagaStartInfo {
+                        time_observed,
+                        source: SagaStartSource::StartChannel,
+                    }
+                )
+                .is_none());
+        }
+        rv
+    }
+
+    /// Update the at-rest state based on the results of a recovery pass.
+    pub fn update_after_pass(&mut self, plan: &Plan, execution: &Execution) {
+        let time_observed = Utc::now();
+
+        for saga_id in plan.sagas_inferred_done() {
+            assert!(self.sagas_started.remove(&saga_id).is_some());
+        }
+
+        for saga_id in execution.sagas_recovered_successfully() {
+            assert!(self
+                .sagas_started
+                .insert(
+                    saga_id,
+                    SagaStartInfo {
+                        time_observed,
+                        source: SagaStartSource::Recovered,
+                    }
+                )
+                .is_none());
+        }
+
+        self.remove_next = plan.sagas_maybe_done().collect();
+    }
+}
+
+/// Read all message that are currently available on the given channel (without
+/// blocking or waiting)
+///
+/// Returns the list of messages (as a `Vec`) plus a boolean that's true iff the
+/// channel is now disconnected.
+fn read_all_from_channel<T>(
+    rx: &mut mpsc::UnboundedReceiver<T>,
+) -> (Vec<T>, bool) {
+    let mut values = Vec::new();
+    let mut disconnected = false;
+
+    loop {
+        match rx.try_recv() {
+            Ok(value) => {
+                values.push(value);
+            }
+
+            Err(mpsc::error::TryRecvError::Empty) => break,
+            Err(mpsc::error::TryRecvError::Disconnected) => {
+                disconnected = true;
+                break;
+            }
+        }
+    }
+
+    (values, disconnected)
+}
+
+/// Describes what should happen during a particular recovery pass
+///
+/// This is constructed by the saga recovery background task via
+/// [`Plan::new()`].
+///
+/// This structure is also much more detailed than it needs to be to support
+/// better observability and testing.
+pub struct Plan {
+    /// sagas that need to be recovered
+    needs_recovery: BTreeMap<SagaId, nexus_db_model::Saga>,
+
+    /// sagas that were found in the database to be in-progress, but that don't
+    /// need to be recovered because they are either already running or have
+    /// actually finished
+    skipped_running: BTreeSet<SagaId>,
+
+    /// sagas that we infer have finished because they were missing from two
+    /// consecutive database queries for in-progress sagas with no intervening
+    /// message indicating that they had been started
+    inferred_done: BTreeSet<SagaId>,
+
+    /// sagas that may be done, but we can't tell yet.  These are sagas where we
+    /// previously had them running in this process and the database state now
+    /// says that they're not running, but the database snapshot was potentially
+    /// from before the time that the saga started, so we cannot tell yet
+    /// whether the saga finished or just started.  We'll be able to tell during
+    /// the next pass and if it's done at that point then these sagas will move
+    /// to `inferred_done`.
+    maybe_done: BTreeSet<SagaId>,
+}
+
+impl Plan {
+    /// For a given saga recovery pass, determine what to do with each found
+    /// saga
+    ///
+    /// This function accepts:
+    ///
+    /// * `rest_state`: the at-rest saga recovery state from the end of the
+    ///   previous pass
+    /// * `running_sagas_found`: a list of sagas that the database reports
+    ///   to be in progress
+    ///
+    /// It determines:
+    ///
+    /// * which in-progresss sagas we don't need to do anything about because
+    ///   they're already running in this process (those sagas that are in both
+    ///   `sagas_started` and `running_sagas_found`)
+    /// * which sagas need to be recovered (those sagas in `running_sagas_found`
+    ///   but not in `sagas_started`)
+    /// * which sagas can be removed from `sagas_started` because they have
+    ///   finished (those in `previously_maybe_done` and *not* in
+    ///   `running_sagas_found`)
+    pub fn new(
+        log: &slog::Logger,
+        rest_state: &RestState,
+        mut running_sagas_found: BTreeMap<SagaId, nexus_db_model::Saga>,
+    ) -> Plan {
+        let mut builder = PlanBuilder::new(log);
+
+        // First of all, remove finished sagas from our "ignore" set.
+        //
+        // `previously_maybe_done` was computed the last time we ran and
+        // contains sagas that either just started or already finished.  We
+        // couldn't tell.  All we knew is that they were running in-memory but
+        // were not included in our database query for in-progress sagas.  At
+        // this point, though, we've done a second database query for
+        // in-progress sagas.  Any items that aren't in that list either cannot
+        // still be running, so we can safely remove them from our ignore set.
+        let previously_maybe_done = &rest_state.remove_next;
+        for saga_id in previously_maybe_done {
+            if !running_sagas_found.contains_key(saga_id) {
+                builder.saga_infer_done(*saga_id);
+            }
+        }
+
+        // Figure out which of the candidate sagas can clearly be skipped.
+        // Correctness here requires that the caller has already updated the set
+        // of sagas that we're ignoring to include any that may have been
+        // created up to the beginning of the database query.  (They do that by
+        // doing the database query first and then updating this set.)  Since we
+        // now have the list of sagas that were not-finished in the database, we
+        // can compare these two sets.
+        let sagas_started = &rest_state.sagas_started;
+        for running_saga_id in sagas_started.keys() {
+            match running_sagas_found.remove(running_saga_id) {
+                None => {
+                    // If this saga is in `previously_maybe_done`, then we
+                    // processed it above already.  We know it's done.
+                    //
+                    // Otherwise, the saga is in the ignore set, but not the
+                    // database list of running sagas.  It's possible that the
+                    // saga has simply finished.  And if the saga is definitely
+                    // not running any more, then we can remove it from the
+                    // ignore set.  This is important to keep that set from
+                    // growing without bound.
+                    //
+                    // But it's also possible that the saga started immediately
+                    // after the database query's snapshot, in which case we
+                    // don't really know if it's still running.
+                    //
+                    // The way to resolve this is to do another database query
+                    // for unfinished sagas.  If it's not in that list, the saga
+                    // must have finished.  Rather than do that now, we'll just
+                    // keep track of this list and take care of it on the next
+                    // pass.
+                    if !previously_maybe_done.contains(running_saga_id) {
+                        builder.saga_maybe_done(*running_saga_id)
+                    }
+                }
+
+                Some(_found_saga) => {
+                    // The saga is in the ignore set and the database list of
+                    // running sagas.  It may have been created in the lifetime
+                    // of this program or we may have recovered it previously,
+                    // but either way, we don't have to do anything else with
+                    // this one.
+                    builder.saga_recovery_not_needed(*running_saga_id);
+                }
+            }
+        }
+
+        // Whatever's left in `running_sagas_found` at this point was found in
+        // the database list of running sagas but is not in the ignore set.  We
+        // must recover it.  (It's not possible that we already did recover it
+        // because we would have added it to our ignore set.  It's not possible
+        // that it was newly started because the starter sends a message to add
+        // this to the ignore set (and waits for it to make it to the channel)
+        // before writing the database record, and we read everything off that
+        // channel and added it to the set before calling this function.
+        for (saga_id, saga) in running_sagas_found.into_iter() {
+            builder.saga_recovery_needed(saga_id, saga);
+        }
+
+        builder.build()
+    }
+
+    /// Iterate over the sagas that need to be recovered
+    pub fn sagas_needing_recovery(
+        &self,
+    ) -> impl Iterator<Item = (&SagaId, &nexus_db_model::Saga)> + '_ {
+        self.needs_recovery.iter()
+    }
+
+    /// Iterate over the sagas that were inferred to be done
+    pub fn sagas_inferred_done(&self) -> impl Iterator<Item = SagaId> + '_ {
+        self.inferred_done.iter().copied()
+    }
+
+    /// Iterate over the sagas that should be checked on the next pass to see if
+    /// they're done
+    pub fn sagas_maybe_done(&self) -> impl Iterator<Item = SagaId> + '_ {
+        self.maybe_done.iter().copied()
+    }
+
+    /// Returns how many in-progress sagas we ignored because they were already
+    /// running
+    pub fn nskipped(&self) -> usize {
+        self.skipped_running.len()
+    }
+
+    /// Returns how many previously-in-progress sagas we now believe are done
+    pub fn ninferred_done(&self) -> usize {
+        self.inferred_done.len()
+    }
+}
+
+/// Internal helper used to construct `Plan`
+struct PlanBuilder<'a> {
+    log: &'a slog::Logger,
+    needs_recovery: BTreeMap<SagaId, nexus_db_model::Saga>,
+    skipped_running: BTreeSet<SagaId>,
+    inferred_done: BTreeSet<SagaId>,
+    maybe_done: BTreeSet<SagaId>,
+}
+
+impl<'a> PlanBuilder<'a> {
+    /// Begin building a `Plan`
+    fn new(log: &'a slog::Logger) -> PlanBuilder {
+        PlanBuilder {
+            log,
+            needs_recovery: BTreeMap::new(),
+            skipped_running: BTreeSet::new(),
+            inferred_done: BTreeSet::new(),
+            maybe_done: BTreeSet::new(),
+        }
+    }
+
+    /// Turn this into a `Plan`
+    fn build(self) -> Plan {
+        Plan {
+            needs_recovery: self.needs_recovery,
+            skipped_running: self.skipped_running,
+            inferred_done: self.inferred_done,
+            maybe_done: self.maybe_done,
+        }
+    }
+
+    /// Record that this saga appears to be done, based on it being missing from
+    /// two different database queries for in-progress sagas with no intervening
+    /// indication that a saga with this id was started in the meantime
+    fn saga_infer_done(&mut self, saga_id: SagaId) {
+        info!(
+            self.log,
+            "found saga that appears to be done \
+             (missing from two database listings)";
+            "saga_id" => %saga_id
+        );
+        assert!(!self.needs_recovery.contains_key(&saga_id));
+        assert!(!self.skipped_running.contains(&saga_id));
+        assert!(!self.maybe_done.contains(&saga_id));
+        assert!(self.inferred_done.insert(saga_id));
+    }
+
+    /// Record that no action is needed for this saga in this recovery pass
+    /// because it appears to already be running
+    fn saga_recovery_not_needed(&mut self, saga_id: SagaId) {
+        debug!(
+            self.log,
+            "found saga that can be ignored (already running)";
+            "saga_id" => %saga_id,
+        );
+        assert!(!self.needs_recovery.contains_key(&saga_id));
+        assert!(!self.inferred_done.contains(&saga_id));
+        assert!(!self.maybe_done.contains(&saga_id));
+        assert!(self.skipped_running.insert(saga_id));
+    }
+
+    /// Record that this saga might be done, but we won't be able to tell for
+    /// sure until we complete the next recovery pass
+    ///
+    /// This sounds a little goofy but there's a race in comparing what our
+    /// in-memory state reports is running vs. what's in the database.  Our
+    /// solution is to only consider sagas done that are missing for two
+    /// consecutive database queries with no intervening report that a saga with
+    /// that id has just started.
+    fn saga_maybe_done(&mut self, saga_id: SagaId) {
+        debug!(
+            self.log,
+            "found saga that may be done (will be sure on the next pass)";
+            "saga_id" => %saga_id
+        );
+        assert!(!self.needs_recovery.contains_key(&saga_id));
+        assert!(!self.skipped_running.contains(&saga_id));
+        assert!(!self.inferred_done.contains(&saga_id));
+        assert!(self.maybe_done.insert(saga_id));
+    }
+
+    /// Record that this saga needs to be recovered, based on it being "in
+    /// progress" according to the database but not yet resumed in this process
+    fn saga_recovery_needed(
+        &mut self,
+        saga_id: SagaId,
+        saga: nexus_db_model::Saga,
+    ) {
+        info!(
+            self.log,
+            "found saga that needs to be recovered";
+            "saga_id" => %saga_id
+        );
+        assert!(!self.skipped_running.contains(&saga_id));
+        assert!(!self.inferred_done.contains(&saga_id));
+        assert!(!self.maybe_done.contains(&saga_id));
+        assert!(self.needs_recovery.insert(saga_id, saga).is_none());
+    }
+}
+
+/// Summarizes the results of executing a single saga recovery pass
+///
+/// This is constructed by the saga recovery background task (in
+/// `recovery_execute()`) via [`ExecutionBuilder::new()`].
+pub struct Execution {
+    /// list of sagas that were successfully recovered
+    pub succeeded: Vec<RecoverySuccess>,
+    /// list of sagas that failed to be recovered
+    pub failed: Vec<RecoveryFailure>,
+}
+
+impl Execution {
+    /// Iterate over the sagas that were successfully recovered during this pass
+    pub fn sagas_recovered_successfully(
+        &self,
+    ) -> impl Iterator<Item = SagaId> + '_ {
+        self.succeeded.iter().map(|s| s.saga_id)
+    }
+
+    pub fn into_results(self) -> (Vec<RecoverySuccess>, Vec<RecoveryFailure>) {
+        (self.succeeded, self.failed)
+    }
+}
+
+pub struct ExecutionBuilder {
+    in_progress: BTreeMap<SagaId, slog::Logger>,
+    succeeded: Vec<RecoverySuccess>,
+    failed: Vec<RecoveryFailure>,
+}
+
+impl ExecutionBuilder {
+    pub fn new() -> ExecutionBuilder {
+        ExecutionBuilder {
+            in_progress: BTreeMap::new(),
+            succeeded: Vec::new(),
+            failed: Vec::new(),
+        }
+    }
+
+    pub fn build(self) -> Execution {
+        assert!(
+            self.in_progress.is_empty(),
+            "attempted to build execution result while some recoveries are \
+            still in progress"
+        );
+        Execution { succeeded: self.succeeded, failed: self.failed }
+    }
+
+    /// Record that we've started recovering this saga
+    pub fn saga_recovery_start(
+        &mut self,
+        saga_id: SagaId,
+        saga_logger: slog::Logger,
+    ) {
+        info!(&saga_logger, "recovering saga: start");
+        assert!(self.in_progress.insert(saga_id, saga_logger).is_none());
+    }
+
+    /// Record that we've successfully recovered this saga
+    pub fn saga_recovery_success(&mut self, saga_id: SagaId) {
+        let saga_logger = self
+            .in_progress
+            .remove(&saga_id)
+            .expect("recovered saga should have previously started");
+        info!(saga_logger, "recovered saga");
+        self.succeeded.push(RecoverySuccess { time: Utc::now(), saga_id });
+    }
+
+    /// Record that we failed to recover this saga
+    pub fn saga_recovery_failure(&mut self, saga_id: SagaId, error: &Error) {
+        let saga_logger = self
+            .in_progress
+            .remove(&saga_id)
+            .expect("recovered saga should have previously started");
+        error!(saga_logger, "failed to recover saga"; error);
+        self.failed.push(RecoveryFailure {
+            time: Utc::now(),
+            saga_id,
+            message: InlineErrorChain::new(error).to_string(),
+        });
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use crate::status;
+    use crate::test::make_fake_saga;
+    use crate::test::make_saga_ids;
+    use omicron_test_utils::dev::test_setup_log;
+
+    #[test]
+    fn test_read_all_from_channel() {
+        let (tx, mut rx) = mpsc::unbounded_channel();
+
+        // If we send nothing on the channel, reading from it should return
+        // immediately, having found nothing.
+        let (numbers, disconnected) = read_all_from_channel::<u32>(&mut rx);
+        assert!(numbers.is_empty());
+        assert!(!disconnected);
+
+        // Send some numbers and make sure we get them back.
+        let expected_numbers = vec![1, 7, 0, 1];
+        for n in &expected_numbers {
+            tx.send(*n).unwrap();
+        }
+        let (numbers, disconnected) = read_all_from_channel(&mut rx);
+        assert_eq!(expected_numbers, numbers);
+        assert!(!disconnected);
+
+        // Send some more numbers and make sure we get them back.
+        let expected_numbers = vec![9, 7, 2, 0, 0, 6];
+        for n in &expected_numbers {
+            tx.send(*n).unwrap();
+        }
+
+        let (numbers, disconnected) = read_all_from_channel(&mut rx);
+        assert_eq!(expected_numbers, numbers);
+        assert!(!disconnected);
+
+        // Send just a few more, then disconnect the channel.
+        tx.send(128).unwrap();
+        drop(tx);
+        let (numbers, disconnected) = read_all_from_channel(&mut rx);
+        assert_eq!(vec![128], numbers);
+        assert!(disconnected);
+
+        // Also exercise the trivial case where the channel is disconnected
+        // before we read anything.
+        let (tx, mut rx) = mpsc::unbounded_channel();
+        drop(tx);
+        let (numbers, disconnected) = read_all_from_channel::<u32>(&mut rx);
+        assert!(numbers.is_empty());
+        assert!(disconnected);
+    }
+
+    /// Creates a `Plan` for testing that covers a variety of cases
+    pub struct BasicPlanTestCase {
+        pub plan: Plan,
+        pub to_recover: Vec<SagaId>,
+        pub to_skip: Vec<SagaId>,
+        pub to_mark_done: Vec<SagaId>,
+        pub to_mark_maybe: Vec<SagaId>,
+    }
+
+    impl BasicPlanTestCase {
+        pub fn new(log: &slog::Logger) -> BasicPlanTestCase {
+            let to_recover = make_saga_ids(4);
+            let to_skip = make_saga_ids(3);
+            let to_mark_done = make_saga_ids(2);
+            let to_mark_maybe = make_saga_ids(1);
+
+            info!(log, "test setup";
+                "to_recover" => ?to_recover,
+                "to_skip" => ?to_skip,
+                "to_mark_done" => ?to_mark_done,
+                "to_mark_maybe" => ?to_mark_maybe,
+            );
+
+            let mut plan_builder = PlanBuilder::new(log);
+            for saga_id in &to_recover {
+                plan_builder
+                    .saga_recovery_needed(*saga_id, make_fake_saga(*saga_id));
+            }
+            for saga_id in &to_skip {
+                plan_builder.saga_recovery_not_needed(*saga_id);
+            }
+            for saga_id in &to_mark_done {
+                plan_builder.saga_infer_done(*saga_id);
+            }
+            for saga_id in &to_mark_maybe {
+                plan_builder.saga_maybe_done(*saga_id);
+            }
+            let plan = plan_builder.build();
+
+            BasicPlanTestCase {
+                plan,
+                to_recover,
+                to_skip,
+                to_mark_done,
+                to_mark_maybe,
+            }
+        }
+    }
+
+    #[test]
+    fn test_plan_basic() {
+        let logctx = test_setup_log("saga_recovery_plan_basic");
+
+        // Trivial initial case
+        let plan_builder = PlanBuilder::new(&logctx.log);
+        let plan = plan_builder.build();
+        assert_eq!(0, plan.sagas_needing_recovery().count());
+        assert_eq!(0, plan.sagas_inferred_done().count());
+        assert_eq!(0, plan.sagas_maybe_done().count());
+
+        // Basic case
+        let BasicPlanTestCase {
+            plan,
+            to_recover,
+            to_skip: _,
+            to_mark_done,
+            to_mark_maybe,
+        } = BasicPlanTestCase::new(&logctx.log);
+
+        let found_to_recover =
+            plan.sagas_needing_recovery().collect::<Vec<_>>();
+        assert_eq!(to_recover.len(), found_to_recover.len());
+        for (expected_saga_id, (found_saga_id, found_saga_record)) in
+            to_recover.into_iter().zip(found_to_recover.into_iter())
+        {
+            assert_eq!(expected_saga_id, *found_saga_id);
+            assert_eq!(expected_saga_id, found_saga_record.id.0);
+            assert_eq!("dummy", found_saga_record.name);
+        }
+        assert_eq!(
+            to_mark_done,
+            plan.sagas_inferred_done().collect::<Vec<_>>(),
+        );
+        assert_eq!(to_mark_maybe, plan.sagas_maybe_done().collect::<Vec<_>>(),);
+
+        logctx.cleanup_successful();
+    }
+
+    #[tokio::test]
+    async fn test_execution_basic() {
+        let logctx = test_setup_log("saga_recovery_execution_basic");
+
+        // Trivial initial case
+        let plan_builder = PlanBuilder::new(&logctx.log);
+        let plan = plan_builder.build();
+        let execution_builder = ExecutionBuilder::new();
+        let execution = execution_builder.build();
+
+        assert_eq!(0, execution.sagas_recovered_successfully().count());
+        let last_pass = status::LastPassSuccess::new(&plan, &execution);
+        assert_eq!(0, last_pass.nfound);
+        assert_eq!(0, last_pass.nrecovered);
+        assert_eq!(0, last_pass.nfailed);
+        assert_eq!(0, last_pass.nskipped);
+        assert_eq!(0, last_pass.nremoved);
+
+        // Test a non-trivial ExecutionDone
+        let BasicPlanTestCase {
+            plan,
+            mut to_recover,
+            to_skip,
+            to_mark_done,
+            to_mark_maybe: _,
+        } = BasicPlanTestCase::new(&logctx.log);
+        let mut execution_builder = ExecutionBuilder::new();
+        assert!(to_recover.len() >= 3, "someone changed the test case");
+
+        // Start recovery backwards, just to make sure there's not some implicit
+        // dependency on the order.  (We could shuffle, but then the test would
+        // be non-deterministic.)
+        for saga_id in to_recover.iter().rev() {
+            execution_builder.saga_recovery_start(*saga_id, logctx.log.clone());
+        }
+
+        // "Finish" recovery, in yet a different order (for the same reason as
+        // above).  We want to test the success and failure cases.
+        //
+        // Act like:
+        // - recovery for the last saga failed
+        // - recovery for the other sagas completes successfully
+        to_recover.rotate_left(2);
+        for (i, saga_id) in to_recover.iter().enumerate() {
+            if i == to_recover.len() - 1 {
+                execution_builder.saga_recovery_failure(
+                    *saga_id,
+                    &Error::internal_error("test error"),
+                );
+            } else {
+                execution_builder.saga_recovery_success(*saga_id);
+            }
+        }
+
+        let execution = execution_builder.build();
+        assert_eq!(
+            to_recover.len() - 1,
+            execution.sagas_recovered_successfully().count()
+        );
+        let last_pass = status::LastPassSuccess::new(&plan, &execution);
+        assert_eq!(to_recover.len() + to_skip.len(), last_pass.nfound);
+        assert_eq!(to_recover.len() - 1, last_pass.nrecovered);
+        assert_eq!(1, last_pass.nfailed);
+        assert_eq!(to_skip.len(), last_pass.nskipped);
+        assert_eq!(to_mark_done.len(), last_pass.nremoved);
+
+        logctx.cleanup_successful();
+    }
+
+    // More interesting tests are done at the crate level because they include
+    // stuff from the `status` module, too.
+}
diff --git a/nexus/saga-recovery/src/status.rs b/nexus/saga-recovery/src/status.rs
new file mode 100644
index 0000000000..d9b0ce242d
--- /dev/null
+++ b/nexus/saga-recovery/src/status.rs
@@ -0,0 +1,175 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+//! Report status for the saga recovery background task
+
+use super::recovery;
+use chrono::{DateTime, Utc};
+use omicron_common::api::external::Error;
+use serde::{Deserialize, Serialize};
+use slog_error_chain::InlineErrorChain;
+use std::collections::VecDeque;
+use steno::SagaId;
+
+// These values are chosen to be large enough to likely cover the complete
+// history of saga recoveries, successful and otherwise.  They just need to be
+// finite so that this system doesn't use an unbounded amount of memory.
+/// Maximum number of successful recoveries to keep track of for debugging
+const N_SUCCESS_SAGA_HISTORY: usize = 128;
+/// Maximum number of recent failures to keep track of for debugging
+const N_FAILED_SAGA_HISTORY: usize = 128;
+
+/// Summarizes the status of saga recovery for debugging
+#[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize)]
+pub struct Report {
+    pub recent_recoveries: DebuggingHistory<RecoverySuccess>,
+    pub recent_failures: DebuggingHistory<RecoveryFailure>,
+    pub last_pass: LastPass,
+
+    pub ntotal_recovered: usize,
+    pub ntotal_failures: usize,
+    pub ntotal_started: usize,
+    pub ntotal_finished: usize,
+    pub ntotal_sec_errors_missing: usize,
+    pub ntotal_sec_errors_bad_state: usize,
+}
+
+impl Report {
+    pub fn new() -> Report {
+        Report {
+            recent_recoveries: DebuggingHistory::new(N_SUCCESS_SAGA_HISTORY),
+            recent_failures: DebuggingHistory::new(N_FAILED_SAGA_HISTORY),
+            last_pass: LastPass::NeverStarted,
+            ntotal_recovered: 0,
+            ntotal_failures: 0,
+            ntotal_started: 0,
+            ntotal_finished: 0,
+            ntotal_sec_errors_missing: 0,
+            ntotal_sec_errors_bad_state: 0,
+        }
+    }
+
+    /// Update the report after a single saga recovery pass where we at least
+    /// successfully constructed a plan
+    pub fn update_after_pass(
+        &mut self,
+        plan: &recovery::Plan,
+        execution: recovery::Execution,
+        nstarted: usize,
+    ) {
+        self.last_pass =
+            LastPass::Success(LastPassSuccess::new(plan, &execution));
+
+        let (succeeded, failed) = execution.into_results();
+
+        for success in succeeded {
+            self.recent_recoveries.append(success);
+            self.ntotal_recovered += 1;
+        }
+
+        for failure in failed {
+            self.recent_failures.append(failure);
+            self.ntotal_failures += 1;
+        }
+
+        self.ntotal_started += nstarted;
+        self.ntotal_finished += plan.ninferred_done();
+    }
+
+    /// Update the report after a saga recovery pass where we couldn't even
+    /// construct a plan (usually because we couldn't load state from the
+    /// database)
+    pub fn update_after_failure(&mut self, error: &Error, nstarted: usize) {
+        self.ntotal_started += nstarted;
+        self.last_pass = LastPass::Failed {
+            message: InlineErrorChain::new(error).to_string(),
+        };
+    }
+}
+
+#[derive(Clone, Debug, Eq, PartialEq, Deserialize, Serialize)]
+pub struct RecoverySuccess {
+    pub time: DateTime<Utc>,
+    pub saga_id: SagaId,
+}
+
+#[derive(Clone, Debug, Eq, PartialEq, Deserialize, Serialize)]
+pub struct RecoveryFailure {
+    pub time: DateTime<Utc>,
+    pub saga_id: SagaId,
+    pub message: String,
+}
+
+/// Describes what happened during the last saga recovery pass
+#[derive(Clone, Debug, Eq, PartialEq, Deserialize, Serialize)]
+pub enum LastPass {
+    /// There has not been a saga recovery pass yet
+    NeverStarted,
+    /// This pass failed to even construct a plan (usually because we couldn't
+    /// load state from the database)
+    Failed { message: String },
+    /// This pass was at least partially successful
+    Success(LastPassSuccess),
+}
+
+/// Describes what happened during a saga recovery pass where we at least
+/// managed to construct a plan
+#[derive(Clone, Debug, Eq, PartialEq, Deserialize, Serialize)]
+pub struct LastPassSuccess {
+    pub nfound: usize,
+    pub nrecovered: usize,
+    pub nfailed: usize,
+    pub nskipped: usize,
+    pub nremoved: usize,
+}
+
+impl LastPassSuccess {
+    pub fn new(
+        plan: &recovery::Plan,
+        execution: &recovery::Execution,
+    ) -> LastPassSuccess {
+        let nfound = plan.sagas_needing_recovery().count() + plan.nskipped();
+        LastPassSuccess {
+            nfound,
+            nrecovered: execution.succeeded.len(),
+            nfailed: execution.failed.len(),
+            nskipped: plan.nskipped(),
+            nremoved: plan.ninferred_done(),
+        }
+    }
+}
+
+/// Debugging ringbuffer, storing arbitrary objects of type `T`
+// There surely exist faster and richer implementations.  At least this one's
+// pretty simple.
+#[derive(Clone, Debug, Eq, PartialEq, Deserialize, Serialize)]
+#[serde(transparent)]
+pub struct DebuggingHistory<T> {
+    ring: VecDeque<T>,
+}
+
+impl<T> DebuggingHistory<T> {
+    pub fn new(size: usize) -> DebuggingHistory<T> {
+        DebuggingHistory { ring: VecDeque::with_capacity(size) }
+    }
+
+    pub fn append(&mut self, t: T) {
+        if self.ring.len() == self.ring.capacity() {
+            let _ = self.ring.pop_front();
+        }
+        self.ring.push_back(t);
+    }
+
+    pub fn len(&self) -> usize {
+        self.ring.len()
+    }
+
+    pub fn is_empty(&self) -> bool {
+        self.ring.is_empty()
+    }
+
+    pub fn iter(&self) -> impl Iterator<Item = &T> {
+        self.ring.iter()
+    }
+}
diff --git a/nexus/src/app/background/driver.rs b/nexus/src/app/background/driver.rs
index c93729a335..be09ccb21f 100644
--- a/nexus/src/app/background/driver.rs
+++ b/nexus/src/app/background/driver.rs
@@ -382,6 +382,9 @@ impl TaskExec {
 
         // Do it!
         let details = self.imp.activate(&self.opctx).await;
+        let details_str = serde_json::to_string(&details).unwrap_or_else(|e| {
+            format!("<<failed to serialize task status: {}>>", e)
+        });
 
         let elapsed = start_instant.elapsed();
 
@@ -407,6 +410,7 @@ impl TaskExec {
             "activation complete";
             "elapsed" => ?elapsed,
             "iteration" => iteration,
+            "status" => details_str,
         );
     }
 }
diff --git a/nexus/src/app/background/init.rs b/nexus/src/app/background/init.rs
index 5f420773e0..4a5d792c80 100644
--- a/nexus/src/app/background/init.rs
+++ b/nexus/src/app/background/init.rs
@@ -106,6 +106,7 @@ use super::tasks::phantom_disks;
 use super::tasks::physical_disk_adoption;
 use super::tasks::region_replacement;
 use super::tasks::region_replacement_driver;
+use super::tasks::saga_recovery;
 use super::tasks::service_firewall_rules;
 use super::tasks::sync_service_zone_nat::ServiceZoneNatTracker;
 use super::tasks::sync_switch_configuration::SwitchPortSettingsManager;
@@ -115,6 +116,7 @@ use super::Activator;
 use super::Driver;
 use crate::app::oximeter::PRODUCER_LEASE_DURATION;
 use crate::app::saga::StartSaga;
+use crate::Nexus;
 use nexus_config::BackgroundTaskConfig;
 use nexus_config::DnsTasksConfig;
 use nexus_db_model::DnsGroup;
@@ -153,6 +155,7 @@ pub struct BackgroundTasks {
     pub task_service_firewall_propagation: Activator,
     pub task_abandoned_vmm_reaper: Activator,
     pub task_vpc_route_manager: Activator,
+    pub task_saga_recovery: Activator,
     pub task_lookup_region_port: Activator,
 
     // Handles to activate background tasks that do not get used by Nexus
@@ -231,6 +234,7 @@ impl BackgroundTasksInitializer {
             task_service_firewall_propagation: Activator::new(),
             task_abandoned_vmm_reaper: Activator::new(),
             task_vpc_route_manager: Activator::new(),
+            task_saga_recovery: Activator::new(),
             task_lookup_region_port: Activator::new(),
 
             task_internal_dns_propagation: Activator::new(),
@@ -246,22 +250,20 @@ impl BackgroundTasksInitializer {
     ///
     /// This function will wire up the `Activator`s in `background_tasks` to the
     /// corresponding tasks once they've been started.
-    #[allow(clippy::too_many_arguments)]
     pub fn start(
         self,
         background_tasks: &'_ BackgroundTasks,
-        opctx: OpContext,
-        datastore: Arc<DataStore>,
-        config: BackgroundTaskConfig,
-        rack_id: Uuid,
-        nexus_id: Uuid,
-        resolver: internal_dns::resolver::Resolver,
-        sagas: Arc<dyn StartSaga>,
-        producer_registry: ProducerRegistry,
+        args: BackgroundTasksData,
     ) -> Driver {
         let mut driver = self.driver;
-        let opctx = &opctx;
-        let producer_registry = &producer_registry;
+        let opctx = &args.opctx;
+        let datastore = args.datastore;
+        let config = args.config;
+        let rack_id = args.rack_id;
+        let nexus_id = args.nexus_id;
+        let resolver = args.resolver;
+        let sagas = args.saga_starter;
+        let producer_registry = &args.producer_registry;
 
         // This "let" construction helps catch mistakes where someone forgets to
         // wire up an activator to its corresponding background task.
@@ -291,6 +293,7 @@ impl BackgroundTasksInitializer {
             task_service_firewall_propagation,
             task_abandoned_vmm_reaper,
             task_vpc_route_manager,
+            task_saga_recovery,
             task_lookup_region_port,
             // Add new background tasks here.  Be sure to use this binding in a
             // call to `Driver::register()` below.  That's what actually wires
@@ -651,6 +654,25 @@ impl BackgroundTasksInitializer {
             activator: task_abandoned_vmm_reaper,
         });
 
+        // Background task: saga recovery
+        {
+            let task_impl = Box::new(saga_recovery::SagaRecovery::new(
+                datastore.clone(),
+                nexus_db_model::SecId(args.nexus_id),
+                args.saga_recovery,
+            ));
+
+            driver.register(TaskDefinition {
+                name: "saga_recovery",
+                description: "recovers sagas assigned to this Nexus",
+                period: config.saga_recovery.period_secs,
+                task_impl,
+                opctx: opctx.child(BTreeMap::new()),
+                watchers: vec![],
+                activator: task_saga_recovery,
+            });
+        }
+
         driver.register(TaskDefinition {
             name: "lookup_region_port",
             description: "fill in missing ports for region records",
@@ -667,6 +689,28 @@ impl BackgroundTasksInitializer {
     }
 }
 
+pub struct BackgroundTasksData {
+    /// root `OpContext` used for background tasks
+    pub opctx: OpContext,
+    /// handle to `DataStore`, provided directly to many background tasks
+    pub datastore: Arc<DataStore>,
+    /// background task configuration
+    pub config: BackgroundTaskConfig,
+    /// rack identifier
+    pub rack_id: Uuid,
+    /// nexus identifier
+    pub nexus_id: Uuid,
+    /// internal DNS DNS resolver, used when tasks need to contact other
+    /// internal services
+    pub resolver: internal_dns::resolver::Resolver,
+    /// handle to saga subsystem for starting sagas
+    pub saga_starter: Arc<dyn StartSaga>,
+    /// Oximeter producer registry (for metrics)
+    pub producer_registry: ProducerRegistry,
+    /// Helpers for saga recovery
+    pub saga_recovery: saga_recovery::SagaRecoveryHelpers<Arc<Nexus>>,
+}
+
 /// Starts the three DNS-propagation-related background tasks for either
 /// internal or external DNS (depending on the arguments)
 #[allow(clippy::too_many_arguments)]
@@ -884,6 +928,7 @@ pub mod test {
                 bind_address: "[::1]:0".parse().unwrap(),
                 request_body_max_bytes: 8 * 1024,
                 default_handler_task_mode: HandlerTaskMode::Detached,
+                log_headers: vec![],
             },
         )
         .await
diff --git a/nexus/src/app/background/mod.rs b/nexus/src/app/background/mod.rs
index 1bd7a323c3..5b24907b0f 100644
--- a/nexus/src/app/background/mod.rs
+++ b/nexus/src/app/background/mod.rs
@@ -136,7 +136,9 @@ mod tasks;
 pub use driver::Activator;
 pub use driver::Driver;
 pub use init::BackgroundTasks;
+pub use init::BackgroundTasksData;
 pub use init::BackgroundTasksInitializer;
+pub use tasks::saga_recovery::SagaRecoveryHelpers;
 
 use futures::future::BoxFuture;
 use nexus_auth::context::OpContext;
diff --git a/nexus/src/app/background/tasks/lookup_region_port.rs b/nexus/src/app/background/tasks/lookup_region_port.rs
index b0f13ac986..fbfc5c5af2 100644
--- a/nexus/src/app/background/tasks/lookup_region_port.rs
+++ b/nexus/src/app/background/tasks/lookup_region_port.rs
@@ -91,26 +91,33 @@ impl BackgroundTask for LookupRegionPort {
                     }
                 };
 
-                let returned_region = match get_region_from_agent(
-                    &dataset.address(),
-                    region.id(),
-                )
-                .await
-                {
-                    Ok(returned_region) => returned_region,
+                let Some(dataset_addr) = dataset.address() else {
+                    let s = format!(
+                        "Missing dataset address for dataset: {dataset_id}"
+                    );
+                    error!(log, "{s}");
+                    status.errors.push(s);
+                    continue;
+                };
 
-                    Err(e) => {
-                        let s = format!(
-                            "could not get region {} from agent: {e}",
-                            region.id(),
-                        );
+                let returned_region =
+                    match get_region_from_agent(&dataset_addr, region.id())
+                        .await
+                    {
+                        Ok(returned_region) => returned_region,
 
-                        error!(log, "{s}");
-                        status.errors.push(s);
+                        Err(e) => {
+                            let s = format!(
+                                "could not get region {} from agent: {e}",
+                                region.id(),
+                            );
 
-                        continue;
-                    }
-                };
+                            error!(log, "{s}");
+                            status.errors.push(s);
+
+                            continue;
+                        }
+                    };
 
                 match self
                     .datastore
diff --git a/nexus/src/app/background/tasks/mod.rs b/nexus/src/app/background/tasks/mod.rs
index 5eb44ed7c3..a5204588d8 100644
--- a/nexus/src/app/background/tasks/mod.rs
+++ b/nexus/src/app/background/tasks/mod.rs
@@ -23,6 +23,7 @@ pub mod phantom_disks;
 pub mod physical_disk_adoption;
 pub mod region_replacement;
 pub mod region_replacement_driver;
+pub mod saga_recovery;
 pub mod service_firewall_rules;
 pub mod sync_service_zone_nat;
 pub mod sync_switch_configuration;
diff --git a/nexus/src/app/background/tasks/saga_recovery.rs b/nexus/src/app/background/tasks/saga_recovery.rs
new file mode 100644
index 0000000000..7b0fe1b331
--- /dev/null
+++ b/nexus/src/app/background/tasks/saga_recovery.rs
@@ -0,0 +1,927 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+//! Saga recovery
+//!
+//! ## Review of distributed sagas
+//!
+//! Nexus uses distributed sagas via [`steno`] to manage multi-step operations
+//! that have their own unwinding or cleanup steps.  While executing sagas,
+//! critical state is durably stored in the **saga log** such that after a
+//! crash, the saga can be resumed while maintaining certain guarantees:
+//!
+//! - During normal execution, each **action** will be executed at least once.
+//! - If an action B depends on action A, then once B has started, A will not
+//!   run again.
+//! - Once any action has failed, the saga is **unwound**, meaning that the undo
+//!   actions are executed for any action that has successfully completed.
+//! - The saga will not come to rest until one of these three things has
+//!   happened:
+//!   1. All actions complete successfully.  This is the normal case of saga
+//!      completion.
+//!   2. Any number of actions complete successfully, at least one action
+//!      failed, and the undo actions complete successfully for any actions that
+//!      *did* run.  This is the normal case of clean saga failure where
+//!      intuitively the state of the world is unwound to match whatever it was
+//!      before the saga ran.
+//!   3. Any number of actions complete successfully, at least one action
+//!      failed, and at least one undo action also failed.  This is a nebulous
+//!      "stuck" state where the world may be partially changed by the saga.
+//!
+//! There's more to all this (see the Steno docs), but the important thing here
+//! is that the persistent state is critical for ensuring these properties
+//! across a Nexus crash.  The process of resuming in-progress sagas after a
+//! crash is called **saga recovery**.  Fortunately, Steno handles the details
+//! of those constraints.  All we have to do is provide Steno with the
+//! persistent state of any sagas that it needs to resume.
+//!
+//!
+//! ## Saga recovery and persistent state
+//!
+//! Everything needed to recover a saga is stored in:
+//!
+//! 1. a **saga** record, which is mostly immutable
+//! 2. the **saga log**, an append-only description of exactly what happened
+//!    during execution
+//!
+//! Responsibility for persisting this state is divided across Steno and Nexus:
+//!
+//! 1. Steno tells its consumer (Nexus) precisely what information needs to be
+//!    stored and when.  It does this by invoking methods on the `SecStore`
+//!    trait at key points in the saga's execution.  Steno does not care how
+//!    this information is stored or where it is stored.
+//!
+//! 2. Nexus serializes the given state and stores it into CockroachDB using
+//!    the `saga` and `saga_node_event` tables.
+//!
+//! After a crash, Nexus is then responsible for:
+//!
+//! 1. Identifying what sagas were in progress before the crash,
+//! 2. Loading all the information about them from the database (namely, the
+//!    `saga` record and the full saga log in the form of records from the
+//!    `saga_node_event` table), and
+//! 3. Providing all this information to Steno so that it can resume running the
+//!    saga.
+//!
+//!
+//! ## Saga recovery: not just at startup
+//!
+//! So far, this is fairly straightforward.  What makes it tricky is that there
+//! are situations where we want to carry out saga recovery after Nexus has
+//! already started and potentially recovered other sagas and started its own
+//! sagas.  Specifically, when a Nexus instance gets **expunged** (removed
+//! permanently), it may have outstanding sagas that need to be re-assigned to
+//! another Nexus instance, which then needs to resume them.  To do this, we run
+//! saga recovery in a Nexus background task so that it runs both periodically
+//! and on-demand when activated.  (This approach is also useful for other
+//! reasons, like retrying recovery for sagas whose recovery failed due to
+//! transient errors.)
+//!
+//! Why does this make things tricky?  When Nexus goes to identify what sagas
+//! it needs to recover, it lists sagas that are (1) assigned to it (as opposed
+//! to a different Nexus) and (2) not yet finished.  But that could include
+//! sagas in one of three groups:
+//!
+//! 1. Sagas from a previous Nexus lifetime (i.e., a different Unix process)
+//!    that have not yet been recovered in this lifetime.  These **should** be
+//!    recovered.
+//! 2. Sagas from a previous Nexus lifetime (i.e., a different Unix process)
+//!    that have already been recovered in this lifetime.  These **should not**
+//!    be recovered.
+//! 3. Sagas that were created in this Nexus lifetime.  These **should not** be
+//!    recovered.
+//!
+//! There are a bunch of ways to attack this problem.  We do it by keeping track
+//! in-memory of the set of sagas that might be running in the current process
+//! and then ignoring those when we do recovery.  Even this is easier said than
+//! done!  It's easy enough to insert new sagas into the set whenever a saga is
+//! successfully recovered as well as any time a saga is created for the first
+//! time (though that requires a structure that's modifiable from multiple
+//! different contexts).  But to avoid this set growing unbounded, we should
+//! remove entries when a saga finishes running.  When exactly can we do that?
+//! We have to be careful of the intrinsic race between when the recovery
+//! process queries the database to list candidate sagas for recovery (i.e.,
+//! unfinished sagas assigned to this Nexus) and when it checks the set of sagas
+//! that should be ignored.  Suppose a saga is running, the recovery process
+//! finds it, then the saga finishes, it gets removed from the set, and then the
+//! recovery process checks the set.  We'll think it wasn't running and start it
+//! again -- very bad.  We can't remove anything from the set until we know that
+//! the saga recovery task _doesn't_ have a stale list of candidate sagas to be
+//! recovered.
+//!
+//! This constraint suggests the solution: the set will be owned and managed
+//! entirely by the task that's doing saga recovery.  We'll use a channel to
+//! trigger inserts when sagas are created elsewhere in Nexus.  What about
+//! deletes?  The recovery process can actually figure out on its own when a
+//! saga can be removed: if a saga that was previously in the list of candidates
+//! to be recovered and is now no longer in that list, then that means it's
+//! finished, and that means it can be deleted from the set.  Care must be taken
+//! to process things in the right order.  These details are mostly handled by
+//! the separate [`nexus_saga_recovery`] crate.
+
+use crate::app::background::BackgroundTask;
+use crate::app::sagas::NexusSagaType;
+use crate::saga_interface::SagaContext;
+use crate::Nexus;
+use futures::future::BoxFuture;
+use futures::FutureExt;
+use nexus_db_queries::context::OpContext;
+use nexus_db_queries::db;
+use nexus_db_queries::db::DataStore;
+use omicron_common::api::external::Error;
+use omicron_common::api::external::InternalContext;
+use std::collections::BTreeMap;
+use std::sync::Arc;
+use steno::SagaId;
+use steno::SagaStateView;
+use tokio::sync::mpsc;
+
+/// Helpers used for saga recovery
+pub struct SagaRecoveryHelpers<N: MakeSagaContext> {
+    pub recovery_opctx: OpContext,
+    pub maker: N,
+    pub sec_client: Arc<steno::SecClient>,
+    pub registry: Arc<steno::ActionRegistry<N::SagaType>>,
+    pub sagas_started_rx: mpsc::UnboundedReceiver<SagaId>,
+}
+
+/// Background task that recovers sagas assigned to this Nexus
+///
+/// Normally, this task only does anything of note once, when Nexus starts up.
+/// But it runs periodically and can be activated explicitly for the rare case
+/// when a saga has been re-assigned to this Nexus (e.g., because some other
+/// Nexus has been expunged) and to handle retries for sagas whose previous
+/// recovery failed.
+pub struct SagaRecovery<N: MakeSagaContext> {
+    datastore: Arc<DataStore>,
+    /// Unique identifier for this Saga Execution Coordinator
+    ///
+    /// This always matches the Nexus id.
+    sec_id: db::SecId,
+    /// OpContext used for saga recovery
+    saga_recovery_opctx: OpContext,
+
+    // state required to resume a saga
+    /// handle to Steno, which actually resumes the saga
+    sec_client: Arc<steno::SecClient>,
+    /// generates the SagaContext for the saga
+    maker: N,
+    /// registry of actions that we need to provide to Steno
+    registry: Arc<steno::ActionRegistry<N::SagaType>>,
+
+    // state that we use during each recovery pass
+    /// channel on which we listen for sagas being started elsewhere in Nexus
+    sagas_started_rx: mpsc::UnboundedReceiver<SagaId>,
+    /// recovery state persisted between passes
+    rest_state: nexus_saga_recovery::RestState,
+
+    /// status reporting
+    status: nexus_saga_recovery::Report,
+}
+
+impl<N: MakeSagaContext> BackgroundTask for SagaRecovery<N> {
+    fn activate<'a>(
+        &'a mut self,
+        opctx: &'a OpContext,
+    ) -> BoxFuture<'a, serde_json::Value> {
+        async {
+            // We don't need the future that's returned by activate_internal().
+            // That's only used by the test suite.
+            let _ = self.activate_internal(opctx).await;
+            serde_json::to_value(&self.status).unwrap()
+        }
+        .boxed()
+    }
+}
+
+impl<N: MakeSagaContext> SagaRecovery<N> {
+    pub fn new(
+        datastore: Arc<DataStore>,
+        sec_id: db::SecId,
+        helpers: SagaRecoveryHelpers<N>,
+    ) -> SagaRecovery<N> {
+        SagaRecovery {
+            datastore,
+            sec_id,
+            saga_recovery_opctx: helpers.recovery_opctx,
+            maker: helpers.maker,
+            sec_client: helpers.sec_client,
+            registry: helpers.registry,
+            sagas_started_rx: helpers.sagas_started_rx,
+            rest_state: nexus_saga_recovery::RestState::new(),
+            status: nexus_saga_recovery::Report::new(),
+        }
+    }
+
+    /// Invoked for each activation of the background task
+    ///
+    /// This internal version exists solely to expose some information about
+    /// what was recovered for testing.
+    async fn activate_internal(
+        &mut self,
+        opctx: &OpContext,
+    ) -> Option<(
+        BoxFuture<'static, Result<(), Error>>,
+        nexus_saga_recovery::LastPassSuccess,
+    )> {
+        let log = &opctx.log;
+        let datastore = &self.datastore;
+
+        // Fetch the list of not-yet-finished sagas that are assigned to
+        // this Nexus instance.
+        let result = list_sagas_in_progress(
+            &self.saga_recovery_opctx,
+            datastore,
+            self.sec_id,
+        )
+        .await;
+
+        // Process any newly-created sagas, adding them to our set of sagas
+        // to ignore during recovery.  We never want to try to recover a
+        // saga that was created within this Nexus's lifetime.
+        //
+        // We do this even if the previous step failed in order to avoid
+        // letting the channel queue build up.  In practice, it shouldn't
+        // really matter.
+        //
+        // But given that we're doing this, it's critical that we do it
+        // *after* having fetched the candidate sagas from the database.
+        // It's okay if one of these newly-created sagas doesn't show up in
+        // the candidate list (because it hadn't actually started at the
+        // point where we fetched the candidate list).  The reverse is not
+        // okay: if we did this step before fetching candidates, and a saga
+        // was immediately created and showed up in our candidate list, we'd
+        // erroneously conclude that it needed to be recovered when in fact
+        // it was already running.
+        let nstarted = self
+            .rest_state
+            .update_started_sagas(log, &mut self.sagas_started_rx);
+
+        match result {
+            Ok(db_sagas) => {
+                let plan = nexus_saga_recovery::Plan::new(
+                    log,
+                    &self.rest_state,
+                    db_sagas,
+                );
+                self.recovery_check_done(log, &plan).await;
+                let (execution, future) =
+                    self.recovery_execute(log, &plan).await;
+                self.rest_state.update_after_pass(&plan, &execution);
+                let last_pass_success =
+                    nexus_saga_recovery::LastPassSuccess::new(
+                        &plan, &execution,
+                    );
+                self.status.update_after_pass(&plan, execution, nstarted);
+                Some((future, last_pass_success))
+            }
+            Err(error) => {
+                self.status.update_after_failure(&error, nstarted);
+                None
+            }
+        }
+    }
+
+    /// Check that for each saga that we inferred was done, Steno agrees
+    ///
+    /// This is not strictly necessary because this should always be true.  But
+    /// if for some reason it's not, that would be a serious issue and we'd want
+    /// to know that.
+    async fn recovery_check_done(
+        &mut self,
+        log: &slog::Logger,
+        plan: &nexus_saga_recovery::Plan,
+    ) {
+        for saga_id in plan.sagas_inferred_done() {
+            match self.sec_client.saga_get(saga_id).await {
+                Err(_) => {
+                    self.status.ntotal_sec_errors_missing += 1;
+                    error!(
+                        log,
+                        "SEC does not know about saga that we thought \
+                        had finished";
+                        "saga_id" => %saga_id
+                    );
+                }
+                Ok(saga_state) => match saga_state.state {
+                    SagaStateView::Done { .. } => (),
+                    _ => {
+                        self.status.ntotal_sec_errors_bad_state += 1;
+                        error!(
+                            log,
+                            "we thought saga was done, but SEC reports a \
+                            different state";
+                            "saga_id" => %saga_id,
+                            "sec_state" => ?saga_state.state
+                        );
+                    }
+                },
+            }
+        }
+    }
+
+    /// Recovers the sagas described in `plan`
+    async fn recovery_execute(
+        &self,
+        bgtask_log: &slog::Logger,
+        plan: &nexus_saga_recovery::Plan,
+    ) -> (nexus_saga_recovery::Execution, BoxFuture<'static, Result<(), Error>>)
+    {
+        let mut builder = nexus_saga_recovery::ExecutionBuilder::new();
+        let mut completion_futures = Vec::new();
+
+        // Load and resume all these sagas serially.  Too much parallelism here
+        // could overload the database.  It wouldn't buy us much anyway to
+        // parallelize this since these operations should generally be quick,
+        // and there shouldn't be too many sagas outstanding, and Nexus has
+        // already crashed so they've experienced a bit of latency already.
+        for (saga_id, saga) in plan.sagas_needing_recovery() {
+            let saga_log = self.maker.make_saga_log(*saga_id, &saga.name);
+            builder.saga_recovery_start(*saga_id, saga_log.clone());
+            match self.recover_one_saga(bgtask_log, &saga_log, saga).await {
+                Ok(completion_future) => {
+                    builder.saga_recovery_success(*saga_id);
+                    completion_futures.push(completion_future);
+                }
+                Err(error) => {
+                    // It's essential that we not bail out early just because we
+                    // hit an error here.  We want to recover all the sagas that
+                    // we can.
+                    builder.saga_recovery_failure(*saga_id, &error);
+                }
+            }
+        }
+
+        let future = async {
+            futures::future::try_join_all(completion_futures).await?;
+            Ok(())
+        }
+        .boxed();
+        (builder.build(), future)
+    }
+
+    async fn recover_one_saga(
+        &self,
+        bgtask_logger: &slog::Logger,
+        saga_logger: &slog::Logger,
+        saga: &nexus_db_model::Saga,
+    ) -> Result<BoxFuture<'static, Result<(), Error>>, Error> {
+        let datastore = &self.datastore;
+        let saga_id: SagaId = saga.id.into();
+
+        let log_events = datastore
+            .saga_fetch_log_batched(&self.saga_recovery_opctx, saga.id)
+            .await
+            .with_internal_context(|| format!("recovering saga {saga_id}"))?;
+        trace!(bgtask_logger, "recovering saga: loaded log";
+            "nevents" => log_events.len(),
+            "saga_id" => %saga_id,
+        );
+
+        let saga_context = self.maker.make_saga_context(saga_logger.clone());
+        let saga_completion = self
+            .sec_client
+            .saga_resume(
+                saga_id,
+                saga_context,
+                saga.saga_dag.clone(),
+                self.registry.clone(),
+                log_events,
+            )
+            .await
+            .map_err(|error| {
+                // TODO-robustness We want to differentiate between retryable and
+                // not here
+                Error::internal_error(&format!(
+                    "failed to resume saga: {:#}",
+                    error
+                ))
+            })?;
+
+        trace!(&bgtask_logger, "recovering saga: starting the saga";
+            "saga_id" => %saga_id
+        );
+        self.sec_client.saga_start(saga_id).await.map_err(|error| {
+            Error::internal_error(&format!("failed to start saga: {:#}", error))
+        })?;
+
+        Ok(async {
+            saga_completion.await.kind.map_err(|e| {
+                Error::internal_error(&format!("Saga failure: {:?}", e))
+            })?;
+            Ok(())
+        }
+        .boxed())
+    }
+}
+
+/// List all in-progress sagas assigned to the given SEC
+async fn list_sagas_in_progress(
+    opctx: &OpContext,
+    datastore: &DataStore,
+    sec_id: db::SecId,
+) -> Result<BTreeMap<SagaId, nexus_db_model::saga_types::Saga>, Error> {
+    let log = &opctx.log;
+    debug!(log, "listing candidate sagas for recovery");
+    let result = datastore
+        .saga_list_recovery_candidates_batched(&opctx, sec_id)
+        .await
+        .internal_context("listing in-progress sagas for saga recovery")
+        .map(|list| {
+            list.into_iter()
+                .map(|saga| (saga.id.into(), saga))
+                .collect::<BTreeMap<SagaId, nexus_db_model::Saga>>()
+        });
+    match &result {
+        Ok(list) => {
+            info!(log, "listed in-progress sagas"; "count" => list.len());
+        }
+        Err(error) => {
+            warn!(log, "failed to list in-progress sagas"; error);
+        }
+    };
+    result
+}
+
+/// Encapsulates the tiny bit of behavior associated with constructing a new
+/// saga context
+///
+/// This type exists so that the rest of the `SagaRecovery` task can avoid
+/// knowing directly about Nexus, which in turn allows us to test it with sagas
+/// that we control.
+pub trait MakeSagaContext: Send + Sync {
+    type SagaType: steno::SagaType;
+
+    fn make_saga_context(
+        &self,
+        log: slog::Logger,
+    ) -> Arc<<Self::SagaType as steno::SagaType>::ExecContextType>;
+
+    fn make_saga_log(&self, id: SagaId, name: &str) -> slog::Logger;
+}
+
+impl MakeSagaContext for Arc<Nexus> {
+    type SagaType = NexusSagaType;
+    fn make_saga_context(&self, log: slog::Logger) -> Arc<Arc<SagaContext>> {
+        // The extra `Arc` is a little ridiculous.  The problem is that Steno
+        // expects (in `sec_client.saga_resume()`) that the user-defined context
+        // will be wrapped in an `Arc`.  But we already use `Arc<SagaContext>`
+        // for our type.  Hence we need two Arcs.
+        Arc::new(Arc::new(SagaContext::new(self.clone(), log)))
+    }
+
+    fn make_saga_log(&self, id: SagaId, name: &str) -> slog::Logger {
+        self.log.new(o!(
+            "saga_name" => name.to_owned(),
+            "saga_id" => id.to_string(),
+        ))
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use nexus_auth::authn;
+    use nexus_db_queries::context::OpContext;
+    use nexus_db_queries::db::test_utils::UnpluggableCockroachDbSecStore;
+    use nexus_test_utils::{
+        db::test_setup_database, resource_helpers::create_project,
+    };
+    use nexus_test_utils_macros::nexus_test;
+    use nexus_types::internal_api::views::LastResult;
+    use omicron_test_utils::dev::{
+        self,
+        poll::{wait_for_condition, CondCheckError},
+    };
+    use once_cell::sync::Lazy;
+    use pretty_assertions::assert_eq;
+    use std::sync::atomic::{AtomicBool, AtomicU32, Ordering};
+    use steno::{
+        new_action_noop_undo, Action, ActionContext, ActionError,
+        ActionRegistry, DagBuilder, Node, SagaDag, SagaId, SagaName,
+        SagaResult, SagaType, SecClient,
+    };
+    use uuid::Uuid;
+    type ControlPlaneTestContext =
+        nexus_test_utils::ControlPlaneTestContext<crate::Server>;
+
+    // Returns a Cockroach DB, as well as a "datastore" interface (which is the
+    // one more frequently used by Nexus).
+    //
+    // The caller is responsible for calling "cleanup().await" on the returned
+    // CockroachInstance - we would normally wrap this in a drop method, but it
+    // is async.
+    async fn new_db(
+        log: &slog::Logger,
+    ) -> (dev::db::CockroachInstance, Arc<db::DataStore>) {
+        let db = test_setup_database(&log).await;
+        let cfg = nexus_db_queries::db::Config { url: db.pg_config().clone() };
+        let pool = Arc::new(db::Pool::new(log, &cfg));
+        let db_datastore = Arc::new(
+            db::DataStore::new(&log, Arc::clone(&pool), None).await.unwrap(),
+        );
+        (db, db_datastore)
+    }
+
+    // The following is our "saga-under-test". It's a simple two-node operation
+    // that tracks how many times it has been called, and provides a mechanism
+    // for detaching storage to simulate power failure (and meaningfully
+    // recover).
+
+    #[derive(Debug)]
+    struct TestContext {
+        log: slog::Logger,
+
+        // Storage, and instructions on whether or not to detach it
+        // when executing the first saga action.
+        storage: Arc<UnpluggableCockroachDbSecStore>,
+        do_unplug: AtomicBool,
+
+        // Tracks of how many times each node has been reached.
+        n1_count: AtomicU32,
+        n2_count: AtomicU32,
+    }
+
+    impl TestContext {
+        fn new(
+            log: &slog::Logger,
+            storage: Arc<UnpluggableCockroachDbSecStore>,
+        ) -> Self {
+            TestContext {
+                log: log.clone(),
+                storage,
+                do_unplug: AtomicBool::new(false),
+
+                // Counters of how many times the nodes have been invoked.
+                n1_count: AtomicU32::new(0),
+                n2_count: AtomicU32::new(0),
+            }
+        }
+    }
+
+    #[derive(Debug)]
+    struct TestOp;
+    impl SagaType for TestOp {
+        type ExecContextType = TestContext;
+    }
+
+    impl MakeSagaContext for Arc<TestContext> {
+        type SagaType = TestOp;
+        fn make_saga_context(&self, _log: slog::Logger) -> Arc<TestContext> {
+            self.clone()
+        }
+
+        fn make_saga_log(&self, id: SagaId, name: &str) -> slog::Logger {
+            self.log.new(o!(
+                "saga_name" => name.to_owned(),
+                "saga_id" => id.to_string(),
+            ))
+        }
+    }
+
+    static ACTION_N1: Lazy<Arc<dyn Action<TestOp>>> =
+        Lazy::new(|| new_action_noop_undo("n1_action", node_one));
+    static ACTION_N2: Lazy<Arc<dyn Action<TestOp>>> =
+        Lazy::new(|| new_action_noop_undo("n2_action", node_two));
+
+    fn registry_create() -> Arc<ActionRegistry<TestOp>> {
+        let mut registry = ActionRegistry::new();
+        registry.register(Arc::clone(&ACTION_N1));
+        registry.register(Arc::clone(&ACTION_N2));
+        Arc::new(registry)
+    }
+
+    fn saga_object_create() -> Arc<SagaDag> {
+        let mut builder = DagBuilder::new(SagaName::new("test-saga"));
+        builder.append(Node::action("n1_out", "NodeOne", ACTION_N1.as_ref()));
+        builder.append(Node::action("n2_out", "NodeTwo", ACTION_N2.as_ref()));
+        let dag = builder.build().unwrap();
+        Arc::new(SagaDag::new(dag, serde_json::Value::Null))
+    }
+
+    async fn node_one(ctx: ActionContext<TestOp>) -> Result<i32, ActionError> {
+        let uctx = ctx.user_data();
+        uctx.n1_count.fetch_add(1, Ordering::SeqCst);
+        info!(&uctx.log, "ACTION: node_one");
+        // If "do_unplug" is true, we detach storage.
+        //
+        // This prevents the SEC from successfully recording that
+        // this node completed, and acts like a crash.
+        if uctx.do_unplug.load(Ordering::SeqCst) {
+            info!(&uctx.log, "Unplugged storage");
+            uctx.storage.set_unplug(true);
+        }
+        Ok(1)
+    }
+
+    async fn node_two(ctx: ActionContext<TestOp>) -> Result<i32, ActionError> {
+        let uctx = ctx.user_data();
+        uctx.n2_count.fetch_add(1, Ordering::SeqCst);
+        info!(&uctx.log, "ACTION: node_two");
+        Ok(2)
+    }
+
+    // Helper function for setting up storage, SEC, and a test context object.
+    fn create_storage_sec_and_context(
+        log: &slog::Logger,
+        db_datastore: Arc<db::DataStore>,
+        sec_id: db::SecId,
+    ) -> (Arc<UnpluggableCockroachDbSecStore>, SecClient, Arc<TestContext>)
+    {
+        let storage = Arc::new(UnpluggableCockroachDbSecStore::new(
+            sec_id,
+            db_datastore,
+            log.new(o!("component" => "SecStore")),
+        ));
+        let sec_client =
+            steno::sec(log.new(o!("component" => "SEC")), storage.clone());
+        let uctx = Arc::new(TestContext::new(&log, storage.clone()));
+        (storage, sec_client, uctx)
+    }
+
+    // Helper function to run a basic saga that we can use to see which nodes
+    // ran and how many times.
+    async fn run_test_saga(
+        uctx: &Arc<TestContext>,
+        sec_client: &SecClient,
+    ) -> (SagaId, SagaResult) {
+        let saga_id = SagaId(Uuid::new_v4());
+        let future = sec_client
+            .saga_create(
+                saga_id,
+                uctx.clone(),
+                saga_object_create(),
+                registry_create(),
+            )
+            .await
+            .unwrap();
+        sec_client.saga_start(saga_id).await.unwrap();
+        (saga_id, future.await)
+    }
+
+    // Tests the basic case: recovery of a saga that appears (from its log) to
+    // be still running, and which is not currently running already.  In Nexus,
+    // this corresponds to the basic case where a saga was created in a previous
+    // Nexus lifetime and the current process knows nothing about it.
+    #[tokio::test]
+    async fn test_failure_during_saga_can_be_recovered() {
+        // Test setup
+        let logctx =
+            dev::test_setup_log("test_failure_during_saga_can_be_recovered");
+        let log = logctx.log.new(o!());
+        let (mut db, db_datastore) = new_db(&log).await;
+        let sec_id = db::SecId(uuid::Uuid::new_v4());
+        let (storage, sec_client, uctx) =
+            create_storage_sec_and_context(&log, db_datastore.clone(), sec_id);
+        let sec_log = log.new(o!("component" => "SEC"));
+        let opctx = OpContext::for_tests(
+            log,
+            Arc::clone(&db_datastore) as Arc<dyn nexus_auth::storage::Storage>,
+        );
+        let saga_recovery_opctx =
+            opctx.child_with_authn(authn::Context::internal_saga_recovery());
+
+        // In order to recover a partially-created saga, we need a partial log.
+        // To create one, we'll run the saga normally, but configure it to
+        // unplug the datastore partway through so that the later log entries
+        // don't get written.  Note that the unplugged datastore completes
+        // operations successfully so that the saga will appeaer to complete
+        // successfully.
+        uctx.do_unplug.store(true, Ordering::SeqCst);
+        let (_, result) = run_test_saga(&uctx, &sec_client).await;
+        let output = result.kind.unwrap();
+        assert_eq!(output.lookup_node_output::<i32>("n1_out").unwrap(), 1);
+        assert_eq!(output.lookup_node_output::<i32>("n2_out").unwrap(), 2);
+        assert_eq!(uctx.n1_count.load(Ordering::SeqCst), 1);
+        assert_eq!(uctx.n2_count.load(Ordering::SeqCst), 1);
+
+        // Simulate a crash by terminating the SEC and creating a new one using
+        // the same storage system.
+        //
+        // Update uctx to prevent the storage system from detaching again.
+        sec_client.shutdown().await;
+        let sec_client = steno::sec(sec_log, storage.clone());
+        uctx.storage.set_unplug(false);
+        uctx.do_unplug.store(false, Ordering::SeqCst);
+
+        // Use our background task to recover the saga.  Observe that it re-runs
+        // operations and completes.
+        let sec_client = Arc::new(sec_client);
+        let (_, sagas_started_rx) = tokio::sync::mpsc::unbounded_channel();
+        let mut task = SagaRecovery::new(
+            db_datastore.clone(),
+            sec_id,
+            SagaRecoveryHelpers {
+                recovery_opctx: saga_recovery_opctx,
+                maker: uctx.clone(),
+                sec_client: sec_client.clone(),
+                registry: registry_create(),
+                sagas_started_rx,
+            },
+        );
+
+        let Some((completion_future, last_pass_success)) =
+            task.activate_internal(&opctx).await
+        else {
+            panic!("saga recovery failed");
+        };
+
+        assert_eq!(last_pass_success.nrecovered, 1);
+        assert_eq!(last_pass_success.nfailed, 0);
+        assert_eq!(last_pass_success.nskipped, 0);
+
+        // Wait for the recovered saga to complete and make sure it re-ran the
+        // operations that we expected it to.
+        completion_future
+            .await
+            .expect("recovered saga to complete successfully");
+        assert_eq!(uctx.n1_count.load(Ordering::SeqCst), 2);
+        assert_eq!(uctx.n2_count.load(Ordering::SeqCst), 2);
+
+        // Test cleanup
+        drop(task);
+        let sec_client = Arc::try_unwrap(sec_client).unwrap();
+        sec_client.shutdown().await;
+        db.cleanup().await.unwrap();
+        logctx.cleanup_successful();
+    }
+
+    // Tests that a saga that has finished (as reflected in the database state)
+    // does not get recovered.
+    #[tokio::test]
+    async fn test_successful_saga_does_not_replay_during_recovery() {
+        // Test setup
+        let logctx = dev::test_setup_log(
+            "test_successful_saga_does_not_replay_during_recovery",
+        );
+        let log = logctx.log.new(o!());
+        let (mut db, db_datastore) = new_db(&log).await;
+        let sec_id = db::SecId(uuid::Uuid::new_v4());
+        let (storage, sec_client, uctx) =
+            create_storage_sec_and_context(&log, db_datastore.clone(), sec_id);
+        let sec_log = log.new(o!("component" => "SEC"));
+        let opctx = OpContext::for_tests(
+            log,
+            Arc::clone(&db_datastore) as Arc<dyn nexus_auth::storage::Storage>,
+        );
+        let saga_recovery_opctx =
+            opctx.child_with_authn(authn::Context::internal_saga_recovery());
+
+        // Create and start a saga, which we expect to complete successfully.
+        let (_, result) = run_test_saga(&uctx, &sec_client).await;
+        let output = result.kind.unwrap();
+        assert_eq!(output.lookup_node_output::<i32>("n1_out").unwrap(), 1);
+        assert_eq!(output.lookup_node_output::<i32>("n2_out").unwrap(), 2);
+        assert_eq!(uctx.n1_count.load(Ordering::SeqCst), 1);
+        assert_eq!(uctx.n2_count.load(Ordering::SeqCst), 1);
+
+        // Simulate a crash by terminating the SEC and creating a new one using
+        // the same storage system.
+        sec_client.shutdown().await;
+        let sec_client = steno::sec(sec_log, storage.clone());
+
+        // Go through recovery.  We should not find or recover this saga.
+        let sec_client = Arc::new(sec_client);
+        let (_, sagas_started_rx) = tokio::sync::mpsc::unbounded_channel();
+        let mut task = SagaRecovery::new(
+            db_datastore.clone(),
+            sec_id,
+            SagaRecoveryHelpers {
+                recovery_opctx: saga_recovery_opctx,
+                maker: uctx.clone(),
+                sec_client: sec_client.clone(),
+                registry: registry_create(),
+                sagas_started_rx,
+            },
+        );
+
+        let Some((_, last_pass_success)) = task.activate_internal(&opctx).await
+        else {
+            panic!("saga recovery failed");
+        };
+
+        assert_eq!(last_pass_success.nrecovered, 0);
+        assert_eq!(last_pass_success.nfailed, 0);
+        assert_eq!(last_pass_success.nskipped, 0);
+
+        // The nodes should not have been replayed.
+        assert_eq!(uctx.n1_count.load(Ordering::SeqCst), 1);
+        assert_eq!(uctx.n2_count.load(Ordering::SeqCst), 1);
+
+        // Test cleanup
+        drop(task);
+        let sec_client = Arc::try_unwrap(sec_client).unwrap();
+        sec_client.shutdown().await;
+        db.cleanup().await.unwrap();
+        logctx.cleanup_successful();
+    }
+
+    // Verify the plumbing that exists between regular saga creation and saga
+    // recovery.
+    #[nexus_test(server = crate::Server)]
+    async fn test_nexus_recovery(cptestctx: &ControlPlaneTestContext) {
+        let nexus = &cptestctx.server.server_context().nexus;
+
+        // This is tricky to do.  We're trying to make sure the plumbing is
+        // hooked up so that when a saga is created, the saga recovery task
+        // learns about it.  The purpose of that plumbing is to ensure that we
+        // don't try to recover a task that's already running.  It'd be ideal to
+        // test that directly, but we can't easily control execution well enough
+        // to ensure that the background task runs while the saga is still
+        // running.  However, even if we miss it (i.e., the background task only
+        // runs after the saga completes successfully), there's a side effect we
+        // can look for: the task should report the completed saga as "maybe
+        // done".  On the next activation, it should report that it's removed a
+        // saga from its internal state (because it saw that it was done).
+
+        // Wait for the task to run once.
+        let driver = nexus.background_tasks_driver.get().unwrap();
+        let task_name = driver
+            .tasks()
+            .find(|task_name| task_name.as_str() == "saga_recovery")
+            .expect("expected background task called \"saga_recovery\"");
+        let first_completed = wait_for_condition(
+            || async {
+                let status = driver.task_status(task_name);
+                let LastResult::Completed(completed) = status.last else {
+                    return Err(CondCheckError::<()>::NotYet);
+                };
+                Ok(completed)
+            },
+            &std::time::Duration::from_millis(250),
+            &std::time::Duration::from_secs(15),
+        )
+        .await
+        .unwrap();
+
+        // Make sure that it didn't find anything to do.
+        let status_raw = first_completed.details;
+        let status: nexus_saga_recovery::Report =
+            serde_json::from_value(status_raw).unwrap();
+        let nexus_saga_recovery::LastPass::Success(last_pass_success) =
+            status.last_pass
+        else {
+            panic!("wrong last pass variant");
+        };
+        assert_eq!(last_pass_success.nfound, 0);
+        assert_eq!(last_pass_success.nrecovered, 0);
+        assert_eq!(last_pass_success.nfailed, 0);
+        assert_eq!(last_pass_success.nskipped, 0);
+
+        // Now kick off a saga -- any saga will do.  We don't even care if it
+        // works or not.  In practice, it will have finished by the time this
+        // call completes.
+        let _ = create_project(&cptestctx.external_client, "test").await;
+
+        // Activate the background task.  Wait for one pass.
+        nexus.background_tasks.task_saga_recovery.activate();
+        let _ = wait_for_condition(
+            || async {
+                let status = driver.task_status(task_name);
+                let LastResult::Completed(completed) = status.last else {
+                    panic!("task had completed before; how has it not now?");
+                };
+                if completed.iteration <= first_completed.iteration {
+                    return Err(CondCheckError::<()>::NotYet);
+                }
+                Ok(completed)
+            },
+            &std::time::Duration::from_millis(250),
+            &std::time::Duration::from_secs(15),
+        )
+        .await
+        .unwrap();
+
+        // Activate it again.  This should be enough for it to report having
+        // removed a saga from its state.
+        nexus.background_tasks.task_saga_recovery.activate();
+        let last_pass_success = wait_for_condition(
+            || async {
+                let status = driver.task_status(task_name);
+                let LastResult::Completed(completed) = status.last else {
+                    panic!("task had completed before; how has it not now?");
+                };
+
+                let status: nexus_saga_recovery::Report =
+                    serde_json::from_value(completed.details).unwrap();
+                let nexus_saga_recovery::LastPass::Success(last_pass_success) =
+                    status.last_pass
+                else {
+                    panic!("wrong last pass variant");
+                };
+                if last_pass_success.nremoved > 0 {
+                    return Ok(last_pass_success);
+                }
+
+                Err(CondCheckError::<()>::NotYet)
+            },
+            &std::time::Duration::from_millis(250),
+            &std::time::Duration::from_secs(15),
+        )
+        .await
+        .unwrap();
+
+        assert!(last_pass_success.nremoved > 0);
+    }
+}
diff --git a/nexus/src/app/crucible.rs b/nexus/src/app/crucible.rs
index caa65255e5..72a5c80baf 100644
--- a/nexus/src/app/crucible.rs
+++ b/nexus/src/app/crucible.rs
@@ -69,11 +69,17 @@ impl super::Nexus {
     fn crucible_agent_client_for_dataset(
         &self,
         dataset: &db::model::Dataset,
-    ) -> CrucibleAgentClient {
-        CrucibleAgentClient::new_with_client(
-            &format!("http://{}", dataset.address()),
+    ) -> Result<CrucibleAgentClient, Error> {
+        let Some(addr) = dataset.address() else {
+            return Err(Error::internal_error(
+                "Missing crucible dataset address",
+            ));
+        };
+
+        Ok(CrucibleAgentClient::new_with_client(
+            &format!("http://{}", addr),
             self.reqwest_client.clone(),
-        )
+        ))
     }
 
     /// Return if the Crucible agent is expected to be there and answer Nexus:
@@ -147,7 +153,7 @@ impl super::Nexus {
         dataset: &db::model::Dataset,
         region: &db::model::Region,
     ) -> Result<Region, Error> {
-        let client = self.crucible_agent_client_for_dataset(dataset);
+        let client = self.crucible_agent_client_for_dataset(dataset)?;
         let dataset_id = dataset.id();
 
         let Ok(extent_count) = u32::try_from(region.extent_count()) else {
@@ -261,7 +267,7 @@ impl super::Nexus {
         dataset: &db::model::Dataset,
         region_id: Uuid,
     ) -> Result<Option<Region>, Error> {
-        let client = self.crucible_agent_client_for_dataset(dataset);
+        let client = self.crucible_agent_client_for_dataset(dataset)?;
         let dataset_id = dataset.id();
 
         let result = ProgenitorOperationRetry::new(
@@ -303,7 +309,7 @@ impl super::Nexus {
         dataset: &db::model::Dataset,
         region_id: Uuid,
     ) -> Result<GetSnapshotResponse, Error> {
-        let client = self.crucible_agent_client_for_dataset(dataset);
+        let client = self.crucible_agent_client_for_dataset(dataset)?;
         let dataset_id = dataset.id();
 
         let result = ProgenitorOperationRetry::new(
@@ -343,7 +349,7 @@ impl super::Nexus {
         dataset: &db::model::Dataset,
         region_id: Uuid,
     ) -> Result<(), Error> {
-        let client = self.crucible_agent_client_for_dataset(dataset);
+        let client = self.crucible_agent_client_for_dataset(dataset)?;
         let dataset_id = dataset.id();
 
         let result = ProgenitorOperationRetry::new(
@@ -386,7 +392,7 @@ impl super::Nexus {
         region_id: Uuid,
         snapshot_id: Uuid,
     ) -> Result<(), Error> {
-        let client = self.crucible_agent_client_for_dataset(dataset);
+        let client = self.crucible_agent_client_for_dataset(dataset)?;
         let dataset_id = dataset.id();
 
         let result = ProgenitorOperationRetry::new(
@@ -435,7 +441,7 @@ impl super::Nexus {
         region_id: Uuid,
         snapshot_id: Uuid,
     ) -> Result<(), Error> {
-        let client = self.crucible_agent_client_for_dataset(dataset);
+        let client = self.crucible_agent_client_for_dataset(dataset)?;
         let dataset_id = dataset.id();
 
         let result = ProgenitorOperationRetry::new(
diff --git a/nexus/src/app/mod.rs b/nexus/src/app/mod.rs
index cee62f1107..60ed611bd7 100644
--- a/nexus/src/app/mod.rs
+++ b/nexus/src/app/mod.rs
@@ -6,11 +6,11 @@
 
 use self::external_endpoints::NexusCertResolver;
 use self::saga::SagaExecutor;
+use crate::app::background::BackgroundTasksData;
 use crate::app::oximeter::LazyTimeseriesClient;
 use crate::populate::populate_start;
 use crate::populate::PopulateArgs;
 use crate::populate::PopulateStatus;
-use crate::saga_interface::SagaContext;
 use crate::DropshotServer;
 use ::oximeter::types::ProducerRegistry;
 use anyhow::anyhow;
@@ -91,8 +91,10 @@ pub(crate) mod sagas;
 
 pub(crate) use nexus_db_queries::db::queries::disk::MAX_DISKS_PER_INSTANCE;
 
+use crate::app::background::SagaRecoveryHelpers;
 use nexus_db_model::AllSchemaVersions;
 pub(crate) use nexus_db_model::MAX_NICS_PER_INSTANCE;
+use tokio::sync::mpsc;
 
 // XXX: Might want to recast as max *floating* IPs, we have at most one
 //      ephemeral (so bounded in saga by design).
@@ -132,12 +134,9 @@ pub struct Nexus {
     /// handle to global authz information
     authz: Arc<authz::Authz>,
 
-    /// saga execution coordinator
+    /// saga execution coordinator (SEC)
     sagas: Arc<SagaExecutor>,
 
-    /// Task representing completion of recovered Sagas
-    recovery_task: std::sync::Mutex<Option<db::RecoveryTask>>,
-
     /// External dropshot servers
     external_server: std::sync::Mutex<Option<DropshotServer>>,
 
@@ -248,9 +247,34 @@ impl Nexus {
             sec_store,
         ));
 
+        // It's a bit of a red flag to use an unbounded channel.
+        //
+        // This particular channel is used to send a Uuid from the saga executor
+        // to the saga recovery background task each time a saga is started.
+        //
+        // The usual argument for keeping a channel bounded is to ensure
+        // backpressure.  But we don't really want that here.  These items don't
+        // represent meaningful work for the saga recovery task, such that if it
+        // were somehow processing these slowly, we'd want to slow down the saga
+        // dispatch process.  Under normal conditions, we'd expect this queue to
+        // grow as we dispatch new sagas until the saga recovery task runs, at
+        // which point the queue will quickly be drained.  The only way this
+        // could really grow without bound is if the saga recovery task gets
+        // completely wedged and stops receiving these messages altogether.  In
+        // this case, the maximum size this queue could grow over time is the
+        // number of sagas we can launch in that time.  That's not ever likely
+        // to be a significant amount of memory.
+        //
+        // We could put our money where our mouth is: pick a sufficiently large
+        // bound and panic if we reach it.  But "sufficiently large" depends on
+        // the saga creation rate and the period of the saga recovery background
+        // task.  If someone changed the config, they'd have to remember to
+        // update this here.  This doesn't seem worth it.
+        let (saga_create_tx, saga_recovery_rx) = mpsc::unbounded_channel();
         let sagas = Arc::new(SagaExecutor::new(
             Arc::clone(&sec_client),
             log.new(o!("component" => "SagaExecutor")),
+            saga_create_tx,
         ));
 
         let client_state = dpd_client::ClientState {
@@ -420,7 +444,6 @@ impl Nexus {
             db_datastore: Arc::clone(&db_datastore),
             authz: Arc::clone(&authz),
             sagas,
-            recovery_task: std::sync::Mutex::new(None),
             external_server: std::sync::Mutex::new(None),
             techport_external_server: std::sync::Mutex::new(None),
             internal_server: std::sync::Mutex::new(None),
@@ -462,26 +485,12 @@ impl Nexus {
         // TODO-cleanup all the extra Arcs here seems wrong
         let nexus = Arc::new(nexus);
         nexus.sagas.set_nexus(nexus.clone());
-        let opctx = OpContext::for_background(
+        let saga_recovery_opctx = OpContext::for_background(
             log.new(o!("component" => "SagaRecoverer")),
             Arc::clone(&authz),
             authn::Context::internal_saga_recovery(),
             Arc::clone(&db_datastore) as Arc<dyn nexus_auth::storage::Storage>,
         );
-        let saga_logger = nexus.log.new(o!("saga_type" => "recovery"));
-        let recovery_task = db::recover(
-            opctx,
-            my_sec_id,
-            Arc::new(Arc::new(SagaContext::new(
-                Arc::clone(&nexus),
-                saga_logger,
-            ))),
-            Arc::clone(&db_datastore),
-            Arc::clone(&sec_client),
-            sagas::ACTION_REGISTRY.clone(),
-        );
-
-        *nexus.recovery_task.lock().unwrap() = Some(recovery_task);
 
         // Wait to start background tasks until after the populate step
         // finishes.  Among other things, the populate step installs role
@@ -508,14 +517,24 @@ impl Nexus {
 
             let driver = background_tasks_initializer.start(
                 &task_nexus.background_tasks,
-                background_ctx,
-                db_datastore,
-                task_config.pkg.background_tasks,
-                rack_id,
-                task_config.deployment.id,
-                resolver,
-                task_nexus.sagas.clone(),
-                task_registry,
+                BackgroundTasksData {
+                    opctx: background_ctx,
+                    datastore: db_datastore,
+                    config: task_config.pkg.background_tasks,
+                    rack_id,
+                    nexus_id: task_config.deployment.id,
+                    resolver,
+                    saga_starter: task_nexus.sagas.clone(),
+                    producer_registry: task_registry,
+
+                    saga_recovery: SagaRecoveryHelpers {
+                        recovery_opctx: saga_recovery_opctx,
+                        maker: task_nexus.clone(),
+                        sec_client: sec_client.clone(),
+                        registry: sagas::ACTION_REGISTRY.clone(),
+                        sagas_started_rx: saga_recovery_rx,
+                    },
+                },
             );
 
             if let Err(_) = task_nexus.background_tasks_driver.set(driver) {
diff --git a/nexus/src/app/rack.rs b/nexus/src/app/rack.rs
index ee3818f40c..13b30fd47a 100644
--- a/nexus/src/app/rack.rs
+++ b/nexus/src/app/rack.rs
@@ -145,7 +145,7 @@ impl super::Nexus {
                 db::model::Dataset::new(
                     dataset.dataset_id,
                     dataset.zpool_id,
-                    dataset.request.address,
+                    Some(dataset.request.address),
                     dataset.request.kind.into(),
                 )
             })
@@ -778,7 +778,7 @@ impl super::Nexus {
                         baseboard: Baseboard {
                             serial: k.serial_number.clone(),
                             part: k.part_number.clone(),
-                            revision: v.baseboard_revision.into(),
+                            revision: v.baseboard_revision,
                         },
                         rack_id: self.rack_id,
                         cubby: v.sp_slot,
diff --git a/nexus/src/app/saga.rs b/nexus/src/app/saga.rs
index ed4ccf44fd..2b510a0f12 100644
--- a/nexus/src/app/saga.rs
+++ b/nexus/src/app/saga.rs
@@ -70,6 +70,7 @@ use steno::SagaDag;
 use steno::SagaId;
 use steno::SagaResult;
 use steno::SagaResultOk;
+use tokio::sync::mpsc;
 use uuid::Uuid;
 
 /// Given a particular kind of Nexus saga (the type parameter `N`) and
@@ -111,14 +112,16 @@ pub(crate) struct SagaExecutor {
     sec_client: Arc<steno::SecClient>,
     log: slog::Logger,
     nexus: OnceLock<Arc<Nexus>>,
+    saga_create_tx: mpsc::UnboundedSender<steno::SagaId>,
 }
 
 impl SagaExecutor {
     pub(crate) fn new(
         sec_client: Arc<steno::SecClient>,
         log: slog::Logger,
+        saga_create_tx: mpsc::UnboundedSender<steno::SagaId>,
     ) -> SagaExecutor {
-        SagaExecutor { sec_client, log, nexus: OnceLock::new() }
+        SagaExecutor { sec_client, log, nexus: OnceLock::new(), saga_create_tx }
     }
 
     // This is a little gross.  We want to hang the SagaExecutor off of Nexus,
@@ -190,6 +193,19 @@ impl SagaExecutor {
             saga_logger.clone(),
         )));
 
+        // Tell the recovery task about this.  It's critical that we send this
+        // message before telling Steno about this saga.  It's not critical that
+        // the task _receive_ this message synchronously.  See the comments in
+        // the recovery task implementation for details.
+        self.saga_create_tx.send(saga_id).map_err(
+            |_: mpsc::error::SendError<SagaId>| {
+                Error::internal_error(
+                    "cannot create saga: recovery task not listening \
+                     (is Nexus shutting down?)",
+                )
+            },
+        )?;
+
         // Tell Steno about it.  This does not start it running yet.
         info!(saga_logger, "preparing saga");
         let saga_completion_future = self
diff --git a/nexus/src/app/sagas/disk_create.rs b/nexus/src/app/sagas/disk_create.rs
index bdccd7f79b..c350534617 100644
--- a/nexus/src/app/sagas/disk_create.rs
+++ b/nexus/src/app/sagas/disk_create.rs
@@ -498,9 +498,17 @@ async fn sdc_regions_ensure(
                     .map(|(dataset, region)| {
                         dataset
                             .address_with_port(region.port_number)
-                            .to_string()
+                            .ok_or_else(|| {
+                                ActionError::action_failed(
+                                    Error::internal_error(&format!(
+                                        "missing IP address for dataset {}",
+                                        dataset.id(),
+                                    )),
+                                )
+                            })
+                            .map(|addr| addr.to_string())
                     })
-                    .collect(),
+                    .collect::<Result<Vec<_>, ActionError>>()?,
 
                 lossy: false,
                 flush_timeout: None,
diff --git a/nexus/src/app/sagas/mod.rs b/nexus/src/app/sagas/mod.rs
index d278fb5600..17f43b4950 100644
--- a/nexus/src/app/sagas/mod.rs
+++ b/nexus/src/app/sagas/mod.rs
@@ -50,7 +50,7 @@ pub mod common_storage;
 mod test_helpers;
 
 #[derive(Debug)]
-pub(crate) struct NexusSagaType;
+pub struct NexusSagaType;
 impl steno::SagaType for NexusSagaType {
     type ExecContextType = Arc<SagaContext>;
 }
diff --git a/nexus/src/app/sagas/region_replacement_start.rs b/nexus/src/app/sagas/region_replacement_start.rs
index a4ba10775a..1297158b24 100644
--- a/nexus/src/app/sagas/region_replacement_start.rs
+++ b/nexus/src/app/sagas/region_replacement_start.rs
@@ -534,12 +534,13 @@ async fn srrs_replace_region_in_volume(
         "ensured_dataset_and_region",
     )?;
 
-    let new_region_address = SocketAddrV6::new(
-        *new_dataset.address().ip(),
-        ensured_region.port_number,
-        0,
-        0,
-    );
+    let Some(new_address) = new_dataset.address() else {
+        return Err(ActionError::action_failed(Error::internal_error(
+            "Dataset missing IP address",
+        )));
+    };
+    let new_region_address =
+        SocketAddrV6::new(*new_address.ip(), ensured_region.port_number, 0, 0);
 
     // If this node is rerun, the forward action will have overwritten
     // db_region's volume id, so get the cached copy.
@@ -611,12 +612,11 @@ async fn srrs_replace_region_in_volume_undo(
         "ensured_dataset_and_region",
     )?;
 
-    let new_region_address = SocketAddrV6::new(
-        *new_dataset.address().ip(),
-        ensured_region.port_number,
-        0,
-        0,
-    );
+    let Some(new_address) = new_dataset.address() else {
+        anyhow::bail!("Dataset missing IP address");
+    };
+    let new_region_address =
+        SocketAddrV6::new(*new_address.ip(), ensured_region.port_number, 0, 0);
 
     // The forward action will have overwritten db_region's volume id, so get
     // the cached copy.
@@ -894,25 +894,25 @@ pub(crate) mod test {
             Dataset::new(
                 Uuid::new_v4(),
                 Uuid::new_v4(),
-                "[fd00:1122:3344:101::1]:12345".parse().unwrap(),
+                Some("[fd00:1122:3344:101::1]:12345".parse().unwrap()),
                 DatasetKind::Crucible,
             ),
             Dataset::new(
                 Uuid::new_v4(),
                 Uuid::new_v4(),
-                "[fd00:1122:3344:102::1]:12345".parse().unwrap(),
+                Some("[fd00:1122:3344:102::1]:12345".parse().unwrap()),
                 DatasetKind::Crucible,
             ),
             Dataset::new(
                 Uuid::new_v4(),
                 Uuid::new_v4(),
-                "[fd00:1122:3344:103::1]:12345".parse().unwrap(),
+                Some("[fd00:1122:3344:103::1]:12345".parse().unwrap()),
                 DatasetKind::Crucible,
             ),
             Dataset::new(
                 Uuid::new_v4(),
                 Uuid::new_v4(),
-                "[fd00:1122:3344:104::1]:12345".parse().unwrap(),
+                Some("[fd00:1122:3344:104::1]:12345".parse().unwrap()),
                 DatasetKind::Crucible,
             ),
         ];
diff --git a/nexus/src/app/sagas/snapshot_create.rs b/nexus/src/app/sagas/snapshot_create.rs
index 9e665a1de1..5a8313229a 100644
--- a/nexus/src/app/sagas/snapshot_create.rs
+++ b/nexus/src/app/sagas/snapshot_create.rs
@@ -411,9 +411,17 @@ async fn ssc_regions_ensure(
                     .map(|(dataset, region)| {
                         dataset
                             .address_with_port(region.port_number)
-                            .to_string()
+                            .ok_or_else(|| {
+                                ActionError::action_failed(
+                                    Error::internal_error(&format!(
+                                        "missing IP address for dataset {}",
+                                        dataset.id(),
+                                    )),
+                                )
+                            })
+                            .map(|addr| addr.to_string())
                     })
-                    .collect(),
+                    .collect::<Result<Vec<_>, ActionError>>()?,
 
                 lossy: false,
                 flush_timeout: None,
@@ -1232,8 +1240,14 @@ async fn ssc_start_running_snapshot(
     let mut map: BTreeMap<String, String> = BTreeMap::new();
 
     for (dataset, region) in datasets_and_regions {
+        let Some(dataset_addr) = dataset.address() else {
+            return Err(ActionError::action_failed(Error::internal_error(
+                &format!("Missing IP address for dataset {}", dataset.id(),),
+            )));
+        };
+
         // Create a Crucible agent client
-        let url = format!("http://{}", dataset.address());
+        let url = format!("http://{}", dataset_addr);
         let client = CrucibleAgentClient::new(&url);
 
         info!(
@@ -1299,11 +1313,21 @@ async fn ssc_start_running_snapshot(
         // Map from the region to the snapshot
         let region_addr = format!(
             "{}",
-            dataset.address_with_port(crucible_region.port_number)
+            SocketAddrV6::new(
+                *dataset_addr.ip(),
+                crucible_region.port_number,
+                0,
+                0
+            )
         );
         let snapshot_addr = format!(
             "{}",
-            dataset.address_with_port(crucible_running_snapshot.port_number)
+            SocketAddrV6::new(
+                *dataset_addr.ip(),
+                crucible_running_snapshot.port_number,
+                0,
+                0
+            )
         );
         info!(log, "map {} to {}", region_addr, snapshot_addr);
         map.insert(region_addr, snapshot_addr.clone());
diff --git a/nexus/src/app/sagas/vpc_create.rs b/nexus/src/app/sagas/vpc_create.rs
index a34b25ceb7..832ca64ace 100644
--- a/nexus/src/app/sagas/vpc_create.rs
+++ b/nexus/src/app/sagas/vpc_create.rs
@@ -8,7 +8,7 @@ use super::NexusSaga;
 use super::ACTION_GENERATE_ID;
 use crate::app::sagas::declare_saga_actions;
 use crate::external_api::params;
-use nexus_db_queries::db::queries::vpc_subnet::SubnetError;
+use nexus_db_queries::db::queries::vpc_subnet::InsertVpcSubnetError;
 use nexus_db_queries::{authn, authz, db};
 use nexus_defaults as defaults;
 use omicron_common::api::external;
@@ -368,7 +368,7 @@ async fn svc_create_subnet(
         .vpc_create_subnet(&opctx, &authz_vpc, subnet)
         .await
         .map_err(|err| match err {
-            SubnetError::OverlappingIpRange(ip) => {
+            InsertVpcSubnetError::OverlappingIpRange(ip) => {
                 let ipv4_block = &defaults::DEFAULT_VPC_SUBNET_IPV4_BLOCK;
                 let log = sagactx.user_data().log();
                 error!(
@@ -388,7 +388,7 @@ async fn svc_create_subnet(
                         found overlapping IP address ranges",
                 )
             }
-            SubnetError::External(e) => e,
+            InsertVpcSubnetError::External(e) => e,
         })
         .map_err(ActionError::action_failed)
 }
diff --git a/nexus/src/app/sled.rs b/nexus/src/app/sled.rs
index fd5341ae80..6e21470368 100644
--- a/nexus/src/app/sled.rs
+++ b/nexus/src/app/sled.rs
@@ -13,7 +13,9 @@ use nexus_db_queries::context::OpContext;
 use nexus_db_queries::db;
 use nexus_db_queries::db::lookup;
 use nexus_db_queries::db::model::DatasetKind;
+use nexus_types::deployment::DiskFilter;
 use nexus_types::deployment::SledFilter;
+use nexus_types::external_api::views::PhysicalDiskPolicy;
 use nexus_types::external_api::views::SledPolicy;
 use nexus_types::external_api::views::SledProvisionPolicy;
 use omicron_common::api::external::DataPageParams;
@@ -186,7 +188,7 @@ impl super::Nexus {
 
     // Physical disks
 
-    pub async fn physical_disk_lookup<'a>(
+    pub fn physical_disk_lookup<'a>(
         &'a self,
         opctx: &'a OpContext,
         disk_selector: &params::PhysicalDiskPath,
@@ -211,7 +213,9 @@ impl super::Nexus {
         opctx: &OpContext,
         pagparams: &DataPageParams<'_, Uuid>,
     ) -> ListResultVec<db::model::PhysicalDisk> {
-        self.db_datastore.physical_disk_list(&opctx, pagparams).await
+        self.db_datastore
+            .physical_disk_list(&opctx, pagparams, DiskFilter::InService)
+            .await
     }
 
     /// Upserts a physical disk into the database, updating it if it already exists.
@@ -240,6 +244,27 @@ impl super::Nexus {
         Ok(())
     }
 
+    /// Mark a physical disk as expunged
+    ///
+    /// This is an irreversible process! It should only be called after
+    /// sufficient warning to the operator.
+    pub(crate) async fn physical_disk_expunge(
+        &self,
+        opctx: &OpContext,
+        disk: params::PhysicalDiskPath,
+    ) -> Result<(), Error> {
+        let physical_disk_lookup = self.physical_disk_lookup(opctx, &disk)?;
+        let (authz_disk,) =
+            physical_disk_lookup.lookup_for(authz::Action::Modify).await?;
+        self.db_datastore
+            .physical_disk_update_policy(
+                opctx,
+                authz_disk.id(),
+                PhysicalDiskPolicy::Expunged.into(),
+            )
+            .await
+    }
+
     // Zpools (contained within sleds)
 
     /// Upserts a Zpool into the database, updating it if it already exists.
@@ -281,7 +306,8 @@ impl super::Nexus {
             "dataset_id" => id.to_string(),
             "address" => address.to_string()
         );
-        let dataset = db::model::Dataset::new(id, zpool_id, address, kind);
+        let dataset =
+            db::model::Dataset::new(id, zpool_id, Some(address), kind);
         self.db_datastore.dataset_upsert(dataset).await?;
         Ok(())
     }
diff --git a/nexus/src/app/vpc_subnet.rs b/nexus/src/app/vpc_subnet.rs
index ce0cd423f4..39b9844799 100644
--- a/nexus/src/app/vpc_subnet.rs
+++ b/nexus/src/app/vpc_subnet.rs
@@ -13,7 +13,7 @@ use nexus_db_queries::db::identity::Resource;
 use nexus_db_queries::db::lookup;
 use nexus_db_queries::db::lookup::LookupPath;
 use nexus_db_queries::db::model::VpcSubnet;
-use nexus_db_queries::db::queries::vpc_subnet::SubnetError;
+use nexus_db_queries::db::queries::vpc_subnet::InsertVpcSubnetError;
 use omicron_common::api::external;
 use omicron_common::api::external::http_pagination::PaginatedBy;
 use omicron_common::api::external::CreateResult;
@@ -24,6 +24,7 @@ use omicron_common::api::external::ListResultVec;
 use omicron_common::api::external::LookupResult;
 use omicron_common::api::external::NameOrId;
 use omicron_common::api::external::UpdateResult;
+use oxnet::IpNet;
 use uuid::Uuid;
 
 impl super::Nexus {
@@ -141,9 +142,9 @@ impl super::Nexus {
                         // Note that we only catch IPv6 overlaps. The client
                         // always specifies the IPv4 range, so we fail the
                         // request if that overlaps with an existing range.
-                        Err(SubnetError::OverlappingIpRange(ip))
-                            if retry <= NUM_RETRIES && ip.is_ipv6() =>
-                        {
+                        Err(InsertVpcSubnetError::OverlappingIpRange(
+                            IpNet::V6(_),
+                        )) if retry <= NUM_RETRIES => {
                             debug!(
                                 self.log,
                                 "autogenerated random IPv6 range overlap";
@@ -157,9 +158,9 @@ impl super::Nexus {
                     }
                 };
                 match result {
-                    Err(SubnetError::OverlappingIpRange(ip))
-                        if ip.is_ipv6() =>
-                    {
+                    Err(InsertVpcSubnetError::OverlappingIpRange(
+                        IpNet::V6(_),
+                    )) => {
                         // TODO-monitoring TODO-debugging
                         //
                         // We should maintain a counter for this occurrence, and
@@ -181,11 +182,11 @@ impl super::Nexus {
                             for VPC Subnet",
                         ))
                     }
-                    Err(SubnetError::OverlappingIpRange(_)) => {
+                    Err(InsertVpcSubnetError::OverlappingIpRange(_)) => {
                         // Overlapping IPv4 ranges, which is always a client error.
                         Err(result.unwrap_err().into_external())
                     }
-                    Err(SubnetError::External(e)) => Err(e),
+                    Err(InsertVpcSubnetError::External(e)) => Err(e),
                     Ok((.., subnet)) => Ok(subnet),
                 }
             }
@@ -210,7 +211,7 @@ impl super::Nexus {
                     .vpc_create_subnet(opctx, &authz_vpc, subnet)
                     .await
                     .map(|(.., subnet)| subnet)
-                    .map_err(SubnetError::into_external)
+                    .map_err(InsertVpcSubnetError::into_external)
             }
         }?;
 
diff --git a/nexus/src/external_api/http_entrypoints.rs b/nexus/src/external_api/http_entrypoints.rs
index d23f0d035a..9d616c7e9c 100644
--- a/nexus/src/external_api/http_entrypoints.rs
+++ b/nexus/src/external_api/http_entrypoints.rs
@@ -6142,7 +6142,7 @@ async fn physical_disk_view(
         let opctx = crate::context::op_context_for_external_api(&rqctx).await?;
 
         let (.., physical_disk) =
-            nexus.physical_disk_lookup(&opctx, &path).await?.fetch().await?;
+            nexus.physical_disk_lookup(&opctx, &path)?.fetch().await?;
         Ok(HttpResponseOk(physical_disk.into()))
     };
     apictx
diff --git a/nexus/src/internal_api/http_entrypoints.rs b/nexus/src/internal_api/http_entrypoints.rs
index f324ea787d..28ff712c24 100644
--- a/nexus/src/internal_api/http_entrypoints.rs
+++ b/nexus/src/internal_api/http_entrypoints.rs
@@ -24,6 +24,7 @@ use nexus_types::deployment::Blueprint;
 use nexus_types::deployment::BlueprintMetadata;
 use nexus_types::deployment::BlueprintTarget;
 use nexus_types::deployment::BlueprintTargetSet;
+use nexus_types::external_api::params::PhysicalDiskPath;
 use nexus_types::external_api::params::SledSelector;
 use nexus_types::external_api::params::UninitializedSledId;
 use nexus_types::external_api::shared::ProbeInfo;
@@ -827,6 +828,24 @@ impl NexusInternalApi for NexusInternalApiImpl {
             .await
     }
 
+    async fn physical_disk_expunge(
+        rqctx: RequestContext<Self::Context>,
+        disk: TypedBody<PhysicalDiskPath>,
+    ) -> Result<HttpResponseUpdatedNoContent, HttpError> {
+        let apictx = &rqctx.context().context;
+        let nexus = &apictx.nexus;
+        let handler = async {
+            let opctx =
+                crate::context::op_context_for_internal_api(&rqctx).await;
+            nexus.physical_disk_expunge(&opctx, disk.into_inner()).await?;
+            Ok(HttpResponseUpdatedNoContent())
+        };
+        apictx
+            .internal_latencies
+            .instrument_dropshot_handler(&rqctx, handler)
+            .await
+    }
+
     async fn probes_get(
         rqctx: RequestContext<Self::Context>,
         path_params: Path<ProbePathParam>,
diff --git a/nexus/src/lib.rs b/nexus/src/lib.rs
index a359ead038..5d5e7d6eba 100644
--- a/nexus/src/lib.rs
+++ b/nexus/src/lib.rs
@@ -57,7 +57,7 @@ extern crate slog;
 /// to stdout.
 pub fn run_openapi_external() -> Result<(), String> {
     external_api()
-        .openapi("Oxide Region API", "20240710.0")
+        .openapi("Oxide Region API", "20240821.0")
         .description("API for interacting with the Oxide control plane")
         .contact_url("https://oxide.computer")
         .contact_email("api@oxide.computer")
diff --git a/nexus/src/saga_interface.rs b/nexus/src/saga_interface.rs
index 5a828ff0ec..aef7044408 100644
--- a/nexus/src/saga_interface.rs
+++ b/nexus/src/saga_interface.rs
@@ -13,7 +13,7 @@ use std::sync::Arc;
 // TODO-design Should this be the same thing as ServerContext?  It's
 // very analogous, but maybe there's utility in having separate views for the
 // HTTP server and sagas.
-pub(crate) struct SagaContext {
+pub struct SagaContext {
     nexus: Arc<Nexus>,
     log: Logger,
 }
diff --git a/nexus/test-utils/src/lib.rs b/nexus/test-utils/src/lib.rs
index 38cdac5fcb..960ded50d5 100644
--- a/nexus/test-utils/src/lib.rs
+++ b/nexus/test-utils/src/lib.rs
@@ -118,7 +118,7 @@ pub struct ControlPlaneTestContext<N> {
     pub sled_agent2: sim::Server,
     pub oximeter: Oximeter,
     pub producer: ProducerServer,
-    pub gateway: HashMap<SwitchLocation, GatewayTestContext>,
+    pub gateway: BTreeMap<SwitchLocation, GatewayTestContext>,
     pub dendrite: HashMap<SwitchLocation, dev::dendrite::DendriteInstance>,
     pub mgd: HashMap<SwitchLocation, dev::maghemite::MgdInstance>,
     pub external_dns_zone_name: String,
@@ -280,7 +280,7 @@ pub struct ControlPlaneTestContextBuilder<'a, N: NexusServer> {
     pub sled_agent2: Option<sim::Server>,
     pub oximeter: Option<Oximeter>,
     pub producer: Option<ProducerServer>,
-    pub gateway: HashMap<SwitchLocation, GatewayTestContext>,
+    pub gateway: BTreeMap<SwitchLocation, GatewayTestContext>,
     pub dendrite: HashMap<SwitchLocation, dev::dendrite::DendriteInstance>,
     pub mgd: HashMap<SwitchLocation, dev::maghemite::MgdInstance>,
 
@@ -330,7 +330,7 @@ impl<'a, N: NexusServer> ControlPlaneTestContextBuilder<'a, N> {
             sled_agent2: None,
             oximeter: None,
             producer: None,
-            gateway: HashMap::new(),
+            gateway: BTreeMap::new(),
             dendrite: HashMap::new(),
             mgd: HashMap::new(),
             nexus_internal: None,
@@ -1575,6 +1575,7 @@ pub async fn start_dns_server(
             bind_address: "[::1]:0".parse().unwrap(),
             request_body_max_bytes: 8 * 1024,
             default_handler_task_mode: HandlerTaskMode::Detached,
+            log_headers: vec![],
         },
     )
     .await
diff --git a/nexus/test-utils/src/resource_helpers.rs b/nexus/test-utils/src/resource_helpers.rs
index 013ce7b630..1a92a6ef8e 100644
--- a/nexus/test-utils/src/resource_helpers.rs
+++ b/nexus/test-utils/src/resource_helpers.rs
@@ -332,7 +332,7 @@ pub async fn create_switch(
     client: &ClientTestContext,
     serial: &str,
     part: &str,
-    revision: i64,
+    revision: u32,
     rack_id: Uuid,
 ) -> views::Switch {
     object_put(
diff --git a/nexus/tests/config.test.toml b/nexus/tests/config.test.toml
index 8415a192b1..415727693b 100644
--- a/nexus/tests/config.test.toml
+++ b/nexus/tests/config.test.toml
@@ -37,7 +37,7 @@ max_vpc_ipv4_subnet_prefix = 29
 [deployment]
 # Identifier for this instance of Nexus.
 # NOTE: The test suite always overrides this.
-id = "e6bff1ff-24fb-49dc-a54e-c6a350cd4d6c"
+id = "913233fe-92a8-4635-9572-183f495429c4"
 rack_id = "c19a698f-c6f9-4a17-ae30-20d711b8f7dc"
 techport_external_server_port = 0
 
@@ -119,6 +119,7 @@ instance_watcher.period_secs = 30
 service_firewall_propagation.period_secs = 300
 v2p_mapping_propagation.period_secs = 30
 abandoned_vmm_reaper.period_secs = 60
+saga_recovery.period_secs = 600
 lookup_region_port.period_secs = 60
 
 [default_region_allocation_strategy]
diff --git a/nexus/tests/integration_tests/commands.rs b/nexus/tests/integration_tests/commands.rs
index 3e133e8681..c2277ba776 100644
--- a/nexus/tests/integration_tests/commands.rs
+++ b/nexus/tests/integration_tests/commands.rs
@@ -109,7 +109,7 @@ fn test_nexus_openapi() {
         .expect("stdout was not valid OpenAPI");
     assert_eq!(spec.openapi, "3.0.3");
     assert_eq!(spec.info.title, "Oxide Region API");
-    assert_eq!(spec.info.version, "20240710.0");
+    assert_eq!(spec.info.version, "20240821.0");
 
     // Spot check a couple of items.
     assert!(!spec.paths.paths.is_empty());
diff --git a/nexus/types/src/deployment/planning_input.rs b/nexus/types/src/deployment/planning_input.rs
index 8a230469d5..a8f3989da4 100644
--- a/nexus/types/src/deployment/planning_input.rs
+++ b/nexus/types/src/deployment/planning_input.rs
@@ -340,7 +340,7 @@ impl SledDisk {
 }
 
 /// Filters that apply to disks.
-#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
+#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash, ValueEnum)]
 pub enum DiskFilter {
     /// All disks
     All,
@@ -355,16 +355,58 @@ impl DiskFilter {
         policy: PhysicalDiskPolicy,
         state: PhysicalDiskState,
     ) -> bool {
+        policy.matches(self) && state.matches(self)
+    }
+}
+
+impl PhysicalDiskPolicy {
+    /// Returns true if self matches the filter
+    pub fn matches(self, filter: DiskFilter) -> bool {
         match self {
-            DiskFilter::All => true,
-            DiskFilter::InService => match (policy, state) {
-                (PhysicalDiskPolicy::InService, PhysicalDiskState::Active) => {
-                    true
-                }
-                _ => false,
+            PhysicalDiskPolicy::InService => match filter {
+                DiskFilter::All => true,
+                DiskFilter::InService => true,
+            },
+            PhysicalDiskPolicy::Expunged => match filter {
+                DiskFilter::All => true,
+                DiskFilter::InService => false,
             },
         }
     }
+
+    /// Returns all policies matching the given filter.
+    ///
+    /// This is meant for database access, and is generally paired with
+    /// [`PhysicalDiskState::all_matching`]. See `ApplyPhysicalDiskFilterExt` in
+    /// nexus-db-model.
+    pub fn all_matching(filter: DiskFilter) -> impl Iterator<Item = Self> {
+        Self::iter().filter(move |state| state.matches(filter))
+    }
+}
+
+impl PhysicalDiskState {
+    /// Returns true if self matches the filter
+    pub fn matches(self, filter: DiskFilter) -> bool {
+        match self {
+            PhysicalDiskState::Active => match filter {
+                DiskFilter::All => true,
+                DiskFilter::InService => true,
+            },
+            PhysicalDiskState::Decommissioned => match filter {
+                DiskFilter::All => true,
+                DiskFilter::InService => false,
+            },
+        }
+    }
+
+    /// Returns all state matching the given filter.
+    ///
+    /// This is meant for database access, and is generally paired with
+    /// [`PhysicalDiskPolicy::all_matching`]. See `ApplyPhysicalDiskFilterExt` in
+    /// nexus-db-model.
+    pub fn all_matching(filter: DiskFilter) -> impl Iterator<Item = Self> {
+        Self::iter().filter(move |state| state.matches(filter))
+    }
 }
 
 /// Filters that apply to zpools.
diff --git a/nexus/types/src/external_api/shared.rs b/nexus/types/src/external_api/shared.rs
index 32d8765a54..9bfa9c8358 100644
--- a/nexus/types/src/external_api/shared.rs
+++ b/nexus/types/src/external_api/shared.rs
@@ -267,7 +267,7 @@ pub enum UpdateableComponentType {
 pub struct Baseboard {
     pub serial: String,
     pub part: String,
-    pub revision: i64,
+    pub revision: u32,
 }
 
 /// A sled that has not been added to an initialized rack yet
diff --git a/openapi/bootstrap-agent.json b/openapi/bootstrap-agent.json
index 370f0fb404..879e8cdc3f 100644
--- a/openapi/bootstrap-agent.json
+++ b/openapi/bootstrap-agent.json
@@ -217,7 +217,8 @@
               },
               "revision": {
                 "type": "integer",
-                "format": "int64"
+                "format": "uint32",
+                "minimum": 0
               },
               "type": {
                 "type": "string",
diff --git a/openapi/dns-server.json b/openapi/dns-server.json
index 1b02199b76..0252c1538a 100644
--- a/openapi/dns-server.json
+++ b/openapi/dns-server.json
@@ -2,7 +2,12 @@
   "openapi": "3.0.3",
   "info": {
     "title": "Internal DNS",
-    "version": "v0.1.0"
+    "description": "API for the internal DNS server",
+    "contact": {
+      "url": "https://oxide.computer",
+      "email": "api@oxide.computer"
+    },
+    "version": "0.0.1"
   },
   "paths": {
     "/config": {
diff --git a/openapi/installinator-artifactd.json b/openapi/installinator.json
similarity index 99%
rename from openapi/installinator-artifactd.json
rename to openapi/installinator.json
index 61f555e10d..0631344b25 100644
--- a/openapi/installinator-artifactd.json
+++ b/openapi/installinator.json
@@ -1,8 +1,8 @@
 {
   "openapi": "3.0.3",
   "info": {
-    "title": "Oxide Installinator Artifact Server",
-    "description": "API for use by the installinator to retrieve artifacts",
+    "title": "Installinator API",
+    "description": "API for installinator to fetch artifacts and report progress",
     "contact": {
       "url": "https://oxide.computer",
       "email": "api@oxide.computer"
diff --git a/openapi/nexus-internal.json b/openapi/nexus-internal.json
index 27430c7599..257a505401 100644
--- a/openapi/nexus-internal.json
+++ b/openapi/nexus-internal.json
@@ -909,6 +909,34 @@
         }
       }
     },
+    "/physical-disk/expunge": {
+      "post": {
+        "summary": "Mark a physical disk as expunged",
+        "description": "This is an irreversible process! It should only be called after sufficient warning to the operator.\nThis is idempotent.",
+        "operationId": "physical_disk_expunge",
+        "requestBody": {
+          "content": {
+            "application/json": {
+              "schema": {
+                "$ref": "#/components/schemas/PhysicalDiskPath"
+              }
+            }
+          },
+          "required": true
+        },
+        "responses": {
+          "204": {
+            "description": "resource updated"
+          },
+          "4XX": {
+            "$ref": "#/components/responses/Error"
+          },
+          "5XX": {
+            "$ref": "#/components/responses/Error"
+          }
+        }
+      }
+    },
     "/probes/{sled}": {
       "get": {
         "summary": "Get all the probes associated with a given sled.",
@@ -1498,7 +1526,8 @@
           },
           "revision": {
             "type": "integer",
-            "format": "int64"
+            "format": "uint32",
+            "minimum": 0
           },
           "serial": {
             "type": "string"
@@ -3794,6 +3823,19 @@
           "u2"
         ]
       },
+      "PhysicalDiskPath": {
+        "type": "object",
+        "properties": {
+          "disk_id": {
+            "description": "ID of the physical disk",
+            "type": "string",
+            "format": "uuid"
+          }
+        },
+        "required": [
+          "disk_id"
+        ]
+      },
       "PhysicalDiskPutRequest": {
         "type": "object",
         "properties": {
diff --git a/openapi/nexus.json b/openapi/nexus.json
index c9d85a8ee3..9e46a92039 100644
--- a/openapi/nexus.json
+++ b/openapi/nexus.json
@@ -7,7 +7,7 @@
       "url": "https://oxide.computer",
       "email": "api@oxide.computer"
     },
-    "version": "20240710.0"
+    "version": "20240821.0"
   },
   "paths": {
     "/device/auth": {
@@ -10013,7 +10013,8 @@
           },
           "revision": {
             "type": "integer",
-            "format": "int64"
+            "format": "uint32",
+            "minimum": 0
           },
           "serial": {
             "type": "string"
diff --git a/openapi/sled-agent.json b/openapi/sled-agent.json
index 8165cfa9d6..27cfe576b7 100644
--- a/openapi/sled-agent.json
+++ b/openapi/sled-agent.json
@@ -710,6 +710,30 @@
         }
       }
     },
+    "/sled-identifiers": {
+      "get": {
+        "summary": "Fetch sled identifiers",
+        "operationId": "sled_identifiers",
+        "responses": {
+          "200": {
+            "description": "successful operation",
+            "content": {
+              "application/json": {
+                "schema": {
+                  "$ref": "#/components/schemas/SledIdentifiers"
+                }
+              }
+            }
+          },
+          "4XX": {
+            "$ref": "#/components/responses/Error"
+          },
+          "5XX": {
+            "$ref": "#/components/responses/Error"
+          }
+        }
+      }
+    },
     "/sled-role": {
       "get": {
         "operationId": "sled_role_get",
@@ -1390,7 +1414,8 @@
               },
               "revision": {
                 "type": "integer",
-                "format": "int64"
+                "format": "uint32",
+                "minimum": 0
               },
               "type": {
                 "type": "string",
@@ -4549,6 +4574,43 @@
         "type": "string",
         "pattern": "^(0|[1-9]\\d*)\\.(0|[1-9]\\d*)\\.(0|[1-9]\\d*)(?:-((?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\\.(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\\+([0-9a-zA-Z-]+(?:\\.[0-9a-zA-Z-]+)*))?$"
       },
+      "SledIdentifiers": {
+        "description": "Identifiers for a single sled.\n\nThis is intended primarily to be used in timeseries, to identify sled from which metric data originates.",
+        "type": "object",
+        "properties": {
+          "model": {
+            "description": "Model name of the sled",
+            "type": "string"
+          },
+          "rack_id": {
+            "description": "Control plane ID of the rack this sled is a member of",
+            "type": "string",
+            "format": "uuid"
+          },
+          "revision": {
+            "description": "Revision number of the sled",
+            "type": "integer",
+            "format": "uint32",
+            "minimum": 0
+          },
+          "serial": {
+            "description": "Serial number of the sled",
+            "type": "string"
+          },
+          "sled_id": {
+            "description": "Control plane ID for the sled itself",
+            "type": "string",
+            "format": "uuid"
+          }
+        },
+        "required": [
+          "model",
+          "rack_id",
+          "revision",
+          "serial",
+          "sled_id"
+        ]
+      },
       "SledInstanceState": {
         "description": "A wrapper type containing a sled's total knowledge of the state of a specific VMM and the instance it incarnates.",
         "type": "object",
diff --git a/openapi/wicketd.json b/openapi/wicketd.json
index 7d50a38268..34e7eadb54 100644
--- a/openapi/wicketd.json
+++ b/openapi/wicketd.json
@@ -85,7 +85,7 @@
     },
     "/baseboard": {
       "get": {
-        "summary": "Report the configured baseboard details",
+        "summary": "Report the configured baseboard details.",
         "operationId": "get_baseboard",
         "responses": {
           "200": {
@@ -214,8 +214,8 @@
     },
     "/inventory": {
       "get": {
-        "summary": "A status endpoint used to report high level information known to wicketd.",
-        "description": "This endpoint can be polled to see if there have been state changes in the system that are useful to report to wicket.\nWicket, and possibly other callers, will retrieve the changed information, with follow up calls.",
+        "summary": "A status endpoint used to report high level information known to",
+        "description": "wicketd.\nThis endpoint can be polled to see if there have been state changes in the system that are useful to report to wicket.\nWicket, and possibly other callers, will retrieve the changed information, with follow up calls.",
         "operationId": "get_inventory",
         "requestBody": {
           "content": {
@@ -274,8 +274,8 @@
     },
     "/preflight/uplink": {
       "get": {
-        "summary": "An endpoint to get the report for the most recent (or still running)",
-        "description": "preflight uplink check.",
+        "summary": "Get the report for the most recent (or still running) preflight uplink",
+        "description": "check.",
         "operationId": "get_preflight_uplink_report",
         "responses": {
           "200": {
@@ -283,7 +283,7 @@
             "content": {
               "application/json": {
                 "schema": {
-                  "$ref": "#/components/schemas/EventReportForGenericSpec"
+                  "$ref": "#/components/schemas/EventReportForUplinkPreflightCheckSpec"
                 }
               }
             }
@@ -297,7 +297,7 @@
         }
       },
       "post": {
-        "summary": "An endpoint to start a preflight check for uplink configuration.",
+        "summary": "Start a preflight check for uplink configuration.",
         "operationId": "post_start_preflight_uplink_check",
         "requestBody": {
           "content": {
@@ -637,7 +637,7 @@
     },
     "/reload-config": {
       "post": {
-        "summary": "An endpoint instructing wicketd to reload its SMF config properties.",
+        "summary": "Instruct wicketd to reload its SMF config properties.",
         "description": "The only expected client of this endpoint is `curl` from wicketd's SMF `refresh` method, but other clients hitting it is harmless.",
         "operationId": "post_reload_config",
         "responses": {
@@ -880,7 +880,8 @@
               },
               "revision": {
                 "type": "integer",
-                "format": "int64"
+                "format": "uint32",
+                "minimum": 0
               },
               "type": {
                 "type": "string",
@@ -1390,7 +1391,7 @@
           "request_id"
         ]
       },
-      "EventReportForGenericSpec": {
+      "EventReportForUplinkPreflightCheckSpec": {
         "description": "A report produced from an [`EventBuffer`](crate::EventBuffer).\n\nRemote reports can be passed into a [`StepContext`](crate::StepContext), in which case they show up as nested events.",
         "type": "object",
         "properties": {
@@ -1405,7 +1406,7 @@
             "description": "A list of progress events, or whether we're currently waiting for a progress event.\n\nCurrently, this produces one progress event for each top-level and nested event in progress.",
             "type": "array",
             "items": {
-              "$ref": "#/components/schemas/ProgressEventForGenericSpec"
+              "$ref": "#/components/schemas/ProgressEventForUplinkPreflightCheckSpec"
             }
           },
           "root_execution_id": {
@@ -1418,7 +1419,7 @@
             "description": "A list of step events.\n\nStep events include success and failure events.",
             "type": "array",
             "items": {
-              "$ref": "#/components/schemas/StepEventForGenericSpec"
+              "$ref": "#/components/schemas/StepEventForUplinkPreflightCheckSpec"
             }
           }
         },
@@ -1547,7 +1548,7 @@
         "type": "object",
         "properties": {
           "force_refresh": {
-            "description": "If true, refresh the state of these SPs from MGS prior to returning (instead of returning cached data).",
+            "description": "Refresh the state of these SPs from MGS prior to returning (instead of returning cached data).",
             "type": "array",
             "items": {
               "$ref": "#/components/schemas/SpIdentifier"
@@ -1892,6 +1893,42 @@
           "total_elapsed"
         ]
       },
+      "ProgressEventForUplinkPreflightCheckSpec": {
+        "type": "object",
+        "properties": {
+          "data": {
+            "description": "The kind of event this is.",
+            "allOf": [
+              {
+                "$ref": "#/components/schemas/ProgressEventKindForUplinkPreflightCheckSpec"
+              }
+            ]
+          },
+          "execution_id": {
+            "description": "The execution ID.",
+            "type": "string",
+            "format": "uuid"
+          },
+          "spec": {
+            "description": "The specification that this event belongs to.\n\nThis is typically the name of the type `S` for which `StepSpec` is implemented.\n\nThis can be used with `Self::from_generic` to deserialize generic metadata.",
+            "type": "string"
+          },
+          "total_elapsed": {
+            "description": "Total time elapsed since the start of execution.",
+            "allOf": [
+              {
+                "$ref": "#/components/schemas/Duration"
+              }
+            ]
+          }
+        },
+        "required": [
+          "data",
+          "execution_id",
+          "spec",
+          "total_elapsed"
+        ]
+      },
       "ProgressEventForWicketdEngineSpec": {
         "type": "object",
         "properties": {
@@ -2114,6 +2151,193 @@
           }
         ]
       },
+      "ProgressEventKindForUplinkPreflightCheckSpec": {
+        "oneOf": [
+          {
+            "description": "The update engine is waiting for a progress message.\n\nThe update engine sends this message immediately after a [`StepEvent`] corresponding to a new step.",
+            "type": "object",
+            "properties": {
+              "attempt": {
+                "description": "The attempt number currently being executed.",
+                "type": "integer",
+                "format": "uint",
+                "minimum": 0
+              },
+              "attempt_elapsed": {
+                "description": "Total time elapsed since the start of the attempt.",
+                "allOf": [
+                  {
+                    "$ref": "#/components/schemas/Duration"
+                  }
+                ]
+              },
+              "kind": {
+                "type": "string",
+                "enum": [
+                  "waiting_for_progress"
+                ]
+              },
+              "step": {
+                "description": "Information about the step.",
+                "allOf": [
+                  {
+                    "$ref": "#/components/schemas/StepInfoWithMetadataForUplinkPreflightCheckSpec"
+                  }
+                ]
+              },
+              "step_elapsed": {
+                "description": "Total time elapsed since the start of the step. Includes prior attempts.",
+                "allOf": [
+                  {
+                    "$ref": "#/components/schemas/Duration"
+                  }
+                ]
+              }
+            },
+            "required": [
+              "attempt",
+              "attempt_elapsed",
+              "kind",
+              "step",
+              "step_elapsed"
+            ]
+          },
+          {
+            "type": "object",
+            "properties": {
+              "attempt": {
+                "description": "The attempt number currently being executed.",
+                "type": "integer",
+                "format": "uint",
+                "minimum": 0
+              },
+              "attempt_elapsed": {
+                "description": "Total time elapsed since the start of the attempt.",
+                "allOf": [
+                  {
+                    "$ref": "#/components/schemas/Duration"
+                  }
+                ]
+              },
+              "kind": {
+                "type": "string",
+                "enum": [
+                  "progress"
+                ]
+              },
+              "metadata": {
+                "description": "Metadata that was returned with progress.",
+                "type": "string"
+              },
+              "progress": {
+                "nullable": true,
+                "description": "Current progress.",
+                "allOf": [
+                  {
+                    "$ref": "#/components/schemas/ProgressCounter"
+                  }
+                ]
+              },
+              "step": {
+                "description": "Information about the step.",
+                "allOf": [
+                  {
+                    "$ref": "#/components/schemas/StepInfoWithMetadataForUplinkPreflightCheckSpec"
+                  }
+                ]
+              },
+              "step_elapsed": {
+                "description": "Total time elapsed since the start of the step. Includes prior attempts.",
+                "allOf": [
+                  {
+                    "$ref": "#/components/schemas/Duration"
+                  }
+                ]
+              }
+            },
+            "required": [
+              "attempt",
+              "attempt_elapsed",
+              "kind",
+              "metadata",
+              "step",
+              "step_elapsed"
+            ]
+          },
+          {
+            "type": "object",
+            "properties": {
+              "attempt": {
+                "description": "The attempt number currently being executed.",
+                "type": "integer",
+                "format": "uint",
+                "minimum": 0
+              },
+              "attempt_elapsed": {
+                "description": "The time it took for this attempt to complete.",
+                "allOf": [
+                  {
+                    "$ref": "#/components/schemas/Duration"
+                  }
+                ]
+              },
+              "event": {
+                "description": "The event that occurred.",
+                "allOf": [
+                  {
+                    "$ref": "#/components/schemas/ProgressEventForGenericSpec"
+                  }
+                ]
+              },
+              "kind": {
+                "type": "string",
+                "enum": [
+                  "nested"
+                ]
+              },
+              "step": {
+                "description": "Information about the step.",
+                "allOf": [
+                  {
+                    "$ref": "#/components/schemas/StepInfoWithMetadataForUplinkPreflightCheckSpec"
+                  }
+                ]
+              },
+              "step_elapsed": {
+                "description": "Total time elapsed since the start of the step. Includes prior attempts.",
+                "allOf": [
+                  {
+                    "$ref": "#/components/schemas/Duration"
+                  }
+                ]
+              }
+            },
+            "required": [
+              "attempt",
+              "attempt_elapsed",
+              "event",
+              "kind",
+              "step",
+              "step_elapsed"
+            ]
+          },
+          {
+            "description": "Future variants that might be unknown.",
+            "type": "object",
+            "properties": {
+              "kind": {
+                "type": "string",
+                "enum": [
+                  "unknown"
+                ]
+              }
+            },
+            "required": [
+              "kind"
+            ]
+          }
+        ]
+      },
       "ProgressEventKindForWicketdEngineSpec": {
         "oneOf": [
           {
@@ -3382,6 +3606,25 @@
           "total_component_steps"
         ]
       },
+      "StepComponentSummaryForUplinkPreflightCheckSpec": {
+        "type": "object",
+        "properties": {
+          "component": {
+            "description": "The component.",
+            "type": "string"
+          },
+          "total_component_steps": {
+            "description": "The number of steps present in this component.",
+            "type": "integer",
+            "format": "uint",
+            "minimum": 0
+          }
+        },
+        "required": [
+          "component",
+          "total_component_steps"
+        ]
+      },
       "StepComponentSummaryForWicketdEngineSpec": {
         "type": "object",
         "properties": {
@@ -3448,14 +3691,14 @@
           "total_elapsed"
         ]
       },
-      "StepEventForWicketdEngineSpec": {
+      "StepEventForUplinkPreflightCheckSpec": {
         "type": "object",
         "properties": {
           "data": {
             "description": "The kind of event this is.",
             "allOf": [
               {
-                "$ref": "#/components/schemas/StepEventKindForWicketdEngineSpec"
+                "$ref": "#/components/schemas/StepEventKindForUplinkPreflightCheckSpec"
               }
             ]
           },
@@ -3491,17 +3734,60 @@
           "total_elapsed"
         ]
       },
-      "StepEventKindForGenericSpec": {
-        "oneOf": [
-          {
-            "description": "No steps were defined, and the executor exited without doing anything.\n\nThis is a terminal event: it is guaranteed that no more events will be seen after this one.",
-            "type": "object",
-            "properties": {
-              "kind": {
-                "type": "string",
-                "enum": [
-                  "no_steps_defined"
-                ]
+      "StepEventForWicketdEngineSpec": {
+        "type": "object",
+        "properties": {
+          "data": {
+            "description": "The kind of event this is.",
+            "allOf": [
+              {
+                "$ref": "#/components/schemas/StepEventKindForWicketdEngineSpec"
+              }
+            ]
+          },
+          "event_index": {
+            "description": "A monotonically increasing index for this `StepEvent`.",
+            "type": "integer",
+            "format": "uint",
+            "minimum": 0
+          },
+          "execution_id": {
+            "description": "The execution ID.",
+            "type": "string",
+            "format": "uuid"
+          },
+          "spec": {
+            "description": "The specification that this event belongs to.\n\nThis is typically the name of the type `S` for which `StepSpec` is implemented.\n\nThis can be used along with `Self::from_generic` to identify which specification to deserialize generic metadata against. For example:\n\n```rust,ignore if event.spec == \"MySpec\" { // event is likely generated from a MySpec engine. let event = Event::<MySpec>::from_generic(event)?; // ... } ```",
+            "type": "string"
+          },
+          "total_elapsed": {
+            "description": "Total time elapsed since the start of execution.",
+            "allOf": [
+              {
+                "$ref": "#/components/schemas/Duration"
+              }
+            ]
+          }
+        },
+        "required": [
+          "data",
+          "event_index",
+          "execution_id",
+          "spec",
+          "total_elapsed"
+        ]
+      },
+      "StepEventKindForGenericSpec": {
+        "oneOf": [
+          {
+            "description": "No steps were defined, and the executor exited without doing anything.\n\nThis is a terminal event: it is guaranteed that no more events will be seen after this one.",
+            "type": "object",
+            "properties": {
+              "kind": {
+                "type": "string",
+                "enum": [
+                  "no_steps_defined"
+                ]
               }
             },
             "required": [
@@ -3976,7 +4262,7 @@
           }
         ]
       },
-      "StepEventKindForWicketdEngineSpec": {
+      "StepEventKindForUplinkPreflightCheckSpec": {
         "oneOf": [
           {
             "description": "No steps were defined, and the executor exited without doing anything.\n\nThis is a terminal event: it is guaranteed that no more events will be seen after this one.",
@@ -4001,14 +4287,14 @@
                 "description": "A list of components, along with the number of items each component has.",
                 "type": "array",
                 "items": {
-                  "$ref": "#/components/schemas/StepComponentSummaryForWicketdEngineSpec"
+                  "$ref": "#/components/schemas/StepComponentSummaryForUplinkPreflightCheckSpec"
                 }
               },
               "first_step": {
                 "description": "Information about the first step.",
                 "allOf": [
                   {
-                    "$ref": "#/components/schemas/StepInfoWithMetadataForWicketdEngineSpec"
+                    "$ref": "#/components/schemas/StepInfoWithMetadataForUplinkPreflightCheckSpec"
                   }
                 ]
               },
@@ -4022,7 +4308,7 @@
                 "description": "The list of steps that will be executed.",
                 "type": "array",
                 "items": {
-                  "$ref": "#/components/schemas/StepInfoForWicketdEngineSpec"
+                  "$ref": "#/components/schemas/StepInfoForUplinkPreflightCheckSpec"
                 }
               }
             },
@@ -4062,13 +4348,14 @@
                 "type": "string"
               },
               "metadata": {
-                "description": "Progress-related metadata associated with this attempt."
+                "description": "Progress-related metadata associated with this attempt.",
+                "type": "string"
               },
               "step": {
                 "description": "Information about the step.",
                 "allOf": [
                   {
-                    "$ref": "#/components/schemas/StepInfoWithMetadataForWicketdEngineSpec"
+                    "$ref": "#/components/schemas/StepInfoWithMetadataForUplinkPreflightCheckSpec"
                   }
                 ]
               },
@@ -4123,7 +4410,7 @@
                 "description": "Information about the step.",
                 "allOf": [
                   {
-                    "$ref": "#/components/schemas/StepInfoWithMetadataForWicketdEngineSpec"
+                    "$ref": "#/components/schemas/StepInfoWithMetadataForUplinkPreflightCheckSpec"
                   }
                 ]
               },
@@ -4173,7 +4460,7 @@
                 "description": "The next step that is being started.",
                 "allOf": [
                   {
-                    "$ref": "#/components/schemas/StepInfoWithMetadataForWicketdEngineSpec"
+                    "$ref": "#/components/schemas/StepInfoWithMetadataForUplinkPreflightCheckSpec"
                   }
                 ]
               },
@@ -4181,7 +4468,7 @@
                 "description": "The outcome of the step.",
                 "allOf": [
                   {
-                    "$ref": "#/components/schemas/StepOutcomeForWicketdEngineSpec"
+                    "$ref": "#/components/schemas/StepOutcomeForUplinkPreflightCheckSpec"
                   }
                 ]
               },
@@ -4189,7 +4476,7 @@
                 "description": "Information about the step that just completed.",
                 "allOf": [
                   {
-                    "$ref": "#/components/schemas/StepInfoWithMetadataForWicketdEngineSpec"
+                    "$ref": "#/components/schemas/StepInfoWithMetadataForUplinkPreflightCheckSpec"
                   }
                 ]
               },
@@ -4240,7 +4527,7 @@
                 "description": "The outcome of the last step.",
                 "allOf": [
                   {
-                    "$ref": "#/components/schemas/StepOutcomeForWicketdEngineSpec"
+                    "$ref": "#/components/schemas/StepOutcomeForUplinkPreflightCheckSpec"
                   }
                 ]
               },
@@ -4248,7 +4535,7 @@
                 "description": "Information about the last step that completed.",
                 "allOf": [
                   {
-                    "$ref": "#/components/schemas/StepInfoWithMetadataForWicketdEngineSpec"
+                    "$ref": "#/components/schemas/StepInfoWithMetadataForUplinkPreflightCheckSpec"
                   }
                 ]
               },
@@ -4293,7 +4580,7 @@
                 "description": "Information about the step that failed.",
                 "allOf": [
                   {
-                    "$ref": "#/components/schemas/StepInfoWithMetadataForWicketdEngineSpec"
+                    "$ref": "#/components/schemas/StepInfoWithMetadataForUplinkPreflightCheckSpec"
                   }
                 ]
               },
@@ -4340,7 +4627,7 @@
                 "description": "Information about the step that was running at the time execution was aborted.",
                 "allOf": [
                   {
-                    "$ref": "#/components/schemas/StepInfoWithMetadataForWicketdEngineSpec"
+                    "$ref": "#/components/schemas/StepInfoWithMetadataForUplinkPreflightCheckSpec"
                   }
                 ]
               },
@@ -4422,7 +4709,7 @@
                 "description": "Information about the step that's occurring.",
                 "allOf": [
                   {
-                    "$ref": "#/components/schemas/StepInfoWithMetadataForWicketdEngineSpec"
+                    "$ref": "#/components/schemas/StepInfoWithMetadataForUplinkPreflightCheckSpec"
                   }
                 ]
               },
@@ -4461,162 +4748,17 @@
           }
         ]
       },
-      "StepInfoForGenericSpec": {
-        "description": "Serializable information about a step.",
-        "type": "object",
-        "properties": {
-          "component": {
-            "description": "The component that this step is part of."
-          },
-          "component_index": {
-            "description": "The index of the step within the component.",
-            "type": "integer",
-            "format": "uint",
-            "minimum": 0
-          },
-          "description": {
-            "description": "The description for this step.",
-            "type": "string"
-          },
-          "id": {
-            "description": "An identifier for this step."
-          },
-          "index": {
-            "description": "The index of the step within all steps to be executed.",
-            "type": "integer",
-            "format": "uint",
-            "minimum": 0
-          },
-          "total_component_steps": {
-            "description": "The total number of steps in this component.",
-            "type": "integer",
-            "format": "uint",
-            "minimum": 0
-          }
-        },
-        "required": [
-          "component",
-          "component_index",
-          "description",
-          "id",
-          "index",
-          "total_component_steps"
-        ]
-      },
-      "StepInfoForWicketdEngineSpec": {
-        "description": "Serializable information about a step.",
-        "type": "object",
-        "properties": {
-          "component": {
-            "description": "The component that this step is part of.",
-            "allOf": [
-              {
-                "$ref": "#/components/schemas/UpdateComponent"
-              }
-            ]
-          },
-          "component_index": {
-            "description": "The index of the step within the component.",
-            "type": "integer",
-            "format": "uint",
-            "minimum": 0
-          },
-          "description": {
-            "description": "The description for this step.",
-            "type": "string"
-          },
-          "id": {
-            "description": "An identifier for this step.",
-            "allOf": [
-              {
-                "$ref": "#/components/schemas/UpdateStepId"
-              }
-            ]
-          },
-          "index": {
-            "description": "The index of the step within all steps to be executed.",
-            "type": "integer",
-            "format": "uint",
-            "minimum": 0
-          },
-          "total_component_steps": {
-            "description": "The total number of steps in this component.",
-            "type": "integer",
-            "format": "uint",
-            "minimum": 0
-          }
-        },
-        "required": [
-          "component",
-          "component_index",
-          "description",
-          "id",
-          "index",
-          "total_component_steps"
-        ]
-      },
-      "StepInfoWithMetadataForGenericSpec": {
-        "description": "Serializable information about a step.",
-        "type": "object",
-        "properties": {
-          "info": {
-            "description": "Information about this step.",
-            "allOf": [
-              {
-                "$ref": "#/components/schemas/StepInfoForGenericSpec"
-              }
-            ]
-          },
-          "metadata": {
-            "nullable": true,
-            "description": "Additional metadata associated with this step."
-          }
-        },
-        "required": [
-          "info"
-        ]
-      },
-      "StepInfoWithMetadataForWicketdEngineSpec": {
-        "description": "Serializable information about a step.",
-        "type": "object",
-        "properties": {
-          "info": {
-            "description": "Information about this step.",
-            "allOf": [
-              {
-                "$ref": "#/components/schemas/StepInfoForWicketdEngineSpec"
-              }
-            ]
-          },
-          "metadata": {
-            "nullable": true,
-            "description": "Additional metadata associated with this step."
-          }
-        },
-        "required": [
-          "info"
-        ]
-      },
-      "StepOutcomeForGenericSpec": {
+      "StepEventKindForWicketdEngineSpec": {
         "oneOf": [
           {
-            "description": "The step completed successfully.",
+            "description": "No steps were defined, and the executor exited without doing anything.\n\nThis is a terminal event: it is guaranteed that no more events will be seen after this one.",
             "type": "object",
             "properties": {
               "kind": {
                 "type": "string",
                 "enum": [
-                  "success"
+                  "no_steps_defined"
                 ]
-              },
-              "message": {
-                "nullable": true,
-                "description": "An optional message associated with this step.",
-                "type": "string"
-              },
-              "metadata": {
-                "nullable": true,
-                "description": "Optional completion metadata associated with the step."
               }
             },
             "required": [
@@ -4624,38 +4766,741 @@
             ]
           },
           {
-            "description": "The step completed with a warning.",
+            "description": "Execution was started.\n\nThis is an initial event -- it is always expected to be the first event received from the event stream.",
             "type": "object",
             "properties": {
+              "components": {
+                "description": "A list of components, along with the number of items each component has.",
+                "type": "array",
+                "items": {
+                  "$ref": "#/components/schemas/StepComponentSummaryForWicketdEngineSpec"
+                }
+              },
+              "first_step": {
+                "description": "Information about the first step.",
+                "allOf": [
+                  {
+                    "$ref": "#/components/schemas/StepInfoWithMetadataForWicketdEngineSpec"
+                  }
+                ]
+              },
               "kind": {
                 "type": "string",
                 "enum": [
-                  "warning"
+                  "execution_started"
                 ]
               },
-              "message": {
-                "description": "A warning message.",
-                "type": "string"
-              },
-              "metadata": {
-                "nullable": true,
-                "description": "Optional completion metadata associated with the step."
+              "steps": {
+                "description": "The list of steps that will be executed.",
+                "type": "array",
+                "items": {
+                  "$ref": "#/components/schemas/StepInfoForWicketdEngineSpec"
+                }
               }
             },
             "required": [
+              "components",
+              "first_step",
               "kind",
-              "message"
+              "steps"
             ]
           },
           {
-            "description": "The step was skipped with a message.",
+            "description": "Progress was reset along an attempt, and this attempt is going down a different path.",
             "type": "object",
             "properties": {
-              "kind": {
-                "type": "string",
-                "enum": [
-                  "skipped"
-                ]
+              "attempt": {
+                "description": "The current attempt number.",
+                "type": "integer",
+                "format": "uint",
+                "minimum": 0
+              },
+              "attempt_elapsed": {
+                "description": "The amount of time this attempt has taken so far.",
+                "allOf": [
+                  {
+                    "$ref": "#/components/schemas/Duration"
+                  }
+                ]
+              },
+              "kind": {
+                "type": "string",
+                "enum": [
+                  "progress_reset"
+                ]
+              },
+              "message": {
+                "description": "A message assocaited with the reset.",
+                "type": "string"
+              },
+              "metadata": {
+                "description": "Progress-related metadata associated with this attempt."
+              },
+              "step": {
+                "description": "Information about the step.",
+                "allOf": [
+                  {
+                    "$ref": "#/components/schemas/StepInfoWithMetadataForWicketdEngineSpec"
+                  }
+                ]
+              },
+              "step_elapsed": {
+                "description": "Total time elapsed since the start of the step. Includes prior attempts.",
+                "allOf": [
+                  {
+                    "$ref": "#/components/schemas/Duration"
+                  }
+                ]
+              }
+            },
+            "required": [
+              "attempt",
+              "attempt_elapsed",
+              "kind",
+              "message",
+              "metadata",
+              "step",
+              "step_elapsed"
+            ]
+          },
+          {
+            "description": "An attempt failed and this step is being retried.",
+            "type": "object",
+            "properties": {
+              "attempt_elapsed": {
+                "description": "The amount of time the previous attempt took.",
+                "allOf": [
+                  {
+                    "$ref": "#/components/schemas/Duration"
+                  }
+                ]
+              },
+              "kind": {
+                "type": "string",
+                "enum": [
+                  "attempt_retry"
+                ]
+              },
+              "message": {
+                "description": "A message associated with the retry.",
+                "type": "string"
+              },
+              "next_attempt": {
+                "description": "The attempt number for the next attempt.",
+                "type": "integer",
+                "format": "uint",
+                "minimum": 0
+              },
+              "step": {
+                "description": "Information about the step.",
+                "allOf": [
+                  {
+                    "$ref": "#/components/schemas/StepInfoWithMetadataForWicketdEngineSpec"
+                  }
+                ]
+              },
+              "step_elapsed": {
+                "description": "Total time elapsed since the start of the step. Includes prior attempts.",
+                "allOf": [
+                  {
+                    "$ref": "#/components/schemas/Duration"
+                  }
+                ]
+              }
+            },
+            "required": [
+              "attempt_elapsed",
+              "kind",
+              "message",
+              "next_attempt",
+              "step",
+              "step_elapsed"
+            ]
+          },
+          {
+            "description": "A step is complete and the next step has been started.",
+            "type": "object",
+            "properties": {
+              "attempt": {
+                "description": "The attempt number that completed.",
+                "type": "integer",
+                "format": "uint",
+                "minimum": 0
+              },
+              "attempt_elapsed": {
+                "description": "The time it took for this attempt to complete.",
+                "allOf": [
+                  {
+                    "$ref": "#/components/schemas/Duration"
+                  }
+                ]
+              },
+              "kind": {
+                "type": "string",
+                "enum": [
+                  "step_completed"
+                ]
+              },
+              "next_step": {
+                "description": "The next step that is being started.",
+                "allOf": [
+                  {
+                    "$ref": "#/components/schemas/StepInfoWithMetadataForWicketdEngineSpec"
+                  }
+                ]
+              },
+              "outcome": {
+                "description": "The outcome of the step.",
+                "allOf": [
+                  {
+                    "$ref": "#/components/schemas/StepOutcomeForWicketdEngineSpec"
+                  }
+                ]
+              },
+              "step": {
+                "description": "Information about the step that just completed.",
+                "allOf": [
+                  {
+                    "$ref": "#/components/schemas/StepInfoWithMetadataForWicketdEngineSpec"
+                  }
+                ]
+              },
+              "step_elapsed": {
+                "description": "Total time elapsed since the start of the step. Includes prior attempts.",
+                "allOf": [
+                  {
+                    "$ref": "#/components/schemas/Duration"
+                  }
+                ]
+              }
+            },
+            "required": [
+              "attempt",
+              "attempt_elapsed",
+              "kind",
+              "next_step",
+              "outcome",
+              "step",
+              "step_elapsed"
+            ]
+          },
+          {
+            "description": "Execution is complete.\n\nThis is a terminal event: it is guaranteed that no more events will be seen after this one.",
+            "type": "object",
+            "properties": {
+              "attempt_elapsed": {
+                "description": "The time it took for this attempt to complete.",
+                "allOf": [
+                  {
+                    "$ref": "#/components/schemas/Duration"
+                  }
+                ]
+              },
+              "kind": {
+                "type": "string",
+                "enum": [
+                  "execution_completed"
+                ]
+              },
+              "last_attempt": {
+                "description": "The attempt number that completed.",
+                "type": "integer",
+                "format": "uint",
+                "minimum": 0
+              },
+              "last_outcome": {
+                "description": "The outcome of the last step.",
+                "allOf": [
+                  {
+                    "$ref": "#/components/schemas/StepOutcomeForWicketdEngineSpec"
+                  }
+                ]
+              },
+              "last_step": {
+                "description": "Information about the last step that completed.",
+                "allOf": [
+                  {
+                    "$ref": "#/components/schemas/StepInfoWithMetadataForWicketdEngineSpec"
+                  }
+                ]
+              },
+              "step_elapsed": {
+                "description": "Total time elapsed since the start of the step. Includes prior attempts.",
+                "allOf": [
+                  {
+                    "$ref": "#/components/schemas/Duration"
+                  }
+                ]
+              }
+            },
+            "required": [
+              "attempt_elapsed",
+              "kind",
+              "last_attempt",
+              "last_outcome",
+              "last_step",
+              "step_elapsed"
+            ]
+          },
+          {
+            "description": "Execution failed.\n\nThis is a terminal event: it is guaranteed that no more events will be seen after this one.",
+            "type": "object",
+            "properties": {
+              "attempt_elapsed": {
+                "description": "The time it took for this attempt to complete.",
+                "allOf": [
+                  {
+                    "$ref": "#/components/schemas/Duration"
+                  }
+                ]
+              },
+              "causes": {
+                "description": "A chain of causes associated with the failure.",
+                "type": "array",
+                "items": {
+                  "type": "string"
+                }
+              },
+              "failed_step": {
+                "description": "Information about the step that failed.",
+                "allOf": [
+                  {
+                    "$ref": "#/components/schemas/StepInfoWithMetadataForWicketdEngineSpec"
+                  }
+                ]
+              },
+              "kind": {
+                "type": "string",
+                "enum": [
+                  "execution_failed"
+                ]
+              },
+              "message": {
+                "description": "A message associated with the failure.",
+                "type": "string"
+              },
+              "step_elapsed": {
+                "description": "Total time elapsed since the start of the step. Includes prior attempts.",
+                "allOf": [
+                  {
+                    "$ref": "#/components/schemas/Duration"
+                  }
+                ]
+              },
+              "total_attempts": {
+                "description": "The total number of attempts that were performed before the step failed.",
+                "type": "integer",
+                "format": "uint",
+                "minimum": 0
+              }
+            },
+            "required": [
+              "attempt_elapsed",
+              "causes",
+              "failed_step",
+              "kind",
+              "message",
+              "step_elapsed",
+              "total_attempts"
+            ]
+          },
+          {
+            "description": "Execution aborted by an external user.\n\nThis is a terminal event: it is guaranteed that no more events will be seen after this one.",
+            "type": "object",
+            "properties": {
+              "aborted_step": {
+                "description": "Information about the step that was running at the time execution was aborted.",
+                "allOf": [
+                  {
+                    "$ref": "#/components/schemas/StepInfoWithMetadataForWicketdEngineSpec"
+                  }
+                ]
+              },
+              "attempt": {
+                "description": "The attempt that was running at the time the step was aborted.",
+                "type": "integer",
+                "format": "uint",
+                "minimum": 0
+              },
+              "attempt_elapsed": {
+                "description": "The time it took for this attempt to complete.",
+                "allOf": [
+                  {
+                    "$ref": "#/components/schemas/Duration"
+                  }
+                ]
+              },
+              "kind": {
+                "type": "string",
+                "enum": [
+                  "execution_aborted"
+                ]
+              },
+              "message": {
+                "description": "A message associated with the abort.",
+                "type": "string"
+              },
+              "step_elapsed": {
+                "description": "Total time elapsed since the start of the step. Includes prior attempts.",
+                "allOf": [
+                  {
+                    "$ref": "#/components/schemas/Duration"
+                  }
+                ]
+              }
+            },
+            "required": [
+              "aborted_step",
+              "attempt",
+              "attempt_elapsed",
+              "kind",
+              "message",
+              "step_elapsed"
+            ]
+          },
+          {
+            "description": "A nested step event occurred.",
+            "type": "object",
+            "properties": {
+              "attempt": {
+                "description": "The current attempt number.",
+                "type": "integer",
+                "format": "uint",
+                "minimum": 0
+              },
+              "attempt_elapsed": {
+                "description": "The time it took for this attempt to complete.",
+                "allOf": [
+                  {
+                    "$ref": "#/components/schemas/Duration"
+                  }
+                ]
+              },
+              "event": {
+                "description": "The event that occurred.",
+                "allOf": [
+                  {
+                    "$ref": "#/components/schemas/StepEventForGenericSpec"
+                  }
+                ]
+              },
+              "kind": {
+                "type": "string",
+                "enum": [
+                  "nested"
+                ]
+              },
+              "step": {
+                "description": "Information about the step that's occurring.",
+                "allOf": [
+                  {
+                    "$ref": "#/components/schemas/StepInfoWithMetadataForWicketdEngineSpec"
+                  }
+                ]
+              },
+              "step_elapsed": {
+                "description": "Total time elapsed since the start of the step. Includes prior attempts.",
+                "allOf": [
+                  {
+                    "$ref": "#/components/schemas/Duration"
+                  }
+                ]
+              }
+            },
+            "required": [
+              "attempt",
+              "attempt_elapsed",
+              "event",
+              "kind",
+              "step",
+              "step_elapsed"
+            ]
+          },
+          {
+            "description": "Future variants that might be unknown.",
+            "type": "object",
+            "properties": {
+              "kind": {
+                "type": "string",
+                "enum": [
+                  "unknown"
+                ]
+              }
+            },
+            "required": [
+              "kind"
+            ]
+          }
+        ]
+      },
+      "StepInfoForGenericSpec": {
+        "description": "Serializable information about a step.",
+        "type": "object",
+        "properties": {
+          "component": {
+            "description": "The component that this step is part of."
+          },
+          "component_index": {
+            "description": "The index of the step within the component.",
+            "type": "integer",
+            "format": "uint",
+            "minimum": 0
+          },
+          "description": {
+            "description": "The description for this step.",
+            "type": "string"
+          },
+          "id": {
+            "description": "An identifier for this step."
+          },
+          "index": {
+            "description": "The index of the step within all steps to be executed.",
+            "type": "integer",
+            "format": "uint",
+            "minimum": 0
+          },
+          "total_component_steps": {
+            "description": "The total number of steps in this component.",
+            "type": "integer",
+            "format": "uint",
+            "minimum": 0
+          }
+        },
+        "required": [
+          "component",
+          "component_index",
+          "description",
+          "id",
+          "index",
+          "total_component_steps"
+        ]
+      },
+      "StepInfoForUplinkPreflightCheckSpec": {
+        "description": "Serializable information about a step.",
+        "type": "object",
+        "properties": {
+          "component": {
+            "description": "The component that this step is part of.",
+            "type": "string"
+          },
+          "component_index": {
+            "description": "The index of the step within the component.",
+            "type": "integer",
+            "format": "uint",
+            "minimum": 0
+          },
+          "description": {
+            "description": "The description for this step.",
+            "type": "string"
+          },
+          "id": {
+            "description": "An identifier for this step.",
+            "allOf": [
+              {
+                "$ref": "#/components/schemas/UplinkPreflightStepId"
+              }
+            ]
+          },
+          "index": {
+            "description": "The index of the step within all steps to be executed.",
+            "type": "integer",
+            "format": "uint",
+            "minimum": 0
+          },
+          "total_component_steps": {
+            "description": "The total number of steps in this component.",
+            "type": "integer",
+            "format": "uint",
+            "minimum": 0
+          }
+        },
+        "required": [
+          "component",
+          "component_index",
+          "description",
+          "id",
+          "index",
+          "total_component_steps"
+        ]
+      },
+      "StepInfoForWicketdEngineSpec": {
+        "description": "Serializable information about a step.",
+        "type": "object",
+        "properties": {
+          "component": {
+            "description": "The component that this step is part of.",
+            "allOf": [
+              {
+                "$ref": "#/components/schemas/UpdateComponent"
+              }
+            ]
+          },
+          "component_index": {
+            "description": "The index of the step within the component.",
+            "type": "integer",
+            "format": "uint",
+            "minimum": 0
+          },
+          "description": {
+            "description": "The description for this step.",
+            "type": "string"
+          },
+          "id": {
+            "description": "An identifier for this step.",
+            "allOf": [
+              {
+                "$ref": "#/components/schemas/UpdateStepId"
+              }
+            ]
+          },
+          "index": {
+            "description": "The index of the step within all steps to be executed.",
+            "type": "integer",
+            "format": "uint",
+            "minimum": 0
+          },
+          "total_component_steps": {
+            "description": "The total number of steps in this component.",
+            "type": "integer",
+            "format": "uint",
+            "minimum": 0
+          }
+        },
+        "required": [
+          "component",
+          "component_index",
+          "description",
+          "id",
+          "index",
+          "total_component_steps"
+        ]
+      },
+      "StepInfoWithMetadataForGenericSpec": {
+        "description": "Serializable information about a step.",
+        "type": "object",
+        "properties": {
+          "info": {
+            "description": "Information about this step.",
+            "allOf": [
+              {
+                "$ref": "#/components/schemas/StepInfoForGenericSpec"
+              }
+            ]
+          },
+          "metadata": {
+            "nullable": true,
+            "description": "Additional metadata associated with this step."
+          }
+        },
+        "required": [
+          "info"
+        ]
+      },
+      "StepInfoWithMetadataForUplinkPreflightCheckSpec": {
+        "description": "Serializable information about a step.",
+        "type": "object",
+        "properties": {
+          "info": {
+            "description": "Information about this step.",
+            "allOf": [
+              {
+                "$ref": "#/components/schemas/StepInfoForUplinkPreflightCheckSpec"
+              }
+            ]
+          },
+          "metadata": {
+            "nullable": true,
+            "description": "Additional metadata associated with this step.",
+            "type": "string",
+            "enum": [
+              null
+            ]
+          }
+        },
+        "required": [
+          "info"
+        ]
+      },
+      "StepInfoWithMetadataForWicketdEngineSpec": {
+        "description": "Serializable information about a step.",
+        "type": "object",
+        "properties": {
+          "info": {
+            "description": "Information about this step.",
+            "allOf": [
+              {
+                "$ref": "#/components/schemas/StepInfoForWicketdEngineSpec"
+              }
+            ]
+          },
+          "metadata": {
+            "nullable": true,
+            "description": "Additional metadata associated with this step."
+          }
+        },
+        "required": [
+          "info"
+        ]
+      },
+      "StepOutcomeForGenericSpec": {
+        "oneOf": [
+          {
+            "description": "The step completed successfully.",
+            "type": "object",
+            "properties": {
+              "kind": {
+                "type": "string",
+                "enum": [
+                  "success"
+                ]
+              },
+              "message": {
+                "nullable": true,
+                "description": "An optional message associated with this step.",
+                "type": "string"
+              },
+              "metadata": {
+                "nullable": true,
+                "description": "Optional completion metadata associated with the step."
+              }
+            },
+            "required": [
+              "kind"
+            ]
+          },
+          {
+            "description": "The step completed with a warning.",
+            "type": "object",
+            "properties": {
+              "kind": {
+                "type": "string",
+                "enum": [
+                  "warning"
+                ]
+              },
+              "message": {
+                "description": "A warning message.",
+                "type": "string"
+              },
+              "metadata": {
+                "nullable": true,
+                "description": "Optional completion metadata associated with the step."
+              }
+            },
+            "required": [
+              "kind",
+              "message"
+            ]
+          },
+          {
+            "description": "The step was skipped with a message.",
+            "type": "object",
+            "properties": {
+              "kind": {
+                "type": "string",
+                "enum": [
+                  "skipped"
+                ]
               },
               "message": {
                 "description": "Message associated with the skip.",
@@ -4673,6 +5518,94 @@
           }
         ]
       },
+      "StepOutcomeForUplinkPreflightCheckSpec": {
+        "oneOf": [
+          {
+            "description": "The step completed successfully.",
+            "type": "object",
+            "properties": {
+              "kind": {
+                "type": "string",
+                "enum": [
+                  "success"
+                ]
+              },
+              "message": {
+                "nullable": true,
+                "description": "An optional message associated with this step.",
+                "type": "string"
+              },
+              "metadata": {
+                "nullable": true,
+                "description": "Optional completion metadata associated with the step.",
+                "type": "array",
+                "items": {
+                  "type": "string"
+                }
+              }
+            },
+            "required": [
+              "kind"
+            ]
+          },
+          {
+            "description": "The step completed with a warning.",
+            "type": "object",
+            "properties": {
+              "kind": {
+                "type": "string",
+                "enum": [
+                  "warning"
+                ]
+              },
+              "message": {
+                "description": "A warning message.",
+                "type": "string"
+              },
+              "metadata": {
+                "nullable": true,
+                "description": "Optional completion metadata associated with the step.",
+                "type": "array",
+                "items": {
+                  "type": "string"
+                }
+              }
+            },
+            "required": [
+              "kind",
+              "message"
+            ]
+          },
+          {
+            "description": "The step was skipped with a message.",
+            "type": "object",
+            "properties": {
+              "kind": {
+                "type": "string",
+                "enum": [
+                  "skipped"
+                ]
+              },
+              "message": {
+                "description": "Message associated with the skip.",
+                "type": "string"
+              },
+              "metadata": {
+                "nullable": true,
+                "description": "Optional metadata associated with the skip.",
+                "type": "string",
+                "enum": [
+                  null
+                ]
+              }
+            },
+            "required": [
+              "kind",
+              "message"
+            ]
+          }
+        ]
+      },
       "StepOutcomeForWicketdEngineSpec": {
         "oneOf": [
           {
@@ -5063,6 +5996,136 @@
           "address"
         ]
       },
+      "UplinkPreflightStepId": {
+        "oneOf": [
+          {
+            "type": "object",
+            "properties": {
+              "id": {
+                "type": "string",
+                "enum": [
+                  "configure_switch"
+                ]
+              }
+            },
+            "required": [
+              "id"
+            ]
+          },
+          {
+            "type": "object",
+            "properties": {
+              "id": {
+                "type": "string",
+                "enum": [
+                  "wait_for_l1_link"
+                ]
+              }
+            },
+            "required": [
+              "id"
+            ]
+          },
+          {
+            "type": "object",
+            "properties": {
+              "id": {
+                "type": "string",
+                "enum": [
+                  "configure_address"
+                ]
+              }
+            },
+            "required": [
+              "id"
+            ]
+          },
+          {
+            "type": "object",
+            "properties": {
+              "id": {
+                "type": "string",
+                "enum": [
+                  "configure_routing"
+                ]
+              }
+            },
+            "required": [
+              "id"
+            ]
+          },
+          {
+            "type": "object",
+            "properties": {
+              "id": {
+                "type": "string",
+                "enum": [
+                  "check_external_dns_connectivity"
+                ]
+              }
+            },
+            "required": [
+              "id"
+            ]
+          },
+          {
+            "type": "object",
+            "properties": {
+              "id": {
+                "type": "string",
+                "enum": [
+                  "check_external_ntp_connectivity"
+                ]
+              }
+            },
+            "required": [
+              "id"
+            ]
+          },
+          {
+            "type": "object",
+            "properties": {
+              "id": {
+                "type": "string",
+                "enum": [
+                  "cleanup_routing"
+                ]
+              }
+            },
+            "required": [
+              "id"
+            ]
+          },
+          {
+            "type": "object",
+            "properties": {
+              "id": {
+                "type": "string",
+                "enum": [
+                  "cleanup_address"
+                ]
+              }
+            },
+            "required": [
+              "id"
+            ]
+          },
+          {
+            "type": "object",
+            "properties": {
+              "id": {
+                "type": "string",
+                "enum": [
+                  "cleanup_l1"
+                ]
+              }
+            },
+            "required": [
+              "id"
+            ]
+          }
+        ]
+      },
       "UserSpecifiedBgpPeerConfig": {
         "description": "User-specified version of [`BgpPeerConfig`].\n\nThis is similar to [`BgpPeerConfig`], except it doesn't have the sensitive `md5_auth_key` parameter, instead requiring that the user provide the key separately.\n\n[`BgpPeerConfig`]: omicron_common::api::internal::shared::BgpPeerConfig",
         "type": "object",
diff --git a/oximeter/db/src/client/mod.rs b/oximeter/db/src/client/mod.rs
index 2d6212971e..517c52f11e 100644
--- a/oximeter/db/src/client/mod.rs
+++ b/oximeter/db/src/client/mod.rs
@@ -43,6 +43,7 @@ use std::collections::btree_map::Entry;
 use std::collections::BTreeMap;
 use std::collections::BTreeSet;
 use std::convert::TryFrom;
+use std::io::ErrorKind;
 use std::net::SocketAddr;
 use std::num::NonZeroU32;
 use std::ops::Bound;
@@ -490,6 +491,29 @@ impl Client {
                 }
             }
         }
+
+        // Check if we have a list of timeseries that should be deleted, and
+        // remove them from the history books.
+        let to_delete = Self::read_timeseries_to_delete(
+            replicated,
+            next_version,
+            schema_dir,
+        )
+        .await?;
+        if to_delete.is_empty() {
+            debug!(
+                self.log,
+                "schema upgrade contained timeseries list file, \
+                but it did not contain any timeseries names",
+            );
+        } else {
+            debug!(
+                self.log,
+                "schema upgrade includes list of timeseries to be deleted";
+                "n_timeseries" => to_delete.len(),
+            );
+            self.expunge_timeseries_by_name(replicated, &to_delete).await?;
+        }
         Ok(())
     }
 
@@ -961,6 +985,128 @@ impl Client {
         }
         Ok(())
     }
+
+    /// Given a list of timeseries by name, delete their schema and any
+    /// associated data records from all tables.
+    async fn expunge_timeseries_by_name(
+        &self,
+        replicated: bool,
+        to_delete: &[TimeseriesName],
+    ) -> Result<(), Error> {
+        // The version table should not have any matching data, but let's avoid
+        // it entirely anyway.
+        let tables = self
+            .list_oximeter_database_tables(ListDetails {
+                include_version: false,
+                replicated,
+            })
+            .await?;
+
+        // This size is arbitrary, and just something to avoid enormous requests
+        // to ClickHouse. It's unlikely that we'll hit this in practice anyway,
+        // given that we have far fewer than 1000 timeseries today.
+        const DELETE_BATCH_SIZE: usize = 1000;
+        let maybe_on_cluster = if replicated {
+            format!("ON CLUSTER {}", crate::CLUSTER_NAME)
+        } else {
+            String::new()
+        };
+        for chunk in to_delete.chunks(DELETE_BATCH_SIZE) {
+            let names = chunk
+                .iter()
+                .map(|name| format!("'{name}'"))
+                .collect::<Vec<_>>()
+                .join(",");
+            debug!(
+                self.log,
+                "deleting chunk of timeseries";
+                "timeseries_names" => &names,
+                "n_timeseries" => chunk.len(),
+            );
+            for table in tables.iter() {
+                let sql = format!(
+                    "ALTER TABLE {}.{} \
+                    {} \
+                    DELETE WHERE timeseries_name in ({})",
+                    crate::DATABASE_NAME,
+                    table,
+                    maybe_on_cluster,
+                    names,
+                );
+                debug!(
+                    self.log,
+                    "deleting timeseries from next table";
+                    "table_name" => table,
+                    "n_timeseries" => chunk.len(),
+                );
+                self.execute(sql).await?;
+            }
+        }
+        Ok(())
+    }
+
+    async fn read_timeseries_to_delete(
+        replicated: bool,
+        next_version: u64,
+        schema_dir: &Path,
+    ) -> Result<Vec<TimeseriesName>, Error> {
+        let version_schema_dir =
+            Self::full_upgrade_path(replicated, next_version, schema_dir);
+        let filename =
+            version_schema_dir.join(crate::TIMESERIES_TO_DELETE_FILE);
+        match fs::read_to_string(&filename).await {
+            Ok(contents) => contents
+                .lines()
+                .map(|line| line.trim().parse().map_err(Error::from))
+                .collect(),
+            Err(e) if e.kind() == ErrorKind::NotFound => Ok(vec![]),
+            Err(err) => Err(Error::ReadTimeseriesToDeleteFile { err }),
+        }
+    }
+
+    /// List tables in the oximeter database.
+    async fn list_oximeter_database_tables(
+        &self,
+        ListDetails { include_version, replicated }: ListDetails,
+    ) -> Result<Vec<String>, Error> {
+        let mut sql = format!(
+            "SELECT name FROM system.tables WHERE database = '{}'",
+            crate::DATABASE_NAME,
+        );
+        if !include_version {
+            sql.push_str(" AND name != '");
+            sql.push_str(crate::VERSION_TABLE_NAME);
+            sql.push('\'');
+        }
+        // On a cluster, we need to operate on the "local" replicated tables.
+        if replicated {
+            sql.push_str(" AND engine = 'ReplicatedMergeTree'");
+        }
+        self.execute_with_body(sql).await.map(|(_summary, body)| {
+            body.lines().map(ToString::to_string).collect()
+        })
+    }
+}
+
+/// Helper argument to `Client::list_oximeter_database_tables`.
+#[derive(Clone, Copy, Debug, PartialEq)]
+struct ListDetails {
+    /// If true, include the version table in the output.
+    include_version: bool,
+    /// If true, list tables to operate on in a replicated cluster configuration.
+    ///
+    /// NOTE: We would like to always operate on the "top-level table", e.g.
+    /// `oximeter.measurements_u64`, regardless of whether we're working on the
+    /// cluster or a single-node setup. Otherwise, we need to know which cluster
+    /// we're working with, and then query either `measurements_u64` or
+    /// `measurements_u64_local` based on that.
+    ///
+    /// However, while that works for the local tables (even replicated ones),
+    /// it does _not_ work for the `Distributed` tables that we use as those
+    /// "top-level tables" in a cluster setup. That table engine does not
+    /// support mutations. Instead, we need to run those operations on the
+    /// `*_local` tables.
+    replicated: bool,
 }
 
 // A regex used to validate supported schema updates.
@@ -4423,4 +4569,316 @@ mod tests {
             })
             .collect()
     }
+
+    // Helper to write a test file containing timeseries to delete.
+    async fn write_timeseries_to_delete_file(
+        schema_dir: &Path,
+        replicated: bool,
+        version: u64,
+        names: &[TimeseriesName],
+    ) {
+        let subdir = schema_dir
+            .join(if replicated { "replicated" } else { "single-node" })
+            .join(version.to_string());
+        tokio::fs::create_dir_all(&subdir)
+            .await
+            .expect("failed to make subdirectories");
+        let filename = subdir.join(crate::TIMESERIES_TO_DELETE_FILE);
+        let contents = names
+            .iter()
+            .map(ToString::to_string)
+            .collect::<Vec<_>>()
+            .join("\n");
+        tokio::fs::write(&filename, contents)
+            .await
+            .expect("failed to write test timeseries to delete file");
+    }
+
+    #[tokio::test]
+    async fn test_read_timeseries_to_delete() {
+        let names: Vec<TimeseriesName> =
+            vec!["a:b".parse().unwrap(), "c:d".parse().unwrap()];
+        let schema_dir =
+            tempfile::TempDir::new().expect("failed to make temp dir");
+        const VERSION: u64 = 7;
+        write_timeseries_to_delete_file(
+            schema_dir.path(),
+            false,
+            VERSION,
+            &names,
+        )
+        .await;
+        let read = Client::read_timeseries_to_delete(
+            false,
+            VERSION,
+            schema_dir.path(),
+        )
+        .await
+        .expect("Failed to read timeseries to delete");
+        assert_eq!(names, read, "Read incorrect list of timeseries to delete",);
+    }
+
+    #[tokio::test]
+    async fn test_read_timeseries_to_delete_empty_file_is_ok() {
+        let schema_dir =
+            tempfile::TempDir::new().expect("failed to make temp dir");
+        const VERSION: u64 = 7;
+        write_timeseries_to_delete_file(schema_dir.path(), false, VERSION, &[])
+            .await;
+        let read = Client::read_timeseries_to_delete(
+            false,
+            VERSION,
+            schema_dir.path(),
+        )
+        .await
+        .expect("Failed to read timeseries to delete");
+        assert!(read.is_empty(), "Read incorrect list of timeseries to delete",);
+    }
+
+    #[tokio::test]
+    async fn test_read_timeseries_to_delete_nonexistent_file_is_ok() {
+        let path = PathBuf::from("/this/file/better/not/exist");
+        let read = Client::read_timeseries_to_delete(false, 1000000, &path)
+            .await
+            .expect("Failed to read timeseries to delete");
+        assert!(read.is_empty(), "Read incorrect list of timeseries to delete",);
+    }
+
+    #[tokio::test]
+    async fn test_expunge_timeseries_by_name_single_node() {
+        const TEST_NAME: &str = "test_expunge_timeseries_by_name_single_node";
+        let logctx = test_setup_log(TEST_NAME);
+        let log = &logctx.log;
+        let mut db = ClickHouseInstance::new_single_node(&logctx, 0)
+            .await
+            .expect("Failed to start ClickHouse");
+        let address = SocketAddr::new(Ipv6Addr::LOCALHOST.into(), db.port());
+        test_expunge_timeseries_by_name_impl(log, address, false).await;
+        db.cleanup().await.expect("Failed to cleanup ClickHouse server");
+        logctx.cleanup_successful();
+    }
+
+    #[tokio::test]
+    async fn test_expunge_timeseries_by_name_replicated() {
+        const TEST_NAME: &str = "test_expunge_timeseries_by_name_replicated";
+        let logctx = test_setup_log(TEST_NAME);
+        let mut cluster = create_cluster(&logctx).await;
+        let address = cluster.replica_1.address;
+        test_expunge_timeseries_by_name_impl(&logctx.log, address, true).await;
+
+        // TODO-cleanup: These should be arrays.
+        // See https://github.com/oxidecomputer/omicron/issues/4460.
+        cluster
+            .keeper_1
+            .cleanup()
+            .await
+            .expect("Failed to cleanup ClickHouse keeper 1");
+        cluster
+            .keeper_2
+            .cleanup()
+            .await
+            .expect("Failed to cleanup ClickHouse keeper 2");
+        cluster
+            .keeper_3
+            .cleanup()
+            .await
+            .expect("Failed to cleanup ClickHouse keeper 3");
+        cluster
+            .replica_1
+            .cleanup()
+            .await
+            .expect("Failed to cleanup ClickHouse server 1");
+        cluster
+            .replica_2
+            .cleanup()
+            .await
+            .expect("Failed to cleanup ClickHouse server 2");
+        logctx.cleanup_successful();
+    }
+
+    // Implementation of the test for expunging timeseries by name during an
+    // upgrade.
+    async fn test_expunge_timeseries_by_name_impl(
+        log: &Logger,
+        address: SocketAddr,
+        replicated: bool,
+    ) {
+        usdt::register_probes().unwrap();
+        let client = Client::new(address, &log);
+
+        const STARTING_VERSION: u64 = 1;
+        const NEXT_VERSION: u64 = 2;
+        const VERSIONS: [u64; 2] = [STARTING_VERSION, NEXT_VERSION];
+
+        // We need to actually have the oximeter DB here, and the version table,
+        // since `ensure_schema()` writes out versions to the DB as they're
+        // applied.
+        client
+            .initialize_db_with_version(replicated, STARTING_VERSION)
+            .await
+            .expect("failed to initialize test DB");
+
+        // Let's insert a few samples from two different timeseries. The
+        // timeseries share some field types and have others that are distinct
+        // between them, so that we can test that we don't touch tables we
+        // shouldn't, and only delete the parts we should.
+        let samples = generate_expunge_timeseries_samples();
+        client
+            .insert_samples(&samples)
+            .await
+            .expect("failed to insert test samples");
+        let all_timeseries: BTreeSet<TimeseriesName> = samples
+            .iter()
+            .map(|s| s.timeseries_name.parse().unwrap())
+            .collect();
+        assert_eq!(all_timeseries.len(), 2);
+
+        // Count the number of records in all tables, by timeseries.
+        let mut records_by_timeseries: BTreeMap<_, Vec<_>> = BTreeMap::new();
+        let all_tables = client
+            .list_oximeter_database_tables(ListDetails {
+                include_version: false,
+                replicated,
+            })
+            .await
+            .unwrap();
+        for table in all_tables.iter() {
+            let sql = format!(
+                "SELECT * FROM {}.{} FORMAT JSONEachRow",
+                crate::DATABASE_NAME,
+                table,
+            );
+            let body = client.execute_with_body(sql).await.unwrap().1;
+            for line in body.lines() {
+                let json: serde_json::Value =
+                    serde_json::from_str(line.trim()).unwrap();
+                let name = json["timeseries_name"].to_string();
+                records_by_timeseries.entry(name).or_default().push(json);
+            }
+        }
+
+        // Even though we don't need SQL, we need the directory for the first
+        // version too.
+        let (schema_dir, _version_dirs) =
+            create_test_upgrade_schema_directory(replicated, &VERSIONS).await;
+
+        // We don't actually need any SQL files in the version we're upgrading
+        // to. The function `ensure_schema` will apply any SQL and any
+        // timeseries to be deleted independently. We're just testing the
+        // latter.
+        let to_delete = vec![all_timeseries.first().unwrap().clone()];
+        write_timeseries_to_delete_file(
+            schema_dir.path(),
+            replicated,
+            NEXT_VERSION,
+            &to_delete,
+        )
+        .await;
+
+        // Let's run the "schema upgrade", which should only delete these
+        // particular timeseries.
+        client
+            .ensure_schema(replicated, NEXT_VERSION, schema_dir.path())
+            .await
+            .unwrap();
+
+        // Look over all tables.
+        //
+        // First, we should have zero mentions of the timeseries we've deleted.
+        for table in all_tables.iter() {
+            let sql = format!(
+                "SELECT COUNT() \
+                FROM {}.{} \
+                WHERE timeseries_name = '{}'
+                FORMAT CSV",
+                crate::DATABASE_NAME,
+                table,
+                &to_delete[0].to_string(),
+            );
+            let count: u64 = client
+                .execute_with_body(sql)
+                .await
+                .expect("failed to get count of timeseries")
+                .1
+                .trim()
+                .parse()
+                .expect("invalid record count from query");
+            assert_eq!(
+                count, 0,
+                "Should not have any rows associated with the deleted \
+                but found {count} records in table {table}",
+            );
+        }
+
+        // We should also still have all the records from the timeseries that we
+        // did _not_ expunge.
+        let mut found: BTreeMap<_, Vec<_>> = BTreeMap::new();
+        for table in all_tables.iter() {
+            let sql = format!(
+                "SELECT * FROM {}.{} FORMAT JSONEachRow",
+                crate::DATABASE_NAME,
+                table,
+            );
+            let body = client.execute_with_body(sql).await.unwrap().1;
+            for line in body.lines() {
+                let json: serde_json::Value =
+                    serde_json::from_str(line.trim()).unwrap();
+                let name = json["timeseries_name"].to_string();
+                found.entry(name).or_default().push(json);
+            }
+        }
+
+        // Check that all records we found exist in the previous set of found
+        // records, and that they are identical.
+        for (name, records) in found.iter() {
+            let existing_records = records_by_timeseries
+                .get(name)
+                .expect("expected to find previous records for timeseries");
+            assert_eq!(
+                records, existing_records,
+                "Some records from timeseries {name} were removed, \
+                but should not have been"
+            );
+        }
+    }
+
+    fn generate_expunge_timeseries_samples() -> Vec<Sample> {
+        #[derive(oximeter::Target)]
+        struct FirstTarget {
+            first_field: String,
+            second_field: Uuid,
+        }
+
+        #[derive(oximeter::Target)]
+        struct SecondTarget {
+            first_field: String,
+            second_field: bool,
+        }
+
+        #[derive(oximeter::Metric)]
+        struct SharedMetric {
+            datum: u64,
+        }
+
+        let ft = FirstTarget {
+            first_field: String::from("foo"),
+            second_field: Uuid::new_v4(),
+        };
+        let st = SecondTarget {
+            first_field: String::from("foo"),
+            second_field: false,
+        };
+        let mut m = SharedMetric { datum: 0 };
+
+        let mut out = Vec::with_capacity(8);
+        for i in 0..4 {
+            m.datum = i;
+            out.push(Sample::new(&ft, &m).unwrap());
+        }
+        for i in 4..8 {
+            m.datum = i;
+            out.push(Sample::new(&st, &m).unwrap());
+        }
+        out
+    }
 }
diff --git a/oximeter/db/src/lib.rs b/oximeter/db/src/lib.rs
index c3d2014ad1..d5cafc84f2 100644
--- a/oximeter/db/src/lib.rs
+++ b/oximeter/db/src/lib.rs
@@ -142,6 +142,12 @@ pub enum Error {
     #[error("Schema update versions must be sequential without gaps")]
     NonSequentialSchemaVersions,
 
+    #[error("Could not read timeseries_to_delete file")]
+    ReadTimeseriesToDeleteFile {
+        #[source]
+        err: io::Error,
+    },
+
     #[cfg(any(feature = "sql", test))]
     #[error("SQL error")]
     Sql(#[from] sql::Error),
@@ -317,6 +323,20 @@ const DATABASE_TIMESTAMP_FORMAT: &str = "%Y-%m-%d %H:%M:%S%.9f";
 // The name of the database storing all metric information.
 const DATABASE_NAME: &str = "oximeter";
 
+// The name of the oximeter cluster, in the case of a replicated database.
+//
+// This must match what is used in the replicated SQL files when created the
+// database itself, and the XML files describing the cluster.
+const CLUSTER_NAME: &str = "oximeter_cluster";
+
+// The name of the table storing database version information.
+const VERSION_TABLE_NAME: &str = "version";
+
+// During schema upgrades, it is possible to list timeseries that should be
+// deleted, rather than deleting the entire database. These must be listed one
+// per line, in the file inside the schema version directory with this name.
+const TIMESERIES_TO_DELETE_FILE: &str = "timeseries-to-delete.txt";
+
 // The output format used for the result of select queries
 //
 // See https://clickhouse.com/docs/en/interfaces/formats/#jsoneachrow for details.
diff --git a/oximeter/impl/Cargo.toml b/oximeter/impl/Cargo.toml
index a8b42d41cd..91277d9d47 100644
--- a/oximeter/impl/Cargo.toml
+++ b/oximeter/impl/Cargo.toml
@@ -32,8 +32,14 @@ uuid.workspace = true
 
 [dev-dependencies]
 approx.workspace = true
+# For benchmark
+criterion.workspace = true
 rand = { workspace = true, features = ["std_rng"] }
 rand_distr.workspace = true
 rstest.workspace = true
 serde_json.workspace = true
 trybuild.workspace = true
+
+[[bench]]
+name = "quantile"
+harness = false
diff --git a/oximeter/impl/benches/quantile.rs b/oximeter/impl/benches/quantile.rs
new file mode 100644
index 0000000000..4540ba8f6a
--- /dev/null
+++ b/oximeter/impl/benches/quantile.rs
@@ -0,0 +1,42 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+//! Benchmarks for the implementation of the P² algorithm with
+//! quantile estimation.
+
+// Copyright 2024 Oxide Computer Company
+
+use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion};
+use oximeter_impl::Quantile;
+use rand_distr::{Distribution, Normal};
+
+/// Emulates baseline code in a Python implementation of the P²
+/// algorithm:
+/// <https://github.com/rfrenoy/psquare/blob/master/tests/test_psquare.py#L47>.
+fn normal_distribution_quantile(size: i32, p: f64) -> f64 {
+    let mu = 500.;
+    let sigma = 100.;
+    let mut q = Quantile::new(p).unwrap();
+    let normal = Normal::new(mu, sigma).unwrap();
+    for _ in 0..size {
+        q.append(normal.sample(&mut rand::thread_rng())).unwrap();
+    }
+    q.estimate().unwrap()
+}
+
+fn baseline_benchmark(c: &mut Criterion) {
+    let mut group = c.benchmark_group("Quantile");
+    let size = 1000;
+    for p in [0.5, 0.9, 0.95, 0.99].iter() {
+        group.bench_with_input(
+            BenchmarkId::new("Estimate on Normal Distribution", p),
+            p,
+            |b, p| b.iter(|| normal_distribution_quantile(size, *p)),
+        );
+    }
+    group.finish();
+}
+
+criterion_group!(benches, baseline_benchmark);
+criterion_main!(benches);
diff --git a/oximeter/impl/src/quantile.rs b/oximeter/impl/src/quantile.rs
index 3e070cc302..fafe9c9ece 100644
--- a/oximeter/impl/src/quantile.rs
+++ b/oximeter/impl/src/quantile.rs
@@ -426,7 +426,6 @@ mod tests {
     use super::*;
     use approx::assert_relative_eq;
     use rand::{Rng, SeedableRng};
-    use rand_distr::{Distribution, Normal};
 
     fn test_quantile_impl(
         p: f64,
@@ -568,37 +567,4 @@ mod tests {
         assert_eq!(q.find_cell(4.), Some(3));
         assert_eq!(q.find_cell(3.5), Some(2));
     }
-
-    /// Emulates baseline test in a basic Python implementation of the P²
-    /// algorithm:
-    /// <https://github.com/rfrenoy/psquare/blob/master/tests/test_psquare.py#L47>.
-    #[test]
-    fn test_against_baseline_normal_distribution() {
-        let mu = 500.;
-        let sigma = 100.;
-        let size = 1000;
-        let p = 0.9;
-
-        let normal = Normal::new(mu, sigma);
-        let mut observations = (0..size)
-            .map(|_| normal.unwrap().sample(&mut rand::thread_rng()))
-            .collect::<Vec<f64>>();
-        float_ord::sort(&mut observations);
-        let idx = ((f64::from(size) - 1.) * p) as usize;
-
-        let base_p_est = observations[idx];
-
-        let mut q = Quantile::new(p).unwrap();
-        for o in observations.iter() {
-            q.append(*o).unwrap();
-        }
-        let p_est = q.estimate().unwrap();
-
-        println!("Base: {}, Est: {}", base_p_est, p_est);
-        assert!(
-            (base_p_est - p_est).abs() < 10.0,
-            "Difference {} is not less than 10",
-            (base_p_est - p_est).abs()
-        );
-    }
 }
diff --git a/oximeter/oximeter/schema/dendrite.toml b/oximeter/oximeter/schema/dendrite.toml
new file mode 100644
index 0000000000..e822069a2f
--- /dev/null
+++ b/oximeter/oximeter/schema/dendrite.toml
@@ -0,0 +1,58 @@
+format_version = 1
+
+[target]
+name = "dendrite"
+description = "Oxide switch management daemon"
+authz_scope = "fleet"
+versions = [
+    { version = 1, fields = [ "rack_id", "sled_model", "sled_revision", "sled_id", "sled_serial" ] },
+]
+
+[[metrics]]
+name = "sample_collection_duration"
+description = "Duration spent collecting all timeseries samples"
+units = "seconds"
+datum_type = "f64"
+versions = [
+    # Note: The sample collection time includes the time spent querying the
+    # switch for its statistics, which is why these fields are included.
+    # Dendrite may eventually report statistics about itself, or other aspects
+    # not related to the switch, so they belong here, not the target.
+    { added_in = 1, fields = [ "switch_model", "switch_revision", "switch_id", "switch_serial" ] }
+]
+
+[fields.rack_id]
+type = "uuid"
+description = "ID of the rack containing the switch"
+
+[fields.sled_model]
+type = "string"
+description = "The model of the sled managing the switch"
+
+[fields.sled_revision]
+type = "u32"
+description = "Revision number of the sled managing the switch"
+
+[fields.sled_id]
+type = "uuid"
+description = "ID of the sled managing the switch"
+
+[fields.sled_serial]
+type = "string"
+description = "Serial number of the sled managing the switch"
+
+[fields.switch_model]
+type = "string"
+description = "The model of the switch being managed"
+
+[fields.switch_revision]
+type = "u32"
+description = "Revision number of the switch being managed"
+
+[fields.switch_id]
+type = "uuid"
+description = "ID of the switch being managed"
+
+[fields.switch_serial]
+type = "string"
+description = "Serial number of the switch being managed"
diff --git a/oximeter/oximeter/schema/switch-rib.toml b/oximeter/oximeter/schema/switch-rib.toml
new file mode 100644
index 0000000000..7cbf020e25
--- /dev/null
+++ b/oximeter/oximeter/schema/switch-rib.toml
@@ -0,0 +1,30 @@
+format_version = 1
+
+[target]
+name = "switch_rib"
+description = "Maghemite router routing information base"
+authz_scope = "fleet"
+versions = [
+    { version = 1, fields = [ "hostname", "sled_id", "rack_id" ] }
+]
+
+[[metrics]]
+name = "active_routes"
+description = "The number of currently active routes"
+units = "count"
+datum_type = "u64"
+versions = [
+    { added_in = 1, fields = [ ] }
+]
+
+[fields.hostname]
+type = "string"
+description = "Name of the host running the Oxide router"
+
+[fields.rack_id]
+type = "uuid"
+description = "ID of the rack of the sled running the Oxide router"
+
+[fields.sled_id]
+type = "uuid"
+description = "ID of the sled running the Oxide router"
diff --git a/oximeter/oximeter/schema/switch-table.toml b/oximeter/oximeter/schema/switch-table.toml
new file mode 100644
index 0000000000..1e8a9b1b93
--- /dev/null
+++ b/oximeter/oximeter/schema/switch-table.toml
@@ -0,0 +1,106 @@
+format_version = 1
+
+[target]
+name = "switch_table"
+description = "A table on a Sidecar switch ASIC"
+authz_scope = "fleet"
+versions = [
+    { version = 1, fields = [ "rack_id", "sled_id", "sidecar_id", "table" ] },
+]
+
+[[metrics]]
+name = "capacity"
+description = "Maximum number of entries in the table"
+units = "count"
+datum_type = "u64"
+versions = [
+    { added_in = 1, fields = [ ] }
+]
+
+[[metrics]]
+name = "collisions"
+description = "Total number of inserts failed due to a collision"
+units = "count"
+datum_type = "cumulative_u64"
+versions = [
+    { added_in = 1, fields = [ ] }
+]
+
+[[metrics]]
+name = "delete_misses"
+description = "Total number of deletes that failed due to a missing entry"
+units = "count"
+datum_type = "cumulative_u64"
+versions = [
+    { added_in = 1, fields = [ ] }
+]
+
+[[metrics]]
+name = "deletes"
+description = "Total number of entries deleted"
+units = "count"
+datum_type = "cumulative_u64"
+versions = [
+    { added_in = 1, fields = [ ] }
+]
+
+[[metrics]]
+name = "exhaustion"
+description = "Total number of inserts that failed due to space exhaustion"
+units = "count"
+datum_type = "cumulative_u64"
+versions = [
+    { added_in = 1, fields = [ ] }
+]
+
+[[metrics]]
+name = "inserts"
+description = "Total number of entries inserted"
+units = "count"
+datum_type = "cumulative_u64"
+versions = [
+    { added_in = 1, fields = [ ] }
+]
+
+[[metrics]]
+name = "occupancy"
+description = "Current number of entries in the table"
+units = "count"
+datum_type = "u64"
+versions = [
+    { added_in = 1, fields = [ ] }
+]
+
+[[metrics]]
+name = "update_misses"
+description = "Total number of updates that failed due to a missing entry"
+units = "count"
+datum_type = "cumulative_u64"
+versions = [
+    { added_in = 1, fields = [ ] }
+]
+
+[[metrics]]
+name = "updates"
+description = "Total number of entries updated in place"
+units = "count"
+datum_type = "cumulative_u64"
+versions = [
+    { added_in = 1, fields = [ ] }
+]
+
+[fields.rack_id]
+type = "uuid"
+description = "ID of the rack containing the switch"
+
+[fields.sled_id]
+type = "uuid"
+description = "ID of the sled responsible for managing the switch"
+
+[fields.sidecar_id]
+type = "uuid"
+description = "ID of the Sidecar switch"
+
+[fields.table]
+type = "string"
+description = "Name of the switch table"
diff --git a/oximeter/producer/src/lib.rs b/oximeter/producer/src/lib.rs
index 36b05d7bb1..e9223b62f3 100644
--- a/oximeter/producer/src/lib.rs
+++ b/oximeter/producer/src/lib.rs
@@ -222,6 +222,7 @@ impl Server {
             bind_address: server_info.address,
             request_body_max_bytes,
             default_handler_task_mode: dropshot::HandlerTaskMode::Detached,
+            log_headers: vec![],
         };
         let server = Self::build_dropshot_server(&log, &registry, &dropshot)?;
 
diff --git a/package-manifest.toml b/package-manifest.toml
index ad7d9a00d7..c54d147f4a 100644
--- a/package-manifest.toml
+++ b/package-manifest.toml
@@ -546,10 +546,10 @@ service_name = "propolis-server"
 only_for_targets.image = "standard"
 source.type = "prebuilt"
 source.repo = "propolis"
-source.commit = "5ebf9626e0ad274eb515d206d102cb09d2d51f15"
+source.commit = "66d1ee7d4a5829dbbf02a152091ea051023b5b8b"
 # The SHA256 digest is automatically posted to:
 # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/propolis/image/<commit>/propolis-server.sha256.txt
-source.sha256 = "0398e7294beaa46636eb1c0703593f67b35366aa3637e9bd0e70ded9f022a9ab"
+source.sha256 = "168d4f061245bae749926104a77d087d144ee4aea8cc6d2a49284ee26ad5ffe9"
 output.type = "zone"
 
 [package.mg-ddm-gz]
@@ -562,10 +562,10 @@ source.repo = "maghemite"
 # `tools/maghemite_openapi_version`. Failing to do so will cause a failure when
 # building `ddm-admin-client` (which will instruct you to update
 # `tools/maghemite_openapi_version`).
-source.commit = "1b385990e8648b221fd11f018f2a7ec425461c6c"
+source.commit = "220dd026e83142b83bd93123f465a64dd4600201"
 # The SHA256 digest is automatically posted to:
 # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/maghemite/image/<commit>/mg-ddm-gz.sha256.txt
-source.sha256 = "280bd6e5c30d8f1076bac9b8dbbdbc45379e76259aa6319da257192fcbf64a54"
+source.sha256 = "58c8fcec6b932f7e602ac82cc28460aa557cabae1b66947ab3cb7334b87c35d4"
 output.type = "tarball"
 
 [package.mg-ddm]
@@ -578,10 +578,10 @@ source.repo = "maghemite"
 # `tools/maghemite_openapi_version`. Failing to do so will cause a failure when
 # building `ddm-admin-client` (which will instruct you to update
 # `tools/maghemite_openapi_version`).
-source.commit = "1b385990e8648b221fd11f018f2a7ec425461c6c"
+source.commit = "220dd026e83142b83bd93123f465a64dd4600201"
 # The SHA256 digest is automatically posted to:
 # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/maghemite/image/<commit>/mg-ddm.sha256.txt
-source.sha256 = "f15f8bb0e13b1a9372c895775dae96b68ff1cc5e395e6bad4389c2a97957354e"
+source.sha256 = "69fa43393a77f19713c7d76a320064e3eb58b3ea0b2953d2079a5c3edebc172e"
 output.type = "zone"
 output.intermediate_only = true
 
@@ -593,10 +593,10 @@ source.repo = "maghemite"
 # `tools/maghemite_openapi_version`. Failing to do so will cause a failure when
 # building `ddm-admin-client` (which will instruct you to update
 # `tools/maghemite_openapi_version`).
-source.commit = "1b385990e8648b221fd11f018f2a7ec425461c6c"
+source.commit = "220dd026e83142b83bd93123f465a64dd4600201"
 # The SHA256 digest is automatically posted to:
 # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/maghemite/image/<commit>/mgd.sha256.txt
-source.sha256 = "d4f2aaca20b312b6716206c335165442d6625b929eb00f0fd23f551e38216ace"
+source.sha256 = "f1103de5dda4830eb653f4d555995d08c31253116448387399a77392c08dfb54"
 output.type = "zone"
 output.intermediate_only = true
 
@@ -638,8 +638,10 @@ only_for_targets.image = "standard"
 # 1. Build the zone image manually
 #   1a. cd <dendrite tree>
 #   1b. cargo build --features=tofino_stub --release
-#   1c. cargo xtask dist -o -r --features tofino_stub
+#   1c. cargo xtask dist --format omicron --release --features tofino_stub
 # 2. Copy dendrite.tar.gz from dendrite/out to omicron/out
+# 3. Change the below `source.type` key to `"manual"` and comment out or remove
+# the other `source.*` keys.
 source.type = "prebuilt"
 source.repo = "dendrite"
 # TODO: Set to images from main branch. Testing out with images from my PR for now.
@@ -664,8 +666,10 @@ only_for_targets.image = "standard"
 # 1. Build the zone image manually
 #   1a. cd <dendrite tree>
 #   1b. cargo build --features=tofino_asic --release
-#   1c. cargo xtask dist -o -r --features tofino_asic
+#   1c. cargo xtask dist --format omicron --release --features tofino_asic
 # 2. Copy the output zone image from dendrite/out to omicron/out
+# 3. Change the below `source.type` key to `"manual"` and comment out or remove
+# the other `source.*` keys.
 source.type = "prebuilt"
 source.repo = "dendrite"
 # TODO: Set to images from main branch. Testing out with images from my PR for now.
@@ -683,8 +687,10 @@ only_for_targets.image = "standard"
 # 1. Build the zone image manually
 #   1a. cd <dendrite tree>
 #   1b. cargo build --features=softnpu --release
-#   1c. cargo xtask dist -o -r --features softnpu
+#   1c. cargo xtask dist --format omicron --release --features softnpu
 # 2. Copy dendrite.tar.gz from dendrite/out to omicron/out/dendrite-softnpu.tar.gz
+# 3. Change the below `source.type` key to `"manual"` and comment out or remove
+# the other `source.*` keys.
 source.type = "prebuilt"
 source.repo = "dendrite"
 # TODO: Set to images from main branch. Testing out with images from my PR for now.
diff --git a/schema/crdb/dataset-address-optional/up01.sql b/schema/crdb/dataset-address-optional/up01.sql
new file mode 100644
index 0000000000..e29215251d
--- /dev/null
+++ b/schema/crdb/dataset-address-optional/up01.sql
@@ -0,0 +1 @@
+ALTER TABLE omicron.public.dataset ALTER COLUMN ip DROP NOT NULL;
diff --git a/schema/crdb/dataset-address-optional/up02.sql b/schema/crdb/dataset-address-optional/up02.sql
new file mode 100644
index 0000000000..997294fa12
--- /dev/null
+++ b/schema/crdb/dataset-address-optional/up02.sql
@@ -0,0 +1 @@
+ALTER TABLE omicron.public.dataset ALTER COLUMN port DROP NOT NULL;
diff --git a/schema/crdb/dataset-address-optional/up03.sql b/schema/crdb/dataset-address-optional/up03.sql
new file mode 100644
index 0000000000..0af212e320
--- /dev/null
+++ b/schema/crdb/dataset-address-optional/up03.sql
@@ -0,0 +1,4 @@
+ALTER TABLE omicron.public.dataset ADD CONSTRAINT IF NOT EXISTS ip_and_port_set_for_crucible CHECK (
+  (kind != 'crucible') OR
+  (kind = 'crucible' AND ip IS NOT NULL and port IS NOT NULL)
+);
diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql
index 7d93a5d5bd..7fc83ad5d0 100644
--- a/schema/crdb/dbinit.sql
+++ b/schema/crdb/dbinit.sql
@@ -525,8 +525,8 @@ CREATE TABLE IF NOT EXISTS omicron.public.dataset (
     pool_id UUID NOT NULL,
 
     /* Contact information for the dataset */
-    ip INET NOT NULL,
-    port INT4 CHECK (port BETWEEN 0 AND 65535) NOT NULL,
+    ip INET,
+    port INT4 CHECK (port BETWEEN 0 AND 65535),
 
     kind omicron.public.dataset_kind NOT NULL,
 
@@ -537,6 +537,11 @@ CREATE TABLE IF NOT EXISTS omicron.public.dataset (
     CONSTRAINT size_used_column_set_for_crucible CHECK (
       (kind != 'crucible') OR
       (kind = 'crucible' AND size_used IS NOT NULL)
+    ),
+
+    CONSTRAINT ip_and_port_set_for_crucible CHECK (
+      (kind != 'crucible') OR
+      (kind = 'crucible' AND ip IS NOT NULL and port IS NOT NULL)
     )
 );
 
@@ -1430,7 +1435,8 @@ CREATE TYPE IF NOT EXISTS omicron.public.network_interface_kind AS ENUM (
     'instance',
 
     /* An interface attached to a service. */
-    'service'
+    'service',
+    'probe'
 );
 
 CREATE TABLE IF NOT EXISTS omicron.public.network_interface (
@@ -1871,6 +1877,8 @@ CREATE TABLE IF NOT EXISTS omicron.public.external_ip (
      */
     state omicron.public.ip_attach_state NOT NULL,
 
+    is_probe BOOL NOT NULL DEFAULT false,
+
     /* The name must be non-NULL iff this is a floating IP. */
     CONSTRAINT null_fip_name CHECK (
         (kind != 'floating' AND name IS NULL) OR
@@ -2618,11 +2626,32 @@ CREATE TABLE IF NOT EXISTS omicron.public.switch_port_settings_port_config (
     geometry omicron.public.switch_port_geometry
 );
 
+CREATE TYPE IF NOT EXISTS omicron.public.switch_link_fec AS ENUM (
+    'Firecode',
+    'None',
+    'Rs'
+);
+
+CREATE TYPE IF NOT EXISTS omicron.public.switch_link_speed AS ENUM (
+    '0G',
+    '1G',
+    '10G',
+    '25G',
+    '40G',
+    '50G',
+    '100G',
+    '200G',
+    '400G'
+);
+
 CREATE TABLE IF NOT EXISTS omicron.public.switch_port_settings_link_config (
     port_settings_id UUID,
     lldp_service_config_id UUID NOT NULL,
     link_name TEXT,
     mtu INT4,
+    fec omicron.public.switch_link_fec,
+    speed omicron.public.switch_link_speed,
+    autoneg BOOL NOT NULL DEFAULT false,
 
     PRIMARY KEY (port_settings_id, link_name)
 );
@@ -3599,27 +3628,6 @@ FROM
 WHERE
     instance.time_deleted IS NULL AND vmm.time_deleted IS NULL;
 
-CREATE TYPE IF NOT EXISTS omicron.public.switch_link_fec AS ENUM (
-    'Firecode',
-    'None',
-    'Rs'
-);
-
-CREATE TYPE IF NOT EXISTS omicron.public.switch_link_speed AS ENUM (
-    '0G',
-    '1G',
-    '10G',
-    '25G',
-    '40G',
-    '50G',
-    '100G',
-    '200G',
-    '400G'
-);
-
-ALTER TABLE omicron.public.switch_port_settings_link_config ADD COLUMN IF NOT EXISTS fec omicron.public.switch_link_fec;
-ALTER TABLE omicron.public.switch_port_settings_link_config ADD COLUMN IF NOT EXISTS speed omicron.public.switch_link_speed;
-
 CREATE SEQUENCE IF NOT EXISTS omicron.public.ipv4_nat_version START 1 INCREMENT 1;
 
 CREATE TABLE IF NOT EXISTS omicron.public.ipv4_nat_entry (
@@ -3696,8 +3704,6 @@ CREATE UNIQUE INDEX IF NOT EXISTS lookup_bfd_session ON omicron.public.bfd_sessi
     switch
 ) WHERE time_deleted IS NULL;
 
-ALTER TABLE omicron.public.switch_port_settings_link_config ADD COLUMN IF NOT EXISTS autoneg BOOL NOT NULL DEFAULT false;
-
 CREATE INDEX IF NOT EXISTS ipv4_nat_lookup_by_vni ON omicron.public.ipv4_nat_entry (
   vni
 )
@@ -3790,10 +3796,6 @@ CREATE UNIQUE INDEX IF NOT EXISTS lookup_probe_by_name ON omicron.public.probe (
 ) WHERE
     time_deleted IS NULL;
 
-ALTER TABLE omicron.public.external_ip ADD COLUMN IF NOT EXISTS is_probe BOOL NOT NULL DEFAULT false;
-
-ALTER TYPE omicron.public.network_interface_kind ADD VALUE IF NOT EXISTS 'probe';
-
 CREATE TYPE IF NOT EXISTS omicron.public.upstairs_repair_notification_type AS ENUM (
   'started',
   'succeeded',
@@ -4143,7 +4145,7 @@ INSERT INTO omicron.public.db_metadata (
     version,
     target_version
 ) VALUES
-    (TRUE, NOW(), NOW(), '82.0.0', NULL)
+    (TRUE, NOW(), NOW(), '83.0.0', NULL)
 ON CONFLICT DO NOTHING;
 
 COMMIT;
diff --git a/schema/rss-sled-plan.json b/schema/rss-sled-plan.json
index cb3c5c8eeb..1abba084ac 100644
--- a/schema/rss-sled-plan.json
+++ b/schema/rss-sled-plan.json
@@ -85,7 +85,8 @@
             },
             "revision": {
               "type": "integer",
-              "format": "int64"
+              "format": "uint32",
+              "minimum": 0.0
             },
             "type": {
               "type": "string",
diff --git a/sled-agent/src/common/instance.rs b/sled-agent/src/common/instance.rs
index ed0aceff82..0fe2e27698 100644
--- a/sled-agent/src/common/instance.rs
+++ b/sled-agent/src/common/instance.rs
@@ -486,9 +486,15 @@ impl InstanceStates {
     /// instance's state in Nexus may become inconsistent. This routine should
     /// therefore only be invoked by callers who know that an instance is not
     /// migrating.
-    pub(crate) fn terminate_rudely(&mut self) {
+    pub(crate) fn terminate_rudely(&mut self, mark_failed: bool) {
+        let vmm_state = if mark_failed {
+            PropolisInstanceState(PropolisApiState::Failed)
+        } else {
+            PropolisInstanceState(PropolisApiState::Destroyed)
+        };
+
         let fake_observed = ObservedPropolisState {
-            vmm_state: PropolisInstanceState(PropolisApiState::Destroyed),
+            vmm_state,
             migration_status: if self.instance.migration_id.is_some() {
                 ObservedMigrationStatus::Failed
             } else {
@@ -893,7 +899,8 @@ mod test {
         assert_eq!(state.propolis_role(), PropolisRole::MigrationTarget);
 
         let prev = state.clone();
-        state.terminate_rudely();
+        let mark_failed = false;
+        state.terminate_rudely(mark_failed);
 
         assert_state_change_has_gen_change(&prev, &state);
         assert_eq!(state.instance.gen, prev.instance.gen);
diff --git a/sled-agent/src/dump_setup.rs b/sled-agent/src/dump_setup.rs
index 02d3d41dd7..02d40195cf 100644
--- a/sled-agent/src/dump_setup.rs
+++ b/sled-agent/src/dump_setup.rs
@@ -100,6 +100,7 @@ use std::ffi::OsString;
 use std::path::{Path, PathBuf};
 use std::time::{Duration, SystemTime, SystemTimeError, UNIX_EPOCH};
 use tokio::sync::mpsc::Receiver;
+use tokio::sync::oneshot;
 use zone::{Zone, ZoneError};
 
 const ZFS_PROP_USED: &str = "used";
@@ -175,6 +176,7 @@ enum DumpSetupCmd {
         dump_slices: Vec<DumpSlicePath>,
         debug_datasets: Vec<DebugZpool>,
         core_datasets: Vec<CoreZpool>,
+        update_complete_tx: oneshot::Sender<()>,
     },
 }
 
@@ -222,6 +224,12 @@ impl DumpSetup {
         Self { tx, mount_config, _poller, log }
     }
 
+    /// Given the set of all managed disks, updates the dump device location
+    /// for logs and dumps.
+    ///
+    /// This function returns only once this request has been handled, which
+    /// can be used as a signal by callers that any "old disks" are no longer
+    /// being used by [DumpSetup].
     pub(crate) async fn update_dumpdev_setup(
         &self,
         disks: impl Iterator<Item = &Disk>,
@@ -279,16 +287,22 @@ impl DumpSetup {
             }
         }
 
+        let (tx, rx) = oneshot::channel();
         if let Err(err) = self
             .tx
             .send(DumpSetupCmd::UpdateDumpdevSetup {
                 dump_slices: m2_dump_slices,
                 debug_datasets: u2_debug_datasets,
                 core_datasets: m2_core_datasets,
+                update_complete_tx: tx,
             })
             .await
         {
             error!(log, "DumpSetup channel closed: {:?}", err.0);
+        };
+
+        if let Err(err) = rx.await {
+            error!(log, "DumpSetup failed to await update"; "err" => ?err);
         }
     }
 }
@@ -504,6 +518,14 @@ impl DumpSetupWorker {
 
     async fn poll_file_archival(mut self) {
         info!(self.log, "DumpSetup poll loop started.");
+
+        // A oneshot which helps callers track when updates have propagated.
+        //
+        // This is particularly useful for disk expungement, when a caller
+        // wants to ensure that the dump device is no longer accessing an
+        // old device.
+        let mut evaluation_and_archiving_complete_tx = None;
+
         loop {
             match tokio::time::timeout(ARCHIVAL_INTERVAL, self.rx.recv()).await
             {
@@ -511,7 +533,10 @@ impl DumpSetupWorker {
                     dump_slices,
                     debug_datasets,
                     core_datasets,
+                    update_complete_tx,
                 })) => {
+                    evaluation_and_archiving_complete_tx =
+                        Some(update_complete_tx);
                     self.update_disk_loadout(
                         dump_slices,
                         debug_datasets,
@@ -537,6 +562,12 @@ impl DumpSetupWorker {
             if let Err(err) = self.archive_files().await {
                 error!(self.log, "Failed to archive debug/dump files: {err:?}");
             }
+
+            if let Some(tx) = evaluation_and_archiving_complete_tx.take() {
+                if let Err(err) = tx.send(()) {
+                    error!(self.log, "DumpDevice failed to notify caller"; "err" => ?err);
+                }
+            }
         }
     }
 
diff --git a/sled-agent/src/http_entrypoints.rs b/sled-agent/src/http_entrypoints.rs
index a21c278699..2612e504f5 100644
--- a/sled-agent/src/http_entrypoints.rs
+++ b/sled-agent/src/http_entrypoints.rs
@@ -32,7 +32,7 @@ use omicron_common::api::internal::nexus::{
     DiskRuntimeState, SledInstanceState, UpdateArtifactId,
 };
 use omicron_common::api::internal::shared::{
-    ResolvedVpcRouteSet, ResolvedVpcRouteState, SwitchPorts,
+    ResolvedVpcRouteSet, ResolvedVpcRouteState, SledIdentifiers, SwitchPorts,
 };
 use omicron_uuid_kinds::{GenericUuid, InstanceUuid};
 use schemars::JsonSchema;
@@ -89,6 +89,7 @@ pub fn api() -> SledApiDescription {
         api.register(host_os_write_status_get)?;
         api.register(host_os_write_status_delete)?;
         api.register(inventory)?;
+        api.register(sled_identifiers)?;
         api.register(bootstore_status)?;
         api.register(list_vpc_routes)?;
         api.register(set_vpc_routes)?;
@@ -1012,6 +1013,17 @@ async fn inventory(
     Ok(HttpResponseOk(sa.inventory().await?))
 }
 
+/// Fetch sled identifiers
+#[endpoint {
+    method = GET,
+    path = "/sled-identifiers",
+}]
+async fn sled_identifiers(
+    request_context: RequestContext<SledAgent>,
+) -> Result<HttpResponseOk<SledIdentifiers>, HttpError> {
+    Ok(HttpResponseOk(request_context.context().sled_identifiers().await))
+}
+
 /// Get the internal state of the local bootstore node
 #[endpoint {
     method = GET,
diff --git a/sled-agent/src/instance.rs b/sled-agent/src/instance.rs
index ec4d503e7b..7b9136e879 100644
--- a/sled-agent/src/instance.rs
+++ b/sled-agent/src/instance.rs
@@ -39,9 +39,10 @@ use omicron_common::api::internal::shared::{
     NetworkInterface, SourceNatConfig,
 };
 use omicron_common::backoff;
+use omicron_common::zpool_name::ZpoolName;
 use omicron_uuid_kinds::{GenericUuid, InstanceUuid, PropolisUuid};
 use propolis_client::Client as PropolisClient;
-use rand::prelude::SliceRandom;
+use rand::prelude::IteratorRandom;
 use rand::SeedableRng;
 use sled_storage::dataset::ZONE_DATASET;
 use sled_storage::manager::StorageHandle;
@@ -214,6 +215,9 @@ enum InstanceRequest {
     RequestZoneBundle {
         tx: oneshot::Sender<Result<ZoneBundleMetadata, BundleError>>,
     },
+    GetFilesystemPool {
+        tx: oneshot::Sender<Option<ZpoolName>>,
+    },
     CurrentState {
         tx: oneshot::Sender<SledInstanceState>,
     },
@@ -227,6 +231,7 @@ enum InstanceRequest {
         tx: oneshot::Sender<Result<SledInstanceState, ManagerError>>,
     },
     Terminate {
+        mark_failed: bool,
         tx: oneshot::Sender<Result<InstanceUnregisterResponse, ManagerError>>,
     },
     IssueSnapshotRequest {
@@ -391,7 +396,8 @@ impl InstanceRunner {
                         // of the sender alive in "self.tx_monitor".
                         None => {
                             warn!(self.log, "Instance 'VMM monitor' channel closed; shutting down");
-                            self.terminate().await;
+                            let mark_failed = true;
+                            self.terminate(mark_failed).await;
                         },
                     }
 
@@ -405,6 +411,10 @@ impl InstanceRunner {
                             tx.send(self.request_zone_bundle().await)
                                 .map_err(|_| Error::FailedSendClientClosed)
                         },
+                        Some(GetFilesystemPool { tx } ) => {
+                            tx.send(self.get_filesystem_zpool())
+                                .map_err(|_| Error::FailedSendClientClosed)
+                        },
                         Some(CurrentState{ tx }) => {
                             tx.send(self.current_state())
                                 .map_err(|_| Error::FailedSendClientClosed)
@@ -424,9 +434,9 @@ impl InstanceRunner {
                             )
                             .map_err(|_| Error::FailedSendClientClosed)
                         },
-                        Some(Terminate { tx }) => {
+                        Some(Terminate { mark_failed, tx }) => {
                             tx.send(Ok(InstanceUnregisterResponse {
-                                updated_runtime: Some(self.terminate().await)
+                                updated_runtime: Some(self.terminate(mark_failed).await)
                             }))
                             .map_err(|_| Error::FailedSendClientClosed)
                         },
@@ -449,7 +459,8 @@ impl InstanceRunner {
                         },
                         None => {
                             warn!(self.log, "Instance request channel closed; shutting down");
-                            self.terminate().await;
+                            let mark_failed = false;
+                            self.terminate(mark_failed).await;
                             break;
                         },
                     };
@@ -609,8 +620,8 @@ impl InstanceRunner {
             Some(InstanceAction::Destroy) => {
                 info!(self.log, "terminating VMM that has exited";
                       "instance_id" => %self.id());
-
-                self.terminate().await;
+                let mark_failed = false;
+                self.terminate(mark_failed).await;
                 Reaction::Terminate
             }
             None => Reaction::Continue,
@@ -651,9 +662,7 @@ impl InstanceRunner {
         let nics = running_zone
             .opte_ports()
             .map(|port| propolis_client::types::NetworkInterfaceRequest {
-                // TODO-correctness: Remove `.vnic()` call when we use the port
-                // directly.
-                name: port.vnic_name().to_string(),
+                name: port.name().to_string(),
                 slot: propolis_client::types::Slot(port.slot()),
             })
             .collect();
@@ -1059,6 +1068,17 @@ impl Instance {
         Ok(())
     }
 
+    pub async fn get_filesystem_zpool(
+        &self,
+    ) -> Result<Option<ZpoolName>, Error> {
+        let (tx, rx) = oneshot::channel();
+        self.tx
+            .send(InstanceRequest::GetFilesystemPool { tx })
+            .await
+            .map_err(|_| Error::FailedSendChannelClosed)?;
+        Ok(rx.await?)
+    }
+
     pub async fn current_state(&self) -> Result<SledInstanceState, Error> {
         let (tx, rx) = oneshot::channel();
         self.tx
@@ -1113,9 +1133,10 @@ impl Instance {
     pub async fn terminate(
         &self,
         tx: oneshot::Sender<Result<InstanceUnregisterResponse, ManagerError>>,
+        mark_failed: bool,
     ) -> Result<(), Error> {
         self.tx
-            .send(InstanceRequest::Terminate { tx })
+            .send(InstanceRequest::Terminate { mark_failed, tx })
             .await
             .map_err(|_| Error::FailedSendChannelClosed)?;
         Ok(())
@@ -1180,6 +1201,13 @@ impl InstanceRunner {
         }
     }
 
+    fn get_filesystem_zpool(&self) -> Option<ZpoolName> {
+        let Some(run_state) = &self.running_state else {
+            return None;
+        };
+        run_state.running_zone.root_zpool().map(|p| p.clone())
+    }
+
     fn current_state(&self) -> SledInstanceState {
         self.state.sled_instance_state()
     }
@@ -1228,7 +1256,8 @@ impl InstanceRunner {
                 // This case is morally equivalent to starting Propolis and then
                 // rudely terminating it before asking it to do anything. Update
                 // the VMM and instance states accordingly.
-                self.state.terminate_rudely();
+                let mark_failed = false;
+                self.state.terminate_rudely(mark_failed);
             }
             setup_result?;
         }
@@ -1255,7 +1284,8 @@ impl InstanceRunner {
                 // this happens, generate an instance record bearing the
                 // "Destroyed" state and return it to the caller.
                 if self.running_state.is_none() {
-                    self.terminate().await;
+                    let mark_failed = false;
+                    self.terminate(mark_failed).await;
                     (None, None)
                 } else {
                     (
@@ -1343,20 +1373,22 @@ impl InstanceRunner {
         // configured VNICs.
         let zname = propolis_zone_name(self.propolis_id());
         let mut rng = rand::rngs::StdRng::from_entropy();
-        let root = self
+        let latest_disks = self
             .storage
             .get_latest_disks()
             .await
-            .all_u2_mountpoints(ZONE_DATASET)
+            .all_u2_mountpoints(ZONE_DATASET);
+
+        let root = latest_disks
+            .into_iter()
             .choose(&mut rng)
-            .ok_or_else(|| Error::U2NotFound)?
-            .clone();
+            .ok_or_else(|| Error::U2NotFound)?;
         let installed_zone = self
             .zone_builder_factory
             .builder()
             .with_log(self.log.clone())
             .with_underlay_vnic_allocator(&self.vnic_allocator)
-            .with_zone_root_path(&root)
+            .with_zone_root_path(root)
             .with_zone_image_paths(&["/opt/oxide".into()])
             .with_zone_type("propolis-server")
             .with_unique_name(self.propolis_id().into_untyped_uuid())
@@ -1453,9 +1485,9 @@ impl InstanceRunner {
         Ok(PropolisSetup { client, running_zone })
     }
 
-    async fn terminate(&mut self) -> SledInstanceState {
+    async fn terminate(&mut self, mark_failed: bool) -> SledInstanceState {
         self.terminate_inner().await;
-        self.state.terminate_rudely();
+        self.state.terminate_rudely(mark_failed);
 
         // This causes the "run" task to exit on the next iteration.
         self.should_terminate = true;
diff --git a/sled-agent/src/instance_manager.rs b/sled-agent/src/instance_manager.rs
index beeb8377d2..cfb96fb8c9 100644
--- a/sled-agent/src/instance_manager.rs
+++ b/sled-agent/src/instance_manager.rs
@@ -24,14 +24,16 @@ use illumos_utils::dladm::Etherstub;
 use illumos_utils::link::VnicAllocator;
 use illumos_utils::opte::PortManager;
 use illumos_utils::running_zone::ZoneBuilderFactory;
+use omicron_common::api::external::Generation;
 use omicron_common::api::internal::nexus::InstanceRuntimeState;
 use omicron_common::api::internal::nexus::SledInstanceState;
 use omicron_common::api::internal::nexus::VmmRuntimeState;
 use omicron_uuid_kinds::InstanceUuid;
 use omicron_uuid_kinds::PropolisUuid;
 use sled_storage::manager::StorageHandle;
+use sled_storage::resources::AllDisks;
 use slog::Logger;
-use std::collections::BTreeMap;
+use std::collections::{BTreeMap, HashSet};
 use std::net::SocketAddr;
 use std::sync::Arc;
 use tokio::sync::{mpsc, oneshot};
@@ -119,6 +121,7 @@ impl InstanceManager {
             instances: BTreeMap::new(),
             vnic_allocator: VnicAllocator::new("Instance", etherstub),
             port_manager,
+            storage_generation: None,
             storage,
             zone_bundler,
             zone_builder_factory,
@@ -325,6 +328,23 @@ impl InstanceManager {
             .map_err(|_| Error::FailedSendInstanceManagerClosed)?;
         rx.await?
     }
+
+    /// Marks instances failed unless they're using storage from `disks`.
+    ///
+    /// This function looks for transient zone filesystem usage on expunged
+    /// zpools.
+    pub async fn use_only_these_disks(
+        &self,
+        disks: AllDisks,
+    ) -> Result<(), Error> {
+        let (tx, rx) = oneshot::channel();
+        self.inner
+            .tx
+            .send(InstanceManagerRequest::OnlyUseDisks { disks, tx })
+            .await
+            .map_err(|_| Error::FailedSendInstanceManagerClosed)?;
+        rx.await?
+    }
 }
 
 // Most requests that can be sent to the "InstanceManagerRunner" task.
@@ -384,6 +404,10 @@ enum InstanceManagerRequest {
         instance_id: InstanceUuid,
         tx: oneshot::Sender<Result<SledInstanceState, Error>>,
     },
+    OnlyUseDisks {
+        disks: AllDisks,
+        tx: oneshot::Sender<Result<(), Error>>,
+    },
 }
 
 // Requests that the instance manager stop processing information about a
@@ -420,6 +444,7 @@ struct InstanceManagerRunner {
 
     vnic_allocator: VnicAllocator<Etherstub>,
     port_manager: PortManager,
+    storage_generation: Option<Generation>,
     storage: StorageHandle,
     zone_bundler: ZoneBundler,
     zone_builder_factory: ZoneBuilderFactory,
@@ -494,6 +519,10 @@ impl InstanceManagerRunner {
                             // the state...
                             self.get_instance_state(tx, instance_id).await
                         },
+                        Some(OnlyUseDisks { disks, tx } ) => {
+                            self.use_only_these_disks(disks).await;
+                            tx.send(Ok(())).map_err(|_| Error::FailedSendClientClosed)
+                        },
                         None => {
                             warn!(self.log, "InstanceManager's request channel closed; shutting down");
                             break;
@@ -638,7 +667,8 @@ impl InstanceManagerRunner {
 
         // Otherwise, we pipeline the request, and send it to the instance,
         // where it can receive an appropriate response.
-        instance.terminate(tx).await?;
+        let mark_failed = false;
+        instance.terminate(tx, mark_failed).await?;
         Ok(())
     }
 
@@ -775,6 +805,56 @@ impl InstanceManagerRunner {
         tx.send(Ok(state)).map_err(|_| Error::FailedSendClientClosed)?;
         Ok(())
     }
+
+    async fn use_only_these_disks(&mut self, disks: AllDisks) {
+        // Consider the generation number on the incoming request to avoid
+        // applying old requests.
+        let requested_generation = *disks.generation();
+        if let Some(last_gen) = self.storage_generation {
+            if last_gen >= requested_generation {
+                // This request looks old, ignore it.
+                info!(self.log, "use_only_these_disks: Ignoring request";
+                    "last_gen" => ?last_gen, "requested_gen" => ?requested_generation);
+                return;
+            }
+        }
+        self.storage_generation = Some(requested_generation);
+        info!(self.log, "use_only_these_disks: Processing new request";
+            "gen" => ?requested_generation);
+
+        let u2_set: HashSet<_> = disks.all_u2_zpools().into_iter().collect();
+
+        let mut to_remove = vec![];
+        for (id, (_, instance)) in self.instances.iter() {
+            // If we can read the filesystem pool, consider it. Otherwise, move
+            // on, to prevent blocking the cleanup of other instances.
+            let Ok(Some(filesystem_pool)) =
+                instance.get_filesystem_zpool().await
+            else {
+                info!(self.log, "use_only_these_disks: Cannot read filesystem pool"; "instance_id" => ?id);
+                continue;
+            };
+            if !u2_set.contains(&filesystem_pool) {
+                to_remove.push(*id);
+            }
+        }
+
+        for id in to_remove {
+            info!(self.log, "use_only_these_disks: Removing instance"; "instance_id" => ?id);
+            if let Some((_, instance)) = self.instances.remove(&id) {
+                let (tx, rx) = oneshot::channel();
+                let mark_failed = true;
+                if let Err(e) = instance.terminate(tx, mark_failed).await {
+                    warn!(self.log, "use_only_these_disks: Failed to request instance removal"; "err" => ?e);
+                    continue;
+                }
+
+                if let Err(e) = rx.await {
+                    warn!(self.log, "use_only_these_disks: Failed while removing instance"; "err" => ?e);
+                }
+            }
+        }
+    }
 }
 
 /// Represents membership of an instance in the [`InstanceManager`].
diff --git a/sled-agent/src/long_running_tasks.rs b/sled-agent/src/long_running_tasks.rs
index faea94f552..e920ffc3fc 100644
--- a/sled-agent/src/long_running_tasks.rs
+++ b/sled-agent/src/long_running_tasks.rs
@@ -20,7 +20,7 @@ use crate::config::Config;
 use crate::hardware_monitor::HardwareMonitor;
 use crate::services::ServiceManager;
 use crate::sled_agent::SledAgent;
-use crate::storage_monitor::StorageMonitor;
+use crate::storage_monitor::{StorageMonitor, StorageMonitorHandle};
 use crate::zone_bundle::{CleanupContext, ZoneBundler};
 use bootstore::schemes::v0 as bootstore;
 use key_manager::{KeyManager, StorageKeyRequester};
@@ -46,6 +46,10 @@ pub struct LongRunningTaskHandles {
     /// for establishing zpools on disks and managing their datasets.
     pub storage_manager: StorageHandle,
 
+    /// A mechanism for talking to the [`StorageMonitor`], which reacts to disk
+    /// changes and updates the dump devices.
+    pub storage_monitor_handle: StorageMonitorHandle,
+
     /// A mechanism for interacting with the hardware device tree
     pub hardware_manager: HardwareManager,
 
@@ -71,7 +75,8 @@ pub async fn spawn_all_longrunning_tasks(
     let mut storage_manager =
         spawn_storage_manager(log, storage_key_requester.clone());
 
-    spawn_storage_monitor(log, storage_manager.clone());
+    let storage_monitor_handle =
+        spawn_storage_monitor(log, storage_manager.clone());
 
     let nongimlet_observed_disks =
         config.nongimlet_observed_disks.clone().unwrap_or(vec![]);
@@ -106,6 +111,7 @@ pub async fn spawn_all_longrunning_tasks(
         LongRunningTaskHandles {
             storage_key_requester,
             storage_manager,
+            storage_monitor_handle,
             hardware_manager,
             bootstore,
             zone_bundler,
@@ -137,13 +143,17 @@ fn spawn_storage_manager(
     handle
 }
 
-fn spawn_storage_monitor(log: &Logger, storage_handle: StorageHandle) {
+fn spawn_storage_monitor(
+    log: &Logger,
+    storage_handle: StorageHandle,
+) -> StorageMonitorHandle {
     info!(log, "Starting StorageMonitor");
-    let storage_monitor =
+    let (storage_monitor, handle) =
         StorageMonitor::new(log, MountConfig::default(), storage_handle);
     tokio::spawn(async move {
         storage_monitor.run().await;
     });
+    handle
 }
 
 async fn spawn_hardware_manager(
diff --git a/sled-agent/src/probe_manager.rs b/sled-agent/src/probe_manager.rs
index 40af604645..9451484f21 100644
--- a/sled-agent/src/probe_manager.rs
+++ b/sled-agent/src/probe_manager.rs
@@ -10,20 +10,21 @@ use nexus_client::types::{
     BackgroundTasksActivateRequest, ProbeExternalIp, ProbeInfo,
 };
 use omicron_common::api::external::{
-    VpcFirewallRuleAction, VpcFirewallRuleDirection, VpcFirewallRulePriority,
-    VpcFirewallRuleStatus,
+    Generation, VpcFirewallRuleAction, VpcFirewallRuleDirection,
+    VpcFirewallRulePriority, VpcFirewallRuleStatus,
 };
 use omicron_common::api::internal::shared::NetworkInterface;
-use rand::prelude::SliceRandom;
+use rand::prelude::IteratorRandom;
 use rand::SeedableRng;
 use sled_storage::dataset::ZONE_DATASET;
 use sled_storage::manager::StorageHandle;
+use sled_storage::resources::AllDisks;
 use slog::{error, warn, Logger};
 use std::collections::{HashMap, HashSet};
 use std::hash::{Hash, Hasher};
 use std::sync::Arc;
 use std::time::Duration;
-use tokio::sync::Mutex;
+use tokio::sync::{Mutex, MutexGuard};
 use tokio::task::JoinHandle;
 use tokio::time::sleep;
 use uuid::Uuid;
@@ -45,6 +46,11 @@ pub(crate) struct ProbeManager {
     inner: Arc<ProbeManagerInner>,
 }
 
+struct RunningProbes {
+    storage_generation: Option<Generation>,
+    zones: HashMap<Uuid, RunningZone>,
+}
+
 pub(crate) struct ProbeManagerInner {
     join_handle: Mutex<Option<JoinHandle<()>>>,
     nexus_client: NexusClientWithResolver,
@@ -53,7 +59,7 @@ pub(crate) struct ProbeManagerInner {
     vnic_allocator: VnicAllocator<Etherstub>,
     storage: StorageHandle,
     port_manager: PortManager,
-    running_probes: Mutex<HashMap<Uuid, RunningZone>>,
+    running_probes: Mutex<RunningProbes>,
 }
 
 impl ProbeManager {
@@ -72,7 +78,10 @@ impl ProbeManager {
                     VNIC_ALLOCATOR_SCOPE,
                     etherstub,
                 ),
-                running_probes: Mutex::new(HashMap::new()),
+                running_probes: Mutex::new(RunningProbes {
+                    storage_generation: None,
+                    zones: HashMap::new(),
+                }),
                 nexus_client,
                 log,
                 sled_id,
@@ -85,6 +94,51 @@ impl ProbeManager {
     pub(crate) async fn run(&self) {
         self.inner.run().await;
     }
+
+    /// Removes any probes using filesystem roots on zpools that are not
+    /// contained in the set of "disks".
+    pub(crate) async fn use_only_these_disks(&self, disks: &AllDisks) {
+        let u2_set: HashSet<_> = disks.all_u2_zpools().into_iter().collect();
+        let mut probes = self.inner.running_probes.lock().await;
+
+        // Consider the generation number on the incoming request to avoid
+        // applying old requests.
+        let requested_generation = *disks.generation();
+        if let Some(last_gen) = probes.storage_generation {
+            if last_gen >= requested_generation {
+                // This request looks old, ignore it.
+                info!(self.inner.log, "use_only_these_disks: Ignoring request";
+                    "last_gen" => ?last_gen, "requested_gen" => ?requested_generation);
+                return;
+            }
+        }
+        probes.storage_generation = Some(requested_generation);
+        info!(self.inner.log, "use_only_these_disks: Processing new request";
+            "gen" => ?requested_generation);
+
+        let to_remove = probes
+            .zones
+            .iter()
+            .filter_map(|(id, probe)| {
+                let Some(probe_pool) = probe.root_zpool() else {
+                    // No known pool for this probe
+                    info!(self.inner.log, "use_only_these_disks: Cannot read filesystem pool"; "id" => ?id);
+                    return None;
+                };
+
+                if !u2_set.contains(probe_pool) {
+                    Some(*id)
+                } else {
+                    None
+                }
+            })
+            .collect::<Vec<_>>();
+
+        for probe_id in to_remove {
+            info!(self.inner.log, "use_only_these_disks: Removing probe"; "probe_id" => ?probe_id);
+            self.inner.remove_probe_locked(&mut probes, probe_id).await;
+        }
+    }
 }
 
 /// State information about a probe. This is a common representation that
@@ -226,14 +280,15 @@ impl ProbeManagerInner {
     /// boots the probe zone.
     async fn add_probe(self: &Arc<Self>, probe: &ProbeState) -> Result<()> {
         let mut rng = rand::rngs::StdRng::from_entropy();
-        let root = self
+        let current_disks = self
             .storage
             .get_latest_disks()
             .await
-            .all_u2_mountpoints(ZONE_DATASET)
+            .all_u2_mountpoints(ZONE_DATASET);
+        let zone_root_path = current_disks
+            .into_iter()
             .choose(&mut rng)
-            .ok_or_else(|| anyhow!("u2 not found"))?
-            .clone();
+            .ok_or_else(|| anyhow!("u2 not found"))?;
 
         let nic = probe
             .interface
@@ -268,7 +323,7 @@ impl ProbeManagerInner {
             .builder()
             .with_log(self.log.clone())
             .with_underlay_vnic_allocator(&self.vnic_allocator)
-            .with_zone_root_path(&root)
+            .with_zone_root_path(zone_root_path)
             .with_zone_image_paths(&["/opt/oxide".into()])
             .with_zone_type("probe")
             .with_unique_name(probe.id)
@@ -290,13 +345,13 @@ impl ProbeManagerInner {
         rz.ensure_address_for_port("overlay", 0).await?;
         info!(self.log, "started probe {}", probe.id);
 
-        self.running_probes.lock().await.insert(probe.id, rz);
+        self.running_probes.lock().await.zones.insert(probe.id, rz);
 
         Ok(())
     }
 
     /// Remove a set of probes from this sled.
-    async fn remove<'a, I>(self: &Arc<Self>, probes: I)
+    async fn remove<'a, I>(&self, probes: I)
     where
         I: Iterator<Item = &'a ProbeState>,
     {
@@ -308,8 +363,17 @@ impl ProbeManagerInner {
 
     /// Remove a probe from this sled. This tears down the zone and it's
     /// network resources.
-    async fn remove_probe(self: &Arc<Self>, id: Uuid) {
-        match self.running_probes.lock().await.remove(&id) {
+    async fn remove_probe(&self, id: Uuid) {
+        let mut probes = self.running_probes.lock().await;
+        self.remove_probe_locked(&mut probes, id).await
+    }
+
+    async fn remove_probe_locked(
+        &self,
+        probes: &mut MutexGuard<'_, RunningProbes>,
+        id: Uuid,
+    ) {
+        match probes.zones.remove(&id) {
             Some(mut running_zone) => {
                 for l in running_zone.links_mut() {
                     if let Err(e) = l.delete() {
diff --git a/sled-agent/src/server.rs b/sled-agent/src/server.rs
index f702e4c67d..ec86066096 100644
--- a/sled-agent/src/server.rs
+++ b/sled-agent/src/server.rs
@@ -68,7 +68,7 @@ impl Server {
 
         let dropshot_config = dropshot::ConfigDropshot {
             bind_address: SocketAddr::V6(sled_address),
-            ..config.dropshot
+            ..config.dropshot.clone()
         };
         let dropshot_log = log.new(o!("component" => "dropshot (SledAgent)"));
         let http_server = dropshot::HttpServerStarter::new(
diff --git a/sled-agent/src/services.rs b/sled-agent/src/services.rs
index 480a41148c..943ff44e06 100644
--- a/sled-agent/src/services.rs
+++ b/sled-agent/src/services.rs
@@ -57,7 +57,7 @@ use illumos_utils::running_zone::{
 };
 use illumos_utils::zfs::ZONE_ZFS_RAMDISK_DATASET_MOUNTPOINT;
 use illumos_utils::zone::AddressRequest;
-use illumos_utils::zpool::ZpoolName;
+use illumos_utils::zpool::{PathInPool, ZpoolName};
 use illumos_utils::{execute, PFEXEC};
 use internal_dns::resolver::Resolver;
 use itertools::Itertools;
@@ -445,6 +445,11 @@ impl OmicronZonesConfigLocal {
 /// Combines the Nexus-provided `OmicronZoneConfig` (which describes what Nexus
 /// wants for this zone) with any locally-determined configuration (like the
 /// path to the root filesystem)
+//
+// NOTE: Although the path to the root filesystem is not exactly equal to the
+// ZpoolName, it is derivable from it, and the ZpoolName for the root filesystem
+// is now being supplied as a part of OmicronZoneConfig. Therefore, this struct
+// is less necessary than it has been historically.
 #[derive(
     Clone,
     Debug,
@@ -548,10 +553,15 @@ impl<'a> ZoneArgs<'a> {
     }
 
     /// Return the root filesystem path for this zone
-    pub fn root(&self) -> &Utf8Path {
+    pub fn root(&self) -> PathInPool {
         match self {
-            ZoneArgs::Omicron(zone_config) => &zone_config.root,
-            ZoneArgs::Switch(zone_request) => &zone_request.root,
+            ZoneArgs::Omicron(zone_config) => PathInPool {
+                pool: zone_config.zone.filesystem_pool.clone(),
+                path: zone_config.root.clone(),
+            },
+            ZoneArgs::Switch(zone_request) => {
+                PathInPool { pool: None, path: zone_request.root.clone() }
+            }
         }
     }
 }
@@ -1372,7 +1382,7 @@ impl ServiceManager {
             })
         })?;
 
-        let opte_interface = port.vnic_name();
+        let opte_interface = port.name();
         let opte_gateway = port.gateway().ip().to_string();
         let opte_ip = port.ip().to_string();
 
@@ -1447,7 +1457,7 @@ impl ServiceManager {
         let all_disks = self.inner.storage.get_latest_disks().await;
         if let Some((_, boot_zpool)) = all_disks.boot_disk() {
             zone_image_paths.push(boot_zpool.dataset_mountpoint(
-                &all_disks.mount_config.root,
+                &all_disks.mount_config().root,
                 INSTALL_DATASET,
             ));
         }
@@ -1473,7 +1483,7 @@ impl ServiceManager {
         let installed_zone = zone_builder
             .with_log(self.inner.log.clone())
             .with_underlay_vnic_allocator(&self.inner.underlay_vnic_allocator)
-            .with_zone_root_path(&request.root())
+            .with_zone_root_path(request.root())
             .with_zone_image_paths(zone_image_paths.as_slice())
             .with_zone_type(&zone_type_str)
             .with_datasets(datasets.as_slice())
@@ -2160,6 +2170,7 @@ impl ServiceManager {
                             request_body_max_bytes: 8192 * 1024,
                             default_handler_task_mode:
                                 HandlerTaskMode::Detached,
+                            log_headers: vec![],
                         },
                     },
                     dropshot_internal: dropshot::ConfigDropshot {
@@ -2170,6 +2181,7 @@ impl ServiceManager {
                         // rack setup.
                         request_body_max_bytes: 10 * 1024 * 1024,
                         default_handler_task_mode: HandlerTaskMode::Detached,
+                        log_headers: vec![],
                     },
                     internal_dns: nexus_config::InternalDns::FromSubnet {
                         subnet: Ipv6Subnet::<RACK_PREFIX>::new(
@@ -2986,7 +2998,8 @@ impl ServiceManager {
             )
             .await?;
 
-        let config = OmicronZoneConfigLocal { zone: zone.clone(), root };
+        let config =
+            OmicronZoneConfigLocal { zone: zone.clone(), root: root.path };
 
         let runtime = self
             .initialize_zone(
@@ -3254,7 +3267,7 @@ impl ServiceManager {
 
         // Collect information that's necessary to start new zones
         let storage = self.inner.storage.get_latest_disks().await;
-        let mount_config = &storage.mount_config;
+        let mount_config = storage.mount_config();
         let all_u2_pools = storage.all_u2_zpools();
         let time_is_synchronized =
             match self.timesync_get_locked(&existing_zones).await {
@@ -3371,7 +3384,7 @@ impl ServiceManager {
         mount_config: &MountConfig,
         zone: &OmicronZoneConfig,
         all_u2_pools: &Vec<ZpoolName>,
-    ) -> Result<Utf8PathBuf, Error> {
+    ) -> Result<PathInPool, Error> {
         let name = zone.zone_name();
 
         // If the caller has requested a specific durable dataset,
@@ -3450,7 +3463,9 @@ impl ServiceManager {
                 device: format!("zpool: {filesystem_pool}"),
             });
         }
-        Ok(filesystem_pool.dataset_mountpoint(&mount_config.root, ZONE_DATASET))
+        let path = filesystem_pool
+            .dataset_mountpoint(&mount_config.root, ZONE_DATASET);
+        Ok(PathInPool { pool: Some(filesystem_pool), path })
     }
 
     pub async fn cockroachdb_initialize(&self) -> Result<(), Error> {
diff --git a/sled-agent/src/sim/instance.rs b/sled-agent/src/sim/instance.rs
index be6c63f53a..e94b3b4984 100644
--- a/sled-agent/src/sim/instance.rs
+++ b/sled-agent/src/sim/instance.rs
@@ -211,7 +211,8 @@ impl SimInstanceInner {
             InstanceStateRequested::Stopped => {
                 match self.next_resting_state() {
                     VmmState::Starting => {
-                        self.state.terminate_rudely();
+                        let mark_failed = false;
+                        self.state.terminate_rudely(mark_failed);
                     }
                     VmmState::Running => self.queue_graceful_stop(),
                     // Idempotently allow requests to stop an instance that is
@@ -363,7 +364,8 @@ impl SimInstanceInner {
     /// Simulates rude termination by moving the instance to the Destroyed state
     /// immediately and clearing the queue of pending state transitions.
     fn terminate(&mut self) -> SledInstanceState {
-        self.state.terminate_rudely();
+        let mark_failed = false;
+        self.state.terminate_rudely(mark_failed);
         self.queue.clear();
         self.destroyed = true;
         self.state.sled_instance_state()
diff --git a/sled-agent/src/sim/storage.rs b/sled-agent/src/sim/storage.rs
index 5077120fdd..0d534b9c4e 100644
--- a/sled-agent/src/sim/storage.rs
+++ b/sled-agent/src/sim/storage.rs
@@ -975,6 +975,7 @@ impl PantryServer {
                 // - bulk writes into disks
                 request_body_max_bytes: 8192 * 1024,
                 default_handler_task_mode: HandlerTaskMode::Detached,
+                log_headers: vec![],
             },
             super::http_entrypoints_pantry::api(),
             pantry.clone(),
diff --git a/sled-agent/src/sled_agent.rs b/sled-agent/src/sled_agent.rs
index 82c16b0b8d..dc946c1bfa 100644
--- a/sled-agent/src/sled_agent.rs
+++ b/sled-agent/src/sled_agent.rs
@@ -27,6 +27,7 @@ use crate::params::{
 };
 use crate::probe_manager::ProbeManager;
 use crate::services::{self, ServiceManager};
+use crate::storage_monitor::StorageMonitorHandle;
 use crate::updates::{ConfigUpdates, UpdateManager};
 use crate::vmm_reservoir::{ReservoirMode, VmmReservoirManager};
 use crate::zone_bundle;
@@ -50,7 +51,7 @@ use omicron_common::api::internal::nexus::{
 };
 use omicron_common::api::internal::shared::{
     HostPortConfig, RackNetworkConfig, ResolvedVpcRouteSet,
-    ResolvedVpcRouteState,
+    ResolvedVpcRouteState, SledIdentifiers,
 };
 use omicron_common::api::{
     internal::nexus::DiskRuntimeState, internal::nexus::InstanceRuntimeState,
@@ -123,6 +124,9 @@ pub enum Error {
     #[error("Error managing storage: {0}")]
     Storage(#[from] sled_storage::error::Error),
 
+    #[error("Error monitoring storage: {0}")]
+    StorageMonitor(#[from] crate::storage_monitor::Error),
+
     #[error("Error updating: {0}")]
     Download(#[from] crate::updates::Error),
 
@@ -152,6 +156,9 @@ pub enum Error {
 
     #[error("Metrics error: {0}")]
     Metrics(#[from] crate::metrics::Error),
+
+    #[error("Expected revision to fit in a u32, but found {0}")]
+    UnexpectedRevision(i64),
 }
 
 impl From<Error> for omicron_common::api::external::Error {
@@ -277,6 +284,10 @@ struct SledAgentInner {
     // Component of Sled Agent responsible for storage and dataset management.
     storage: StorageHandle,
 
+    // Component of Sled Agent responsible for monitoring storage and updating
+    // dump devices.
+    storage_monitor: StorageMonitorHandle,
+
     // Component of Sled Agent responsible for managing Propolis instances.
     instances: InstanceManager,
 
@@ -562,6 +573,9 @@ impl SledAgent {
                 subnet: request.body.subnet,
                 start_request: request,
                 storage: long_running_task_handles.storage_manager.clone(),
+                storage_monitor: long_running_task_handles
+                    .storage_monitor_handle
+                    .clone(),
                 instances,
                 probes,
                 hardware: long_running_task_handles.hardware_manager.clone(),
@@ -808,7 +822,60 @@ impl SledAgent {
         &self,
         config: OmicronPhysicalDisksConfig,
     ) -> Result<DisksManagementResult, Error> {
-        Ok(self.storage().omicron_physical_disks_ensure(config).await?)
+        info!(self.log, "physical disks ensure");
+        // Tell the storage subsystem which disks should be managed.
+        let disk_result =
+            self.storage().omicron_physical_disks_ensure(config).await?;
+        info!(self.log, "physical disks ensure: Updated storage");
+
+        // Grab a view of the latest set of disks, alongside a generation
+        // number.
+        //
+        // This generation is at LEAST as high as our last call through
+        // omicron_physical_disks_ensure. It may actually be higher, if a
+        // concurrent operation occurred.
+        //
+        // "latest_disks" has a generation number, which is important for other
+        // subcomponents of Sled Agent to consider. If multiple requests to
+        // ensure disks arrive concurrently, it's important to "only advance
+        // forward" as requested by Nexus.
+        //
+        // For example: if we receive the following requests concurrently:
+        // - Use Disks {A, B, C}, generation = 1
+        // - Use Disks {A, B, C, D}, generation = 2
+        //
+        // If we ignore generation numbers, it's possible that we start using
+        // "disk D" -- e.g., for instance filesystems -- and then immediately
+        // delete it when we process the request with "generation 1".
+        //
+        // By keeping these requests ordered, we prevent this thrashing, and
+        // ensure that we always progress towards the last-requested state.
+        let latest_disks = self.storage().get_latest_disks().await;
+        let our_gen = latest_disks.generation();
+        info!(self.log, "physical disks ensure: Propagating new generation of disks"; "generation" => ?our_gen);
+
+        // Ensure that the StorageMonitor, and the dump devices, have committed
+        // to start using new disks and stop using old ones.
+        self.inner.storage_monitor.await_generation(*our_gen).await?;
+        info!(self.log, "physical disks ensure: Updated storage monitor");
+
+        // Ensure that the ZoneBundler, if it was creating a bundle referencing
+        // the old U.2s, has stopped using them.
+        self.inner.zone_bundler.await_completion_of_prior_bundles().await;
+        info!(self.log, "physical disks ensure: Updated zone bundler");
+
+        // Ensure that all probes, at least after our call to
+        // "omicron_physical_disks_ensure", stop using any disks that
+        // may have been in-service from before that request.
+        self.inner.probes.use_only_these_disks(&latest_disks).await;
+        info!(self.log, "physical disks ensure: Updated probes");
+
+        // Do the same for instances - mark them failed if they were using
+        // expunged disks.
+        self.inner.instances.use_only_these_disks(latest_disks).await?;
+        info!(self.log, "physical disks ensure: Updated instances");
+
+        Ok(disk_result)
     }
 
     /// List the Omicron zone configuration that's currently running
@@ -1121,6 +1188,26 @@ impl SledAgent {
         &self.inner.boot_disk_os_writer
     }
 
+    /// Return identifiers for this sled.
+    ///
+    /// This is mostly used to identify timeseries data with the originating
+    /// sled.
+    ///
+    /// NOTE: This only returns the identifiers for the _sled_ itself. If you're
+    /// interested in the switch identifiers, MGS is the current best way to do
+    /// that, by asking for the local switch's slot, and then that switch's SP
+    /// state.
+    pub(crate) async fn sled_identifiers(&self) -> SledIdentifiers {
+        let baseboard = self.inner.hardware.baseboard();
+        SledIdentifiers {
+            rack_id: self.inner.start_request.body.rack_id,
+            sled_id: self.inner.id,
+            model: baseboard.model().to_string(),
+            revision: baseboard.revision(),
+            serial: baseboard.identifier().to_string(),
+        }
+    }
+
     /// Return basic information about ourselves: identity and status
     ///
     /// This is basically a GET version of the information we push to Nexus on
diff --git a/sled-agent/src/storage_monitor.rs b/sled-agent/src/storage_monitor.rs
index 8cb63e31f8..11883adcd2 100644
--- a/sled-agent/src/storage_monitor.rs
+++ b/sled-agent/src/storage_monitor.rs
@@ -7,10 +7,18 @@
 //! code.
 
 use crate::dump_setup::DumpSetup;
+use omicron_common::api::external::Generation;
 use sled_storage::config::MountConfig;
 use sled_storage::manager::StorageHandle;
 use sled_storage::resources::AllDisks;
 use slog::Logger;
+use tokio::sync::watch;
+
+#[derive(thiserror::Error, Debug)]
+pub enum Error {
+    #[error("Storage Monitor no longer running")]
+    NotRunning,
+}
 
 pub struct StorageMonitor {
     log: Logger,
@@ -18,6 +26,46 @@ pub struct StorageMonitor {
 
     // Invokes dumpadm(8) and savecore(8) when new disks are encountered
     dump_setup: DumpSetup,
+
+    tx: watch::Sender<StorageMonitorStatus>,
+}
+
+/// Emits status about storage monitoring.
+#[derive(Debug, Clone)]
+pub struct StorageMonitorStatus {
+    /// The latest generation of physical disks to be processed
+    /// by the storage monitor.
+    pub latest_gen: Option<Generation>,
+}
+
+impl StorageMonitorStatus {
+    fn new() -> Self {
+        Self { latest_gen: None }
+    }
+}
+
+#[derive(Clone)]
+pub struct StorageMonitorHandle {
+    rx: watch::Receiver<StorageMonitorStatus>,
+}
+
+impl StorageMonitorHandle {
+    pub async fn await_generation(
+        &self,
+        wanted: Generation,
+    ) -> Result<(), Error> {
+        self.rx
+            .clone()
+            .wait_for(|status| {
+                let Some(observed) = status.latest_gen else {
+                    return false;
+                };
+                return observed >= wanted;
+            })
+            .await
+            .map_err(|_| Error::NotRunning)?;
+        Ok(())
+    }
 }
 
 impl StorageMonitor {
@@ -25,10 +73,14 @@ impl StorageMonitor {
         log: &Logger,
         mount_config: MountConfig,
         storage_manager: StorageHandle,
-    ) -> StorageMonitor {
+    ) -> (StorageMonitor, StorageMonitorHandle) {
         let dump_setup = DumpSetup::new(&log, mount_config);
         let log = log.new(o!("component" => "StorageMonitor"));
-        StorageMonitor { log, storage_manager, dump_setup }
+        let (tx, rx) = watch::channel(StorageMonitorStatus::new());
+        (
+            StorageMonitor { log, storage_manager, dump_setup, tx },
+            StorageMonitorHandle { rx },
+        )
     }
 
     /// Run the main receive loop of the `StorageMonitor`
@@ -50,10 +102,14 @@ impl StorageMonitor {
     }
 
     async fn handle_resource_update(&mut self, updated_disks: AllDisks) {
+        let generation = updated_disks.generation();
         self.dump_setup
             .update_dumpdev_setup(
                 updated_disks.iter_managed().map(|(_id, disk)| disk),
             )
             .await;
+        self.tx.send_replace(StorageMonitorStatus {
+            latest_gen: Some(*generation),
+        });
     }
 }
diff --git a/sled-agent/src/zone_bundle.rs b/sled-agent/src/zone_bundle.rs
index 16147e5957..088e7b356f 100644
--- a/sled-agent/src/zone_bundle.rs
+++ b/sled-agent/src/zone_bundle.rs
@@ -256,6 +256,9 @@ impl Inner {
     // exist; and returns those.
     async fn bundle_directories(&self) -> Vec<Utf8PathBuf> {
         let resources = self.storage_handle.get_latest_disks().await;
+        // NOTE: These bundle directories are always stored on M.2s, so we don't
+        // need to worry about synchronizing with U.2 disk expungement at the
+        // callsite.
         let expected = resources.all_zone_bundle_directories();
         let mut out = Vec::with_capacity(expected.len());
         for each in expected.into_iter() {
@@ -426,12 +429,17 @@ impl ZoneBundler {
         zone: &RunningZone,
         cause: ZoneBundleCause,
     ) -> Result<ZoneBundleMetadata, BundleError> {
+        // NOTE: [Self::await_completion_of_prior_bundles] relies on this lock
+        // being held across this whole function. If we want more concurrency,
+        // we'll need to add a barrier-like mechanism to let callers know when
+        // prior bundles have completed.
         let inner = self.inner.lock().await;
         let storage_dirs = inner.bundle_directories().await;
         let resources = inner.storage_handle.get_latest_disks().await;
         let extra_log_dirs = resources
             .all_u2_mountpoints(U2_DEBUG_DATASET)
             .into_iter()
+            .map(|pool_path| pool_path.path)
             .collect();
         let context = ZoneBundleContext { cause, storage_dirs, extra_log_dirs };
         info!(
@@ -443,6 +451,14 @@ impl ZoneBundler {
         create(&self.log, zone, &context).await
     }
 
+    /// Awaits the completion of all prior calls to [ZoneBundler::create].
+    ///
+    /// This is critical for disk expungement, which wants to ensure that the
+    /// Sled Agent is no longer using devices after they have been expunged.
+    pub async fn await_completion_of_prior_bundles(&self) {
+        let _ = self.inner.lock().await;
+    }
+
     /// Return the paths for all bundles of the provided zone and ID.
     pub async fn bundle_paths(
         &self,
diff --git a/sled-hardware/src/illumos/mod.rs b/sled-hardware/src/illumos/mod.rs
index ff627f3e6e..65f439eaeb 100644
--- a/sled-hardware/src/illumos/mod.rs
+++ b/sled-hardware/src/illumos/mod.rs
@@ -9,6 +9,7 @@ use crate::{
 use camino::Utf8PathBuf;
 use gethostname::gethostname;
 use illumos_devinfo::{DevInfo, DevLinkType, DevLinks, Node, Property};
+use libnvme::{controller::Controller, Nvme};
 use omicron_common::disk::DiskIdentity;
 use sled_hardware_types::Baseboard;
 use slog::debug;
@@ -58,6 +59,9 @@ enum Error {
     #[error("Failed to issue request to sysconf: {0}")]
     SysconfError(#[from] sysconf::Error),
 
+    #[error("Node {node} missing device instance")]
+    MissingNvmeDevinfoInstance { node: String },
+
     #[error("Failed to init nvme handle: {0}")]
     NvmeHandleInit(#[from] libnvme::NvmeInitError),
 
@@ -67,6 +71,9 @@ enum Error {
     #[error("libnvme controller error: {0}")]
     NvmeController(#[from] libnvme::controller::NvmeControllerError),
 
+    #[error("Unable to grab NVMe Controller lock")]
+    NvmeControllerLocked,
+
     #[error("Failed to get NVMe Controller's firmware log page: {0}")]
     FirmwareLogPage(#[from] libnvme::firmware::FirmwareLogPageError),
 }
@@ -155,7 +162,7 @@ impl HardwareSnapshot {
         let baseboard = Baseboard::new_gimlet(
             string_from_property(&properties[0])?,
             string_from_property(&properties[1])?,
-            i64_from_property(&properties[2])?,
+            u32_from_property(&properties[2])?,
         );
         let boot_storage_unit =
             BootStorageUnit::try_from(i64_from_property(&properties[3])?)?;
@@ -405,6 +412,21 @@ fn get_parent_node<'a>(
     Ok(parent)
 }
 
+/// Convert a property to a `u32` if possible, passing through an `i64`.
+///
+/// Returns an error if either:
+///
+/// - The actual devinfo property isn't an integer type.
+/// - The value does not fit in a `u32`.
+fn u32_from_property(prop: &Property<'_>) -> Result<u32, Error> {
+    i64_from_property(prop).and_then(|val| {
+        u32::try_from(val).map_err(|_| Error::UnexpectedPropertyType {
+            name: prop.name(),
+            ty: "u32".to_string(),
+        })
+    })
+}
+
 fn i64_from_property(prop: &Property<'_>) -> Result<i64, Error> {
     prop.as_i64().ok_or_else(|| Error::UnexpectedPropertyType {
         name: prop.name(),
@@ -493,6 +515,13 @@ fn poll_blkdev_node(
 
     // We expect that the parent of the "blkdev" node is an "nvme" driver.
     let nvme_node = get_parent_node(&node, "nvme")?;
+    // Importantly we grab the NVMe instance and not the blkdev instance.
+    // Eventually we should switch the logic here to search for nvme instances
+    // and confirm that we only have one blkdev sibling:
+    // https://github.com/oxidecomputer/omicron/issues/5241
+    let nvme_instance = nvme_node
+        .instance()
+        .ok_or(Error::MissingNvmeDevinfoInstance { node: node.node_name() })?;
 
     let vendor_id =
         i64_from_property(&find_properties(&nvme_node, ["vendor-id"])?[0])?;
@@ -526,10 +555,31 @@ fn poll_blkdev_node(
         return Err(Error::UnrecognizedSlot { slot });
     };
 
-    // XXX See https://github.com/oxidecomputer/meta/issues/443
-    // Temporarily providing static data until the issue is resolved.
-    let firmware =
-        DiskFirmware::new(1, None, true, vec![Some("meta-443".to_string())]);
+    let nvme = Nvme::new()?;
+    let controller = Controller::init_by_instance(&nvme, nvme_instance)?;
+    let controller_lock = match controller.try_read_lock() {
+        libnvme::controller::TryLockResult::Ok(locked) => locked,
+        // We should only hit this if something in the system has locked the
+        // controller in question for writing.
+        libnvme::controller::TryLockResult::Locked(_) => {
+            warn!(
+                log,
+                "NVMe Controller is already locked so we will try again
+                in the next hardware snapshot"
+            );
+            return Err(Error::NvmeControllerLocked);
+        }
+        libnvme::controller::TryLockResult::Err(err) => {
+            return Err(Error::from(err))
+        }
+    };
+    let firmware_log_page = controller_lock.get_firmware_log_page()?;
+    let firmware = DiskFirmware::new(
+        firmware_log_page.active_slot,
+        firmware_log_page.next_active_slot,
+        firmware_log_page.slot1_is_read_only,
+        firmware_log_page.slot_iter().map(|s| s.map(str::to_string)).collect(),
+    );
 
     let disk = UnparsedDisk::new(
         Utf8PathBuf::from(&devfs_path),
diff --git a/sled-hardware/types/src/lib.rs b/sled-hardware/types/src/lib.rs
index e589498ff8..b34b5b1f42 100644
--- a/sled-hardware/types/src/lib.rs
+++ b/sled-hardware/types/src/lib.rs
@@ -22,7 +22,7 @@ pub mod underlay;
 )]
 #[serde(tag = "type", rename_all = "snake_case")]
 pub enum Baseboard {
-    Gimlet { identifier: String, model: String, revision: i64 },
+    Gimlet { identifier: String, model: String, revision: u32 },
 
     Unknown,
 
@@ -34,7 +34,7 @@ impl Baseboard {
     pub fn new_gimlet(
         identifier: String,
         model: String,
-        revision: i64,
+        revision: u32,
     ) -> Self {
         Self::Gimlet { identifier, model, revision }
     }
@@ -73,7 +73,7 @@ impl Baseboard {
         }
     }
 
-    pub fn revision(&self) -> i64 {
+    pub fn revision(&self) -> u32 {
         match self {
             Self::Gimlet { revision, .. } => *revision,
             Self::Pc { .. } => 0,
diff --git a/sled-storage/src/manager.rs b/sled-storage/src/manager.rs
index 9e31568e00..e081bc5034 100644
--- a/sled-storage/src/manager.rs
+++ b/sled-storage/src/manager.rs
@@ -8,9 +8,7 @@ use std::collections::HashSet;
 
 use crate::config::MountConfig;
 use crate::dataset::{DatasetName, CONFIG_DATASET};
-use crate::disk::{
-    OmicronPhysicalDiskConfig, OmicronPhysicalDisksConfig, RawDisk,
-};
+use crate::disk::{OmicronPhysicalDisksConfig, RawDisk};
 use crate::error::Error;
 use crate::resources::{AllDisks, DisksManagementResult, StorageResources};
 use camino::Utf8PathBuf;
@@ -586,94 +584,14 @@ impl StorageManager {
             // Identify which disks should be managed by the control
             // plane, and adopt all requested disks into the control plane
             // in a background task (see: [Self::manage_disks]).
-            self.resources.set_config(&ledger.data().disks);
+            self.resources.set_config(&ledger.data());
         } else {
             info!(self.log, "KeyManager ready, but no ledger detected");
-            let mut synthetic_config =
-                self.resources.get_config().values().cloned().collect();
-            // TODO(https://github.com/oxidecomputer/omicron/issues/5328): Once
-            // we are confident that we have migrated to a world where this
-            // ledger is universally used, we should remove the following
-            // kludge. The sled agent should not need to "self-manage" anything!
-            let changed = self
-                .self_manage_disks_with_zpools(&mut synthetic_config)
-                .await?;
-            if !changed {
-                info!(self.log, "No disks to be automatically managed");
-                return Ok(());
-            }
-            info!(self.log, "auto-managed disks"; "count" => synthetic_config.len());
-            self.resources.set_config(&synthetic_config);
         }
 
         Ok(())
     }
 
-    // NOTE: What follows is an exceptional case: one where we have
-    // no record of "Control Plane Physical Disks", but we have zpools
-    // on our U.2s, and we want to use them regardless.
-    //
-    // THIS WOULD NORMALLY BE INCORRECT BEHAVIOR. In the future, these
-    // zpools will not be "automatically imported", and instead, we'll
-    // let Nexus decide whether or not to reformat the disks.
-    //
-    // However, because we are transitioning from "the set of disks /
-    // zpools is implicit" to a world where that set is explicit, this
-    // is a necessary transitional tool.
-    //
-    // Returns "true" if the synthetic_config has changed.
-    async fn self_manage_disks_with_zpools(
-        &mut self,
-        synthetic_config: &mut Vec<OmicronPhysicalDiskConfig>,
-    ) -> Result<bool, Error> {
-        let mut changed = false;
-        for (identity, disk) in self.resources.disks().values.iter() {
-            match disk {
-                crate::resources::ManagedDisk::Unmanaged(raw) => {
-                    let zpool_path = match raw.u2_zpool_path() {
-                        Ok(zpool_path) => zpool_path,
-                        Err(err) => {
-                            info!(self.log, "Cannot find zpool path"; "identity" => ?identity, "err" => ?err);
-                            continue;
-                        }
-                    };
-
-                    let zpool_name =
-                        match sled_hardware::disk::check_if_zpool_exists(
-                            &zpool_path,
-                        ) {
-                            Ok(zpool_name) => zpool_name,
-                            Err(err) => {
-                                info!(self.log, "Zpool does not exist"; "identity" => ?identity, "err" => ?err);
-                                continue;
-                            }
-                        };
-
-                    info!(self.log, "Found existing zpool on device without ledger";
-                        "identity" => ?identity,
-                        "zpool" => ?zpool_name);
-
-                    // We found an unmanaged disk with a zpool, even though
-                    // we have no prior record of a ledger of control-plane
-                    // disks.
-                    synthetic_config.push(
-                        // These disks don't have a control-plane UUID --
-                        // report "nil" until they're overwritten with real
-                        // values.
-                        OmicronPhysicalDiskConfig {
-                            identity: identity.clone(),
-                            id: Uuid::nil(),
-                            pool_id: zpool_name.id(),
-                        },
-                    );
-                    changed = true;
-                }
-                _ => continue,
-            }
-        }
-        Ok(changed)
-    }
-
     // Makes an U.2 disk managed by the control plane within [`StorageResources`].
     async fn omicron_physical_disks_ensure(
         &mut self,
@@ -763,7 +681,7 @@ impl StorageManager {
 
         // Identify which disks should be managed by the control
         // plane, and adopt all requested disks into the control plane.
-        self.resources.set_config(&config.disks);
+        self.resources.set_config(&config);
 
         // Actually try to "manage" those disks, which may involve formatting
         // zpools and conforming partitions to those expected by the control
@@ -907,14 +825,12 @@ mod tests {
     use crate::dataset::DatasetKind;
     use crate::disk::RawSyntheticDisk;
     use crate::manager_test_harness::StorageManagerTestHarness;
-    use crate::resources::{DiskManagementError, ManagedDisk};
+    use crate::resources::DiskManagementError;
 
     use super::*;
     use camino_tempfile::tempdir_in;
-    use omicron_common::api::external::Generation;
     use omicron_common::ledger;
     use omicron_test_utils::dev::test_setup_log;
-    use omicron_uuid_kinds::ZpoolUuid;
     use sled_hardware::DiskFirmware;
     use std::sync::atomic::Ordering;
     use uuid::Uuid;
@@ -1083,21 +999,17 @@ mod tests {
 
         // Now let's verify we saw the correct firmware update.
         for rd in &raw_disks {
-            let managed =
-                all_disks_gen2.values.get(rd.identity()).expect("disk exists");
-            match managed {
-                ManagedDisk::ExplicitlyManaged(disk)
-                | ManagedDisk::ImplicitlyManaged(disk) => {
-                    assert_eq!(
-                        disk.firmware(),
-                        rd.firmware(),
-                        "didn't see firmware update"
-                    );
-                }
-                ManagedDisk::Unmanaged(disk) => {
-                    assert_eq!(disk, rd, "didn't see firmware update");
-                }
-            }
+            let firmware = all_disks_gen2
+                .iter_all()
+                .find_map(|(identity, _, _, fw)| {
+                    if identity == rd.identity() {
+                        Some(fw)
+                    } else {
+                        None
+                    }
+                })
+                .expect("disk exists");
+            assert_eq!(firmware, rd.firmware(), "didn't see firmware update");
         }
 
         harness.cleanup().await;
@@ -1320,7 +1232,8 @@ mod tests {
 
         let expected: HashSet<_> =
             disks.iter().skip(1).take(3).map(|d| d.identity()).collect();
-        let actual: HashSet<_> = all_disks.values.keys().collect();
+        let actual: HashSet<_> =
+            all_disks.iter_all().map(|(identity, _, _, _)| identity).collect();
         assert_eq!(expected, actual);
 
         // Ensure the same set of disks and make sure no change occurs
@@ -1335,7 +1248,10 @@ mod tests {
             .await
             .unwrap();
         let all_disks2 = harness.handle().get_latest_disks().await;
-        assert_eq!(all_disks.values, all_disks2.values);
+        assert_eq!(
+            all_disks.iter_all().collect::<Vec<_>>(),
+            all_disks2.iter_all().collect::<Vec<_>>()
+        );
 
         // Add a disjoint set of disks and see that only they come through
         harness
@@ -1350,7 +1266,8 @@ mod tests {
         let all_disks = harness.handle().get_latest_disks().await;
         let expected: HashSet<_> =
             disks.iter().skip(4).take(5).map(|d| d.identity()).collect();
-        let actual: HashSet<_> = all_disks.values.keys().collect();
+        let actual: HashSet<_> =
+            all_disks.iter_all().map(|(identity, _, _, _)| identity).collect();
         assert_eq!(expected, actual);
 
         harness.cleanup().await;
@@ -1390,154 +1307,6 @@ mod tests {
         harness.cleanup().await;
         logctx.cleanup_successful();
     }
-
-    #[tokio::test]
-    async fn ledgerless_to_ledgered_migration() {
-        illumos_utils::USE_MOCKS.store(false, Ordering::SeqCst);
-        let logctx = test_setup_log("ledgerless_to_ledgered_migration");
-        let mut harness = StorageManagerTestHarness::new(&logctx.log).await;
-
-        // Test setup: Create two U.2s and an M.2
-        let raw_disks = harness
-            .add_vdevs(&[
-                "u2_under_test.vdev",
-                "u2_that_shows_up_late.vdev",
-                "m2_helping.vdev",
-            ])
-            .await;
-
-        // First, we format the U.2s to have a zpool. This should work, even
-        // without looping in the StorageManager.
-        let first_u2 = &raw_disks[0];
-        let first_pool_id = ZpoolUuid::new_v4();
-        let _disk = crate::disk::Disk::new(
-            &logctx.log,
-            &harness.mount_config(),
-            first_u2.clone(),
-            Some(first_pool_id),
-            Some(harness.key_requester()),
-        )
-        .await
-        .expect("Failed to format U.2");
-
-        let second_u2 = &raw_disks[1];
-        let second_pool_id = ZpoolUuid::new_v4();
-        let _disk = crate::disk::Disk::new(
-            &logctx.log,
-            &harness.mount_config(),
-            second_u2.clone(),
-            Some(second_pool_id),
-            Some(harness.key_requester()),
-        )
-        .await
-        .expect("Failed to format U.2");
-
-        // Because we did that formatting "behind the back" of the
-        // StorageManager, we should see no evidence of the U.2 being managed.
-        //
-        // This currently matches the format of "existing systems, which were
-        // initialized before the storage ledger was created".
-
-        // We should still see no ledger.
-        let result = harness.handle().omicron_physical_disks_list().await;
-        assert!(matches!(result, Err(Error::LedgerNotFound)), "{:?}", result);
-
-        // We should also not see any managed U.2s.
-        let disks = harness.handle().get_latest_disks().await;
-        assert!(disks.all_u2_zpools().is_empty());
-
-        // Leave one of the U.2s attached, but "remove" the other one.
-        harness.remove_vdev(second_u2).await;
-
-        // When the system activates, we should see a single Zpool, and
-        // "auto-manage" it.
-        harness.handle().key_manager_ready().await;
-
-        // It might take a moment for synchronization to be handled by the
-        // background task, but we'll eventually see the U.2 zpool.
-        //
-        // This is the equivalent of us "loading a zpool, even though
-        // it was not backed by a ledger".
-        let tt = TimeTravel::new();
-        tt.enough_to_start_synchronization().await;
-        while harness
-            .handle_mut()
-            .wait_for_changes()
-            .await
-            .all_u2_zpools()
-            .is_empty()
-        {
-            info!(&logctx.log, "Waiting for U.2 to automatically show up");
-        }
-        let u2s = harness.handle().get_latest_disks().await.all_u2_zpools();
-        assert_eq!(u2s.len(), 1, "{:?}", u2s);
-
-        // If we attach the second U.2 -- the equivalent of it appearing after
-        // the key manager is ready -- it'll also be included in the set of
-        // auto-maanged U.2s.
-        harness.add_vdev_as(second_u2.clone()).await;
-        tt.enough_to_start_synchronization().await;
-        while harness
-            .handle_mut()
-            .wait_for_changes()
-            .await
-            .all_u2_zpools()
-            .len()
-            == 1
-        {
-            info!(&logctx.log, "Waiting for U.2 to automatically show up");
-        }
-        let u2s = harness.handle().get_latest_disks().await.all_u2_zpools();
-        assert_eq!(u2s.len(), 2, "{:?}", u2s);
-
-        // This is the equivalent of the "/omicron-physical-disks GET" API,
-        // which Nexus might use to contact this sled.
-        //
-        // This means that we'll bootstrap the sled successfully, but report a
-        // 404 if nexus asks us for the latest configuration.
-        let result = harness.handle().omicron_physical_disks_list().await;
-        assert!(matches!(result, Err(Error::LedgerNotFound),), "{:?}", result);
-
-        // At this point, Nexus may want to explicitly tell sled agent which
-        // disks it should use. This is the equivalent of invoking
-        // "/omicron-physical-disks PUT".
-        let mut disks = vec![
-            OmicronPhysicalDiskConfig {
-                identity: first_u2.identity().clone(),
-                id: Uuid::new_v4(),
-                pool_id: first_pool_id,
-            },
-            OmicronPhysicalDiskConfig {
-                identity: second_u2.identity().clone(),
-                id: Uuid::new_v4(),
-                pool_id: second_pool_id,
-            },
-        ];
-        // Sort the disks to ensure the "output" matches the "input" when we
-        // query later.
-        disks.sort_by(|a, b| a.identity.partial_cmp(&b.identity).unwrap());
-        let config =
-            OmicronPhysicalDisksConfig { generation: Generation::new(), disks };
-        let result = harness
-            .handle()
-            .omicron_physical_disks_ensure(config.clone())
-            .await
-            .expect("Failed to ensure disks with 'new' Config");
-        assert!(!result.has_error(), "{:?}", result);
-
-        let observed_config = harness
-            .handle()
-            .omicron_physical_disks_list()
-            .await
-            .expect("Failed to retreive config after ensuring it");
-        assert_eq!(observed_config, config);
-
-        let u2s = harness.handle().get_latest_disks().await.all_u2_zpools();
-        assert_eq!(u2s.len(), 2, "{:?}", u2s);
-
-        harness.cleanup().await;
-        logctx.cleanup_successful();
-    }
 }
 
 #[cfg(test)]
diff --git a/sled-storage/src/resources.rs b/sled-storage/src/resources.rs
index 5cc4672e1e..f02f62e0a6 100644
--- a/sled-storage/src/resources.rs
+++ b/sled-storage/src/resources.rs
@@ -6,12 +6,16 @@
 
 use crate::config::MountConfig;
 use crate::dataset::{DatasetError, M2_DEBUG_DATASET};
-use crate::disk::{Disk, DiskError, OmicronPhysicalDiskConfig, RawDisk};
+use crate::disk::{
+    Disk, DiskError, OmicronPhysicalDiskConfig, OmicronPhysicalDisksConfig,
+    RawDisk,
+};
 use crate::error::Error;
 use camino::Utf8PathBuf;
 use cfg_if::cfg_if;
-use illumos_utils::zpool::ZpoolName;
+use illumos_utils::zpool::{PathInPool, ZpoolName};
 use key_manager::StorageKeyRequester;
+use omicron_common::api::external::Generation;
 use omicron_common::disk::DiskIdentity;
 use omicron_uuid_kinds::ZpoolUuid;
 use schemars::JsonSchema;
@@ -102,7 +106,7 @@ impl DisksManagementResult {
 // the request of the broader control plane. This enum encompasses that duality,
 // by representing all disks that can exist, managed or not.
 #[derive(Debug, Clone, PartialEq, Eq)]
-pub enum ManagedDisk {
+pub(crate) enum ManagedDisk {
     // A disk explicitly managed by the control plane.
     //
     // This includes U.2s which Nexus has told us to format and use.
@@ -121,6 +125,11 @@ pub enum ManagedDisk {
     Unmanaged(RawDisk),
 }
 
+#[derive(Debug, Clone, Eq, PartialEq)]
+struct AllDisksInner {
+    values: BTreeMap<DiskIdentity, ManagedDisk>,
+}
+
 /// The disks, keyed by their identity, managed by the sled agent.
 ///
 /// This state is owned by [`crate::manager::StorageManager`], through
@@ -139,16 +148,28 @@ pub enum ManagedDisk {
 /// gets cloned or dropped.
 #[derive(Debug, Clone, Eq, PartialEq)]
 pub struct AllDisks {
-    pub values: Arc<BTreeMap<DiskIdentity, ManagedDisk>>,
-    pub mount_config: MountConfig,
+    // This generation corresponds to the generation supplied in
+    // [OmicronPhysicalDisksConfig].
+    generation: Generation,
+    inner: Arc<AllDisksInner>,
+    mount_config: MountConfig,
 }
 
 impl AllDisks {
+    /// Returns the latest generation number of this set of disks.
+    pub fn generation(&self) -> &Generation {
+        &self.generation
+    }
+
+    pub fn mount_config(&self) -> &MountConfig {
+        &self.mount_config
+    }
+
     /// Returns the identity of the boot disk.
     ///
     /// If this returns `None`, we have not processed the boot disk yet.
     pub fn boot_disk(&self) -> Option<(DiskIdentity, ZpoolName)> {
-        for (id, disk) in self.values.iter() {
+        for (id, disk) in self.inner.values.iter() {
             if let ManagedDisk::ImplicitlyManaged(disk) = disk {
                 if disk.is_boot_disk() {
                     return Some((id.clone(), disk.zpool_name().clone()));
@@ -179,18 +200,21 @@ impl AllDisks {
     }
 
     /// Returns all mountpoints within all U.2s for a particular dataset.
-    pub fn all_u2_mountpoints(&self, dataset: &str) -> Vec<Utf8PathBuf> {
+    pub fn all_u2_mountpoints(&self, dataset: &str) -> Vec<PathInPool> {
         self.all_u2_zpools()
-            .iter()
-            .map(|zpool| {
-                zpool.dataset_mountpoint(&self.mount_config.root, dataset)
+            .into_iter()
+            .map(|pool| {
+                let path =
+                    pool.dataset_mountpoint(&self.mount_config.root, dataset);
+                PathInPool { pool: Some(pool), path }
             })
             .collect()
     }
 
     /// Returns all zpools managed by the control plane
     pub fn get_all_zpools(&self) -> Vec<(ZpoolName, DiskVariant)> {
-        self.values
+        self.inner
+            .values
             .values()
             .filter_map(|disk| match disk {
                 ManagedDisk::ExplicitlyManaged(disk)
@@ -206,7 +230,8 @@ impl AllDisks {
     //
     // Only returns zpools from disks actively being managed.
     fn all_zpools(&self, variant: DiskVariant) -> Vec<ZpoolName> {
-        self.values
+        self.inner
+            .values
             .values()
             .filter_map(|disk| match disk {
                 ManagedDisk::ExplicitlyManaged(disk)
@@ -231,7 +256,7 @@ impl AllDisks {
 
     /// Returns an iterator over all managed disks.
     pub fn iter_managed(&self) -> impl Iterator<Item = (&DiskIdentity, &Disk)> {
-        self.values.iter().filter_map(|(identity, disk)| match disk {
+        self.inner.values.iter().filter_map(|(identity, disk)| match disk {
             ManagedDisk::ExplicitlyManaged(disk) => Some((identity, disk)),
             ManagedDisk::ImplicitlyManaged(disk) => Some((identity, disk)),
             _ => None,
@@ -243,7 +268,7 @@ impl AllDisks {
         &self,
     ) -> impl Iterator<Item = (&DiskIdentity, DiskVariant, i64, &DiskFirmware)>
     {
-        self.values.iter().map(|(identity, disk)| match disk {
+        self.inner.values.iter().map(|(identity, disk)| match disk {
             ManagedDisk::ExplicitlyManaged(disk) => {
                 (identity, disk.variant(), disk.slot(), disk.firmware())
             }
@@ -284,8 +309,11 @@ impl StorageResources {
         mount_config: MountConfig,
         key_requester: StorageKeyRequester,
     ) -> Self {
-        let disks =
-            AllDisks { values: Arc::new(BTreeMap::new()), mount_config };
+        let disks = AllDisks {
+            generation: Generation::new(),
+            inner: Arc::new(AllDisksInner { values: BTreeMap::new() }),
+            mount_config,
+        };
         Self {
             log: log.new(o!("component" => "StorageResources")),
             key_requester,
@@ -310,8 +338,14 @@ impl StorageResources {
     /// Does not attempt to manage any of the physical disks previously
     /// observed. To synchronize the "set of requested disks" with the "set of
     /// observed disks", call [Self::synchronize_disk_management].
-    pub fn set_config(&mut self, config: &Vec<OmicronPhysicalDiskConfig>) {
+    pub fn set_config(&mut self, config: &OmicronPhysicalDisksConfig) {
+        let our_gen = &mut self.disks.generation;
+        if *our_gen > config.generation {
+            return;
+        }
+        *our_gen = config.generation;
         self.control_plane_disks = config
+            .disks
             .iter()
             .map(|disk| (disk.identity.clone(), disk.clone()))
             .collect();
@@ -336,14 +370,14 @@ impl StorageResources {
         &mut self,
     ) -> DisksManagementResult {
         let mut updated = false;
-        let disks = Arc::make_mut(&mut self.disks.values);
+        let disks = Arc::make_mut(&mut self.disks.inner);
         info!(self.log, "Synchronizing disk managment");
 
         // "Unmanage" all disks no longer requested by the control plane.
         //
         // This updates the reported sets of "managed" disks, and performs no
         // other modifications to the underlying storage.
-        for (identity, managed_disk) in &mut *disks {
+        for (identity, managed_disk) in &mut disks.values {
             match managed_disk {
                 // This leaves the presence of the disk still in "Self", but
                 // downgrades the disk to an unmanaged status.
@@ -365,7 +399,7 @@ impl StorageResources {
         // configuration.
         let mut result = DisksManagementResult::default();
         for (identity, config) in &self.control_plane_disks {
-            let Some(managed_disk) = disks.get_mut(identity) else {
+            let Some(managed_disk) = disks.values.get_mut(identity) else {
                 warn!(
                     self.log,
                     "Control plane disk requested, but not detected within sled";
@@ -496,11 +530,11 @@ impl StorageResources {
 
         // This is a trade-off for simplicity even though we may be potentially
         // cloning data before we know if there is a write action to perform.
-        let disks = Arc::make_mut(&mut self.disks.values);
+        let disks = Arc::make_mut(&mut self.disks.inner);
 
         // First check if there are any updates we need to apply to existing
         // managed disks.
-        if let Some(managed) = disks.get_mut(&disk_identity) {
+        if let Some(managed) = disks.values.get_mut(&disk_identity) {
             let mut updated = false;
             match managed {
                 ManagedDisk::ExplicitlyManaged(mdisk)
@@ -532,7 +566,9 @@ impl StorageResources {
         // If there's no update then we are inserting a new disk.
         match disk.variant() {
             DiskVariant::U2 => {
-                disks.insert(disk_identity, ManagedDisk::Unmanaged(disk));
+                disks
+                    .values
+                    .insert(disk_identity, ManagedDisk::Unmanaged(disk));
             }
             DiskVariant::M2 => {
                 let managed_disk = Disk::new(
@@ -543,12 +579,13 @@ impl StorageResources {
                     Some(&self.key_requester),
                 )
                 .await?;
-                disks.insert(
+                disks.values.insert(
                     disk_identity,
                     ManagedDisk::ImplicitlyManaged(managed_disk),
                 );
             }
         }
+
         self.disk_updates.send_replace(self.disks.clone());
 
         Ok(())
@@ -562,7 +599,7 @@ impl StorageResources {
     /// are only added once.
     pub(crate) fn remove_disk(&mut self, id: &DiskIdentity) {
         info!(self.log, "Removing disk"; "identity" => ?id);
-        let Some(entry) = self.disks.values.get(id) else {
+        let Some(entry) = self.disks.inner.values.get(id) else {
             info!(self.log, "Disk not found by id, exiting"; "identity" => ?id);
             return;
         };
@@ -589,7 +626,9 @@ impl StorageResources {
         }
 
         // Safe to unwrap as we just checked the key existed above
-        Arc::make_mut(&mut self.disks.values).remove(id).unwrap();
+        let disks = Arc::make_mut(&mut self.disks.inner);
+        disks.values.remove(id).unwrap();
+
         self.disk_updates.send_replace(self.disks.clone());
     }
 }
diff --git a/smf/nexus/multi-sled/config-partial.toml b/smf/nexus/multi-sled/config-partial.toml
index 92d3d6e392..50f9bf646e 100644
--- a/smf/nexus/multi-sled/config-partial.toml
+++ b/smf/nexus/multi-sled/config-partial.toml
@@ -61,6 +61,7 @@ service_firewall_propagation.period_secs = 300
 v2p_mapping_propagation.period_secs = 30
 instance_watcher.period_secs = 30
 abandoned_vmm_reaper.period_secs = 60
+saga_recovery.period_secs = 600
 lookup_region_port.period_secs = 60
 
 [default_region_allocation_strategy]
diff --git a/smf/nexus/single-sled/config-partial.toml b/smf/nexus/single-sled/config-partial.toml
index 8de9b6cb79..31db278616 100644
--- a/smf/nexus/single-sled/config-partial.toml
+++ b/smf/nexus/single-sled/config-partial.toml
@@ -61,6 +61,7 @@ service_firewall_propagation.period_secs = 300
 v2p_mapping_propagation.period_secs = 30
 instance_watcher.period_secs = 30
 abandoned_vmm_reaper.period_secs = 60
+saga_recovery.period_secs = 600
 lookup_region_port.period_secs = 60
 
 [default_region_allocation_strategy]
diff --git a/sp-sim/Cargo.toml b/sp-sim/Cargo.toml
index 35cb791f4c..7270db1a67 100644
--- a/sp-sim/Cargo.toml
+++ b/sp-sim/Cargo.toml
@@ -20,7 +20,6 @@ omicron-common.workspace = true
 serde.workspace = true
 slog.workspace = true
 slog-dtrace.workspace = true
-sprockets-rot.workspace = true
 thiserror.workspace = true
 tokio = { workspace = true, features = [ "full" ] }
 toml.workspace = true
diff --git a/sp-sim/src/gimlet.rs b/sp-sim/src/gimlet.rs
index 4e0b264e64..ac465cb217 100644
--- a/sp-sim/src/gimlet.rs
+++ b/sp-sim/src/gimlet.rs
@@ -6,7 +6,6 @@ use crate::config::GimletConfig;
 use crate::config::SpComponentConfig;
 use crate::helpers::rot_slot_id_from_u16;
 use crate::helpers::rot_slot_id_to_u16;
-use crate::rot::RotSprocketExt;
 use crate::serial_number_padded;
 use crate::server;
 use crate::server::SimSpHandler;
@@ -38,9 +37,6 @@ use gateway_messages::{version, MessageKind};
 use gateway_messages::{ComponentDetails, Message, MgsError, StartupOptions};
 use gateway_messages::{DiscoverResponse, IgnitionState, PowerState};
 use slog::{debug, error, info, warn, Logger};
-use sprockets_rot::common::msgs::{RotRequestV1, RotResponseV1};
-use sprockets_rot::common::Ed25519PublicKey;
-use sprockets_rot::{RotSprocket, RotSprocketError};
 use std::cell::Cell;
 use std::collections::HashMap;
 use std::iter;
@@ -88,8 +84,6 @@ pub enum SimSpHandledRequest {
 }
 
 pub struct Gimlet {
-    rot: Mutex<RotSprocket>,
-    manufacturing_public_key: Ed25519PublicKey,
     local_addrs: Option<[SocketAddrV6; 2]>,
     handler: Option<Arc<TokioMutex<Handler>>>,
     serial_console_addrs: HashMap<String, SocketAddrV6>,
@@ -116,10 +110,6 @@ impl SimulatedSp for Gimlet {
         )
     }
 
-    fn manufacturing_public_key(&self) -> Ed25519PublicKey {
-        self.manufacturing_public_key
-    }
-
     fn local_addr(&self, port: SpPort) -> Option<SocketAddrV6> {
         let i = match port {
             SpPort::One => 0,
@@ -135,13 +125,6 @@ impl SimulatedSp for Gimlet {
         }
     }
 
-    fn rot_request(
-        &self,
-        request: RotRequestV1,
-    ) -> Result<RotResponseV1, RotSprocketError> {
-        self.rot.lock().unwrap().handle_deserialized(request)
-    }
-
     async fn last_sp_update_data(&self) -> Option<Box<[u8]>> {
         let handler = self.handler.as_ref()?;
         let handler = handler.lock().await;
@@ -201,16 +184,11 @@ impl Gimlet {
         let (commands, commands_rx) = mpsc::unbounded_channel();
         let last_request_handled = Arc::default();
 
-        let (manufacturing_public_key, rot) =
-            RotSprocket::bootstrap_from_config(&gimlet.common);
-
         // Weird case - if we don't have any bind addresses, we're only being
         // created to simulate an RoT, so go ahead and return without actually
         // starting a simulated SP.
         let Some(bind_addrs) = gimlet.common.bind_addrs else {
             return Ok(Self {
-                rot: Mutex::new(rot),
-                manufacturing_public_key,
                 local_addrs: None,
                 handler: None,
                 serial_console_addrs,
@@ -299,8 +277,6 @@ impl Gimlet {
             .push(task::spawn(async move { inner.run().await.unwrap() }));
 
         Ok(Self {
-            rot: Mutex::new(rot),
-            manufacturing_public_key,
             local_addrs: Some(local_addrs),
             handler: Some(handler),
             serial_console_addrs,
diff --git a/sp-sim/src/lib.rs b/sp-sim/src/lib.rs
index ca9231bec0..868d7ded2c 100644
--- a/sp-sim/src/lib.rs
+++ b/sp-sim/src/lib.rs
@@ -5,7 +5,6 @@
 pub mod config;
 mod gimlet;
 mod helpers;
-mod rot;
 mod server;
 mod sidecar;
 mod update;
@@ -21,10 +20,6 @@ pub use server::logger;
 pub use sidecar::Sidecar;
 pub use sidecar::SIM_SIDECAR_BOARD;
 pub use slog::Logger;
-pub use sprockets_rot::common::msgs::RotRequestV1;
-pub use sprockets_rot::common::msgs::RotResponseV1;
-use sprockets_rot::common::Ed25519PublicKey;
-pub use sprockets_rot::RotSprocketError;
 use std::net::SocketAddrV6;
 use tokio::sync::mpsc;
 use tokio::sync::watch;
@@ -43,9 +38,6 @@ pub trait SimulatedSp {
     /// Serial number.
     async fn state(&self) -> omicron_gateway::http_entrypoints::SpState;
 
-    /// Public key for the manufacturing cert used to sign this SP's RoT certs.
-    fn manufacturing_public_key(&self) -> Ed25519PublicKey;
-
     /// Listening UDP address of the given port of this simulated SP, if it was
     /// configured to listen.
     fn local_addr(&self, port: SpPort) -> Option<SocketAddrV6>;
@@ -54,12 +46,6 @@ pub trait SimulatedSp {
     /// messages.
     async fn set_responsiveness(&self, r: Responsiveness);
 
-    /// Send a request to the (simulated) RoT.
-    fn rot_request(
-        &self,
-        request: RotRequestV1,
-    ) -> Result<RotResponseV1, RotSprocketError>;
-
     /// Get the last completed update delivered to this simulated SP.
     ///
     /// Only returns data after a simulated reset of the SP.
diff --git a/sp-sim/src/rot.rs b/sp-sim/src/rot.rs
deleted file mode 100644
index 9f0bf61cc0..0000000000
--- a/sp-sim/src/rot.rs
+++ /dev/null
@@ -1,46 +0,0 @@
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this
-// file, You can obtain one at https://mozilla.org/MPL/2.0/.
-
-//! Simualting a Root of Trust
-
-use crate::config::SpCommonConfig;
-use sprockets_rot::common::certificates::SerialNumber;
-use sprockets_rot::common::Ed25519PublicKey;
-use sprockets_rot::salty;
-use sprockets_rot::RotConfig;
-use sprockets_rot::RotSprocket;
-
-pub(crate) trait RotSprocketExt {
-    // Returns the (derived-from-config) manufacturing public key and the
-    // `RotSprocket`.
-    fn bootstrap_from_config(
-        config: &SpCommonConfig,
-    ) -> (Ed25519PublicKey, Self);
-}
-
-impl RotSprocketExt for RotSprocket {
-    fn bootstrap_from_config(
-        config: &SpCommonConfig,
-    ) -> (Ed25519PublicKey, Self) {
-        let mut serial_number = [0; 16];
-        serial_number
-            .get_mut(0..config.serial_number.len())
-            .expect("simulated serial number too long")
-            .copy_from_slice(config.serial_number.as_bytes());
-
-        let manufacturing_keypair =
-            salty::Keypair::from(&config.manufacturing_root_cert_seed);
-        let device_id_keypair =
-            salty::Keypair::from(&config.device_id_cert_seed);
-        let serial_number = SerialNumber(serial_number);
-        let config = RotConfig::bootstrap_for_testing(
-            &manufacturing_keypair,
-            device_id_keypair,
-            serial_number,
-        );
-        let manufacturing_public_key =
-            Ed25519PublicKey(manufacturing_keypair.public.to_bytes());
-        (manufacturing_public_key, Self::new(config))
-    }
-}
diff --git a/sp-sim/src/sidecar.rs b/sp-sim/src/sidecar.rs
index 696989f791..a6bc49e609 100644
--- a/sp-sim/src/sidecar.rs
+++ b/sp-sim/src/sidecar.rs
@@ -8,7 +8,6 @@ use crate::config::SimulatedSpsConfig;
 use crate::config::SpComponentConfig;
 use crate::helpers::rot_slot_id_from_u16;
 use crate::helpers::rot_slot_id_to_u16;
-use crate::rot::RotSprocketExt;
 use crate::serial_number_padded;
 use crate::server;
 use crate::server::SimSpHandler;
@@ -49,16 +48,10 @@ use slog::debug;
 use slog::info;
 use slog::warn;
 use slog::Logger;
-use sprockets_rot::common::msgs::RotRequestV1;
-use sprockets_rot::common::msgs::RotResponseV1;
-use sprockets_rot::common::Ed25519PublicKey;
-use sprockets_rot::RotSprocket;
-use sprockets_rot::RotSprocketError;
 use std::iter;
 use std::net::SocketAddrV6;
 use std::pin::Pin;
 use std::sync::Arc;
-use std::sync::Mutex;
 use tokio::select;
 use tokio::sync::mpsc;
 use tokio::sync::oneshot;
@@ -70,8 +63,6 @@ use tokio::task::JoinHandle;
 pub const SIM_SIDECAR_BOARD: &str = "SimSidecarSp";
 
 pub struct Sidecar {
-    rot: Mutex<RotSprocket>,
-    manufacturing_public_key: Ed25519PublicKey,
     local_addrs: Option<[SocketAddrV6; 2]>,
     handler: Option<Arc<TokioMutex<Handler>>>,
     commands: mpsc::UnboundedSender<Command>,
@@ -96,10 +87,6 @@ impl SimulatedSp for Sidecar {
         )
     }
 
-    fn manufacturing_public_key(&self) -> Ed25519PublicKey {
-        self.manufacturing_public_key
-    }
-
     fn local_addr(&self, port: SpPort) -> Option<SocketAddrV6> {
         let i = match port {
             SpPort::One => 0,
@@ -117,13 +104,6 @@ impl SimulatedSp for Sidecar {
         rx.await.unwrap();
     }
 
-    fn rot_request(
-        &self,
-        request: RotRequestV1,
-    ) -> Result<RotResponseV1, RotSprocketError> {
-        self.rot.lock().unwrap().handle_deserialized(request)
-    }
-
     async fn last_sp_update_data(&self) -> Option<Box<[u8]>> {
         let handler = self.handler.as_ref()?;
         let handler = handler.lock().await;
@@ -224,11 +204,7 @@ impl Sidecar {
                 (None, None, None, None)
             };
 
-        let (manufacturing_public_key, rot) =
-            RotSprocket::bootstrap_from_config(&sidecar.common);
         Ok(Self {
-            rot: Mutex::new(rot),
-            manufacturing_public_key,
             local_addrs,
             handler,
             commands,
diff --git a/tools/cargo_hack_checksum b/tools/cargo_hack_checksum
new file mode 100644
index 0000000000..12ed33c12e
--- /dev/null
+++ b/tools/cargo_hack_checksum
@@ -0,0 +1,3 @@
+CIDL_SHA256_DARWIN="ee00750378126c7e14402a45c34f95ed1ba4be2ae505b0c0020bb39b5b3467a4"
+CIDL_SHA256_ILLUMOS="f80d281343368bf7a027e2a7e94ae98a19e085c0666bff8d15264f39b42997bc"
+CIDL_SHA256_LINUX="ffecd932fc7569975eb77d70f2e299f07b57220868bedeb5867062a4a95a0376"
diff --git a/tools/cargo_hack_version b/tools/cargo_hack_version
new file mode 100644
index 0000000000..cb180fda59
--- /dev/null
+++ b/tools/cargo_hack_version
@@ -0,0 +1 @@
+0.6.29
diff --git a/tools/console_version b/tools/console_version
index 626464c23d..4f67064733 100644
--- a/tools/console_version
+++ b/tools/console_version
@@ -1,2 +1,2 @@
-COMMIT="4377d01585ef87981ed51a4cd1f07376e8502d39"
-SHA2="3e0707dcd6a350ecc3bd62e8e7485a773eebf52f5ffd0db4e8cfb01251e28374"
+COMMIT="17ae890c68a5277fbefe773694e790a8f1b178b4"
+SHA2="273a31ba14546305bfafeb9aedb2d9a7530328a0359cda363380c9ca3240b948"
diff --git a/tools/generate-wicketd-api.sh b/tools/generate-wicketd-api.sh
deleted file mode 100755
index 3fbddee5af..0000000000
--- a/tools/generate-wicketd-api.sh
+++ /dev/null
@@ -1,3 +0,0 @@
-#!/usr/bin/env bash
-
-./target/debug/wicketd openapi > openapi/wicketd.json
diff --git a/tools/maghemite_ddm_openapi_version b/tools/maghemite_ddm_openapi_version
index f5848af24f..40d39b3dd0 100644
--- a/tools/maghemite_ddm_openapi_version
+++ b/tools/maghemite_ddm_openapi_version
@@ -1,2 +1,2 @@
-COMMIT="1b385990e8648b221fd11f018f2a7ec425461c6c"
+COMMIT="220dd026e83142b83bd93123f465a64dd4600201"
 SHA2="007bfb717ccbc077c0250dee3121aeb0c5bb0d1c16795429a514fa4f8635a5ef"
diff --git a/tools/maghemite_mg_openapi_version b/tools/maghemite_mg_openapi_version
index a5f027ec9f..172c5c6f3d 100644
--- a/tools/maghemite_mg_openapi_version
+++ b/tools/maghemite_mg_openapi_version
@@ -1,2 +1,2 @@
-COMMIT="1b385990e8648b221fd11f018f2a7ec425461c6c"
+COMMIT="220dd026e83142b83bd93123f465a64dd4600201"
 SHA2="e4b42ab9daad90f0c561a830b62a9d17e294b4d0da0a6d44b4030929b0c37b7e"
diff --git a/tools/maghemite_mgd_checksums b/tools/maghemite_mgd_checksums
index 9c4f94c6cd..5479623d30 100644
--- a/tools/maghemite_mgd_checksums
+++ b/tools/maghemite_mgd_checksums
@@ -1,2 +1,2 @@
-CIDL_SHA256="d4f2aaca20b312b6716206c335165442d6625b929eb00f0fd23f551e38216ace"
-MGD_LINUX_SHA256="2a9484345e6cba6587f71c1ee75048e2ee45a18a6628a7d88ccb9b9fb7b07faf"
\ No newline at end of file
+CIDL_SHA256="f1103de5dda4830eb653f4d555995d08c31253116448387399a77392c08dfb54"
+MGD_LINUX_SHA256="b4469b8ec3b2193f3eff2886fe1c7ac17dc135b8d7572e1a6c765811738402bd"
\ No newline at end of file
diff --git a/tools/opte_version b/tools/opte_version
index ff992938ae..dfbb589f24 100644
--- a/tools/opte_version
+++ b/tools/opte_version
@@ -1 +1 @@
-0.32.265
+0.33.277
diff --git a/wicket-common/Cargo.toml b/wicket-common/Cargo.toml
index 9a82b3d8bd..3c24cea805 100644
--- a/wicket-common/Cargo.toml
+++ b/wicket-common/Cargo.toml
@@ -9,6 +9,8 @@ workspace = true
 
 [dependencies]
 anyhow.workspace = true
+dpd-client.workspace = true
+dropshot.workspace = true
 gateway-client.workspace = true
 maplit.workspace = true
 omicron-common.workspace = true
@@ -20,7 +22,9 @@ serde.workspace = true
 serde_json.workspace = true
 sha2.workspace = true
 sled-hardware-types.workspace = true
+slog.workspace = true
 thiserror.workspace = true
+tokio.workspace = true
 update-engine.workspace = true
 
 [dev-dependencies]
diff --git a/wicket-common/src/example.rs b/wicket-common/src/example.rs
index 16b5df6768..bb70273b45 100644
--- a/wicket-common/src/example.rs
+++ b/wicket-common/src/example.rs
@@ -6,7 +6,6 @@
 
 use std::{collections::BTreeSet, net::Ipv6Addr};
 
-use gateway_client::types::{SpIdentifier, SpType};
 use maplit::{btreemap, btreeset};
 use omicron_common::{
     address::{IpRange, Ipv4Range},
@@ -19,11 +18,14 @@ use omicron_common::{
 };
 use sled_hardware_types::Baseboard;
 
-use crate::rack_setup::{
-    BgpAuthKeyId, BootstrapSledDescription, CurrentRssUserConfigInsensitive,
-    PutRssUserConfigInsensitive, UserSpecifiedBgpPeerConfig,
-    UserSpecifiedImportExportPolicy, UserSpecifiedPortConfig,
-    UserSpecifiedRackNetworkConfig,
+use crate::{
+    inventory::{SpIdentifier, SpType},
+    rack_setup::{
+        BgpAuthKeyId, BootstrapSledDescription,
+        CurrentRssUserConfigInsensitive, PutRssUserConfigInsensitive,
+        UserSpecifiedBgpPeerConfig, UserSpecifiedImportExportPolicy,
+        UserSpecifiedPortConfig, UserSpecifiedRackNetworkConfig,
+    },
 };
 
 /// A collection of example data structures.
diff --git a/wicketd/src/inventory.rs b/wicket-common/src/inventory.rs
similarity index 77%
rename from wicketd/src/inventory.rs
rename to wicket-common/src/inventory.rs
index e1465147b5..f7b42e4ec0 100644
--- a/wicketd/src/inventory.rs
+++ b/wicket-common/src/inventory.rs
@@ -2,17 +2,25 @@
 // License, v. 2.0. If a copy of the MPL was not distributed with this
 // file, You can obtain one at https://mozilla.org/MPL/2.0/.
 
-//! Rack inventory for display by wicket
-
-use gateway_client::types::{
-    RotSlot, SpComponentCaboose, SpComponentInfo, SpIdentifier, SpIgnition,
-    SpState,
+// Re-export these types from gateway_client, so that users are oblivious to
+// where these types come from.
+pub use gateway_client::types::{
+    RotSlot, RotState, SpComponentCaboose, SpComponentInfo,
+    SpComponentPresence, SpIdentifier, SpIgnition, SpIgnitionSystemType,
+    SpState, SpType,
 };
 use schemars::JsonSchema;
-use serde::Serialize;
+use serde::{Deserialize, Serialize};
+
+/// The current state of the v1 Rack as known to wicketd
+#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema)]
+#[serde(tag = "inventory", rename_all = "snake_case")]
+pub struct RackV1Inventory {
+    pub sps: Vec<SpInventory>,
+}
 
 /// SP-related data
-#[derive(Debug, Clone, Serialize, JsonSchema)]
+#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
 #[serde(tag = "sp_inventory", rename_all = "snake_case")]
 pub struct SpInventory {
     pub id: SpIdentifier,
@@ -42,7 +50,7 @@ impl SpInventory {
 }
 
 /// RoT-related data that isn't already supplied in [`SpState`].
-#[derive(Debug, Clone, Serialize, JsonSchema)]
+#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
 #[serde(tag = "sp_inventory", rename_all = "snake_case")]
 pub struct RotInventory {
     pub active: RotSlot,
@@ -53,10 +61,3 @@ pub struct RotInventory {
     pub caboose_stage0: Option<Option<SpComponentCaboose>>,
     pub caboose_stage0next: Option<Option<SpComponentCaboose>>,
 }
-
-/// The current state of the v1 Rack as known to wicketd
-#[derive(Clone, Debug, Serialize, JsonSchema)]
-#[serde(tag = "inventory", rename_all = "snake_case")]
-pub struct RackV1Inventory {
-    pub sps: Vec<SpInventory>,
-}
diff --git a/wicket-common/src/lib.rs b/wicket-common/src/lib.rs
index c5ddabdb1c..7ad676e976 100644
--- a/wicket-common/src/lib.rs
+++ b/wicket-common/src/lib.rs
@@ -7,6 +7,8 @@
 use std::time::Duration;
 
 pub mod example;
+pub mod inventory;
+pub mod preflight_check;
 pub mod rack_setup;
 pub mod rack_update;
 pub mod update_events;
diff --git a/wicket-common/src/preflight_check.rs b/wicket-common/src/preflight_check.rs
new file mode 100644
index 0000000000..e13be0a9d7
--- /dev/null
+++ b/wicket-common/src/preflight_check.rs
@@ -0,0 +1,79 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+use std::net::IpAddr;
+
+use dpd_client::types::PortId;
+use oxnet::IpNet;
+use schemars::JsonSchema;
+use serde::{Deserialize, Serialize};
+use thiserror::Error;
+use update_engine::StepSpec;
+
+#[derive(Debug, Error)]
+pub enum UplinkPreflightTerminalError {
+    #[error("invalid port name: {0}")]
+    InvalidPortName(String),
+    #[error("failed to connect to dpd to check for current configuration")]
+    GetCurrentConfig(#[source] DpdError),
+    #[error("uplink already configured - is rack already initialized?")]
+    UplinkAlreadyConfigured,
+    #[error("failed to create port {port_id:?}")]
+    ConfigurePort {
+        #[source]
+        err: DpdError,
+        port_id: PortId,
+    },
+    #[error(
+        "failed to remove host OS route {destination} -> {nexthop}: {err}"
+    )]
+    RemoveHostRoute { err: String, destination: IpNet, nexthop: IpAddr },
+    #[error("failed to remove uplink SMF property {property:?}: {err}")]
+    RemoveSmfProperty { property: String, err: String },
+    #[error("failed to refresh uplink service config: {0}")]
+    RefreshUplinkSmf(String),
+    #[error("failed to clear settings for port {port_id:?}")]
+    UnconfigurePort {
+        #[source]
+        err: DpdError,
+        port_id: PortId,
+    },
+}
+
+impl update_engine::AsError for UplinkPreflightTerminalError {
+    fn as_error(&self) -> &(dyn std::error::Error + 'static) {
+        self
+    }
+}
+
+type DpdError = dpd_client::Error<dpd_client::types::Error>;
+
+#[derive(JsonSchema)]
+pub enum UplinkPreflightCheckSpec {}
+
+impl StepSpec for UplinkPreflightCheckSpec {
+    type Component = String;
+    type StepId = UplinkPreflightStepId;
+    type StepMetadata = ();
+    type ProgressMetadata = String;
+    type CompletionMetadata = Vec<String>;
+    type SkippedMetadata = ();
+    type Error = UplinkPreflightTerminalError;
+}
+
+#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize, JsonSchema)]
+#[serde(tag = "id", rename_all = "snake_case")]
+pub enum UplinkPreflightStepId {
+    ConfigureSwitch,
+    WaitForL1Link,
+    ConfigureAddress,
+    ConfigureRouting,
+    CheckExternalDnsConnectivity,
+    CheckExternalNtpConnectivity,
+    CleanupRouting,
+    CleanupAddress,
+    CleanupL1,
+}
+
+update_engine::define_update_engine!(pub UplinkPreflightCheckSpec);
diff --git a/wicket-common/src/rack_setup.rs b/wicket-common/src/rack_setup.rs
index 33fbcb65b3..7fd83e522a 100644
--- a/wicket-common/src/rack_setup.rs
+++ b/wicket-common/src/rack_setup.rs
@@ -4,8 +4,6 @@
 
 // Copyright 2024 Oxide Computer Company
 
-pub use gateway_client::types::SpIdentifier as GatewaySpIdentifier;
-pub use gateway_client::types::SpType as GatewaySpType;
 use omicron_common::address;
 use omicron_common::api::external::ImportExportPolicy;
 use omicron_common::api::external::Name;
@@ -36,6 +34,8 @@ use std::net::Ipv4Addr;
 use std::net::Ipv6Addr;
 use std::str::FromStr;
 
+use crate::inventory::SpIdentifier;
+
 /// The subset of `RackInitializeRequest` that the user fills in as clear text
 /// (e.g., via an uploaded config file).
 #[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize, JsonSchema)]
@@ -84,10 +84,7 @@ pub struct PutRssUserConfigInsensitive {
     Ord,
 )]
 pub struct BootstrapSledDescription {
-    // TODO: We currently use gateway-client's SpIdentifier here, not our own,
-    // to avoid wicketd-client getting an "SpIdentifier2". We really do need to
-    // unify this type once and forever.
-    pub id: GatewaySpIdentifier,
+    pub id: SpIdentifier,
     pub baseboard: Baseboard,
     /// The sled's bootstrap address, if the host is on and we've discovered it
     /// on the bootstrap network.
diff --git a/wicket-common/src/rack_update.rs b/wicket-common/src/rack_update.rs
index 4fa3ea371c..e5d96db726 100644
--- a/wicket-common/src/rack_update.rs
+++ b/wicket-common/src/rack_update.rs
@@ -4,90 +4,79 @@
 
 // Copyright 2023 Oxide Computer Company
 
-use std::{collections::BTreeSet, fmt};
+use std::{collections::BTreeSet, time::Duration};
 
+use dropshot::HttpError;
 use schemars::JsonSchema;
 use serde::{Deserialize, Serialize};
 
-// TODO: unify this with the one in gateway http_entrypoints.rs.
-#[derive(
-    Debug,
-    Clone,
-    Copy,
-    PartialEq,
-    Eq,
-    PartialOrd,
-    Ord,
-    Serialize,
-    Deserialize,
-    JsonSchema,
-)]
-pub struct SpIdentifier {
-    #[serde(rename = "type")]
-    pub type_: SpType,
-    pub slot: u32,
-}
+use crate::inventory::SpIdentifier;
 
-impl From<SpIdentifier> for gateway_client::types::SpIdentifier {
-    fn from(value: SpIdentifier) -> Self {
-        Self { type_: value.type_.into(), slot: value.slot }
-    }
-}
+#[derive(Clone, Debug, Default, JsonSchema, Deserialize, Serialize)]
+pub struct StartUpdateOptions {
+    /// If passed in, fails the update with a simulated error.
+    pub test_error: Option<UpdateTestError>,
 
-impl From<gateway_client::types::SpIdentifier> for SpIdentifier {
-    fn from(value: gateway_client::types::SpIdentifier) -> Self {
-        Self { type_: value.type_.into(), slot: value.slot }
-    }
-}
+    /// If passed in, creates a test step that lasts these many seconds long.
+    ///
+    /// This is used for testing.
+    pub test_step_seconds: Option<u64>,
 
-#[derive(
-    Debug,
-    Clone,
-    Copy,
-    PartialEq,
-    Eq,
-    PartialOrd,
-    Ord,
-    Hash,
-    Serialize,
-    Deserialize,
-    JsonSchema,
-)]
-#[serde(rename_all = "lowercase")]
-pub enum SpType {
-    Switch,
-    Sled,
-    Power,
+    /// If passed in, simulates a result for the RoT Bootloader update.
+    ///
+    /// This is used for testing.
+    pub test_simulate_rot_bootloader_result: Option<UpdateSimulatedResult>,
+
+    /// If passed in, simulates a result for the RoT update.
+    ///
+    /// This is used for testing.
+    pub test_simulate_rot_result: Option<UpdateSimulatedResult>,
+
+    /// If passed in, simulates a result for the SP update.
+    ///
+    /// This is used for testing.
+    pub test_simulate_sp_result: Option<UpdateSimulatedResult>,
+
+    /// If true, skip the check on the current RoT version and always update it
+    /// regardless of whether the update appears to be neeeded.
+    pub skip_rot_bootloader_version_check: bool,
+
+    /// If true, skip the check on the current RoT version and always update it
+    /// regardless of whether the update appears to be neeeded.
+    pub skip_rot_version_check: bool,
+
+    /// If true, skip the check on the current SP version and always update it
+    /// regardless of whether the update appears to be neeeded.
+    pub skip_sp_version_check: bool,
 }
 
-impl From<SpType> for gateway_client::types::SpType {
-    fn from(value: SpType) -> Self {
-        match value {
-            SpType::Switch => Self::Switch,
-            SpType::Sled => Self::Sled,
-            SpType::Power => Self::Power,
-        }
-    }
+/// A simulated result for a component update.
+///
+/// Used by [`StartUpdateOptions`].
+#[derive(Clone, Debug, JsonSchema, Deserialize, Serialize)]
+#[serde(rename_all = "snake_case")]
+pub enum UpdateSimulatedResult {
+    Success,
+    Warning,
+    Skipped,
+    Failure,
 }
 
-impl From<gateway_client::types::SpType> for SpType {
-    fn from(value: gateway_client::types::SpType) -> Self {
-        match value {
-            gateway_client::types::SpType::Switch => Self::Switch,
-            gateway_client::types::SpType::Sled => Self::Sled,
-            gateway_client::types::SpType::Power => Self::Power,
-        }
-    }
+#[derive(Clone, Debug, JsonSchema, Deserialize, Serialize)]
+pub struct ClearUpdateStateOptions {
+    /// If passed in, fails the clear update state operation with a simulated
+    /// error.
+    pub test_error: Option<UpdateTestError>,
 }
 
-impl fmt::Display for SpType {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        match self {
-            SpType::Switch => write!(f, "switch"),
-            SpType::Sled => write!(f, "sled"),
-            SpType::Power => write!(f, "power"),
-        }
-    }
+#[derive(Clone, Debug, JsonSchema, Deserialize, Serialize)]
+pub struct AbortUpdateOptions {
+    /// The message to abort the update with.
+    pub message: String,
+
+    /// If passed in, fails the force cancel update operation with a simulated
+    /// error.
+    pub test_error: Option<UpdateTestError>,
 }
 
 #[derive(
@@ -100,3 +89,47 @@ pub struct ClearUpdateStateResponse {
     /// The SPs that had no update state to clear.
     pub no_update_data: BTreeSet<SpIdentifier>,
 }
+
+#[derive(
+    Copy, Clone, Debug, JsonSchema, Deserialize, Serialize, PartialEq, Eq,
+)]
+#[serde(rename_all = "snake_case", tag = "kind", content = "content")]
+pub enum UpdateTestError {
+    /// Simulate an error where the operation fails to complete.
+    Fail,
+
+    /// Simulate an issue where the operation times out.
+    Timeout {
+        /// The number of seconds to time out after.
+        secs: u64,
+    },
+}
+
+impl UpdateTestError {
+    pub async fn into_http_error(
+        self,
+        log: &slog::Logger,
+        reason: &str,
+    ) -> HttpError {
+        let message = self.into_error_string(log, reason).await;
+        HttpError::for_bad_request(None, message)
+    }
+
+    pub async fn into_error_string(
+        self,
+        log: &slog::Logger,
+        reason: &str,
+    ) -> String {
+        match self {
+            UpdateTestError::Fail => {
+                format!("Simulated failure while {reason}")
+            }
+            UpdateTestError::Timeout { secs } => {
+                slog::info!(log, "Simulating timeout while {reason}");
+                // 15 seconds should be enough to cause a timeout.
+                tokio::time::sleep(Duration::from_secs(secs)).await;
+                "XXX request should time out before this is hit".into()
+            }
+        }
+    }
+}
diff --git a/wicket/src/cli/preflight.rs b/wicket/src/cli/preflight.rs
index 29b6d2a5cb..3e8d5027ba 100644
--- a/wicket/src/cli/preflight.rs
+++ b/wicket/src/cli/preflight.rs
@@ -17,12 +17,11 @@ use std::borrow::Cow;
 use std::fmt::Display;
 use std::net::SocketAddrV6;
 use std::time::Duration;
-use update_engine::events::StepEvent;
-use update_engine::events::StepEventKind;
-use update_engine::events::StepInfo;
-use update_engine::events::StepInfoWithMetadata;
-use update_engine::events::StepOutcome;
-use update_engine::StepSpec;
+use wicket_common::preflight_check::StepEvent;
+use wicket_common::preflight_check::StepEventKind;
+use wicket_common::preflight_check::StepInfo;
+use wicket_common::preflight_check::StepInfoWithMetadata;
+use wicket_common::preflight_check::StepOutcome;
 use wicketd_client::types::PreflightUplinkCheckOptions;
 use wicketd_client::Client;
 
@@ -141,12 +140,10 @@ async fn poll_uplink_status_until_complete(client: Client) -> Result<()> {
     }
 }
 
-fn print_completed_steps<
-    S: StepSpec<CompletionMetadata = serde_json::Value>,
->(
-    step_events: Vec<StepEvent<S>>,
+fn print_completed_steps(
+    step_events: Vec<StepEvent>,
     last_seen: &mut Option<usize>,
-    all_steps: &mut Option<Vec<StepInfo<S>>>,
+    all_steps: &mut Option<Vec<StepInfo>>,
     progress_bar: &mut Option<ProgressBar>,
     execution_failed: &mut bool,
 ) -> Result<()> {
@@ -228,9 +225,9 @@ fn print_completed_steps<
     Ok(())
 }
 
-fn print_completed_step<S: StepSpec<CompletionMetadata = serde_json::Value>>(
-    info: &StepInfoWithMetadata<S>,
-    outcome: &StepOutcome<S>,
+fn print_completed_step(
+    info: &StepInfoWithMetadata,
+    outcome: &StepOutcome,
     step_elapsed: Duration,
 ) {
     let icon = icon_for_outcome(outcome);
@@ -243,8 +240,8 @@ fn print_completed_step<S: StepSpec<CompletionMetadata = serde_json::Value>>(
     );
 }
 
-fn print_failed_step<S: StepSpec>(
-    info: &StepInfoWithMetadata<S>,
+fn print_failed_step(
+    info: &StepInfoWithMetadata,
     step_elapsed: Duration,
     message: String,
 ) {
@@ -252,27 +249,22 @@ fn print_failed_step<S: StepSpec>(
     print_step(icon, info, step_elapsed, None, Some(&Cow::from(message)));
 }
 
-fn print_step<S: StepSpec>(
+fn print_step(
     icon: impl Display,
-    info: &StepInfoWithMetadata<S>,
+    info: &StepInfoWithMetadata,
     step_elapsed: Duration,
-    outcome_metadata: Option<&serde_json::Value>,
+    outcome_metadata: Option<&Vec<String>>,
     message: Option<&Cow<'static, str>>,
 ) {
     println!("{icon} {} ({:?})", info.info.description, step_elapsed);
     if let Some(metadata) = outcome_metadata {
-        if let Some(array) = metadata.as_array() {
-            for element in array {
-                if let Some(s) = element.as_str() {
-                    println!("    {s}");
-                } else {
-                    println!("    unexpected metadata type: {element:?}");
-                }
-            }
-        } else {
-            println!("    unexpected metadata type: {metadata:?}");
+        for element in metadata {
+            println!("    {element}");
         }
+    } else {
+        println!("    missing metadata");
     }
+
     if let Some(message) = message {
         for line in message.split('\n') {
             println!("    {line}");
@@ -280,7 +272,7 @@ fn print_step<S: StepSpec>(
     }
 }
 
-fn icon_for_outcome<S: StepSpec>(outcome: &StepOutcome<S>) -> Box<dyn Display> {
+fn icon_for_outcome(outcome: &StepOutcome) -> Box<dyn Display> {
     match outcome {
         StepOutcome::Success { .. } => Box::new('✔'.green()),
         StepOutcome::Warning { .. } => Box::new('⚠'.red()),
diff --git a/wicket/src/cli/rack_setup/config_toml.rs b/wicket/src/cli/rack_setup/config_toml.rs
index cef3746ff9..68485815a8 100644
--- a/wicket/src/cli/rack_setup/config_toml.rs
+++ b/wicket/src/cli/rack_setup/config_toml.rs
@@ -23,9 +23,9 @@ use toml_edit::InlineTable;
 use toml_edit::Item;
 use toml_edit::Table;
 use toml_edit::Value;
+use wicket_common::inventory::SpType;
 use wicket_common::rack_setup::BootstrapSledDescription;
 use wicket_common::rack_setup::CurrentRssUserConfigInsensitive;
-use wicket_common::rack_setup::GatewaySpType;
 use wicket_common::rack_setup::UserSpecifiedBgpPeerConfig;
 use wicket_common::rack_setup::UserSpecifiedImportExportPolicy;
 use wicket_common::rack_setup::UserSpecifiedPortConfig;
@@ -206,7 +206,7 @@ fn build_sleds_array(sleds: &BTreeSet<BootstrapSledDescription>) -> Array {
 
     for sled in sleds {
         // We should never get a non-sled from wicketd; if we do, filter it out.
-        if sled.id.type_ != GatewaySpType::Sled {
+        if sled.id.type_ != SpType::Sled {
             continue;
         }
 
diff --git a/wicket/src/events.rs b/wicket/src/events.rs
index fd0ac086ad..36480d261f 100644
--- a/wicket/src/events.rs
+++ b/wicket/src/events.rs
@@ -8,10 +8,11 @@ use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
 use std::fs::File;
 use std::time::{Duration, SystemTime};
+use wicket_common::inventory::RackV1Inventory;
 use wicket_common::update_events::EventReport;
 use wicketd_client::types::{
     ArtifactId, CurrentRssUserConfig, GetLocationResponse, IgnitionCommand,
-    RackOperationStatus, RackV1Inventory, SemverVersion,
+    RackOperationStatus, SemverVersion,
 };
 
 /// Event report type returned by the get_artifacts_and_event_reports API call.
diff --git a/wicket/src/helpers.rs b/wicket/src/helpers.rs
index 564b7e9348..bb4155231c 100644
--- a/wicket/src/helpers.rs
+++ b/wicket/src/helpers.rs
@@ -7,7 +7,7 @@
 use std::env::VarError;
 
 use anyhow::{bail, Context};
-use wicketd_client::types::{UpdateSimulatedResult, UpdateTestError};
+use wicket_common::rack_update::{UpdateSimulatedResult, UpdateTestError};
 
 pub(crate) fn get_update_test_error(
     env_var: &str,
diff --git a/wicket/src/runner.rs b/wicket/src/runner.rs
index 77fbb82df8..3af68ccbec 100644
--- a/wicket/src/runner.rs
+++ b/wicket/src/runner.rs
@@ -22,7 +22,7 @@ use tokio::sync::mpsc::{
     unbounded_channel, UnboundedReceiver, UnboundedSender,
 };
 use tokio::time::{interval, Duration};
-use wicketd_client::types::AbortUpdateOptions;
+use wicket_common::rack_update::AbortUpdateOptions;
 
 use crate::events::EventReportMap;
 use crate::helpers::get_update_test_error;
diff --git a/wicket/src/state/inventory.rs b/wicket/src/state/inventory.rs
index 0ab187cc48..8155efb606 100644
--- a/wicket/src/state/inventory.rs
+++ b/wicket/src/state/inventory.rs
@@ -11,10 +11,9 @@ use serde::{Deserialize, Serialize};
 use std::collections::BTreeMap;
 use std::fmt::Display;
 use std::iter::Iterator;
-use wicket_common::rack_update::SpType;
-use wicketd_client::types::{
+use wicket_common::inventory::{
     RackV1Inventory, RotInventory, RotSlot, SpComponentCaboose,
-    SpComponentInfo, SpIgnition, SpState,
+    SpComponentInfo, SpIgnition, SpState, SpType,
 };
 
 pub static ALL_COMPONENT_IDS: Lazy<Vec<ComponentId>> = Lazy::new(|| {
@@ -173,18 +172,23 @@ impl Component {
     }
 
     pub fn stage0_version(&self) -> String {
-        version_or_unknown(
-            self.sp().rot.as_ref().and_then(|rot| rot.caboose_stage0.as_ref()),
-        )
+        version_or_unknown(self.sp().rot.as_ref().and_then(|rot| {
+            // caboose_stage0 is an Option<Option<SpComponentCaboose>>, so we
+            // need to unwrap it twice, effectively. flatten would be nice but
+            // it doesn't work on Option<&Option<T>>, which is what we end up
+            // with.
+            rot.caboose_stage0.as_ref().map_or(None, |x| x.as_ref())
+        }))
     }
 
     pub fn stage0next_version(&self) -> String {
-        version_or_unknown(
-            self.sp()
-                .rot
-                .as_ref()
-                .and_then(|rot| rot.caboose_stage0next.as_ref()),
-        )
+        version_or_unknown(self.sp().rot.as_ref().and_then(|rot| {
+            // caboose_stage0next is an Option<Option<SpComponentCaboose>>, so we
+            // need to unwrap it twice, effectively. flatten would be nice but
+            // it doesn't work on Option<&Option<T>>, which is what we end up
+            // with.
+            rot.caboose_stage0next.as_ref().map_or(None, |x| x.as_ref())
+        }))
     }
 }
 
diff --git a/wicket/src/state/update.rs b/wicket/src/state/update.rs
index 31876365e2..3e0c89e83e 100644
--- a/wicket/src/state/update.rs
+++ b/wicket/src/state/update.rs
@@ -4,6 +4,7 @@
 
 use anyhow::Result;
 use ratatui::style::Style;
+use wicket_common::rack_update::{ClearUpdateStateOptions, StartUpdateOptions};
 use wicket_common::update_events::{
     EventReport, ProgressEventKind, StepEventKind, UpdateComponent,
     UpdateStepId,
@@ -18,9 +19,7 @@ use serde::{Deserialize, Serialize};
 use slog::Logger;
 use std::collections::BTreeMap;
 use std::fmt::Display;
-use wicketd_client::types::{
-    ArtifactId, ClearUpdateStateOptions, SemverVersion, StartUpdateOptions,
-};
+use wicketd_client::types::{ArtifactId, SemverVersion};
 
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct RackUpdateState {
diff --git a/wicket/src/ui/panes/overview.rs b/wicket/src/ui/panes/overview.rs
index 45d02311aa..00da6396c2 100644
--- a/wicket/src/ui/panes/overview.rs
+++ b/wicket/src/ui/panes/overview.rs
@@ -22,12 +22,12 @@ use ratatui::style::Style;
 use ratatui::text::{Line, Span, Text};
 use ratatui::widgets::{Block, BorderType, Borders, Paragraph};
 use ratatui::Frame;
-use wicketd_client::types::RotState;
-use wicketd_client::types::SpComponentCaboose;
-use wicketd_client::types::SpComponentInfo;
-use wicketd_client::types::SpComponentPresence;
-use wicketd_client::types::SpIgnition;
-use wicketd_client::types::SpState;
+use wicket_common::inventory::RotState;
+use wicket_common::inventory::SpComponentCaboose;
+use wicket_common::inventory::SpComponentInfo;
+use wicket_common::inventory::SpComponentPresence;
+use wicket_common::inventory::SpIgnition;
+use wicket_common::inventory::SpState;
 
 enum PopupKind {
     Ignition,
@@ -844,9 +844,9 @@ fn inventory_description(component: &Component) -> Text {
                     ]
                     .into(),
                 );
-                if let Some(caboose) =
-                    sp.rot().and_then(|r| r.caboose_stage0.as_ref())
-                {
+                if let Some(caboose) = sp.rot().and_then(|r| {
+                    r.caboose_stage0.as_ref().map_or(None, |x| x.as_ref())
+                }) {
                     append_caboose(&mut spans, nest_bullet(), caboose);
                 } else {
                     spans.push(
@@ -889,9 +889,9 @@ fn inventory_description(component: &Component) -> Text {
                     ]
                     .into(),
                 );
-                if let Some(caboose) =
-                    sp.rot().and_then(|r| r.caboose_stage0next.as_ref())
-                {
+                if let Some(caboose) = sp.rot().and_then(|r| {
+                    r.caboose_stage0next.as_ref().map_or(None, |x| x.as_ref())
+                }) {
                     append_caboose(&mut spans, nest_bullet(), caboose);
                 } else {
                     spans.push(
diff --git a/wicket/src/ui/panes/update.rs b/wicket/src/ui/panes/update.rs
index 09b119443d..3a61e25a3a 100644
--- a/wicket/src/ui/panes/update.rs
+++ b/wicket/src/ui/panes/update.rs
@@ -33,11 +33,12 @@ use update_engine::{
     AbortReason, CompletionReason, ExecutionStatus, FailureReason, StepKey,
     TerminalKind, WillNotBeRunReason,
 };
+use wicket_common::inventory::RotSlot;
 use wicket_common::update_events::{
     EventBuffer, EventReport, ProgressEvent, StepOutcome, StepStatus,
     UpdateComponent,
 };
-use wicketd_client::types::{RotSlot, SemverVersion};
+use wicketd_client::types::SemverVersion;
 
 const MAX_COLUMN_WIDTH: u16 = 25;
 
diff --git a/wicket/src/ui/widgets/rack.rs b/wicket/src/ui/widgets/rack.rs
index 7aa0c7d652..42ebf39d02 100644
--- a/wicket/src/ui/widgets/rack.rs
+++ b/wicket/src/ui/widgets/rack.rs
@@ -17,7 +17,7 @@ use ratatui::widgets::Borders;
 use ratatui::widgets::Paragraph;
 use ratatui::widgets::Widget;
 use std::collections::BTreeMap;
-use wicketd_client::types::SpIgnition;
+use wicket_common::inventory::SpIgnition;
 
 #[derive(Debug, Clone)]
 pub struct Rack<'a> {
diff --git a/wicket/src/wicketd.rs b/wicket/src/wicketd.rs
index c0ee3d9b14..dce1c7d286 100644
--- a/wicket/src/wicketd.rs
+++ b/wicket/src/wicketd.rs
@@ -9,12 +9,14 @@ use std::convert::From;
 use std::net::SocketAddrV6;
 use tokio::sync::mpsc::{self, Sender, UnboundedSender};
 use tokio::time::{interval, Duration, MissedTickBehavior};
-use wicket_common::rack_update::{SpIdentifier, SpType};
+use wicket_common::inventory::{SpIdentifier, SpType};
+use wicket_common::rack_update::{
+    AbortUpdateOptions, ClearUpdateStateOptions, StartUpdateOptions,
+};
 use wicket_common::WICKETD_TIMEOUT;
 use wicketd_client::types::{
-    AbortUpdateOptions, ClearUpdateStateOptions, ClearUpdateStateParams,
-    GetInventoryParams, GetInventoryResponse, GetLocationResponse,
-    IgnitionCommand, StartUpdateOptions, StartUpdateParams,
+    ClearUpdateStateParams, GetInventoryParams, GetInventoryResponse,
+    GetLocationResponse, IgnitionCommand, StartUpdateParams,
 };
 
 use crate::events::EventReportMap;
diff --git a/wicketd-api/Cargo.toml b/wicketd-api/Cargo.toml
new file mode 100644
index 0000000000..ba1d862a40
--- /dev/null
+++ b/wicketd-api/Cargo.toml
@@ -0,0 +1,20 @@
+[package]
+name = "wicketd-api"
+version = "0.1.0"
+edition = "2021"
+
+[lints]
+workspace = true
+
+[dependencies]
+bootstrap-agent-client.workspace = true
+dropshot.workspace = true
+gateway-client.workspace = true
+omicron-common.workspace = true
+omicron-passwords.workspace = true
+omicron-workspace-hack.workspace = true
+schemars.workspace = true
+serde.workspace = true
+sled-hardware-types.workspace = true
+slog.workspace = true
+wicket-common.workspace = true
diff --git a/wicketd-api/src/lib.rs b/wicketd-api/src/lib.rs
new file mode 100644
index 0000000000..9192578305
--- /dev/null
+++ b/wicketd-api/src/lib.rs
@@ -0,0 +1,545 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+use bootstrap_agent_client::types::RackInitId;
+use bootstrap_agent_client::types::RackOperationStatus;
+use bootstrap_agent_client::types::RackResetId;
+use dropshot::HttpError;
+use dropshot::HttpResponseOk;
+use dropshot::HttpResponseUpdatedNoContent;
+use dropshot::Path;
+use dropshot::RequestContext;
+use dropshot::StreamingBody;
+use dropshot::TypedBody;
+use gateway_client::types::IgnitionCommand;
+use omicron_common::api::external::SemverVersion;
+use omicron_common::update::ArtifactHashId;
+use omicron_common::update::ArtifactId;
+use schemars::JsonSchema;
+use serde::Deserialize;
+use serde::Serialize;
+use sled_hardware_types::Baseboard;
+use std::collections::BTreeMap;
+use std::collections::BTreeSet;
+use std::net::Ipv6Addr;
+use std::time::Duration;
+use wicket_common::inventory::RackV1Inventory;
+use wicket_common::inventory::SpIdentifier;
+use wicket_common::inventory::SpType;
+use wicket_common::preflight_check;
+use wicket_common::rack_setup::BgpAuthKey;
+use wicket_common::rack_setup::BgpAuthKeyId;
+use wicket_common::rack_setup::CurrentRssUserConfigInsensitive;
+use wicket_common::rack_setup::GetBgpAuthKeyInfoResponse;
+use wicket_common::rack_setup::PutRssUserConfigInsensitive;
+use wicket_common::rack_update::AbortUpdateOptions;
+use wicket_common::rack_update::ClearUpdateStateOptions;
+use wicket_common::rack_update::ClearUpdateStateResponse;
+use wicket_common::rack_update::StartUpdateOptions;
+use wicket_common::update_events::EventReport;
+
+#[dropshot::api_description {
+    module = "wicketd_api_mod",
+}]
+pub trait WicketdApi {
+    type Context;
+
+    /// Get wicketd's current view of all sleds visible on the bootstrap network.
+    #[endpoint {
+        method = GET,
+        path = "/bootstrap-sleds"
+    }]
+    async fn get_bootstrap_sleds(
+        rqctx: RequestContext<Self::Context>,
+    ) -> Result<HttpResponseOk<BootstrapSledIps>, HttpError>;
+
+    /// Get the current status of the user-provided (or system-default-provided, in
+    /// some cases) RSS configuration.
+    #[endpoint {
+        method = GET,
+        path = "/rack-setup/config"
+    }]
+    async fn get_rss_config(
+        rqctx: RequestContext<Self::Context>,
+    ) -> Result<HttpResponseOk<CurrentRssUserConfig>, HttpError>;
+
+    /// Update (a subset of) the current RSS configuration.
+    ///
+    /// Sensitive values (certificates and password hash) are not set through
+    /// this endpoint.
+    #[endpoint {
+        method = PUT,
+        path = "/rack-setup/config"
+    }]
+    async fn put_rss_config(
+        rqctx: RequestContext<Self::Context>,
+        body: TypedBody<PutRssUserConfigInsensitive>,
+    ) -> Result<HttpResponseUpdatedNoContent, HttpError>;
+
+    /// Add an external certificate.
+    ///
+    /// This must be paired with its private key. They may be posted in either
+    /// order, but one cannot post two certs in a row (or two keys in a row).
+    #[endpoint {
+        method = POST,
+        path = "/rack-setup/config/cert"
+    }]
+    async fn post_rss_config_cert(
+        rqctx: RequestContext<Self::Context>,
+        body: TypedBody<String>,
+    ) -> Result<HttpResponseOk<CertificateUploadResponse>, HttpError>;
+
+    /// Add the private key of an external certificate.
+    ///
+    /// This must be paired with its certificate. They may be posted in either
+    /// order, but one cannot post two keys in a row (or two certs in a row).
+    #[endpoint {
+        method = POST,
+        path = "/rack-setup/config/key"
+    }]
+    async fn post_rss_config_key(
+        rqctx: RequestContext<Self::Context>,
+        body: TypedBody<String>,
+    ) -> Result<HttpResponseOk<CertificateUploadResponse>, HttpError>;
+
+    // -- BGP authentication key management
+
+    /// Return information about BGP authentication keys, including checking
+    /// validity of keys.
+    ///
+    /// Produces an error if the rack setup config wasn't set, or if any of the
+    /// requested key IDs weren't found.
+    #[endpoint(
+        method = GET,
+        path = "/rack-setup/config/bgp/auth-key"
+    )]
+    async fn get_bgp_auth_key_info(
+        rqctx: RequestContext<Self::Context>,
+        // A bit weird for a GET request to have a TypedBody, but there's no other
+        // nice way to transmit this information as a batch.
+        params: TypedBody<GetBgpAuthKeyParams>,
+    ) -> Result<HttpResponseOk<GetBgpAuthKeyInfoResponse>, HttpError>;
+
+    /// Set the BGP authentication key for a particular key ID.
+    #[endpoint {
+        method = PUT,
+        path = "/rack-setup/config/bgp/auth-key/{key_id}"
+    }]
+    async fn put_bgp_auth_key(
+        rqctx: RequestContext<Self::Context>,
+        params: Path<PutBgpAuthKeyParams>,
+        body: TypedBody<PutBgpAuthKeyBody>,
+    ) -> Result<HttpResponseOk<PutBgpAuthKeyResponse>, HttpError>;
+
+    /// Update the RSS config recovery silo user password hash.
+    #[endpoint {
+        method = PUT,
+        path = "/rack-setup/config/recovery-user-password-hash"
+    }]
+    async fn put_rss_config_recovery_user_password_hash(
+        rqctx: RequestContext<Self::Context>,
+        body: TypedBody<PutRssRecoveryUserPasswordHash>,
+    ) -> Result<HttpResponseUpdatedNoContent, HttpError>;
+
+    /// Reset all RSS configuration to their default values.
+    #[endpoint {
+        method = DELETE,
+        path = "/rack-setup/config"
+    }]
+    async fn delete_rss_config(
+        rqctx: RequestContext<Self::Context>,
+    ) -> Result<HttpResponseUpdatedNoContent, HttpError>;
+
+    /// Query current state of rack setup.
+    #[endpoint {
+        method = GET,
+        path = "/rack-setup"
+    }]
+    async fn get_rack_setup_state(
+        rqctx: RequestContext<Self::Context>,
+    ) -> Result<HttpResponseOk<RackOperationStatus>, HttpError>;
+
+    /// Run rack setup.
+    ///
+    /// Will return an error if not all of the rack setup configuration has
+    /// been populated.
+    #[endpoint {
+        method = POST,
+        path = "/rack-setup"
+    }]
+    async fn post_run_rack_setup(
+        rqctx: RequestContext<Self::Context>,
+    ) -> Result<HttpResponseOk<RackInitId>, HttpError>;
+
+    /// Run rack reset.
+    #[endpoint {
+        method = DELETE,
+        path = "/rack-setup"
+    }]
+    async fn post_run_rack_reset(
+        rqctx: RequestContext<Self::Context>,
+    ) -> Result<HttpResponseOk<RackResetId>, HttpError>;
+
+    /// A status endpoint used to report high level information known to
+    /// wicketd.
+    ///
+    /// This endpoint can be polled to see if there have been state changes in
+    /// the system that are useful to report to wicket.
+    ///
+    /// Wicket, and possibly other callers, will retrieve the changed
+    /// information, with follow up calls.
+    #[endpoint {
+        method = GET,
+        path = "/inventory"
+    }]
+    async fn get_inventory(
+        rqctx: RequestContext<Self::Context>,
+        body_params: TypedBody<GetInventoryParams>,
+    ) -> Result<HttpResponseOk<GetInventoryResponse>, HttpError>;
+
+    /// Upload a TUF repository to the server.
+    ///
+    /// At any given time, wicketd will keep at most one TUF repository in
+    /// memory. Any previously-uploaded repositories will be discarded.
+    #[endpoint {
+        method = PUT,
+        path = "/repository",
+    }]
+    async fn put_repository(
+        rqctx: RequestContext<Self::Context>,
+        body: StreamingBody,
+    ) -> Result<HttpResponseUpdatedNoContent, HttpError>;
+
+    /// An endpoint used to report all available artifacts and event reports.
+    ///
+    /// The order of the returned artifacts is unspecified, and may change between
+    /// calls even if the total set of artifacts has not.
+    #[endpoint {
+        method = GET,
+        path = "/artifacts-and-event-reports",
+    }]
+    async fn get_artifacts_and_event_reports(
+        rqctx: RequestContext<Self::Context>,
+    ) -> Result<HttpResponseOk<GetArtifactsAndEventReportsResponse>, HttpError>;
+
+    /// Report the configured baseboard details.
+    #[endpoint {
+        method = GET,
+        path = "/baseboard",
+    }]
+    async fn get_baseboard(
+        rqctx: RequestContext<Self::Context>,
+    ) -> Result<HttpResponseOk<GetBaseboardResponse>, HttpError>;
+
+    /// Report the identity of the sled and switch we're currently running on /
+    /// connected to.
+    #[endpoint {
+        method = GET,
+        path = "/location",
+    }]
+    async fn get_location(
+        rqctx: RequestContext<Self::Context>,
+    ) -> Result<HttpResponseOk<GetLocationResponse>, HttpError>;
+
+    /// An endpoint to start updating one or more sleds, switches and PSCs.
+    #[endpoint {
+        method = POST,
+        path = "/update",
+    }]
+    async fn post_start_update(
+        rqctx: RequestContext<Self::Context>,
+        params: TypedBody<StartUpdateParams>,
+    ) -> Result<HttpResponseUpdatedNoContent, HttpError>;
+
+    /// An endpoint to get the status of any update being performed or recently
+    /// completed on a single SP.
+    #[endpoint {
+        method = GET,
+        path = "/update/{type}/{slot}",
+    }]
+    async fn get_update_sp(
+        rqctx: RequestContext<Self::Context>,
+        target: Path<SpIdentifier>,
+    ) -> Result<HttpResponseOk<EventReport>, HttpError>;
+
+    /// Forcibly cancels a running update.
+    ///
+    /// This is a potentially dangerous operation, but one that is sometimes
+    /// required. A machine reset might be required after this operation completes.
+    #[endpoint {
+        method = POST,
+        path = "/abort-update/{type}/{slot}",
+    }]
+    async fn post_abort_update(
+        rqctx: RequestContext<Self::Context>,
+        target: Path<SpIdentifier>,
+        opts: TypedBody<AbortUpdateOptions>,
+    ) -> Result<HttpResponseUpdatedNoContent, HttpError>;
+
+    /// Resets update state for a sled.
+    ///
+    /// Use this to clear update state after a failed update.
+    #[endpoint {
+        method = POST,
+        path = "/clear-update-state",
+    }]
+    async fn post_clear_update_state(
+        rqctx: RequestContext<Self::Context>,
+        params: TypedBody<ClearUpdateStateParams>,
+    ) -> Result<HttpResponseOk<ClearUpdateStateResponse>, HttpError>;
+
+    /// Send an ignition command targeting a specific SP.
+    ///
+    /// This endpoint acts as a proxy to the MGS endpoint performing the same
+    /// function, allowing wicket to communicate exclusively with wicketd (even
+    /// though wicketd adds no meaningful functionality here beyond what MGS
+    /// offers).
+    #[endpoint {
+        method = POST,
+        path = "/ignition/{type}/{slot}/{command}",
+    }]
+    async fn post_ignition_command(
+        rqctx: RequestContext<Self::Context>,
+        path: Path<PathSpIgnitionCommand>,
+    ) -> Result<HttpResponseUpdatedNoContent, HttpError>;
+
+    /// Start a preflight check for uplink configuration.
+    #[endpoint {
+        method = POST,
+        path = "/preflight/uplink",
+    }]
+    async fn post_start_preflight_uplink_check(
+        rqctx: RequestContext<Self::Context>,
+        body: TypedBody<PreflightUplinkCheckOptions>,
+    ) -> Result<HttpResponseUpdatedNoContent, HttpError>;
+
+    /// Get the report for the most recent (or still running) preflight uplink
+    /// check.
+    #[endpoint {
+        method = GET,
+        path = "/preflight/uplink",
+    }]
+    async fn get_preflight_uplink_report(
+        rqctx: RequestContext<Self::Context>,
+    ) -> Result<HttpResponseOk<preflight_check::EventReport>, HttpError>;
+
+    /// Instruct wicketd to reload its SMF config properties.
+    ///
+    /// The only expected client of this endpoint is `curl` from wicketd's SMF
+    /// `refresh` method, but other clients hitting it is harmless.
+    #[endpoint {
+        method = POST,
+        path = "/reload-config",
+    }]
+    async fn post_reload_config(
+        rqctx: RequestContext<Self::Context>,
+    ) -> Result<HttpResponseUpdatedNoContent, HttpError>;
+}
+
+#[derive(
+    Clone,
+    Debug,
+    Serialize,
+    Deserialize,
+    JsonSchema,
+    PartialEq,
+    Eq,
+    PartialOrd,
+    Ord,
+)]
+pub struct BootstrapSledIp {
+    pub baseboard: Baseboard,
+    pub ip: Ipv6Addr,
+}
+
+#[derive(
+    Clone,
+    Debug,
+    Serialize,
+    Deserialize,
+    JsonSchema,
+    PartialEq,
+    Eq,
+    PartialOrd,
+    Ord,
+)]
+pub struct BootstrapSledIps {
+    pub sleds: Vec<BootstrapSledIp>,
+}
+
+// This is a summary of the subset of `RackInitializeRequest` that is sensitive;
+// we only report a summary instead of returning actual data.
+#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema, PartialEq)]
+pub struct CurrentRssUserConfigSensitive {
+    pub num_external_certificates: usize,
+    pub recovery_silo_password_set: bool,
+    // We define GetBgpAuthKeyInfoResponse in wicket-common and use a
+    // progenitor replace directive for it, because we don't want typify to
+    // turn the BTreeMap into a HashMap. Use the same struct here to piggyback
+    // on that.
+    pub bgp_auth_keys: GetBgpAuthKeyInfoResponse,
+}
+
+#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema, PartialEq)]
+pub struct CurrentRssUserConfig {
+    pub sensitive: CurrentRssUserConfigSensitive,
+    pub insensitive: CurrentRssUserConfigInsensitive,
+}
+
+#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema, PartialEq)]
+#[serde(tag = "status", rename_all = "snake_case")]
+pub enum CertificateUploadResponse {
+    /// The key has been uploaded, but we're waiting on its corresponding
+    /// certificate chain.
+    WaitingOnCert,
+    /// The cert chain has been uploaded, but we're waiting on its corresponding
+    /// private key.
+    WaitingOnKey,
+    /// A cert chain and its key have been accepted.
+    CertKeyAccepted,
+    /// A cert chain and its key are valid, but have already been uploaded.
+    CertKeyDuplicateIgnored,
+}
+
+#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Eq)]
+pub struct GetBgpAuthKeyParams {
+    /// Checks that these keys are valid.
+    pub check_valid: BTreeSet<BgpAuthKeyId>,
+}
+
+#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema, PartialEq)]
+pub struct PutBgpAuthKeyParams {
+    pub key_id: BgpAuthKeyId,
+}
+
+#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Eq)]
+pub struct PutBgpAuthKeyBody {
+    pub key: BgpAuthKey,
+}
+
+#[derive(Clone, Debug, Serialize, JsonSchema, PartialEq)]
+pub struct PutBgpAuthKeyResponse {
+    pub status: SetBgpAuthKeyStatus,
+}
+
+#[derive(Clone, Debug, Serialize, JsonSchema, PartialEq)]
+#[serde(rename_all = "snake_case")]
+pub enum SetBgpAuthKeyStatus {
+    /// The key was accepted and replaced an old key.
+    Replaced,
+
+    /// The key was accepted, and is the same as the existing key.
+    Unchanged,
+
+    /// The key was accepted and is new.
+    Added,
+}
+
+#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema, PartialEq)]
+pub struct PutRssRecoveryUserPasswordHash {
+    pub hash: omicron_passwords::NewPasswordHash,
+}
+
+#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema, PartialEq)]
+pub struct GetInventoryParams {
+    /// Refresh the state of these SPs from MGS prior to returning (instead of
+    /// returning cached data).
+    pub force_refresh: Vec<SpIdentifier>,
+}
+
+/// The response to a `get_inventory` call: the inventory known to wicketd, or a
+/// notification that data is unavailable.
+#[derive(Clone, Debug, JsonSchema, Serialize)]
+#[serde(rename_all = "snake_case", tag = "type", content = "data")]
+pub enum GetInventoryResponse {
+    Response { inventory: RackV1Inventory, mgs_last_seen: Duration },
+    Unavailable,
+}
+
+#[derive(Clone, Debug, JsonSchema, Serialize)]
+#[serde(rename_all = "snake_case")]
+pub struct InstallableArtifacts {
+    pub artifact_id: ArtifactId,
+    pub installable: Vec<ArtifactHashId>,
+}
+
+/// The response to a `get_artifacts` call: the system version, and the list of
+/// all artifacts currently held by wicketd.
+#[derive(Clone, Debug, JsonSchema, Serialize)]
+#[serde(rename_all = "snake_case")]
+pub struct GetArtifactsAndEventReportsResponse {
+    pub system_version: Option<SemverVersion>,
+
+    /// Map of artifacts we ingested from the most-recently-uploaded TUF
+    /// repository to a list of artifacts we're serving over the bootstrap
+    /// network. In some cases the list of artifacts being served will have
+    /// length 1 (when we're serving the artifact directly); in other cases the
+    /// artifact in the TUF repo contains multiple nested artifacts inside it
+    /// (e.g., RoT artifacts contain both A and B images), and we serve the list
+    /// of extracted artifacts but not the original combination.
+    ///
+    /// Conceptually, this is a `BTreeMap<ArtifactId, Vec<ArtifactHashId>>`, but
+    /// JSON requires string keys for maps, so we give back a vec of pairs
+    /// instead.
+    pub artifacts: Vec<InstallableArtifacts>,
+
+    pub event_reports: BTreeMap<SpType, BTreeMap<u32, EventReport>>,
+}
+
+#[derive(Clone, Debug, JsonSchema, Deserialize)]
+pub struct StartUpdateParams {
+    /// The SP identifiers to start the update with. Must be non-empty.
+    pub targets: BTreeSet<SpIdentifier>,
+
+    /// Options for the update.
+    pub options: StartUpdateOptions,
+}
+
+#[derive(Clone, Debug, JsonSchema, Deserialize)]
+pub struct ClearUpdateStateParams {
+    /// The SP identifiers to clear the update state for. Must be non-empty.
+    pub targets: BTreeSet<SpIdentifier>,
+
+    /// Options for clearing update state
+    pub options: ClearUpdateStateOptions,
+}
+
+#[derive(Clone, Debug, JsonSchema, Serialize)]
+#[serde(rename_all = "snake_case")]
+pub struct GetBaseboardResponse {
+    pub baseboard: Option<Baseboard>,
+}
+
+/// All the fields of this response are optional, because it's possible we don't
+/// know any of them (yet) if MGS has not yet finished discovering its location
+/// or (ever) if we're running in a dev environment that doesn't support
+/// MGS-location / baseboard mapping.
+#[derive(Clone, Debug, JsonSchema, Serialize)]
+#[serde(rename_all = "snake_case")]
+pub struct GetLocationResponse {
+    /// The identity of our sled (where wicketd is running).
+    pub sled_id: Option<SpIdentifier>,
+    /// The baseboard of our sled (where wicketd is running).
+    pub sled_baseboard: Option<Baseboard>,
+    /// The baseboard of the switch our sled is physically connected to.
+    pub switch_baseboard: Option<Baseboard>,
+    /// The identity of the switch our sled is physically connected to.
+    pub switch_id: Option<SpIdentifier>,
+}
+
+#[derive(Serialize, Deserialize, JsonSchema)]
+pub struct PathSpIgnitionCommand {
+    #[serde(rename = "type")]
+    pub type_: SpType,
+    pub slot: u32,
+    pub command: IgnitionCommand,
+}
+
+/// Options provided to the preflight uplink check.
+#[derive(Clone, Debug, JsonSchema, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub struct PreflightUplinkCheckOptions {
+    /// DNS name to query.
+    pub dns_name_to_query: Option<String>,
+}
diff --git a/wicketd/Cargo.toml b/wicketd/Cargo.toml
index bfd8a4cf45..d2e870226b 100644
--- a/wicketd/Cargo.toml
+++ b/wicketd/Cargo.toml
@@ -52,7 +52,7 @@ uuid.workspace = true
 bootstrap-agent-client.workspace = true
 omicron-ddm-admin-client.workspace = true
 gateway-client.workspace = true
-installinator-artifactd.workspace = true
+installinator-api.workspace = true
 installinator-common.workspace = true
 omicron-certificates.workspace = true
 omicron-common.workspace = true
@@ -62,6 +62,7 @@ tufaceous-lib.workspace = true
 update-common.workspace = true
 update-engine.workspace = true
 wicket-common.workspace = true
+wicketd-api.workspace = true
 wicketd-client.workspace = true
 omicron-workspace-hack.workspace = true
 
@@ -76,7 +77,7 @@ fs-err.workspace = true
 gateway-test-utils.workspace = true
 http.workspace = true
 installinator.workspace = true
-installinator-artifact-client.workspace = true
+installinator-client.workspace = true
 maplit.workspace = true
 omicron-test-utils.workspace = true
 openapi-lint.workspace = true
diff --git a/wicketd/src/artifacts.rs b/wicketd/src/artifacts.rs
index 3e5854d17e..59981b2ac3 100644
--- a/wicketd/src/artifacts.rs
+++ b/wicketd/src/artifacts.rs
@@ -5,5 +5,6 @@
 mod server;
 mod store;
 
-pub(crate) use self::server::WicketdArtifactServer;
+pub(crate) use self::server::WicketdInstallinatorApiImpl;
+pub(crate) use self::server::WicketdInstallinatorContext;
 pub(crate) use self::store::WicketdArtifactStore;
diff --git a/wicketd/src/artifacts/server.rs b/wicketd/src/artifacts/server.rs
index 3808f01753..6d677c7b4f 100644
--- a/wicketd/src/artifacts/server.rs
+++ b/wicketd/src/artifacts/server.rs
@@ -2,62 +2,99 @@
 // License, v. 2.0. If a copy of the MPL was not distributed with this
 // file, You can obtain one at https://mozilla.org/MPL/2.0/.
 
-use super::store::WicketdArtifactStore;
 use crate::installinator_progress::IprArtifactServer;
-use async_trait::async_trait;
+use dropshot::FreeformBody;
 use dropshot::HttpError;
+use dropshot::HttpResponseHeaders;
+use dropshot::HttpResponseOk;
+use dropshot::HttpResponseUpdatedNoContent;
+use dropshot::Path;
+use dropshot::RequestContext;
+use dropshot::TypedBody;
 use hyper::Body;
-use installinator_artifactd::ArtifactGetter;
-use installinator_artifactd::EventReportStatus;
+use installinator_api::body_to_artifact_response;
+use installinator_api::InstallinatorApi;
+use installinator_api::ReportQuery;
+use installinator_common::EventReport;
 use omicron_common::update::ArtifactHashId;
 use slog::error;
 use slog::Logger;
-use uuid::Uuid;
+
+use super::WicketdArtifactStore;
+
+pub(crate) enum WicketdInstallinatorApiImpl {}
 
 /// The artifact server interface for wicketd.
 #[derive(Debug)]
-pub(crate) struct WicketdArtifactServer {
-    #[allow(dead_code)]
+pub struct WicketdInstallinatorContext {
     log: Logger,
     store: WicketdArtifactStore,
     ipr_artifact: IprArtifactServer,
 }
 
-impl WicketdArtifactServer {
+impl WicketdInstallinatorContext {
     pub(crate) fn new(
         log: &Logger,
         store: WicketdArtifactStore,
         ipr_artifact: IprArtifactServer,
     ) -> Self {
-        let log = log.new(slog::o!("component" => "wicketd artifact server"));
-        Self { log, store, ipr_artifact }
+        Self {
+            log: log
+                .new(slog::o!("component" => "wicketd installinator server")),
+            store,
+            ipr_artifact,
+        }
     }
 }
 
-#[async_trait]
-impl ArtifactGetter for WicketdArtifactServer {
-    async fn get_by_hash(&self, id: &ArtifactHashId) -> Option<(u64, Body)> {
-        let data_handle = self.store.get_by_hash(id)?;
-        let size = data_handle.file_size() as u64;
-        let data_stream = match data_handle.reader_stream().await {
-            Ok(stream) => stream,
-            Err(err) => {
-                error!(
-                    self.log, "failed to open extracted archive on demand";
-                    "error" => #%err,
-                );
-                return None;
-            }
-        };
+impl InstallinatorApi for WicketdInstallinatorApiImpl {
+    type Context = WicketdInstallinatorContext;
+
+    async fn get_artifact_by_hash(
+        rqctx: RequestContext<Self::Context>,
+        path: Path<ArtifactHashId>,
+    ) -> Result<HttpResponseHeaders<HttpResponseOk<FreeformBody>>, HttpError>
+    {
+        let context = rqctx.context();
+        match context.store.get_by_hash(&path.into_inner()) {
+            Some(data_handle) => {
+                let size = data_handle.file_size() as u64;
+                let data_stream = match data_handle.reader_stream().await {
+                    Ok(stream) => stream,
+                    Err(err) => {
+                        error!(
+                            context.log, "failed to open extracted archive on demand";
+                            "error" => #%err,
+                        );
+                        return Err(HttpError::for_internal_error(format!(
+                            // TODO: print error chain
+                            "Artifact not found: {err}"
+                        )));
+                    }
+                };
 
-        Some((size, Body::wrap_stream(data_stream)))
+                Ok(body_to_artifact_response(
+                    size,
+                    Body::wrap_stream(data_stream),
+                ))
+            }
+            None => {
+                Err(HttpError::for_not_found(None, "Artifact not found".into()))
+            }
+        }
     }
 
     async fn report_progress(
-        &self,
-        update_id: Uuid,
-        report: installinator_common::EventReport,
-    ) -> Result<EventReportStatus, HttpError> {
-        Ok(self.ipr_artifact.report_progress(update_id, report))
+        rqctx: RequestContext<Self::Context>,
+        path: Path<ReportQuery>,
+        report: TypedBody<EventReport>,
+    ) -> Result<HttpResponseUpdatedNoContent, HttpError> {
+        let context = rqctx.context();
+        let update_id = path.into_inner().update_id;
+
+        context
+            .ipr_artifact
+            .report_progress(update_id, report.into_inner())
+            .to_http_result(update_id)
     }
 }
diff --git a/wicketd/src/artifacts/store.rs b/wicketd/src/artifacts/store.rs
index 01543432a2..98a6abcaad 100644
--- a/wicketd/src/artifacts/store.rs
+++ b/wicketd/src/artifacts/store.rs
@@ -2,7 +2,6 @@
 // License, v. 2.0. If a copy of the MPL was not distributed with this
 // file, You can obtain one at https://mozilla.org/MPL/2.0/.
 
-use crate::http_entrypoints::InstallableArtifacts;
 use omicron_common::api::external::SemverVersion;
 use omicron_common::update::ArtifactHashId;
 use slog::Logger;
@@ -11,6 +10,7 @@ use std::sync::Mutex;
 use update_common::artifacts::ArtifactsWithPlan;
 use update_common::artifacts::ExtractedArtifactDataHandle;
 use update_common::artifacts::UpdatePlan;
+use wicketd_api::InstallableArtifacts;
 
 /// The artifact store for wicketd.
 ///
diff --git a/wicketd/src/bin/wicketd.rs b/wicketd/src/bin/wicketd.rs
index 4037bc4c23..bc23362daf 100644
--- a/wicketd/src/bin/wicketd.rs
+++ b/wicketd/src/bin/wicketd.rs
@@ -14,14 +14,11 @@ use omicron_common::{
 use sled_hardware_types::Baseboard;
 use std::net::{Ipv6Addr, SocketAddrV6};
 use std::path::PathBuf;
-use wicketd::{self, run_openapi, Config, Server, SmfConfigValues};
+use wicketd::{Config, Server, SmfConfigValues};
 
 #[derive(Debug, Parser)]
 #[clap(name = "wicketd", about = "See README.adoc for more information")]
 enum Args {
-    /// Print the external OpenAPI Spec document and exit
-    Openapi,
-
     /// Start a wicketd server
     Run {
         #[clap(name = "CONFIG_FILE_PATH", action)]
@@ -84,9 +81,6 @@ async fn do_run() -> Result<(), CmdError> {
     let args = Args::parse();
 
     match args {
-        Args::Openapi => {
-            run_openapi().map_err(|err| CmdError::Failure(anyhow!(err)))
-        }
         Args::Run {
             config_file_path,
             address,
@@ -144,9 +138,8 @@ async fn do_run() -> Result<(), CmdError> {
                 .to_logger("wicketd")
                 .context("failed to initialize logger")
                 .map_err(CmdError::Failure)?;
-            let server = Server::start(log, args)
-                .await
-                .map_err(|err| CmdError::Failure(anyhow!(err)))?;
+            let server =
+                Server::start(log, args).await.map_err(CmdError::Failure)?;
             server
                 .wait_for_finish()
                 .await
diff --git a/wicketd/src/context.rs b/wicketd/src/context.rs
index 68d04f35dc..8f4dfb451b 100644
--- a/wicketd/src/context.rs
+++ b/wicketd/src/context.rs
@@ -12,7 +12,6 @@ use crate::MgsHandle;
 use anyhow::anyhow;
 use anyhow::bail;
 use anyhow::Result;
-use gateway_client::types::SpIdentifier;
 use internal_dns::resolver::Resolver;
 use sled_hardware_types::Baseboard;
 use slog::info;
@@ -21,6 +20,7 @@ use std::net::SocketAddrV6;
 use std::sync::Arc;
 use std::sync::Mutex;
 use std::sync::OnceLock;
+use wicket_common::inventory::SpIdentifier;
 
 /// Shared state used by API handlers
 pub struct ServerContext {
diff --git a/wicketd/src/helpers.rs b/wicketd/src/helpers.rs
index a8b47d4f12..8cc0d3330d 100644
--- a/wicketd/src/helpers.rs
+++ b/wicketd/src/helpers.rs
@@ -6,8 +6,8 @@
 
 use std::fmt;
 
-use gateway_client::types::{SpIdentifier, SpType};
 use itertools::Itertools;
+use wicket_common::inventory::{SpIdentifier, SpType};
 
 #[derive(Clone, Debug, Eq, PartialEq, PartialOrd, Ord, Hash)]
 pub(crate) struct SpIdentifierDisplay(pub(crate) SpIdentifier);
diff --git a/wicketd/src/http_entrypoints.rs b/wicketd/src/http_entrypoints.rs
index 4a4374b312..5661843c23 100644
--- a/wicketd/src/http_entrypoints.rs
+++ b/wicketd/src/http_entrypoints.rs
@@ -7,18 +7,13 @@
 use crate::helpers::sps_to_string;
 use crate::helpers::SpIdentifierDisplay;
 use crate::mgs::GetInventoryError;
-use crate::mgs::GetInventoryResponse;
 use crate::mgs::MgsHandle;
 use crate::mgs::ShutdownInProgress;
-use crate::preflight_check::UplinkEventReport;
-use crate::RackV1Inventory;
 use crate::SmfConfigValues;
 use bootstrap_agent_client::types::RackInitId;
 use bootstrap_agent_client::types::RackOperationStatus;
 use bootstrap_agent_client::types::RackResetId;
-use dropshot::endpoint;
 use dropshot::ApiDescription;
-use dropshot::ApiDescriptionRegisterError;
 use dropshot::HttpError;
 use dropshot::HttpResponseOk;
 use dropshot::HttpResponseUpdatedNoContent;
@@ -26,31 +21,23 @@ use dropshot::Path;
 use dropshot::RequestContext;
 use dropshot::StreamingBody;
 use dropshot::TypedBody;
-use gateway_client::types::IgnitionCommand;
-use gateway_client::types::SpIdentifier;
-use gateway_client::types::SpType;
 use http::StatusCode;
 use internal_dns::resolver::Resolver;
-use omicron_common::api::external::SemverVersion;
 use omicron_common::api::internal::shared::SwitchLocation;
-use omicron_common::update::ArtifactHashId;
-use omicron_common::update::ArtifactId;
-use schemars::JsonSchema;
-use serde::Deserialize;
-use serde::Serialize;
 use sled_hardware_types::Baseboard;
 use slog::o;
 use std::collections::BTreeMap;
 use std::collections::BTreeSet;
-use std::net::Ipv6Addr;
-use std::time::Duration;
-use wicket_common::rack_setup::BgpAuthKey;
-use wicket_common::rack_setup::BgpAuthKeyId;
-use wicket_common::rack_setup::CurrentRssUserConfigInsensitive;
+use wicket_common::inventory::RackV1Inventory;
+use wicket_common::inventory::SpIdentifier;
+use wicket_common::inventory::SpType;
 use wicket_common::rack_setup::GetBgpAuthKeyInfoResponse;
 use wicket_common::rack_setup::PutRssUserConfigInsensitive;
+use wicket_common::rack_update::AbortUpdateOptions;
+use wicket_common::rack_update::ClearUpdateStateResponse;
 use wicket_common::update_events::EventReport;
 use wicket_common::WICKETD_TIMEOUT;
+use wicketd_api::*;
 
 use crate::ServerContext;
 
@@ -58,1258 +45,753 @@ type WicketdApiDescription = ApiDescription<ServerContext>;
 
 /// Return a description of the wicketd api for use in generating an OpenAPI spec
 pub fn api() -> WicketdApiDescription {
-    fn register_endpoints(
-        api: &mut WicketdApiDescription,
-    ) -> Result<(), ApiDescriptionRegisterError> {
-        api.register(get_bootstrap_sleds)?;
-        api.register(get_rss_config)?;
-        api.register(put_rss_config)?;
-        api.register(put_rss_config_recovery_user_password_hash)?;
-        api.register(post_rss_config_cert)?;
-        api.register(post_rss_config_key)?;
-        api.register(get_bgp_auth_key_info)?;
-        api.register(put_bgp_auth_key)?;
-        api.register(delete_rss_config)?;
-        api.register(get_rack_setup_state)?;
-        api.register(post_run_rack_setup)?;
-        api.register(post_run_rack_reset)?;
-        api.register(get_inventory)?;
-        api.register(get_location)?;
-        api.register(put_repository)?;
-        api.register(get_artifacts_and_event_reports)?;
-        api.register(get_baseboard)?;
-        api.register(post_start_update)?;
-        api.register(post_abort_update)?;
-        api.register(post_clear_update_state)?;
-        api.register(get_update_sp)?;
-        api.register(post_ignition_command)?;
-        api.register(post_start_preflight_uplink_check)?;
-        api.register(get_preflight_uplink_report)?;
-        api.register(post_reload_config)?;
-        Ok(())
-    }
-
-    let mut api = WicketdApiDescription::new();
-    if let Err(err) = register_endpoints(&mut api) {
-        panic!("failed to register entrypoints: {}", err);
-    }
-    api
+    wicketd_api_mod::api_description::<WicketdApiImpl>()
+        .expect("failed to register entrypoints")
 }
 
-#[derive(
-    Clone,
-    Debug,
-    Serialize,
-    Deserialize,
-    JsonSchema,
-    PartialEq,
-    Eq,
-    PartialOrd,
-    Ord,
-)]
-pub struct BootstrapSledIp {
-    pub baseboard: Baseboard,
-    pub ip: Ipv6Addr,
-}
+pub enum WicketdApiImpl {}
 
-#[derive(
-    Clone,
-    Debug,
-    Serialize,
-    Deserialize,
-    JsonSchema,
-    PartialEq,
-    Eq,
-    PartialOrd,
-    Ord,
-)]
-pub struct BootstrapSledIps {
-    pub sleds: Vec<BootstrapSledIp>,
-}
+impl WicketdApi for WicketdApiImpl {
+    type Context = ServerContext;
 
-/// Get wicketd's current view of all sleds visible on the bootstrap network.
-#[endpoint {
-    method = GET,
-    path = "/bootstrap-sleds"
-}]
-async fn get_bootstrap_sleds(
-    rqctx: RequestContext<ServerContext>,
-) -> Result<HttpResponseOk<BootstrapSledIps>, HttpError> {
-    let ctx = rqctx.context();
-
-    let sleds = ctx
-        .bootstrap_peers
-        .sleds()
-        .into_iter()
-        .map(|(baseboard, ip)| BootstrapSledIp { baseboard, ip })
-        .collect();
-
-    Ok(HttpResponseOk(BootstrapSledIps { sleds }))
-}
-
-// This is a summary of the subset of `RackInitializeRequest` that is sensitive;
-// we only report a summary instead of returning actual data.
-#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema, PartialEq)]
-pub struct CurrentRssUserConfigSensitive {
-    pub num_external_certificates: usize,
-    pub recovery_silo_password_set: bool,
-    // We define GetBgpAuthKeyInfoResponse in wicket-common and use a
-    // progenitor replace directive for it, because we don't want typify to
-    // turn the BTreeMap into a HashMap. Use the same struct here to piggyback
-    // on that.
-    pub bgp_auth_keys: GetBgpAuthKeyInfoResponse,
-}
+    async fn get_bootstrap_sleds(
+        rqctx: RequestContext<Self::Context>,
+    ) -> Result<HttpResponseOk<BootstrapSledIps>, HttpError> {
+        let ctx = rqctx.context();
 
-#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema, PartialEq)]
-pub struct CurrentRssUserConfig {
-    pub sensitive: CurrentRssUserConfigSensitive,
-    pub insensitive: CurrentRssUserConfigInsensitive,
-}
+        let sleds = ctx
+            .bootstrap_peers
+            .sleds()
+            .into_iter()
+            .map(|(baseboard, ip)| BootstrapSledIp { baseboard, ip })
+            .collect();
 
-// Get the current inventory or return a 503 Unavailable.
-async fn inventory_or_unavail(
-    mgs_handle: &MgsHandle,
-) -> Result<RackV1Inventory, HttpError> {
-    match mgs_handle.get_cached_inventory().await {
-        Ok(GetInventoryResponse::Response { inventory, .. }) => Ok(inventory),
-        Ok(GetInventoryResponse::Unavailable) => Err(HttpError::for_unavail(
-            None,
-            "Rack inventory not yet available".into(),
-        )),
-        Err(ShutdownInProgress) => {
-            Err(HttpError::for_unavail(None, "Server is shutting down".into()))
-        }
+        Ok(HttpResponseOk(BootstrapSledIps { sleds }))
     }
-}
-
-/// Get the current status of the user-provided (or system-default-provided, in
-/// some cases) RSS configuration.
-#[endpoint {
-    method = GET,
-    path = "/rack-setup/config"
-}]
-async fn get_rss_config(
-    rqctx: RequestContext<ServerContext>,
-) -> Result<HttpResponseOk<CurrentRssUserConfig>, HttpError> {
-    let ctx = rqctx.context();
-
-    // We can't run RSS if we don't have an inventory from MGS yet; we always
-    // need to fill in the bootstrap sleds first.
-    let inventory = inventory_or_unavail(&ctx.mgs_handle).await?;
-
-    let mut config = ctx.rss_config.lock().unwrap();
-    config.update_with_inventory_and_bootstrap_peers(
-        &inventory,
-        &ctx.bootstrap_peers,
-    );
-
-    Ok(HttpResponseOk((&*config).into()))
-}
-
-/// Update (a subset of) the current RSS configuration.
-///
-/// Sensitive values (certificates and password hash) are not set through this
-/// endpoint.
-#[endpoint {
-    method = PUT,
-    path = "/rack-setup/config"
-}]
-async fn put_rss_config(
-    rqctx: RequestContext<ServerContext>,
-    body: TypedBody<PutRssUserConfigInsensitive>,
-) -> Result<HttpResponseUpdatedNoContent, HttpError> {
-    let ctx = rqctx.context();
-
-    // We can't run RSS if we don't have an inventory from MGS yet; we always
-    // need to fill in the bootstrap sleds first.
-    let inventory = inventory_or_unavail(&ctx.mgs_handle).await?;
-
-    let mut config = ctx.rss_config.lock().unwrap();
-    config.update_with_inventory_and_bootstrap_peers(
-        &inventory,
-        &ctx.bootstrap_peers,
-    );
-    config
-        .update(body.into_inner(), ctx.baseboard.as_ref())
-        .map_err(|err| HttpError::for_bad_request(None, err))?;
-
-    Ok(HttpResponseUpdatedNoContent())
-}
-
-#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema, PartialEq)]
-#[serde(tag = "status", rename_all = "snake_case")]
-pub enum CertificateUploadResponse {
-    /// The key has been uploaded, but we're waiting on its corresponding
-    /// certificate chain.
-    WaitingOnCert,
-    /// The cert chain has been uploaded, but we're waiting on its corresponding
-    /// private key.
-    WaitingOnKey,
-    /// A cert chain and its key have been accepted.
-    CertKeyAccepted,
-    /// A cert chain and its key are valid, but have already been uploaded.
-    CertKeyDuplicateIgnored,
-}
-
-/// Add an external certificate.
-///
-/// This must be paired with its private key. They may be posted in either
-/// order, but one cannot post two certs in a row (or two keys in a row).
-#[endpoint {
-    method = POST,
-    path = "/rack-setup/config/cert"
-}]
-async fn post_rss_config_cert(
-    rqctx: RequestContext<ServerContext>,
-    body: TypedBody<String>,
-) -> Result<HttpResponseOk<CertificateUploadResponse>, HttpError> {
-    let ctx = rqctx.context();
-
-    let mut config = ctx.rss_config.lock().unwrap();
-    let response = config
-        .push_cert(body.into_inner())
-        .map_err(|err| HttpError::for_bad_request(None, err))?;
-
-    Ok(HttpResponseOk(response))
-}
-
-/// Add the private key of an external certificate.
-///
-/// This must be paired with its certificate. They may be posted in either
-/// order, but one cannot post two keys in a row (or two certs in a row).
-#[endpoint {
-    method = POST,
-    path = "/rack-setup/config/key"
-}]
-async fn post_rss_config_key(
-    rqctx: RequestContext<ServerContext>,
-    body: TypedBody<String>,
-) -> Result<HttpResponseOk<CertificateUploadResponse>, HttpError> {
-    let ctx = rqctx.context();
-
-    let mut config = ctx.rss_config.lock().unwrap();
-    let response = config
-        .push_key(body.into_inner())
-        .map_err(|err| HttpError::for_bad_request(None, err))?;
-
-    Ok(HttpResponseOk(response))
-}
-
-// -- BGP authentication key management
 
-#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Eq)]
-pub(crate) struct GetBgpAuthKeyParams {
-    /// Checks that these keys are valid.
-    check_valid: BTreeSet<BgpAuthKeyId>,
-}
+    async fn get_rss_config(
+        rqctx: RequestContext<Self::Context>,
+    ) -> Result<HttpResponseOk<CurrentRssUserConfig>, HttpError> {
+        let ctx = rqctx.context();
 
-/// Return information about BGP authentication keys, including checking
-/// validity of keys.
-///
-/// Produces an error if the rack setup config wasn't set, or if any of the
-/// requested key IDs weren't found.
-#[endpoint(
-    method = GET,
-    path = "/rack-setup/config/bgp/auth-key"
-)]
-async fn get_bgp_auth_key_info(
-    rqctx: RequestContext<ServerContext>,
-    // A bit weird for a GET request to have a TypedBody, but there's no other
-    // nice way to transmit this information as a batch.
-    params: TypedBody<GetBgpAuthKeyParams>,
-) -> Result<HttpResponseOk<GetBgpAuthKeyInfoResponse>, HttpError> {
-    let ctx = rqctx.context();
-    let params = params.into_inner();
-
-    let config = ctx.rss_config.lock().unwrap();
-    config
-        .check_bgp_auth_keys_valid(&params.check_valid)
-        .map_err(|err| HttpError::for_bad_request(None, err.to_string()))?;
-    let data = config.get_bgp_auth_key_data();
-
-    Ok(HttpResponseOk(GetBgpAuthKeyInfoResponse { data }))
-}
+        // We can't run RSS if we don't have an inventory from MGS yet; we always
+        // need to fill in the bootstrap sleds first.
+        let inventory = inventory_or_unavail(&ctx.mgs_handle).await?;
 
-#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema, PartialEq)]
-struct PutBgpAuthKeyParams {
-    key_id: BgpAuthKeyId,
-}
-
-#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Eq)]
-struct PutBgpAuthKeyBody {
-    key: BgpAuthKey,
-}
+        let mut config = ctx.rss_config.lock().unwrap();
+        config.update_with_inventory_and_bootstrap_peers(
+            &inventory,
+            &ctx.bootstrap_peers,
+        );
 
-#[derive(Clone, Debug, Serialize, JsonSchema, PartialEq)]
-struct PutBgpAuthKeyResponse {
-    status: SetBgpAuthKeyStatus,
-}
+        Ok(HttpResponseOk((&*config).into()))
+    }
 
-#[derive(Clone, Debug, Serialize, JsonSchema, PartialEq)]
-#[serde(rename_all = "snake_case")]
-pub(crate) enum SetBgpAuthKeyStatus {
-    /// The key was accepted and replaced an old key.
-    Replaced,
+    async fn put_rss_config(
+        rqctx: RequestContext<Self::Context>,
+        body: TypedBody<PutRssUserConfigInsensitive>,
+    ) -> Result<HttpResponseUpdatedNoContent, HttpError> {
+        let ctx = rqctx.context();
 
-    /// The key was accepted, and is the same as the existing key.
-    Unchanged,
+        // We can't run RSS if we don't have an inventory from MGS yet; we always
+        // need to fill in the bootstrap sleds first.
+        let inventory = inventory_or_unavail(&ctx.mgs_handle).await?;
 
-    /// The key was accepted and is new.
-    Added,
-}
+        let mut config = ctx.rss_config.lock().unwrap();
+        config.update_with_inventory_and_bootstrap_peers(
+            &inventory,
+            &ctx.bootstrap_peers,
+        );
+        config
+            .update(body.into_inner(), ctx.baseboard.as_ref())
+            .map_err(|err| HttpError::for_bad_request(None, err))?;
+
+        Ok(HttpResponseUpdatedNoContent())
+    }
 
-/// Set the BGP authentication key for a particular key ID.
-#[endpoint {
-    method = PUT,
-    path = "/rack-setup/config/bgp/auth-key/{key_id}"
-}]
-async fn put_bgp_auth_key(
-    rqctx: RequestContext<ServerContext>,
-    params: Path<PutBgpAuthKeyParams>,
-    body: TypedBody<PutBgpAuthKeyBody>,
-) -> Result<HttpResponseOk<PutBgpAuthKeyResponse>, HttpError> {
-    let ctx = rqctx.context();
-    let params = params.into_inner();
-
-    let mut config = ctx.rss_config.lock().unwrap();
-    let status = config
-        .set_bgp_auth_key(params.key_id, body.into_inner().key)
-        .map_err(|err| HttpError::for_bad_request(None, err.to_string()))?;
-
-    Ok(HttpResponseOk(PutBgpAuthKeyResponse { status }))
-}
+    async fn post_rss_config_cert(
+        rqctx: RequestContext<Self::Context>,
+        body: TypedBody<String>,
+    ) -> Result<HttpResponseOk<CertificateUploadResponse>, HttpError> {
+        let ctx = rqctx.context();
 
-#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema, PartialEq)]
-pub struct PutRssRecoveryUserPasswordHash {
-    pub hash: omicron_passwords::NewPasswordHash,
-}
+        let mut config = ctx.rss_config.lock().unwrap();
+        let response = config
+            .push_cert(body.into_inner())
+            .map_err(|err| HttpError::for_bad_request(None, err))?;
 
-/// Update the RSS config recovery silo user password hash.
-#[endpoint {
-    method = PUT,
-    path = "/rack-setup/config/recovery-user-password-hash"
-}]
-async fn put_rss_config_recovery_user_password_hash(
-    rqctx: RequestContext<ServerContext>,
-    body: TypedBody<PutRssRecoveryUserPasswordHash>,
-) -> Result<HttpResponseUpdatedNoContent, HttpError> {
-    let ctx = rqctx.context();
-
-    let mut config = ctx.rss_config.lock().unwrap();
-    config.set_recovery_user_password_hash(body.into_inner().hash);
-
-    Ok(HttpResponseUpdatedNoContent())
-}
+        Ok(HttpResponseOk(response))
+    }
 
-/// Reset all RSS configuration to their default values.
-#[endpoint {
-    method = DELETE,
-    path = "/rack-setup/config"
-}]
-async fn delete_rss_config(
-    rqctx: RequestContext<ServerContext>,
-) -> Result<HttpResponseUpdatedNoContent, HttpError> {
-    let ctx = rqctx.context();
+    async fn post_rss_config_key(
+        rqctx: RequestContext<Self::Context>,
+        body: TypedBody<String>,
+    ) -> Result<HttpResponseOk<CertificateUploadResponse>, HttpError> {
+        let ctx = rqctx.context();
 
-    let mut config = ctx.rss_config.lock().unwrap();
-    *config = Default::default();
+        let mut config = ctx.rss_config.lock().unwrap();
+        let response = config
+            .push_key(body.into_inner())
+            .map_err(|err| HttpError::for_bad_request(None, err))?;
 
-    Ok(HttpResponseUpdatedNoContent())
-}
+        Ok(HttpResponseOk(response))
+    }
 
-/// Query current state of rack setup.
-#[endpoint {
-    method = GET,
-    path = "/rack-setup"
-}]
-async fn get_rack_setup_state(
-    rqctx: RequestContext<ServerContext>,
-) -> Result<HttpResponseOk<RackOperationStatus>, HttpError> {
-    let ctx = rqctx.context();
-
-    let sled_agent_addr = ctx
-        .bootstrap_agent_addr()
-        .map_err(|err| HttpError::for_bad_request(None, format!("{err:#}")))?;
-
-    let client = bootstrap_agent_client::Client::new(
-        &format!("http://{}", sled_agent_addr),
-        ctx.log.new(slog::o!("component" => "bootstrap client")),
-    );
-
-    let op_status = client
-        .rack_initialization_status()
-        .await
-        .map_err(|err| {
-            use bootstrap_agent_client::Error as BaError;
-            match err {
-                BaError::CommunicationError(err) => {
-                    let message =
-                        format!("Failed to send rack setup request: {err}");
-                    HttpError {
-                        status_code: http::StatusCode::SERVICE_UNAVAILABLE,
-                        error_code: None,
-                        external_message: message.clone(),
-                        internal_message: message,
-                    }
-                }
-                other => HttpError::for_bad_request(
-                    None,
-                    format!("Rack setup request failed: {other}"),
-                ),
-            }
-        })?
-        .into_inner();
+    async fn get_bgp_auth_key_info(
+        rqctx: RequestContext<ServerContext>,
+        // A bit weird for a GET request to have a TypedBody, but there's no other
+        // nice way to transmit this information as a batch.
+        params: TypedBody<GetBgpAuthKeyParams>,
+    ) -> Result<HttpResponseOk<GetBgpAuthKeyInfoResponse>, HttpError> {
+        let ctx = rqctx.context();
+        let params = params.into_inner();
+
+        let config = ctx.rss_config.lock().unwrap();
+        config
+            .check_bgp_auth_keys_valid(&params.check_valid)
+            .map_err(|err| HttpError::for_bad_request(None, err.to_string()))?;
+        let data = config.get_bgp_auth_key_data();
+
+        Ok(HttpResponseOk(GetBgpAuthKeyInfoResponse { data }))
+    }
 
-    Ok(HttpResponseOk(op_status))
-}
+    async fn put_bgp_auth_key(
+        rqctx: RequestContext<ServerContext>,
+        params: Path<PutBgpAuthKeyParams>,
+        body: TypedBody<PutBgpAuthKeyBody>,
+    ) -> Result<HttpResponseOk<PutBgpAuthKeyResponse>, HttpError> {
+        let ctx = rqctx.context();
+        let params = params.into_inner();
 
-/// Run rack setup.
-///
-/// Will return an error if not all of the rack setup configuration has been
-/// populated.
-#[endpoint {
-    method = POST,
-    path = "/rack-setup"
-}]
-async fn post_run_rack_setup(
-    rqctx: RequestContext<ServerContext>,
-) -> Result<HttpResponseOk<RackInitId>, HttpError> {
-    let ctx = rqctx.context();
-    let log = &rqctx.log;
-
-    let sled_agent_addr = ctx
-        .bootstrap_agent_addr()
-        .map_err(|err| HttpError::for_bad_request(None, format!("{err:#}")))?;
-
-    let request = {
         let mut config = ctx.rss_config.lock().unwrap();
-        config.start_rss_request(&ctx.bootstrap_peers, log).map_err(|err| {
-            HttpError::for_bad_request(None, format!("{err:#}"))
-        })?
-    };
-
-    slog::info!(
-        ctx.log,
-        "Sending RSS initialize request to {}",
-        sled_agent_addr
-    );
-    let client = bootstrap_agent_client::Client::new(
-        &format!("http://{}", sled_agent_addr),
-        ctx.log.new(slog::o!("component" => "bootstrap client")),
-    );
-
-    let init_id = client
-        .rack_initialize(&request)
-        .await
-        .map_err(|err| {
-            use bootstrap_agent_client::Error as BaError;
-            match err {
-                BaError::CommunicationError(err) => {
-                    let message =
-                        format!("Failed to send rack setup request: {err}");
-                    HttpError {
-                        status_code: http::StatusCode::SERVICE_UNAVAILABLE,
-                        error_code: None,
-                        external_message: message.clone(),
-                        internal_message: message,
-                    }
-                }
-                other => HttpError::for_bad_request(
-                    None,
-                    format!("Rack setup request failed: {other}"),
-                ),
-            }
-        })?
-        .into_inner();
-
-    Ok(HttpResponseOk(init_id))
-}
+        let status = config
+            .set_bgp_auth_key(params.key_id, body.into_inner().key)
+            .map_err(|err| HttpError::for_bad_request(None, err.to_string()))?;
 
-/// Run rack reset.
-#[endpoint {
-    method = DELETE,
-    path = "/rack-setup"
-}]
-async fn post_run_rack_reset(
-    rqctx: RequestContext<ServerContext>,
-) -> Result<HttpResponseOk<RackResetId>, HttpError> {
-    let ctx = rqctx.context();
-
-    let sled_agent_addr = ctx
-        .bootstrap_agent_addr()
-        .map_err(|err| HttpError::for_bad_request(None, format!("{err:#}")))?;
-
-    slog::info!(ctx.log, "Sending RSS reset request to {}", sled_agent_addr);
-    let client = bootstrap_agent_client::Client::new(
-        &format!("http://{}", sled_agent_addr),
-        ctx.log.new(slog::o!("component" => "bootstrap client")),
-    );
-
-    let reset_id = client
-        .rack_reset()
-        .await
-        .map_err(|err| {
-            use bootstrap_agent_client::Error as BaError;
-            match err {
-                BaError::CommunicationError(err) => {
-                    let message =
-                        format!("Failed to send rack reset request: {err}");
-                    HttpError {
-                        status_code: http::StatusCode::SERVICE_UNAVAILABLE,
-                        error_code: None,
-                        external_message: message.clone(),
-                        internal_message: message,
-                    }
-                }
-                other => HttpError::for_bad_request(
-                    None,
-                    format!("Rack setup request failed: {other}"),
-                ),
-            }
-        })?
-        .into_inner();
+        Ok(HttpResponseOk(PutBgpAuthKeyResponse { status }))
+    }
 
-    Ok(HttpResponseOk(reset_id))
-}
+    async fn put_rss_config_recovery_user_password_hash(
+        rqctx: RequestContext<ServerContext>,
+        body: TypedBody<PutRssRecoveryUserPasswordHash>,
+    ) -> Result<HttpResponseUpdatedNoContent, HttpError> {
+        let ctx = rqctx.context();
 
-#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema, PartialEq)]
-pub struct GetInventoryParams {
-    /// If true, refresh the state of these SPs from MGS prior to returning
-    /// (instead of returning cached data).
-    pub force_refresh: Vec<SpIdentifier>,
-}
+        let mut config = ctx.rss_config.lock().unwrap();
+        config.set_recovery_user_password_hash(body.into_inner().hash);
 
-/// A status endpoint used to report high level information known to wicketd.
-///
-/// This endpoint can be polled to see if there have been state changes in the
-/// system that are useful to report to wicket.
-///
-/// Wicket, and possibly other callers, will retrieve the changed information,
-/// with follow up calls.
-#[endpoint {
-    method = GET,
-    path = "/inventory"
-}]
-async fn get_inventory(
-    rqctx: RequestContext<ServerContext>,
-    body_params: TypedBody<GetInventoryParams>,
-) -> Result<HttpResponseOk<GetInventoryResponse>, HttpError> {
-    let GetInventoryParams { force_refresh } = body_params.into_inner();
-    match rqctx
-        .context()
-        .mgs_handle
-        .get_inventory_refreshing_sps(force_refresh)
-        .await
-    {
-        Ok(response) => Ok(HttpResponseOk(response)),
-        Err(GetInventoryError::InvalidSpIdentifier) => {
-            Err(HttpError::for_unavail(
-                None,
-                "Invalid SP identifier in request".into(),
-            ))
-        }
-        Err(GetInventoryError::ShutdownInProgress) => {
-            Err(HttpError::for_unavail(None, "Server is shutting down".into()))
-        }
+        Ok(HttpResponseUpdatedNoContent())
     }
-}
-
-/// Upload a TUF repository to the server.
-///
-/// At any given time, wicketd will keep at most one TUF repository in memory.
-/// Any previously-uploaded repositories will be discarded.
-#[endpoint {
-    method = PUT,
-    path = "/repository",
-}]
-async fn put_repository(
-    rqctx: RequestContext<ServerContext>,
-    body: StreamingBody,
-) -> Result<HttpResponseUpdatedNoContent, HttpError> {
-    let rqctx = rqctx.context();
-
-    rqctx.update_tracker.put_repository(body.into_stream()).await?;
-
-    Ok(HttpResponseUpdatedNoContent())
-}
 
-#[derive(Clone, Debug, JsonSchema, Serialize)]
-#[serde(rename_all = "snake_case")]
-pub struct InstallableArtifacts {
-    pub artifact_id: ArtifactId,
-    pub installable: Vec<ArtifactHashId>,
-}
+    async fn delete_rss_config(
+        rqctx: RequestContext<ServerContext>,
+    ) -> Result<HttpResponseUpdatedNoContent, HttpError> {
+        let ctx = rqctx.context();
 
-/// The response to a `get_artifacts` call: the system version, and the list of
-/// all artifacts currently held by wicketd.
-#[derive(Clone, Debug, JsonSchema, Serialize)]
-#[serde(rename_all = "snake_case")]
-pub struct GetArtifactsAndEventReportsResponse {
-    pub system_version: Option<SemverVersion>,
-
-    /// Map of artifacts we ingested from the most-recently-uploaded TUF
-    /// repository to a list of artifacts we're serving over the bootstrap
-    /// network. In some cases the list of artifacts being served will have
-    /// length 1 (when we're serving the artifact directly); in other cases the
-    /// artifact in the TUF repo contains multiple nested artifacts inside it
-    /// (e.g., RoT artifacts contain both A and B images), and we serve the list
-    /// of extracted artifacts but not the original combination.
-    ///
-    /// Conceptually, this is a `BTreeMap<ArtifactId, Vec<ArtifactHashId>>`, but
-    /// JSON requires string keys for maps, so we give back a vec of pairs
-    /// instead.
-    pub artifacts: Vec<InstallableArtifacts>,
-
-    pub event_reports: BTreeMap<SpType, BTreeMap<u32, EventReport>>,
-}
+        let mut config = ctx.rss_config.lock().unwrap();
+        *config = Default::default();
 
-/// An endpoint used to report all available artifacts and event reports.
-///
-/// The order of the returned artifacts is unspecified, and may change between
-/// calls even if the total set of artifacts has not.
-#[endpoint {
-    method = GET,
-    path = "/artifacts-and-event-reports",
-}]
-async fn get_artifacts_and_event_reports(
-    rqctx: RequestContext<ServerContext>,
-) -> Result<HttpResponseOk<GetArtifactsAndEventReportsResponse>, HttpError> {
-    let response =
-        rqctx.context().update_tracker.artifacts_and_event_reports().await;
-    Ok(HttpResponseOk(response))
-}
+        Ok(HttpResponseUpdatedNoContent())
+    }
 
-#[derive(Clone, Debug, JsonSchema, Deserialize)]
-pub(crate) struct StartUpdateParams {
-    /// The SP identifiers to start the update with. Must be non-empty.
-    pub(crate) targets: BTreeSet<SpIdentifier>,
+    async fn get_rack_setup_state(
+        rqctx: RequestContext<ServerContext>,
+    ) -> Result<HttpResponseOk<RackOperationStatus>, HttpError> {
+        let ctx = rqctx.context();
 
-    /// Options for the update.
-    pub(crate) options: StartUpdateOptions,
-}
+        let sled_agent_addr = ctx.bootstrap_agent_addr().map_err(|err| {
+            HttpError::for_bad_request(None, format!("{err:#}"))
+        })?;
 
-#[derive(Clone, Debug, JsonSchema, Deserialize)]
-pub(crate) struct StartUpdateOptions {
-    /// If passed in, fails the update with a simulated error.
-    pub(crate) test_error: Option<UpdateTestError>,
-
-    /// If passed in, creates a test step that lasts these many seconds long.
-    ///
-    /// This is used for testing.
-    pub(crate) test_step_seconds: Option<u64>,
-
-    /// If passed in, simulates a result for the RoT Bootloader update.
-    ///
-    /// This is used for testing.
-    pub(crate) test_simulate_rot_bootloader_result:
-        Option<UpdateSimulatedResult>,
-
-    /// If passed in, simulates a result for the RoT update.
-    ///
-    /// This is used for testing.
-    pub(crate) test_simulate_rot_result: Option<UpdateSimulatedResult>,
-
-    /// If passed in, simulates a result for the SP update.
-    ///
-    /// This is used for testing.
-    pub(crate) test_simulate_sp_result: Option<UpdateSimulatedResult>,
-
-    /// If true, skip the check on the current RoT version and always update it
-    /// regardless of whether the update appears to be neeeded.
-    pub(crate) skip_rot_bootloader_version_check: bool,
-
-    /// If true, skip the check on the current RoT version and always update it
-    /// regardless of whether the update appears to be neeeded.
-    pub(crate) skip_rot_version_check: bool,
-
-    /// If true, skip the check on the current SP version and always update it
-    /// regardless of whether the update appears to be neeeded.
-    pub(crate) skip_sp_version_check: bool,
-}
+        let client = bootstrap_agent_client::Client::new(
+            &format!("http://{}", sled_agent_addr),
+            ctx.log.new(slog::o!("component" => "bootstrap client")),
+        );
+
+        let op_status = client
+            .rack_initialization_status()
+            .await
+            .map_err(|err| {
+                use bootstrap_agent_client::Error as BaError;
+                match err {
+                    BaError::CommunicationError(err) => {
+                        let message =
+                            format!("Failed to send rack setup request: {err}");
+                        HttpError {
+                            status_code: http::StatusCode::SERVICE_UNAVAILABLE,
+                            error_code: None,
+                            external_message: message.clone(),
+                            internal_message: message,
+                        }
+                    }
+                    other => HttpError::for_bad_request(
+                        None,
+                        format!("Rack setup request failed: {other}"),
+                    ),
+                }
+            })?
+            .into_inner();
 
-/// A simulated result for a component update.
-///
-/// Used by [`StartUpdateOptions`].
-#[derive(Clone, Debug, JsonSchema, Deserialize)]
-#[serde(rename_all = "snake_case")]
-pub(crate) enum UpdateSimulatedResult {
-    Success,
-    Warning,
-    Skipped,
-    Failure,
-}
+        Ok(HttpResponseOk(op_status))
+    }
 
-#[derive(Clone, Debug, JsonSchema, Deserialize)]
-pub(crate) struct ClearUpdateStateParams {
-    /// The SP identifiers to clear the update state for. Must be non-empty.
-    pub(crate) targets: BTreeSet<SpIdentifier>,
+    async fn post_run_rack_setup(
+        rqctx: RequestContext<ServerContext>,
+    ) -> Result<HttpResponseOk<RackInitId>, HttpError> {
+        let ctx = rqctx.context();
+        let log = &rqctx.log;
 
-    /// Options for clearing update state
-    pub(crate) options: ClearUpdateStateOptions,
-}
+        let sled_agent_addr = ctx.bootstrap_agent_addr().map_err(|err| {
+            HttpError::for_bad_request(None, format!("{err:#}"))
+        })?;
 
-#[derive(Clone, Debug, JsonSchema, Deserialize)]
-pub(crate) struct ClearUpdateStateOptions {
-    /// If passed in, fails the clear update state operation with a simulated
-    /// error.
-    pub(crate) test_error: Option<UpdateTestError>,
-}
+        let request = {
+            let mut config = ctx.rss_config.lock().unwrap();
+            config.start_rss_request(&ctx.bootstrap_peers, log).map_err(
+                |err| HttpError::for_bad_request(None, format!("{err:#}")),
+            )?
+        };
 
-#[derive(Clone, Debug, Default, JsonSchema, Serialize)]
-pub(crate) struct ClearUpdateStateResponse {
-    /// The SPs for which update data was cleared.
-    pub(crate) cleared: BTreeSet<SpIdentifier>,
+        slog::info!(
+            ctx.log,
+            "Sending RSS initialize request to {}",
+            sled_agent_addr
+        );
+        let client = bootstrap_agent_client::Client::new(
+            &format!("http://{}", sled_agent_addr),
+            ctx.log.new(slog::o!("component" => "bootstrap client")),
+        );
+
+        let init_id = client
+            .rack_initialize(&request)
+            .await
+            .map_err(|err| {
+                use bootstrap_agent_client::Error as BaError;
+                match err {
+                    BaError::CommunicationError(err) => {
+                        let message =
+                            format!("Failed to send rack setup request: {err}");
+                        HttpError {
+                            status_code: http::StatusCode::SERVICE_UNAVAILABLE,
+                            error_code: None,
+                            external_message: message.clone(),
+                            internal_message: message,
+                        }
+                    }
+                    other => HttpError::for_bad_request(
+                        None,
+                        format!("Rack setup request failed: {other}"),
+                    ),
+                }
+            })?
+            .into_inner();
 
-    /// The SPs that had no update state to clear.
-    pub(crate) no_update_data: BTreeSet<SpIdentifier>,
-}
+        Ok(HttpResponseOk(init_id))
+    }
 
-#[derive(Clone, Debug, JsonSchema, Deserialize)]
-pub(crate) struct AbortUpdateOptions {
-    /// The message to abort the update with.
-    pub(crate) message: String,
+    async fn post_run_rack_reset(
+        rqctx: RequestContext<ServerContext>,
+    ) -> Result<HttpResponseOk<RackResetId>, HttpError> {
+        let ctx = rqctx.context();
 
-    /// If passed in, fails the force cancel update operation with a simulated
-    /// error.
-    pub(crate) test_error: Option<UpdateTestError>,
-}
+        let sled_agent_addr = ctx.bootstrap_agent_addr().map_err(|err| {
+            HttpError::for_bad_request(None, format!("{err:#}"))
+        })?;
 
-#[derive(Copy, Clone, Debug, JsonSchema, Deserialize, PartialEq, Eq)]
-#[serde(rename_all = "snake_case", tag = "kind", content = "content")]
-pub(crate) enum UpdateTestError {
-    /// Simulate an error where the operation fails to complete.
-    Fail,
-
-    /// Simulate an issue where the operation times out.
-    Timeout {
-        /// The number of seconds to time out after.
-        secs: u64,
-    },
-}
+        slog::info!(
+            ctx.log,
+            "Sending RSS reset request to {}",
+            sled_agent_addr
+        );
+        let client = bootstrap_agent_client::Client::new(
+            &format!("http://{}", sled_agent_addr),
+            ctx.log.new(slog::o!("component" => "bootstrap client")),
+        );
+
+        let reset_id = client
+            .rack_reset()
+            .await
+            .map_err(|err| {
+                use bootstrap_agent_client::Error as BaError;
+                match err {
+                    BaError::CommunicationError(err) => {
+                        let message =
+                            format!("Failed to send rack reset request: {err}");
+                        HttpError {
+                            status_code: http::StatusCode::SERVICE_UNAVAILABLE,
+                            error_code: None,
+                            external_message: message.clone(),
+                            internal_message: message,
+                        }
+                    }
+                    other => HttpError::for_bad_request(
+                        None,
+                        format!("Rack setup request failed: {other}"),
+                    ),
+                }
+            })?
+            .into_inner();
 
-impl UpdateTestError {
-    pub(crate) async fn into_http_error(
-        self,
-        log: &slog::Logger,
-        reason: &str,
-    ) -> HttpError {
-        let message = self.into_error_string(log, reason).await;
-        HttpError::for_bad_request(None, message)
+        Ok(HttpResponseOk(reset_id))
     }
 
-    pub(crate) async fn into_error_string(
-        self,
-        log: &slog::Logger,
-        reason: &str,
-    ) -> String {
-        match self {
-            UpdateTestError::Fail => {
-                format!("Simulated failure while {reason}")
-            }
-            UpdateTestError::Timeout { secs } => {
-                slog::info!(log, "Simulating timeout while {reason}");
-                // 15 seconds should be enough to cause a timeout.
-                tokio::time::sleep(Duration::from_secs(secs)).await;
-                "XXX request should time out before this is hit".into()
+    async fn get_inventory(
+        rqctx: RequestContext<ServerContext>,
+        body_params: TypedBody<GetInventoryParams>,
+    ) -> Result<HttpResponseOk<GetInventoryResponse>, HttpError> {
+        let GetInventoryParams { force_refresh } = body_params.into_inner();
+        match rqctx
+            .context()
+            .mgs_handle
+            .get_inventory_refreshing_sps(force_refresh)
+            .await
+        {
+            Ok(response) => Ok(HttpResponseOk(response)),
+            Err(GetInventoryError::InvalidSpIdentifier) => {
+                Err(HttpError::for_unavail(
+                    None,
+                    "Invalid SP identifier in request".into(),
+                ))
             }
+            Err(GetInventoryError::ShutdownInProgress) => Err(
+                HttpError::for_unavail(None, "Server is shutting down".into()),
+            ),
         }
     }
-}
 
-#[derive(Clone, Debug, JsonSchema, Serialize)]
-#[serde(rename_all = "snake_case")]
-pub struct GetBaseboardResponse {
-    pub baseboard: Option<Baseboard>,
-}
+    async fn put_repository(
+        rqctx: RequestContext<ServerContext>,
+        body: StreamingBody,
+    ) -> Result<HttpResponseUpdatedNoContent, HttpError> {
+        let rqctx = rqctx.context();
 
-/// Report the configured baseboard details
-#[endpoint {
-    method = GET,
-    path = "/baseboard",
-}]
-async fn get_baseboard(
-    rqctx: RequestContext<ServerContext>,
-) -> Result<HttpResponseOk<GetBaseboardResponse>, HttpError> {
-    let rqctx = rqctx.context();
-    Ok(HttpResponseOk(GetBaseboardResponse {
-        baseboard: rqctx.baseboard.clone(),
-    }))
-}
+        rqctx.update_tracker.put_repository(body.into_stream()).await?;
 
-/// All the fields of this response are optional, because it's possible we don't
-/// know any of them (yet) if MGS has not yet finished discovering its location
-/// or (ever) if we're running in a dev environment that doesn't support
-/// MGS-location / baseboard mapping.
-#[derive(Clone, Debug, JsonSchema, Serialize)]
-#[serde(rename_all = "snake_case")]
-pub struct GetLocationResponse {
-    /// The identity of our sled (where wicketd is running).
-    pub sled_id: Option<SpIdentifier>,
-    /// The baseboard of our sled (where wicketd is running).
-    pub sled_baseboard: Option<Baseboard>,
-    /// The baseboard of the switch our sled is physically connected to.
-    pub switch_baseboard: Option<Baseboard>,
-    /// The identity of the switch our sled is physically connected to.
-    pub switch_id: Option<SpIdentifier>,
-}
+        Ok(HttpResponseUpdatedNoContent())
+    }
 
-/// Report the identity of the sled and switch we're currently running on /
-/// connected to.
-#[endpoint {
-    method = GET,
-    path = "/location",
-}]
-async fn get_location(
-    rqctx: RequestContext<ServerContext>,
-) -> Result<HttpResponseOk<GetLocationResponse>, HttpError> {
-    let rqctx = rqctx.context();
-    let inventory = inventory_or_unavail(&rqctx.mgs_handle).await?;
-
-    let switch_id = rqctx.local_switch_id().await;
-    let sled_baseboard = rqctx.baseboard.clone();
-
-    let mut switch_baseboard = None;
-    let mut sled_id = None;
-
-    for sp in &inventory.sps {
-        if Some(sp.id) == switch_id {
-            switch_baseboard = sp.state.as_ref().map(|state| {
-                // TODO-correctness `new_gimlet` isn't the right name: this is a
-                // sidecar baseboard.
-                Baseboard::new_gimlet(
-                    state.serial_number.clone(),
-                    state.model.clone(),
-                    i64::from(state.revision),
-                )
-            });
-        } else if let (Some(sled_baseboard), Some(state)) =
-            (sled_baseboard.as_ref(), sp.state.as_ref())
-        {
-            if sled_baseboard.identifier() == state.serial_number
-                && sled_baseboard.model() == state.model
-                && sled_baseboard.revision() == i64::from(state.revision)
+    async fn get_artifacts_and_event_reports(
+        rqctx: RequestContext<ServerContext>,
+    ) -> Result<HttpResponseOk<GetArtifactsAndEventReportsResponse>, HttpError>
+    {
+        let response =
+            rqctx.context().update_tracker.artifacts_and_event_reports().await;
+        Ok(HttpResponseOk(response))
+    }
+
+    async fn get_baseboard(
+        rqctx: RequestContext<ServerContext>,
+    ) -> Result<HttpResponseOk<GetBaseboardResponse>, HttpError> {
+        let rqctx = rqctx.context();
+        Ok(HttpResponseOk(GetBaseboardResponse {
+            baseboard: rqctx.baseboard.clone(),
+        }))
+    }
+
+    async fn get_location(
+        rqctx: RequestContext<ServerContext>,
+    ) -> Result<HttpResponseOk<GetLocationResponse>, HttpError> {
+        let rqctx = rqctx.context();
+        let inventory = inventory_or_unavail(&rqctx.mgs_handle).await?;
+
+        let switch_id = rqctx.local_switch_id().await;
+        let sled_baseboard = rqctx.baseboard.clone();
+
+        let mut switch_baseboard = None;
+        let mut sled_id = None;
+
+        for sp in &inventory.sps {
+            if Some(sp.id) == switch_id {
+                switch_baseboard = sp.state.as_ref().map(|state| {
+                    // TODO-correctness `new_gimlet` isn't the right name: this is a
+                    // sidecar baseboard.
+                    Baseboard::new_gimlet(
+                        state.serial_number.clone(),
+                        state.model.clone(),
+                        state.revision,
+                    )
+                });
+            } else if let (Some(sled_baseboard), Some(state)) =
+                (sled_baseboard.as_ref(), sp.state.as_ref())
             {
-                sled_id = Some(sp.id);
+                if sled_baseboard.identifier() == state.serial_number
+                    && sled_baseboard.model() == state.model
+                    && sled_baseboard.revision() == state.revision
+                {
+                    sled_id = Some(sp.id);
+                }
             }
         }
-    }
-
-    Ok(HttpResponseOk(GetLocationResponse {
-        sled_id,
-        sled_baseboard,
-        switch_baseboard,
-        switch_id,
-    }))
-}
 
-/// An endpoint to start updating one or more sleds, switches and PSCs.
-#[endpoint {
-    method = POST,
-    path = "/update",
-}]
-async fn post_start_update(
-    rqctx: RequestContext<ServerContext>,
-    params: TypedBody<StartUpdateParams>,
-) -> Result<HttpResponseUpdatedNoContent, HttpError> {
-    let log = &rqctx.log;
-    let rqctx = rqctx.context();
-    let params = params.into_inner();
-
-    if params.targets.is_empty() {
-        return Err(HttpError::for_bad_request(
-            None,
-            "No update targets specified".into(),
-        ));
+        Ok(HttpResponseOk(GetLocationResponse {
+            sled_id,
+            sled_baseboard,
+            switch_baseboard,
+            switch_id,
+        }))
     }
 
-    // Can we update the target SPs? We refuse to update if, for any target SP:
-    //
-    // 1. We haven't pulled its state in our inventory (most likely cause: the
-    //    cubby is empty; less likely cause: the SP is misbehaving, which will
-    //    make updating it very unlikely to work anyway)
-    // 2. We have pulled its state but our hardware manager says we can't
-    //    update it (most likely cause: the target is the sled we're currently
-    //    running on; less likely cause: our hardware manager failed to get our
-    //    local identifying information, and it refuses to update this target
-    //    out of an abundance of caution).
-    //
-    // First, get our most-recently-cached inventory view. (Only wait 80% of
-    // WICKETD_TIMEOUT for this: if even a cached inventory isn't available,
-    // it's because we've never established contact with MGS. In that case, we
-    // should produce a useful error message rather than timing out on the
-    // client.)
-    let inventory = match tokio::time::timeout(
-        WICKETD_TIMEOUT.mul_f32(0.8),
-        rqctx.mgs_handle.get_cached_inventory(),
-    )
-    .await
-    {
-        Ok(Ok(inventory)) => inventory,
-        Ok(Err(ShutdownInProgress)) => {
-            return Err(HttpError::for_unavail(
+    async fn post_start_update(
+        rqctx: RequestContext<ServerContext>,
+        params: TypedBody<StartUpdateParams>,
+    ) -> Result<HttpResponseUpdatedNoContent, HttpError> {
+        let log = &rqctx.log;
+        let rqctx = rqctx.context();
+        let params = params.into_inner();
+
+        if params.targets.is_empty() {
+            return Err(HttpError::for_bad_request(
                 None,
-                "Server is shutting down".into(),
+                "No update targets specified".into(),
             ));
         }
-        Err(_) => {
-            // Have to construct an HttpError manually because
-            // HttpError::for_unavail doesn't accept an external message.
-            let message =
-                "Rack inventory not yet available (is MGS alive?)".to_owned();
-            return Err(HttpError {
-                status_code: http::StatusCode::SERVICE_UNAVAILABLE,
-                error_code: None,
-                external_message: message.clone(),
-                internal_message: message,
-            });
-        }
-    };
 
-    // Error cases.
-    let mut inventory_absent = BTreeSet::new();
-    let mut self_update = None;
-    let mut maybe_self_update = BTreeSet::new();
+        // Can we update the target SPs? We refuse to update if, for any target SP:
+        //
+        // 1. We haven't pulled its state in our inventory (most likely cause: the
+        //    cubby is empty; less likely cause: the SP is misbehaving, which will
+        //    make updating it very unlikely to work anyway)
+        // 2. We have pulled its state but our hardware manager says we can't
+        //    update it (most likely cause: the target is the sled we're currently
+        //    running on; less likely cause: our hardware manager failed to get our
+        //    local identifying information, and it refuses to update this target
+        //    out of an abundance of caution).
+        //
+        // First, get our most-recently-cached inventory view. (Only wait 80% of
+        // WICKETD_TIMEOUT for this: if even a cached inventory isn't available,
+        // it's because we've never established contact with MGS. In that case, we
+        // should produce a useful error message rather than timing out on the
+        // client.)
+        let inventory = match tokio::time::timeout(
+            WICKETD_TIMEOUT.mul_f32(0.8),
+            rqctx.mgs_handle.get_cached_inventory(),
+        )
+        .await
+        {
+            Ok(Ok(inventory)) => inventory,
+            Ok(Err(ShutdownInProgress)) => {
+                return Err(HttpError::for_unavail(
+                    None,
+                    "Server is shutting down".into(),
+                ));
+            }
+            Err(_) => {
+                // Have to construct an HttpError manually because
+                // HttpError::for_unavail doesn't accept an external message.
+                let message =
+                    "Rack inventory not yet available (is MGS alive?)"
+                        .to_owned();
+                return Err(HttpError {
+                    status_code: http::StatusCode::SERVICE_UNAVAILABLE,
+                    error_code: None,
+                    external_message: message.clone(),
+                    internal_message: message,
+                });
+            }
+        };
 
-    // Next, do we have the states of the target SP?
-    let sp_states = match inventory {
-        GetInventoryResponse::Response { inventory, .. } => inventory
-            .sps
-            .into_iter()
-            .filter_map(|sp| {
-                if params.targets.contains(&sp.id) {
-                    if let Some(sp_state) = sp.state {
-                        Some((sp.id, sp_state))
+        // Error cases.
+        let mut inventory_absent = BTreeSet::new();
+        let mut self_update = None;
+        let mut maybe_self_update = BTreeSet::new();
+
+        // Next, do we have the states of the target SP?
+        let sp_states = match inventory {
+            GetInventoryResponse::Response { inventory, .. } => inventory
+                .sps
+                .into_iter()
+                .filter_map(|sp| {
+                    if params.targets.contains(&sp.id) {
+                        if let Some(sp_state) = sp.state {
+                            Some((sp.id, sp_state))
+                        } else {
+                            None
+                        }
                     } else {
                         None
                     }
-                } else {
-                    None
-                }
-            })
-            .collect(),
-        GetInventoryResponse::Unavailable => BTreeMap::new(),
-    };
-
-    for target in &params.targets {
-        let sp_state = match sp_states.get(target) {
-            Some(sp_state) => sp_state,
-            None => {
-                // The state isn't present, so add to inventory_absent.
-                inventory_absent.insert(*target);
-                continue;
-            }
+                })
+                .collect(),
+            GetInventoryResponse::Unavailable => BTreeMap::new(),
         };
 
-        // If we have the state of the SP, are we allowed to update it? We
-        // refuse to try to update our own sled.
-        match &rqctx.baseboard {
-            Some(baseboard) => {
-                if baseboard.identifier() == sp_state.serial_number
-                    && baseboard.model() == sp_state.model
-                    && baseboard.revision() == i64::from(sp_state.revision)
-                {
-                    self_update = Some(*target);
+        for target in &params.targets {
+            let sp_state = match sp_states.get(target) {
+                Some(sp_state) => sp_state,
+                None => {
+                    // The state isn't present, so add to inventory_absent.
+                    inventory_absent.insert(*target);
                     continue;
                 }
-            }
-            None => {
-                // We don't know our own baseboard, which is a very questionable
-                // state to be in! For now, we will hard-code the possibly
-                // locations where we could be running: scrimlets can only be in
-                // cubbies 14 or 16, so we refuse to update either of those.
-                let target_is_scrimlet = matches!(
-                    (target.type_, target.slot),
-                    (SpType::Sled, 14 | 16)
-                );
-                if target_is_scrimlet {
-                    maybe_self_update.insert(*target);
-                    continue;
+            };
+
+            // If we have the state of the SP, are we allowed to update it? We
+            // refuse to try to update our own sled.
+            match &rqctx.baseboard {
+                Some(baseboard) => {
+                    if baseboard.identifier() == sp_state.serial_number
+                        && baseboard.model() == sp_state.model
+                        && baseboard.revision() == sp_state.revision
+                    {
+                        self_update = Some(*target);
+                        continue;
+                    }
+                }
+                None => {
+                    // We don't know our own baseboard, which is a very questionable
+                    // state to be in! For now, we will hard-code the possibly
+                    // locations where we could be running: scrimlets can only be in
+                    // cubbies 14 or 16, so we refuse to update either of those.
+                    let target_is_scrimlet = matches!(
+                        (target.type_, target.slot),
+                        (SpType::Sled, 14 | 16)
+                    );
+                    if target_is_scrimlet {
+                        maybe_self_update.insert(*target);
+                        continue;
+                    }
                 }
             }
         }
-    }
 
-    // Do we have any errors?
-    let mut errors = Vec::new();
-    if !inventory_absent.is_empty() {
-        errors.push(format!(
-            "cannot update sleds (no inventory state present for {})",
-            sps_to_string(&inventory_absent)
-        ));
-    }
-    if let Some(self_update) = self_update {
-        errors.push(format!(
-            "cannot update sled where wicketd is running ({})",
-            SpIdentifierDisplay(self_update)
-        ));
-    }
-    if !maybe_self_update.is_empty() {
-        errors.push(format!(
-            "wicketd does not know its own baseboard details: \
+        // Do we have any errors?
+        let mut errors = Vec::new();
+        if !inventory_absent.is_empty() {
+            errors.push(format!(
+                "cannot update sleds (no inventory state present for {})",
+                sps_to_string(&inventory_absent)
+            ));
+        }
+        if let Some(self_update) = self_update {
+            errors.push(format!(
+                "cannot update sled where wicketd is running ({})",
+                SpIdentifierDisplay(self_update)
+            ));
+        }
+        if !maybe_self_update.is_empty() {
+            errors.push(format!(
+                "wicketd does not know its own baseboard details: \
              refusing to update either scrimlet ({})",
-            sps_to_string(&inventory_absent)
-        ));
+                sps_to_string(&inventory_absent)
+            ));
+        }
+
+        if let Some(test_error) = &params.options.test_error {
+            errors.push(
+                test_error.into_error_string(log, "starting update").await,
+            );
+        }
+
+        let start_update_errors = if errors.is_empty() {
+            // No errors: we can try and proceed with this update.
+            match rqctx
+                .update_tracker
+                .start(params.targets, params.options)
+                .await
+            {
+                Ok(()) => return Ok(HttpResponseUpdatedNoContent {}),
+                Err(errors) => errors,
+            }
+        } else {
+            // We've already found errors, so all we want to do is to check whether
+            // the update tracker thinks there are any errors as well.
+            match rqctx.update_tracker.update_pre_checks(params.targets).await {
+                Ok(()) => Vec::new(),
+                Err(errors) => errors,
+            }
+        };
+
+        errors
+            .extend(start_update_errors.iter().map(|error| error.to_string()));
+
+        // If we get here, we have errors to report.
+
+        match errors.len() {
+            0 => {
+                unreachable!(
+                    "we already returned Ok(_) above if there were no errors"
+                )
+            }
+            1 => {
+                return Err(HttpError::for_bad_request(
+                    None,
+                    errors.pop().unwrap(),
+                ));
+            }
+            _ => {
+                return Err(HttpError::for_bad_request(
+                    None,
+                    format!(
+                        "multiple errors encountered:\n - {}",
+                        itertools::join(errors, "\n - ")
+                    ),
+                ));
+            }
+        }
     }
 
-    if let Some(test_error) = &params.options.test_error {
-        errors.push(test_error.into_error_string(log, "starting update").await);
+    async fn get_update_sp(
+        rqctx: RequestContext<ServerContext>,
+        target: Path<SpIdentifier>,
+    ) -> Result<HttpResponseOk<EventReport>, HttpError> {
+        let event_report = rqctx
+            .context()
+            .update_tracker
+            .event_report(target.into_inner())
+            .await;
+        Ok(HttpResponseOk(event_report))
     }
 
-    let start_update_errors = if errors.is_empty() {
-        // No errors: we can try and proceed with this update.
-        match rqctx.update_tracker.start(params.targets, params.options).await {
-            Ok(()) => return Ok(HttpResponseUpdatedNoContent {}),
-            Err(errors) => errors,
-        }
-    } else {
-        // We've already found errors, so all we want to do is to check whether
-        // the update tracker thinks there are any errors as well.
-        match rqctx.update_tracker.update_pre_checks(params.targets).await {
-            Ok(()) => Vec::new(),
-            Err(errors) => errors,
+    async fn post_abort_update(
+        rqctx: RequestContext<ServerContext>,
+        target: Path<SpIdentifier>,
+        opts: TypedBody<AbortUpdateOptions>,
+    ) -> Result<HttpResponseUpdatedNoContent, HttpError> {
+        let log = &rqctx.log;
+        let target = target.into_inner();
+
+        let opts = opts.into_inner();
+        if let Some(test_error) = opts.test_error {
+            return Err(test_error
+                .into_http_error(log, "aborting update")
+                .await);
         }
-    };
 
-    errors.extend(start_update_errors.iter().map(|error| error.to_string()));
+        match rqctx
+            .context()
+            .update_tracker
+            .abort_update(target, opts.message)
+            .await
+        {
+            Ok(()) => Ok(HttpResponseUpdatedNoContent {}),
+            Err(err) => Err(err.to_http_error()),
+        }
+    }
 
-    // If we get here, we have errors to report.
+    async fn post_clear_update_state(
+        rqctx: RequestContext<ServerContext>,
+        params: TypedBody<ClearUpdateStateParams>,
+    ) -> Result<HttpResponseOk<ClearUpdateStateResponse>, HttpError> {
+        let log = &rqctx.log;
+        let rqctx = rqctx.context();
+        let params = params.into_inner();
 
-    match errors.len() {
-        0 => {
-            unreachable!(
-                "we already returned Ok(_) above if there were no errors"
-            )
-        }
-        1 => {
+        if params.targets.is_empty() {
             return Err(HttpError::for_bad_request(
                 None,
-                errors.pop().unwrap(),
+                "No targets specified".into(),
             ));
         }
-        _ => {
-            return Err(HttpError::for_bad_request(
-                None,
-                format!(
-                    "multiple errors encountered:\n - {}",
-                    itertools::join(errors, "\n - ")
-                ),
-            ));
-        }
-    }
-}
 
-/// An endpoint to get the status of any update being performed or recently
-/// completed on a single SP.
-#[endpoint {
-    method = GET,
-    path = "/update/{type}/{slot}",
-}]
-async fn get_update_sp(
-    rqctx: RequestContext<ServerContext>,
-    target: Path<SpIdentifier>,
-) -> Result<HttpResponseOk<EventReport>, HttpError> {
-    let event_report =
-        rqctx.context().update_tracker.event_report(target.into_inner()).await;
-    Ok(HttpResponseOk(event_report))
-}
-
-/// Forcibly cancels a running update.
-///
-/// This is a potentially dangerous operation, but one that is sometimes
-/// required. A machine reset might be required after this operation completes.
-#[endpoint {
-    method = POST,
-    path = "/abort-update/{type}/{slot}",
-}]
-async fn post_abort_update(
-    rqctx: RequestContext<ServerContext>,
-    target: Path<SpIdentifier>,
-    opts: TypedBody<AbortUpdateOptions>,
-) -> Result<HttpResponseUpdatedNoContent, HttpError> {
-    let log = &rqctx.log;
-    let target = target.into_inner();
-
-    let opts = opts.into_inner();
-    if let Some(test_error) = opts.test_error {
-        return Err(test_error.into_http_error(log, "aborting update").await);
-    }
+        if let Some(test_error) = params.options.test_error {
+            return Err(test_error
+                .into_http_error(log, "clearing update state")
+                .await);
+        }
 
-    match rqctx
-        .context()
-        .update_tracker
-        .abort_update(target, opts.message)
-        .await
-    {
-        Ok(()) => Ok(HttpResponseUpdatedNoContent {}),
-        Err(err) => Err(err.to_http_error()),
+        match rqctx.update_tracker.clear_update_state(params.targets).await {
+            Ok(response) => Ok(HttpResponseOk(response)),
+            Err(err) => Err(err.to_http_error()),
+        }
     }
-}
 
-/// Resets update state for a sled.
-///
-/// Use this to clear update state after a failed update.
-#[endpoint {
-    method = POST,
-    path = "/clear-update-state",
-}]
-async fn post_clear_update_state(
-    rqctx: RequestContext<ServerContext>,
-    params: TypedBody<ClearUpdateStateParams>,
-) -> Result<HttpResponseOk<ClearUpdateStateResponse>, HttpError> {
-    let log = &rqctx.log;
-    let rqctx = rqctx.context();
-    let params = params.into_inner();
-
-    if params.targets.is_empty() {
-        return Err(HttpError::for_bad_request(
-            None,
-            "No targets specified".into(),
-        ));
-    }
+    async fn post_ignition_command(
+        rqctx: RequestContext<ServerContext>,
+        path: Path<PathSpIgnitionCommand>,
+    ) -> Result<HttpResponseUpdatedNoContent, HttpError> {
+        let apictx = rqctx.context();
+        let PathSpIgnitionCommand { type_, slot, command } = path.into_inner();
 
-    if let Some(test_error) = params.options.test_error {
-        return Err(test_error
-            .into_http_error(log, "clearing update state")
-            .await);
-    }
+        apictx
+            .mgs_client
+            .ignition_command(type_, slot, command)
+            .await
+            .map_err(http_error_from_client_error)?;
 
-    match rqctx.update_tracker.clear_update_state(params.targets).await {
-        Ok(response) => Ok(HttpResponseOk(response)),
-        Err(err) => Err(err.to_http_error()),
+        Ok(HttpResponseUpdatedNoContent())
     }
-}
 
-#[derive(Serialize, Deserialize, JsonSchema)]
-struct PathSpIgnitionCommand {
-    #[serde(rename = "type")]
-    type_: SpType,
-    slot: u32,
-    command: IgnitionCommand,
-}
-
-/// Send an ignition command targeting a specific SP.
-///
-/// This endpoint acts as a proxy to the MGS endpoint performing the same
-/// function, allowing wicket to communicate exclusively with wicketd (even
-/// though wicketd adds no meaningful functionality here beyond what MGS
-/// offers).
-#[endpoint {
-    method = POST,
-    path = "/ignition/{type}/{slot}/{command}",
-}]
-async fn post_ignition_command(
-    rqctx: RequestContext<ServerContext>,
-    path: Path<PathSpIgnitionCommand>,
-) -> Result<HttpResponseUpdatedNoContent, HttpError> {
-    let apictx = rqctx.context();
-    let PathSpIgnitionCommand { type_, slot, command } = path.into_inner();
-
-    apictx
-        .mgs_client
-        .ignition_command(type_, slot, command)
-        .await
-        .map_err(http_error_from_client_error)?;
-
-    Ok(HttpResponseUpdatedNoContent())
-}
-
-/// Options provided to the preflight uplink check.
-#[derive(Clone, Debug, JsonSchema, Serialize, Deserialize)]
-#[serde(rename_all = "snake_case")]
-pub struct PreflightUplinkCheckOptions {
-    /// DNS name to query.
-    pub dns_name_to_query: Option<String>,
-}
-
-/// An endpoint to start a preflight check for uplink configuration.
-#[endpoint {
-    method = POST,
-    path = "/preflight/uplink",
-}]
-async fn post_start_preflight_uplink_check(
-    rqctx: RequestContext<ServerContext>,
-    body: TypedBody<PreflightUplinkCheckOptions>,
-) -> Result<HttpResponseUpdatedNoContent, HttpError> {
-    let rqctx = rqctx.context();
-    let options = body.into_inner();
-
-    let our_switch_location = match rqctx.local_switch_id().await {
-        Some(SpIdentifier { slot, type_: SpType::Switch }) => match slot {
-            0 => SwitchLocation::Switch0,
-            1 => SwitchLocation::Switch1,
-            _ => {
+    async fn post_start_preflight_uplink_check(
+        rqctx: RequestContext<ServerContext>,
+        body: TypedBody<PreflightUplinkCheckOptions>,
+    ) -> Result<HttpResponseUpdatedNoContent, HttpError> {
+        let rqctx = rqctx.context();
+        let options = body.into_inner();
+
+        let our_switch_location = match rqctx.local_switch_id().await {
+            Some(SpIdentifier { slot, type_: SpType::Switch }) => match slot {
+                0 => SwitchLocation::Switch0,
+                1 => SwitchLocation::Switch1,
+                _ => {
+                    return Err(HttpError::for_internal_error(format!(
+                        "unexpected switch slot {slot}"
+                    )));
+                }
+            },
+            Some(other) => {
                 return Err(HttpError::for_internal_error(format!(
-                    "unexpected switch slot {slot}"
+                    "unexpected switch SP identifier {other:?}"
                 )));
             }
-        },
-        Some(other) => {
-            return Err(HttpError::for_internal_error(format!(
-                "unexpected switch SP identifier {other:?}"
-            )));
-        }
-        None => {
-            return Err(HttpError::for_unavail(
-                Some("UnknownSwitchLocation".to_string()),
-                "local switch location not yet determined".to_string(),
-            ));
-        }
-    };
+            None => {
+                return Err(HttpError::for_unavail(
+                    Some("UnknownSwitchLocation".to_string()),
+                    "local switch location not yet determined".to_string(),
+                ));
+            }
+        };
 
-    let (network_config, dns_servers, ntp_servers) = {
-        let rss_config = rqctx.rss_config.lock().unwrap();
+        let (network_config, dns_servers, ntp_servers) = {
+            let rss_config = rqctx.rss_config.lock().unwrap();
 
-        let network_config = rss_config
-            .user_specified_rack_network_config()
-            .cloned()
-            .ok_or_else(|| {
-                HttpError::for_bad_request(
-                    None,
-                    "uplink preflight check requires setting \
+            let network_config = rss_config
+                .user_specified_rack_network_config()
+                .cloned()
+                .ok_or_else(|| {
+                    HttpError::for_bad_request(
+                        None,
+                        "uplink preflight check requires setting \
                      the uplink config for RSS"
-                        .to_string(),
-                )
-            })?;
+                            .to_string(),
+                    )
+                })?;
+
+            (
+                network_config,
+                rss_config.dns_servers().to_vec(),
+                rss_config.ntp_servers().to_vec(),
+            )
+        };
 
-        (
-            network_config,
-            rss_config.dns_servers().to_vec(),
-            rss_config.ntp_servers().to_vec(),
-        )
-    };
-
-    match rqctx
-        .preflight_checker
-        .uplink_start(
-            network_config,
-            dns_servers,
-            ntp_servers,
-            our_switch_location,
-            options.dns_name_to_query,
-        )
-        .await
-    {
-        Ok(()) => Ok(HttpResponseUpdatedNoContent {}),
-        Err(err) => Err(HttpError::for_client_error(
-            None,
-            StatusCode::TOO_MANY_REQUESTS,
-            err.to_string(),
-        )),
+        match rqctx
+            .preflight_checker
+            .uplink_start(
+                network_config,
+                dns_servers,
+                ntp_servers,
+                our_switch_location,
+                options.dns_name_to_query,
+            )
+            .await
+        {
+            Ok(()) => Ok(HttpResponseUpdatedNoContent {}),
+            Err(err) => Err(HttpError::for_client_error(
+                None,
+                StatusCode::TOO_MANY_REQUESTS,
+                err.to_string(),
+            )),
+        }
     }
-}
 
-/// An endpoint to get the report for the most recent (or still running)
-/// preflight uplink check.
-#[endpoint {
-    method = GET,
-    path = "/preflight/uplink",
-}]
-async fn get_preflight_uplink_report(
-    rqctx: RequestContext<ServerContext>,
-) -> Result<HttpResponseOk<UplinkEventReport>, HttpError> {
-    let rqctx = rqctx.context();
-
-    match rqctx.preflight_checker.uplink_event_report() {
+    async fn get_preflight_uplink_report(
+        rqctx: RequestContext<ServerContext>,
+    ) -> Result<
+        HttpResponseOk<wicket_common::preflight_check::EventReport>,
+        HttpError,
+    > {
+        let rqctx = rqctx.context();
+
+        match rqctx.preflight_checker.uplink_event_report() {
         Some(report) => Ok(HttpResponseOk(report)),
         None => Err(HttpError::for_bad_request(
             None,
@@ -1317,53 +799,62 @@ async fn get_preflight_uplink_report(
                 .to_string(),
         )),
     }
-}
-
-/// An endpoint instructing wicketd to reload its SMF config properties.
-///
-/// The only expected client of this endpoint is `curl` from wicketd's SMF
-/// `refresh` method, but other clients hitting it is harmless.
-#[endpoint {
-    method = POST,
-    path = "/reload-config",
-}]
-async fn post_reload_config(
-    rqctx: RequestContext<ServerContext>,
-) -> Result<HttpResponseUpdatedNoContent, HttpError> {
-    let smf_values = SmfConfigValues::read_current().map_err(|err| {
-        HttpError::for_unavail(
-            None,
-            format!("failed to read SMF values: {err}"),
-        )
-    })?;
-
-    let rqctx = rqctx.context();
-
-    // We do not allow a config reload to change our bound address; return an
-    // error if the caller is attempting to do so.
-    if rqctx.bind_address != smf_values.address {
-        return Err(HttpError::for_bad_request(
-            None,
-            "listening address cannot be reconfigured".to_string(),
-        ));
     }
 
-    if let Some(rack_subnet) = smf_values.rack_subnet {
-        let resolver = Resolver::new_from_subnet(
-            rqctx.log.new(o!("component" => "InternalDnsResolver")),
-            rack_subnet,
-        )
-        .map_err(|err| {
+    async fn post_reload_config(
+        rqctx: RequestContext<ServerContext>,
+    ) -> Result<HttpResponseUpdatedNoContent, HttpError> {
+        let smf_values = SmfConfigValues::read_current().map_err(|err| {
             HttpError::for_unavail(
                 None,
-                format!("failed to create internal DNS resolver: {err}"),
+                format!("failed to read SMF values: {err}"),
             )
         })?;
 
-        *rqctx.internal_dns_resolver.lock().unwrap() = Some(resolver);
+        let rqctx = rqctx.context();
+
+        // We do not allow a config reload to change our bound address; return an
+        // error if the caller is attempting to do so.
+        if rqctx.bind_address != smf_values.address {
+            return Err(HttpError::for_bad_request(
+                None,
+                "listening address cannot be reconfigured".to_string(),
+            ));
+        }
+
+        if let Some(rack_subnet) = smf_values.rack_subnet {
+            let resolver = Resolver::new_from_subnet(
+                rqctx.log.new(o!("component" => "InternalDnsResolver")),
+                rack_subnet,
+            )
+            .map_err(|err| {
+                HttpError::for_unavail(
+                    None,
+                    format!("failed to create internal DNS resolver: {err}"),
+                )
+            })?;
+
+            *rqctx.internal_dns_resolver.lock().unwrap() = Some(resolver);
+        }
+
+        Ok(HttpResponseUpdatedNoContent())
     }
+}
 
-    Ok(HttpResponseUpdatedNoContent())
+// Get the current inventory or return a 503 Unavailable.
+async fn inventory_or_unavail(
+    mgs_handle: &MgsHandle,
+) -> Result<RackV1Inventory, HttpError> {
+    match mgs_handle.get_cached_inventory().await {
+        Ok(GetInventoryResponse::Response { inventory, .. }) => Ok(inventory),
+        Ok(GetInventoryResponse::Unavailable) => Err(HttpError::for_unavail(
+            None,
+            "Rack inventory not yet available".into(),
+        )),
+        Err(ShutdownInProgress) => {
+            Err(HttpError::for_unavail(None, "Server is shutting down".into()))
+        }
+    }
 }
 
 fn http_error_from_client_error(
diff --git a/wicketd/src/installinator_progress.rs b/wicketd/src/installinator_progress.rs
index 77baec2c94..7d076e7b0e 100644
--- a/wicketd/src/installinator_progress.rs
+++ b/wicketd/src/installinator_progress.rs
@@ -12,7 +12,7 @@ use std::{
     sync::{Arc, Mutex},
 };
 
-use installinator_artifactd::EventReportStatus;
+use installinator_api::EventReportStatus;
 use tokio::sync::{oneshot, watch};
 use update_engine::events::StepEventIsTerminal;
 use uuid::Uuid;
diff --git a/wicketd/src/lib.rs b/wicketd/src/lib.rs
index 5926fc468d..907d8754f8 100644
--- a/wicketd/src/lib.rs
+++ b/wicketd/src/lib.rs
@@ -9,15 +9,17 @@ mod context;
 mod helpers;
 mod http_entrypoints;
 mod installinator_progress;
-mod inventory;
 pub mod mgs;
 mod nexus_proxy;
 mod preflight_check;
 mod rss_config;
 mod update_tracker;
 
-use anyhow::{anyhow, Context, Result};
-use artifacts::{WicketdArtifactServer, WicketdArtifactStore};
+use anyhow::{anyhow, bail, Context, Result};
+use artifacts::{
+    WicketdArtifactStore, WicketdInstallinatorApiImpl,
+    WicketdInstallinatorContext,
+};
 use bootstrap_addrs::BootstrapPeers;
 pub use config::Config;
 pub(crate) use context::ServerContext;
@@ -25,7 +27,6 @@ use display_error_chain::DisplayErrorChain;
 use dropshot::{ConfigDropshot, HandlerTaskMode, HttpServer};
 pub use installinator_progress::{IprUpdateTracker, RunningUpdateState};
 use internal_dns::resolver::Resolver;
-pub use inventory::{RackV1Inventory, SpInventory};
 use mgs::make_mgs_client;
 pub(crate) use mgs::{MgsHandle, MgsManager};
 use nexus_proxy::NexusTcpProxy;
@@ -42,18 +43,6 @@ use std::{
 };
 pub use update_tracker::{StartUpdateError, UpdateTracker};
 
-/// Run the OpenAPI generator for the API; which emits the OpenAPI spec
-/// to stdout.
-pub fn run_openapi() -> Result<(), String> {
-    http_entrypoints::api()
-        .openapi("Oxide Technician Port Control Service", "0.0.1")
-        .description("API for use by the technician port TUI: wicket")
-        .contact_url("https://oxide.computer")
-        .contact_email("api@oxide.computer")
-        .write(&mut std::io::stdout())
-        .map_err(|e| e.to_string())
-}
-
 /// Command line arguments for wicketd
 pub struct Args {
     pub address: SocketAddrV6,
@@ -118,7 +107,7 @@ impl SmfConfigValues {
 
 pub struct Server {
     pub wicketd_server: HttpServer<ServerContext>,
-    pub artifact_server: HttpServer<installinator_artifactd::ServerContext>,
+    pub installinator_server: HttpServer<WicketdInstallinatorContext>,
     pub artifact_store: WicketdArtifactStore,
     pub update_tracker: Arc<UpdateTracker>,
     pub ipr_update_tracker: IprUpdateTracker,
@@ -127,14 +116,14 @@ pub struct Server {
 
 impl Server {
     /// Run an instance of the wicketd server
-    pub async fn start(log: slog::Logger, args: Args) -> Result<Self, String> {
+    pub async fn start(log: slog::Logger, args: Args) -> anyhow::Result<Self> {
         let (drain, registration) = slog_dtrace::with_drain(log);
 
         let log = slog::Logger::root(drain.fuse(), slog::o!(FileKv));
         if let slog_dtrace::ProbeRegistration::Failed(e) = registration {
             let msg = format!("failed to register DTrace probes: {}", e);
             error!(log, "{}", msg);
-            return Err(msg);
+            bail!(msg);
         } else {
             debug!(log, "registered DTrace probes");
         };
@@ -146,6 +135,7 @@ impl Server {
             // some endpoints.
             request_body_max_bytes: 4 << 30,
             default_handler_task_mode: HandlerTaskMode::Detached,
+            log_headers: vec![],
         };
 
         let mgs_manager = MgsManager::new(&log, args.mgs_address);
@@ -174,7 +164,8 @@ impl Server {
                     addr,
                 )
                 .map_err(|err| {
-                    format!("Could not create internal DNS resolver: {err}")
+                    anyhow!(err)
+                        .context("Could not create internal DNS resolver")
                 })
             })
             .transpose()?;
@@ -186,7 +177,9 @@ impl Server {
             &log,
         )
         .await
-        .map_err(|err| format!("failed to start Nexus TCP proxy: {err}"))?;
+        .map_err(|err| {
+            anyhow!(err).context("failed to start Nexus TCP proxy")
+        })?;
 
         let wicketd_server = {
             let ds_log = log.new(o!("component" => "dropshot (wicketd)"));
@@ -209,25 +202,39 @@ impl Server {
                 },
                 &ds_log,
             )
-            .map_err(|err| format!("initializing http server: {}", err))?
+            .map_err(|err| anyhow!(err).context("initializing http server"))?
             .start()
         };
 
-        let server =
-            WicketdArtifactServer::new(&log, store.clone(), ipr_artifact);
-        let artifact_server = installinator_artifactd::ArtifactServer::new(
-            server,
-            args.artifact_address,
-            &log,
-        )
-        .start()
-        .map_err(|error| {
-            format!("failed to start artifact server: {error:?}")
-        })?;
+        let installinator_server = {
+            let installinator_config = installinator_api::default_config(
+                SocketAddr::V6(args.artifact_address),
+            );
+            let api_description =
+                installinator_api::installinator_api::api_description::<
+                    WicketdInstallinatorApiImpl,
+                >()?;
+
+            dropshot::HttpServerStarter::new(
+                &installinator_config,
+                api_description,
+                WicketdInstallinatorContext::new(
+                    &log,
+                    store.clone(),
+                    ipr_artifact,
+                ),
+                &log,
+            )
+            .map_err(|err| {
+                anyhow!(err)
+                    .context("failed to create installinator artifact server")
+            })?
+            .start()
+        };
 
         Ok(Self {
             wicketd_server,
-            artifact_server,
+            installinator_server,
             artifact_store: store,
             update_tracker,
             ipr_update_tracker,
@@ -240,7 +247,7 @@ impl Server {
         self.wicketd_server.close().await.map_err(|error| {
             anyhow!("error closing wicketd server: {error}")
         })?;
-        self.artifact_server.close().await.map_err(|error| {
+        self.installinator_server.close().await.map_err(|error| {
             anyhow!("error closing artifact server: {error}")
         })?;
         self.nexus_tcp_proxy.shutdown();
@@ -257,7 +264,7 @@ impl Server {
                     Err(err) => Err(format!("running wicketd server: {err}")),
                 }
             }
-            res = self.artifact_server => {
+            res = self.installinator_server => {
                 match res {
                     Ok(()) => Err("artifact server exited unexpectedly".to_owned()),
                     // The artifact server returns an anyhow::Error, which has a
diff --git a/wicketd/src/mgs.rs b/wicketd/src/mgs.rs
index 8cc773786c..da09ac5802 100644
--- a/wicketd/src/mgs.rs
+++ b/wicketd/src/mgs.rs
@@ -5,17 +5,16 @@
 //! The collection of tasks used for interacting with MGS and maintaining
 //! runtime state.
 
-use crate::{RackV1Inventory, SpInventory};
 use futures::StreamExt;
-use gateway_client::types::{SpIdentifier, SpIgnition};
-use schemars::JsonSchema;
-use serde::Serialize;
+use gateway_client::types::SpIgnition;
 use slog::{info, o, warn, Logger};
 use std::collections::{BTreeMap, BTreeSet};
 use std::net::SocketAddrV6;
 use tokio::sync::{mpsc, oneshot};
 use tokio::time::{Duration, Instant};
 use tokio_stream::StreamMap;
+use wicket_common::inventory::{RackV1Inventory, SpIdentifier, SpInventory};
+use wicketd_api::GetInventoryResponse;
 
 use self::inventory::{
     FetchedIgnitionState, FetchedSpData, IgnitionPresence,
@@ -52,15 +51,6 @@ pub struct MgsHandle {
     tx: tokio::sync::mpsc::Sender<MgsRequest>,
 }
 
-/// The response to a `get_inventory` call: the inventory known to wicketd, or a
-/// notification that data is unavailable.
-#[derive(Clone, Debug, JsonSchema, Serialize)]
-#[serde(rename_all = "snake_case", tag = "type", content = "data")]
-pub enum GetInventoryResponse {
-    Response { inventory: RackV1Inventory, mgs_last_seen: Duration },
-    Unavailable,
-}
-
 /// Channel errors result only from system shutdown.
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
 pub struct ShutdownInProgress;
diff --git a/wicketd/src/mgs/inventory.rs b/wicketd/src/mgs/inventory.rs
index a9805b4c1d..7d9aa586ad 100644
--- a/wicketd/src/mgs/inventory.rs
+++ b/wicketd/src/mgs/inventory.rs
@@ -7,7 +7,6 @@
 use gateway_client::types::RotState;
 use gateway_client::types::SpComponentCaboose;
 use gateway_client::types::SpComponentInfo;
-use gateway_client::types::SpIdentifier;
 use gateway_client::types::SpIgnition;
 use gateway_client::types::SpState;
 use gateway_messages::SpComponent;
@@ -21,8 +20,8 @@ use tokio::time::interval;
 use tokio::time::Duration;
 use tokio::time::Instant;
 use tokio_stream::wrappers::ReceiverStream;
-
-use crate::inventory::RotInventory;
+use wicket_common::inventory::RotInventory;
+use wicket_common::inventory::SpIdentifier;
 
 // Frequency at which we fetch state from our local ignition controller (via our
 // local sidecar SP) for the ignition state of all ignition targets in the rack.
diff --git a/wicketd/src/preflight_check.rs b/wicketd/src/preflight_check.rs
index 4cd17604a0..6863e41a84 100644
--- a/wicketd/src/preflight_check.rs
+++ b/wicketd/src/preflight_check.rs
@@ -9,19 +9,16 @@ use std::net::IpAddr;
 use std::sync::Arc;
 use std::sync::Mutex;
 use tokio::sync::oneshot;
-use update_engine::events::EventReport;
-use update_engine::GenericSpec;
+use wicket_common::preflight_check::EventBuffer;
+use wicket_common::preflight_check::EventReport;
 use wicket_common::rack_setup::UserSpecifiedRackNetworkConfig;
 
 mod uplink;
 
-pub(crate) type UplinkEventReport =
-    EventReport<GenericSpec<uplink::UplinkPreflightTerminalError>>;
-
 #[derive(Debug)]
 pub(crate) struct PreflightCheckerHandler {
     request_tx: flume::Sender<PreflightCheck>,
-    uplink_event_buffer: Arc<Mutex<Option<uplink::EventBuffer>>>,
+    uplink_event_buffer: Arc<Mutex<Option<EventBuffer>>>,
 }
 
 impl PreflightCheckerHandler {
@@ -78,12 +75,12 @@ impl PreflightCheckerHandler {
         Ok(())
     }
 
-    pub(crate) fn uplink_event_report(&self) -> Option<UplinkEventReport> {
+    pub(crate) fn uplink_event_report(&self) -> Option<EventReport> {
         self.uplink_event_buffer
             .lock()
             .unwrap()
             .as_ref()
-            .map(|event_buffer| event_buffer.generate_report().into_generic())
+            .map(|event_buffer| event_buffer.generate_report())
     }
 }
 
@@ -105,7 +102,7 @@ enum PreflightCheck {
 
 async fn preflight_task_main(
     request_rx: flume::Receiver<PreflightCheck>,
-    uplink_event_buffer: Arc<Mutex<Option<uplink::EventBuffer>>>,
+    uplink_event_buffer: Arc<Mutex<Option<EventBuffer>>>,
     log: Logger,
 ) {
     while let Ok(request) = request_rx.recv_async().await {
@@ -120,7 +117,7 @@ async fn preflight_task_main(
             } => {
                 // New preflight check: create a new event buffer.
                 *uplink_event_buffer.lock().unwrap() =
-                    Some(uplink::EventBuffer::new(16));
+                    Some(EventBuffer::new(16));
 
                 // We've cleared the shared event buffer; release our caller
                 // (they can now lock and check the event buffer while we run
diff --git a/wicketd/src/preflight_check/uplink.rs b/wicketd/src/preflight_check/uplink.rs
index 395fb8c795..36a4f61779 100644
--- a/wicketd/src/preflight_check/uplink.rs
+++ b/wicketd/src/preflight_check/uplink.rs
@@ -22,9 +22,6 @@ use omicron_common::api::internal::shared::PortSpeed as OmicronPortSpeed;
 use omicron_common::api::internal::shared::SwitchLocation;
 use omicron_common::OMICRON_DPD_TAG;
 use oxnet::IpNet;
-use schemars::JsonSchema;
-use serde::Deserialize;
-use serde::Serialize;
 use slog::error;
 use slog::o;
 use slog::Logger;
@@ -36,7 +33,6 @@ use std::sync::Arc;
 use std::sync::Mutex;
 use std::time::Duration;
 use std::time::Instant;
-use thiserror::Error;
 use tokio::process::Command;
 use tokio::sync::mpsc;
 use trust_dns_resolver::config::NameServerConfigGroup;
@@ -45,7 +41,16 @@ use trust_dns_resolver::config::ResolverOpts;
 use trust_dns_resolver::error::ResolveError;
 use trust_dns_resolver::error::ResolveErrorKind;
 use trust_dns_resolver::TokioAsyncResolver;
-use update_engine::StepSpec;
+use wicket_common::preflight_check::EventBuffer;
+use wicket_common::preflight_check::StepContext;
+use wicket_common::preflight_check::StepProgress;
+use wicket_common::preflight_check::StepResult;
+use wicket_common::preflight_check::StepSkipped;
+use wicket_common::preflight_check::StepSuccess;
+use wicket_common::preflight_check::StepWarning;
+use wicket_common::preflight_check::UpdateEngine;
+use wicket_common::preflight_check::UplinkPreflightStepId;
+use wicket_common::preflight_check::UplinkPreflightTerminalError;
 use wicket_common::rack_setup::UserSpecifiedPortConfig;
 use wicket_common::rack_setup::UserSpecifiedRackNetworkConfig;
 
@@ -865,73 +870,6 @@ struct RoutingSuccess {
     level2: L2Success,
 }
 
-#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize, JsonSchema)]
-#[serde(tag = "id", rename_all = "snake_case")]
-pub(super) enum UplinkPreflightStepId {
-    ConfigureSwitch,
-    WaitForL1Link,
-    ConfigureAddress,
-    ConfigureRouting,
-    CheckExternalDnsConnectivity,
-    CheckExternalNtpConnectivity,
-    CleanupRouting,
-    CleanupAddress,
-    CleanupL1,
-}
-
-type DpdError = dpd_client::Error<dpd_client::types::Error>;
-
-#[derive(Debug, Error)]
-pub(crate) enum UplinkPreflightTerminalError {
-    #[error("invalid port name: {0}")]
-    InvalidPortName(String),
-    #[error("failed to connect to dpd to check for current configuration")]
-    GetCurrentConfig(#[source] DpdError),
-    #[error("uplink already configured - is rack already initialized?")]
-    UplinkAlreadyConfigured,
-    #[error("failed to create port {port_id:?}")]
-    ConfigurePort {
-        #[source]
-        err: DpdError,
-        port_id: PortId,
-    },
-    #[error(
-        "failed to remove host OS route {destination} -> {nexthop}: {err}"
-    )]
-    RemoveHostRoute { err: String, destination: IpNet, nexthop: IpAddr },
-    #[error("failed to remove uplink SMF property {property:?}: {err}")]
-    RemoveSmfProperty { property: String, err: String },
-    #[error("failed to refresh uplink service config: {0}")]
-    RefreshUplinkSmf(String),
-    #[error("failed to clear settings for port {port_id:?}")]
-    UnconfigurePort {
-        #[source]
-        err: DpdError,
-        port_id: PortId,
-    },
-}
-
-impl update_engine::AsError for UplinkPreflightTerminalError {
-    fn as_error(&self) -> &(dyn std::error::Error + 'static) {
-        self
-    }
-}
-
-#[derive(JsonSchema)]
-pub(super) enum UplinkPreflightCheckSpec {}
-
-impl StepSpec for UplinkPreflightCheckSpec {
-    type Component = String;
-    type StepId = UplinkPreflightStepId;
-    type StepMetadata = ();
-    type ProgressMetadata = String;
-    type CompletionMetadata = Vec<String>;
-    type SkippedMetadata = ();
-    type Error = UplinkPreflightTerminalError;
-}
-
-update_engine::define_update_engine!(pub(super) UplinkPreflightCheckSpec);
-
 #[derive(Debug, Default)]
 struct DnsLookupStep {
     // Usable output of this step: IP addrs of the NTP servers to use.
diff --git a/wicketd/src/rss_config.rs b/wicketd/src/rss_config.rs
index dde6d35da5..b7545af19d 100644
--- a/wicketd/src/rss_config.rs
+++ b/wicketd/src/rss_config.rs
@@ -5,11 +5,6 @@
 //! Support for user-provided RSS configuration options.
 
 use crate::bootstrap_addrs::BootstrapPeers;
-use crate::http_entrypoints::CertificateUploadResponse;
-use crate::http_entrypoints::CurrentRssUserConfig;
-use crate::http_entrypoints::CurrentRssUserConfigSensitive;
-use crate::http_entrypoints::SetBgpAuthKeyStatus;
-use crate::RackV1Inventory;
 use anyhow::anyhow;
 use anyhow::bail;
 use anyhow::Context;
@@ -22,7 +17,6 @@ use bootstrap_agent_client::types::RackInitializeRequest;
 use bootstrap_agent_client::types::RecoverySiloConfig;
 use bootstrap_agent_client::types::UserId;
 use display_error_chain::DisplayErrorChain;
-use gateway_client::types::SpType;
 use omicron_certificates::CertificateError;
 use omicron_common::address;
 use omicron_common::address::Ipv4Range;
@@ -40,6 +34,8 @@ use std::mem;
 use std::net::IpAddr;
 use std::net::Ipv6Addr;
 use thiserror::Error;
+use wicket_common::inventory::RackV1Inventory;
+use wicket_common::inventory::SpType;
 use wicket_common::rack_setup::BgpAuthKey;
 use wicket_common::rack_setup::BgpAuthKeyId;
 use wicket_common::rack_setup::BgpAuthKeyStatus;
@@ -50,6 +46,10 @@ use wicket_common::rack_setup::GetBgpAuthKeyInfoResponse;
 use wicket_common::rack_setup::PutRssUserConfigInsensitive;
 use wicket_common::rack_setup::UserSpecifiedPortConfig;
 use wicket_common::rack_setup::UserSpecifiedRackNetworkConfig;
+use wicketd_api::CertificateUploadResponse;
+use wicketd_api::CurrentRssUserConfig;
+use wicketd_api::CurrentRssUserConfigSensitive;
+use wicketd_api::SetBgpAuthKeyStatus;
 
 // TODO-correctness For now, we always use the same rack subnet when running
 // RSS. When we get to multirack, this will be wrong, but there are many other
@@ -130,7 +130,7 @@ impl CurrentRssConfig {
                 let baseboard = Baseboard::new_gimlet(
                     state.serial_number.clone(),
                     state.model.clone(),
-                    state.revision.into(),
+                    state.revision,
                 );
                 let bootstrap_ip = bootstrap_sleds.get(&baseboard).copied();
                 Some(BootstrapSledDescription {
diff --git a/wicketd/src/update_tracker.rs b/wicketd/src/update_tracker.rs
index 6de7090ce4..dee22f70c0 100644
--- a/wicketd/src/update_tracker.rs
+++ b/wicketd/src/update_tracker.rs
@@ -6,10 +6,6 @@
 
 use crate::artifacts::WicketdArtifactStore;
 use crate::helpers::sps_to_string;
-use crate::http_entrypoints::ClearUpdateStateResponse;
-use crate::http_entrypoints::GetArtifactsAndEventReportsResponse;
-use crate::http_entrypoints::StartUpdateOptions;
-use crate::http_entrypoints::UpdateSimulatedResult;
 use crate::installinator_progress::IprStartReceiver;
 use crate::installinator_progress::IprUpdateTracker;
 use crate::mgs::make_mgs_client;
@@ -33,8 +29,6 @@ use gateway_client::types::RotCfpaSlot;
 use gateway_client::types::RotImageError;
 use gateway_client::types::RotState;
 use gateway_client::types::SpComponentFirmwareSlot;
-use gateway_client::types::SpIdentifier;
-use gateway_client::types::SpType;
 use gateway_client::types::SpUpdateStatus;
 use gateway_messages::SpComponent;
 use gateway_messages::ROT_PAGE_SIZE;
@@ -74,6 +68,11 @@ use update_engine::events::ProgressUnits;
 use update_engine::AbortHandle;
 use update_engine::StepSpec;
 use uuid::Uuid;
+use wicket_common::inventory::SpIdentifier;
+use wicket_common::inventory::SpType;
+use wicket_common::rack_update::ClearUpdateStateResponse;
+use wicket_common::rack_update::StartUpdateOptions;
+use wicket_common::rack_update::UpdateSimulatedResult;
 use wicket_common::update_events::ComponentRegistrar;
 use wicket_common::update_events::EventBuffer;
 use wicket_common::update_events::EventReport;
@@ -96,6 +95,7 @@ use wicket_common::update_events::UpdateComponent;
 use wicket_common::update_events::UpdateEngine;
 use wicket_common::update_events::UpdateStepId;
 use wicket_common::update_events::UpdateTerminalError;
+use wicketd_api::GetArtifactsAndEventReportsResponse;
 
 #[derive(Debug)]
 struct SpUpdateData {
diff --git a/wicketd/tests/integration_tests/commands.rs b/wicketd/tests/integration_tests/commands.rs
deleted file mode 100644
index f14d608879..0000000000
--- a/wicketd/tests/integration_tests/commands.rs
+++ /dev/null
@@ -1,43 +0,0 @@
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this
-// file, You can obtain one at https://mozilla.org/MPL/2.0/.
-
-//! Tests for the executable commands in this repo. Most functionality is tested
-//! elsewhere, so this really just sanity checks argument parsing, bad args, and
-//! the --openapi mode.
-
-use std::path::PathBuf;
-
-use expectorate::assert_contents;
-use omicron_test_utils::dev::test_cmds::{
-    assert_exit_code, path_to_executable, run_command, EXIT_SUCCESS,
-};
-use openapiv3::OpenAPI;
-use subprocess::Exec;
-
-// name of wicketd executable
-const CMD_WICKETD: &str = env!("CARGO_BIN_EXE_wicketd");
-
-fn path_to_wicketd() -> PathBuf {
-    path_to_executable(CMD_WICKETD)
-}
-
-#[test]
-fn test_wicketd_openapi() {
-    let exec = Exec::cmd(path_to_wicketd()).arg("openapi");
-    let (exit_status, stdout_text, stderr_text) = run_command(exec);
-    assert_exit_code(exit_status, EXIT_SUCCESS, &stderr_text);
-    assert_contents("tests/output/cmd-wicketd-openapi-stderr", &stderr_text);
-
-    let spec: OpenAPI = serde_json::from_str(&stdout_text)
-        .expect("stdout was not valid OpenAPI");
-
-    // Check for lint errors.
-    let errors = openapi_lint::validate(&spec);
-    assert!(errors.is_empty(), "{}", errors.join("\n\n"));
-
-    // Confirm that the output hasn't changed. It's expected that we'll change
-    // this file as the API evolves, but pay attention to the diffs to ensure
-    // that the changes match your expectations.
-    assert_contents("../openapi/wicketd.json", &stdout_text);
-}
diff --git a/wicketd/tests/integration_tests/mod.rs b/wicketd/tests/integration_tests/mod.rs
index c73f49805c..656dd79818 100644
--- a/wicketd/tests/integration_tests/mod.rs
+++ b/wicketd/tests/integration_tests/mod.rs
@@ -2,7 +2,6 @@
 // License, v. 2.0. If a copy of the MPL was not distributed with this
 // file, You can obtain one at https://mozilla.org/MPL/2.0/.
 
-mod commands;
 mod inventory;
 mod setup;
 mod updates;
diff --git a/wicketd/tests/integration_tests/setup.rs b/wicketd/tests/integration_tests/setup.rs
index 62682a73ab..01f01e21e1 100644
--- a/wicketd/tests/integration_tests/setup.rs
+++ b/wicketd/tests/integration_tests/setup.rs
@@ -16,7 +16,7 @@ pub struct WicketdTestContext {
     // this way.
     pub wicketd_raw_client: ClientTestContext,
     pub artifact_addr: SocketAddrV6,
-    pub artifact_client: installinator_artifact_client::Client,
+    pub artifact_client: installinator_client::Client,
     pub server: wicketd::Server,
     pub gateway: GatewayTestContext,
 }
@@ -62,14 +62,15 @@ impl WicketdTestContext {
             )
         };
 
-        let artifact_addr = assert_ipv6(server.artifact_server.local_addr());
+        let artifact_addr =
+            assert_ipv6(server.installinator_server.local_addr());
         let artifact_client = {
             let endpoint = format!(
                 "http://[{}]:{}",
                 artifact_addr.ip(),
                 artifact_addr.port()
             );
-            installinator_artifact_client::Client::new(
+            installinator_client::Client::new(
                 &endpoint,
                 log.new(slog::o!("component" => "artifact test client")),
             )
diff --git a/wicketd/tests/integration_tests/updates.rs b/wicketd/tests/integration_tests/updates.rs
index 611d81c7f5..af3bbfe656 100644
--- a/wicketd/tests/integration_tests/updates.rs
+++ b/wicketd/tests/integration_tests/updates.rs
@@ -22,13 +22,13 @@ use update_engine::NestedError;
 use uuid::Uuid;
 use wicket::OutputKind;
 use wicket_common::{
-    rack_update::{ClearUpdateStateResponse, SpIdentifier, SpType},
+    inventory::{SpIdentifier, SpType},
+    rack_update::{ClearUpdateStateResponse, StartUpdateOptions},
     update_events::{StepEventKind, UpdateComponent},
 };
 use wicketd::{RunningUpdateState, StartUpdateError};
 use wicketd_client::types::{
-    GetInventoryParams, GetInventoryResponse, StartUpdateOptions,
-    StartUpdateParams,
+    GetInventoryParams, GetInventoryResponse, StartUpdateParams,
 };
 
 // See documentation for extract_nested_artifact_pair in update_plan.rs for why
@@ -430,10 +430,7 @@ async fn test_update_races() {
         .expect("bytes read and archived");
 
     // Now start an update.
-    let sp = gateway_client::types::SpIdentifier {
-        slot: 0,
-        type_: gateway_client::types::SpType::Sled,
-    };
+    let sp = SpIdentifier { slot: 0, type_: SpType::Sled };
     let sps: BTreeSet<_> = vec![sp].into_iter().collect();
 
     let (sender, receiver) = oneshot::channel();
diff --git a/wicketd/tests/output/cmd-wicketd-openapi-stderr b/wicketd/tests/output/cmd-wicketd-openapi-stderr
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/workspace-hack/Cargo.toml b/workspace-hack/Cargo.toml
index 796cf0bf63..498f25d017 100644
--- a/workspace-hack/Cargo.toml
+++ b/workspace-hack/Cargo.toml
@@ -31,8 +31,8 @@ byteorder = { version = "1.5.0" }
 bytes = { version = "1.6.0", features = ["serde"] }
 chrono = { version = "0.4.38", features = ["serde"] }
 cipher = { version = "0.4.4", default-features = false, features = ["block-padding", "zeroize"] }
-clap = { version = "4.5.4", features = ["cargo", "derive", "env", "wrap_help"] }
-clap_builder = { version = "4.5.2", default-features = false, features = ["cargo", "color", "env", "std", "suggestions", "usage", "wrap_help"] }
+clap = { version = "4.5.9", features = ["cargo", "derive", "env", "wrap_help"] }
+clap_builder = { version = "4.5.9", default-features = false, features = ["cargo", "color", "env", "std", "suggestions", "usage", "wrap_help"] }
 console = { version = "0.15.8" }
 const-oid = { version = "0.9.6", default-features = false, features = ["db", "std"] }
 crossbeam-epoch = { version = "0.9.18" }
@@ -99,24 +99,25 @@ sha2 = { version = "0.10.8", features = ["oid"] }
 similar = { version = "2.5.0", features = ["bytes", "inline", "unicode"] }
 slog = { version = "2.7.0", features = ["dynamic-keys", "max_level_trace", "release_max_level_debug", "release_max_level_trace"] }
 smallvec = { version = "1.13.2", default-features = false, features = ["const_new"] }
+socket2 = { version = "0.5.7", default-features = false, features = ["all"] }
 spin = { version = "0.9.8" }
 string_cache = { version = "0.8.7" }
 subtle = { version = "2.5.0" }
-syn-f595c2ba2a3f28df = { package = "syn", version = "2.0.68", features = ["extra-traits", "fold", "full", "visit", "visit-mut"] }
+syn-f595c2ba2a3f28df = { package = "syn", version = "2.0.71", features = ["extra-traits", "fold", "full", "visit", "visit-mut"] }
 time = { version = "0.3.36", features = ["formatting", "local-offset", "macros", "parsing"] }
-tokio = { version = "1.38.0", features = ["full", "test-util"] }
+tokio = { version = "1.38.1", features = ["full", "test-util"] }
 tokio-postgres = { version = "0.7.10", features = ["with-chrono-0_4", "with-serde_json-1", "with-uuid-1"] }
 tokio-stream = { version = "0.1.15", features = ["net"] }
 tokio-util = { version = "0.7.11", features = ["codec", "io-util"] }
 toml = { version = "0.7.8" }
-toml_edit-3c51e837cfc5589a = { package = "toml_edit", version = "0.22.14", features = ["serde"] }
+toml_edit-3c51e837cfc5589a = { package = "toml_edit", version = "0.22.16", features = ["serde"] }
 tracing = { version = "0.1.40", features = ["log"] }
 trust-dns-proto = { version = "0.22.0" }
 unicode-bidi = { version = "0.3.15" }
 unicode-normalization = { version = "0.1.23" }
 usdt = { version = "0.5.0" }
 usdt-impl = { version = "0.5.0", default-features = false, features = ["asm", "des"] }
-uuid = { version = "1.9.1", features = ["serde", "v4"] }
+uuid = { version = "1.10.0", features = ["serde", "v4"] }
 yasna = { version = "0.5.2", features = ["bit-vec", "num-bigint", "std", "time"] }
 zerocopy = { version = "0.7.34", features = ["derive", "simd"] }
 zeroize = { version = "1.7.0", features = ["std", "zeroize_derive"] }
@@ -136,8 +137,8 @@ byteorder = { version = "1.5.0" }
 bytes = { version = "1.6.0", features = ["serde"] }
 chrono = { version = "0.4.38", features = ["serde"] }
 cipher = { version = "0.4.4", default-features = false, features = ["block-padding", "zeroize"] }
-clap = { version = "4.5.4", features = ["cargo", "derive", "env", "wrap_help"] }
-clap_builder = { version = "4.5.2", default-features = false, features = ["cargo", "color", "env", "std", "suggestions", "usage", "wrap_help"] }
+clap = { version = "4.5.9", features = ["cargo", "derive", "env", "wrap_help"] }
+clap_builder = { version = "4.5.9", default-features = false, features = ["cargo", "color", "env", "std", "suggestions", "usage", "wrap_help"] }
 console = { version = "0.15.8" }
 const-oid = { version = "0.9.6", default-features = false, features = ["db", "std"] }
 crossbeam-epoch = { version = "0.9.18" }
@@ -204,19 +205,20 @@ sha2 = { version = "0.10.8", features = ["oid"] }
 similar = { version = "2.5.0", features = ["bytes", "inline", "unicode"] }
 slog = { version = "2.7.0", features = ["dynamic-keys", "max_level_trace", "release_max_level_debug", "release_max_level_trace"] }
 smallvec = { version = "1.13.2", default-features = false, features = ["const_new"] }
+socket2 = { version = "0.5.7", default-features = false, features = ["all"] }
 spin = { version = "0.9.8" }
 string_cache = { version = "0.8.7" }
 subtle = { version = "2.5.0" }
 syn-dff4ba8e3ae991db = { package = "syn", version = "1.0.109", features = ["extra-traits", "fold", "full", "visit"] }
-syn-f595c2ba2a3f28df = { package = "syn", version = "2.0.68", features = ["extra-traits", "fold", "full", "visit", "visit-mut"] }
+syn-f595c2ba2a3f28df = { package = "syn", version = "2.0.71", features = ["extra-traits", "fold", "full", "visit", "visit-mut"] }
 time = { version = "0.3.36", features = ["formatting", "local-offset", "macros", "parsing"] }
 time-macros = { version = "0.2.18", default-features = false, features = ["formatting", "parsing"] }
-tokio = { version = "1.38.0", features = ["full", "test-util"] }
+tokio = { version = "1.38.1", features = ["full", "test-util"] }
 tokio-postgres = { version = "0.7.10", features = ["with-chrono-0_4", "with-serde_json-1", "with-uuid-1"] }
 tokio-stream = { version = "0.1.15", features = ["net"] }
 tokio-util = { version = "0.7.11", features = ["codec", "io-util"] }
 toml = { version = "0.7.8" }
-toml_edit-3c51e837cfc5589a = { package = "toml_edit", version = "0.22.14", features = ["serde"] }
+toml_edit-3c51e837cfc5589a = { package = "toml_edit", version = "0.22.16", features = ["serde"] }
 tracing = { version = "0.1.40", features = ["log"] }
 trust-dns-proto = { version = "0.22.0" }
 unicode-bidi = { version = "0.3.15" }
@@ -224,7 +226,7 @@ unicode-normalization = { version = "0.1.23" }
 unicode-xid = { version = "0.2.4" }
 usdt = { version = "0.5.0" }
 usdt-impl = { version = "0.5.0", default-features = false, features = ["asm", "des"] }
-uuid = { version = "1.9.1", features = ["serde", "v4"] }
+uuid = { version = "1.10.0", features = ["serde", "v4"] }
 yasna = { version = "0.5.2", features = ["bit-vec", "num-bigint", "std", "time"] }
 zerocopy = { version = "0.7.34", features = ["derive", "simd"] }
 zeroize = { version = "1.7.0", features = ["std", "zeroize_derive"] }