From 1efad9b483b8949edeb89f9c542552128ec4fb7d Mon Sep 17 00:00:00 2001
From: Salah Al Saleh <sg.alsaleh@gmail.com>
Date: Thu, 31 Oct 2024 17:18:15 -0700
Subject: [PATCH] Implement EC install CLI command integration test (#1420)

* Implement EC install CLI command integration test
---
 .github/workflows/ci.yaml                     |  15 ++
 Makefile                                      |  14 +
 cmd/embedded-cluster/main.go                  |  23 +-
 e2e/cluster/docker/cluster.go                 |   3 +
 .../embeddedclusteroperator.go                |  33 +--
 .../cmd}/admin_console.go                     |   2 +-
 pkg/cmd/app.go                                |  28 ++
 .../cmd}/assets/resource-modifiers.yaml       |   0
 {cmd/embedded-cluster => pkg/cmd}/flags.go    |   2 +-
 {cmd/embedded-cluster => pkg/cmd}/install.go  |  58 +++--
 .../cmd}/install_test.go                      |   2 +-
 {cmd/embedded-cluster => pkg/cmd}/join.go     |  11 +-
 .../embedded-cluster => pkg/cmd}/join_test.go |   2 +-
 {cmd/embedded-cluster => pkg/cmd}/k0s.go      |   2 +-
 .../cmd}/list_images.go                       |   2 +-
 .../cmd}/materialize.go                       |   2 +-
 {cmd/embedded-cluster => pkg/cmd}/metadata.go |   2 +-
 {cmd/embedded-cluster => pkg/cmd}/network.go  |   2 +-
 .../cmd}/network_test.go                      |   2 +-
 {cmd/embedded-cluster => pkg/cmd}/node.go     |   2 +-
 .../cmd}/preflights.go                        |   2 +-
 {cmd/embedded-cluster => pkg/cmd}/provider.go |   2 +-
 {cmd/embedded-cluster => pkg/cmd}/proxy.go    |   2 +-
 .../cmd}/proxy_test.go                        |   2 +-
 {cmd/embedded-cluster => pkg/cmd}/reset.go    |  25 +-
 {cmd/embedded-cluster => pkg/cmd}/restore.go  |   9 +-
 {cmd/embedded-cluster => pkg/cmd}/shell.go    |   2 +-
 .../cmd}/support_bundle.go                    |   6 +-
 ...join-command-response-empty-overrides.yaml |   0
 ...mand-response-multiple-port-overrides.yaml |   0
 ...and-response-multiple-unused-overlays.yaml |   0
 .../join-command-response-no-overrides.yaml   |   0
 .../join-command-response-override-ports.yaml |   0
 ...d-response-storage-and-port-overrides.yaml |   0
 ...ch-k0s-config-change-external-address.yaml |   0
 .../patch-k0s-config-change-sans.yaml         |   0
 .../testdata/patch-k0s-config-extra-args.yaml |   0
 .../patch-k0s-config-k0s-storage.yaml         |   0
 ...k0s-config-no-embedded-cluster-config.yaml |   0
 .../patch-k0s-config-no-overrides.yaml        |   0
 {cmd/embedded-cluster => pkg/cmd}/update.go   |   2 +-
 {cmd/embedded-cluster => pkg/cmd}/util.go     |  21 +-
 {cmd/embedded-cluster => pkg/cmd}/version.go  |   2 +-
 pkg/dryrun/dryrun.go                          | 101 ++++++++
 pkg/dryrun/helpers.go                         |  29 +++
 pkg/dryrun/kubeutils.go                       |  81 ++++++
 pkg/dryrun/metrics.go                         |  23 ++
 pkg/dryrun/types/types.go                     | 224 ++++++++++++++++
 pkg/helm/values.go                            |  12 +
 pkg/helm/values_test.go                       | 108 ++++++++
 pkg/helpers/command.go                        |  27 +-
 pkg/helpers/interface.go                      |  54 ++++
 pkg/helpers/systemd.go                        |   2 +-
 pkg/kotscli/kotscli.go                        |   4 +-
 pkg/kubeutils/client.go                       |  24 --
 pkg/kubeutils/interface.go                    | 108 ++++++++
 pkg/kubeutils/kubeutils.go                    |  81 ++++--
 pkg/metrics/interface.go                      |  41 +++
 pkg/metrics/reporter.go                       |  30 ++-
 pkg/metrics/sender.go                         |  39 +--
 pkg/metrics/sender_test.go                    |  18 +-
 pkg/metrics/{events.go => types/types.go}     |   2 +-
 pkg/metrics/util.go                           |  24 ++
 scripts/dryrun-tests.sh                       |  56 ++++
 tests/dryrun/Dockerfile                       |   5 +
 tests/dryrun/assets/install-license.yaml      |  36 +++
 tests/dryrun/assets/install-release.yaml      |   5 +
 tests/dryrun/install_test.go                  | 242 ++++++++++++++++++
 tests/dryrun/util.go                          | 178 +++++++++++++
 69 files changed, 1594 insertions(+), 242 deletions(-)
 rename {cmd/embedded-cluster => pkg/cmd}/admin_console.go (99%)
 create mode 100644 pkg/cmd/app.go
 rename {cmd/embedded-cluster => pkg/cmd}/assets/resource-modifiers.yaml (100%)
 rename {cmd/embedded-cluster => pkg/cmd}/flags.go (99%)
 rename {cmd/embedded-cluster => pkg/cmd}/install.go (96%)
 rename {cmd/embedded-cluster => pkg/cmd}/install_test.go (99%)
 rename {cmd/embedded-cluster => pkg/cmd}/join.go (98%)
 rename {cmd/embedded-cluster => pkg/cmd}/join_test.go (99%)
 rename {cmd/embedded-cluster => pkg/cmd}/k0s.go (98%)
 rename {cmd/embedded-cluster => pkg/cmd}/list_images.go (98%)
 rename {cmd/embedded-cluster => pkg/cmd}/materialize.go (98%)
 rename {cmd/embedded-cluster => pkg/cmd}/metadata.go (99%)
 rename {cmd/embedded-cluster => pkg/cmd}/network.go (99%)
 rename {cmd/embedded-cluster => pkg/cmd}/network_test.go (99%)
 rename {cmd/embedded-cluster => pkg/cmd}/node.go (95%)
 rename {cmd/embedded-cluster => pkg/cmd}/preflights.go (99%)
 rename {cmd/embedded-cluster => pkg/cmd}/provider.go (99%)
 rename {cmd/embedded-cluster => pkg/cmd}/proxy.go (99%)
 rename {cmd/embedded-cluster => pkg/cmd}/proxy_test.go (99%)
 rename {cmd/embedded-cluster => pkg/cmd}/reset.go (95%)
 rename {cmd/embedded-cluster => pkg/cmd}/restore.go (99%)
 rename {cmd/embedded-cluster => pkg/cmd}/shell.go (99%)
 rename {cmd/embedded-cluster => pkg/cmd}/support_bundle.go (97%)
 rename {cmd/embedded-cluster => pkg/cmd}/testdata/join-command-response-empty-overrides.yaml (100%)
 rename {cmd/embedded-cluster => pkg/cmd}/testdata/join-command-response-multiple-port-overrides.yaml (100%)
 rename {cmd/embedded-cluster => pkg/cmd}/testdata/join-command-response-multiple-unused-overlays.yaml (100%)
 rename {cmd/embedded-cluster => pkg/cmd}/testdata/join-command-response-no-overrides.yaml (100%)
 rename {cmd/embedded-cluster => pkg/cmd}/testdata/join-command-response-override-ports.yaml (100%)
 rename {cmd/embedded-cluster => pkg/cmd}/testdata/join-command-response-storage-and-port-overrides.yaml (100%)
 rename {cmd/embedded-cluster => pkg/cmd}/testdata/patch-k0s-config-change-external-address.yaml (100%)
 rename {cmd/embedded-cluster => pkg/cmd}/testdata/patch-k0s-config-change-sans.yaml (100%)
 rename {cmd/embedded-cluster => pkg/cmd}/testdata/patch-k0s-config-extra-args.yaml (100%)
 rename {cmd/embedded-cluster => pkg/cmd}/testdata/patch-k0s-config-k0s-storage.yaml (100%)
 rename {cmd/embedded-cluster => pkg/cmd}/testdata/patch-k0s-config-no-embedded-cluster-config.yaml (100%)
 rename {cmd/embedded-cluster => pkg/cmd}/testdata/patch-k0s-config-no-overrides.yaml (100%)
 rename {cmd/embedded-cluster => pkg/cmd}/update.go (99%)
 rename {cmd/embedded-cluster => pkg/cmd}/util.go (83%)
 rename {cmd/embedded-cluster => pkg/cmd}/version.go (99%)
 create mode 100644 pkg/dryrun/dryrun.go
 create mode 100644 pkg/dryrun/helpers.go
 create mode 100644 pkg/dryrun/kubeutils.go
 create mode 100644 pkg/dryrun/metrics.go
 create mode 100644 pkg/dryrun/types/types.go
 create mode 100644 pkg/helpers/interface.go
 delete mode 100644 pkg/kubeutils/client.go
 create mode 100644 pkg/kubeutils/interface.go
 create mode 100644 pkg/metrics/interface.go
 rename pkg/metrics/{events.go => types/types.go} (99%)
 create mode 100644 pkg/metrics/util.go
 create mode 100755 scripts/dryrun-tests.sh
 create mode 100644 tests/dryrun/Dockerfile
 create mode 100644 tests/dryrun/assets/install-license.yaml
 create mode 100644 tests/dryrun/assets/install-release.yaml
 create mode 100644 tests/dryrun/install_test.go
 create mode 100644 tests/dryrun/util.go

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 3f768046d..0b0921006 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -59,6 +59,21 @@ jobs:
         run: |
           make unit-tests
 
+  dryrun-tests:
+    name: Dryrun tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Setup go
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+          cache-dependency-path: "**/*.sum"
+      - name: Dryrun tests
+        run: |
+          make dryrun-tests
+
   check-operator-crds:
     name: Check operator CRDs
     runs-on: ubuntu-latest
diff --git a/Makefile b/Makefile
index 6271adbd5..ac62776e5 100644
--- a/Makefile
+++ b/Makefile
@@ -202,6 +202,16 @@ static: pkg/goods/bins/k0s \
 	pkg/goods/bins/fio \
 	pkg/goods/internal/bins/kubectl-kots
 
+.PHONY: static-dryrun
+static-dryrun:
+	@mkdir -p pkg/goods/bins pkg/goods/internal/bins
+	@touch pkg/goods/bins/k0s \
+		pkg/goods/bins/kubectl-preflight \
+		pkg/goods/bins/kubectl-support_bundle \
+		pkg/goods/bins/local-artifact-mirror \
+		pkg/goods/bins/fio \
+		pkg/goods/internal/bins/kubectl-kots
+
 .PHONY: embedded-cluster-linux-amd64
 embedded-cluster-linux-amd64: export OS = linux
 embedded-cluster-linux-amd64: export ARCH = amd64
@@ -250,6 +260,10 @@ e2e-tests: embedded-release
 e2e-test:
 	go test -timeout 60m -ldflags="$(LD_FLAGS)" -v ./e2e -run ^$(TEST_NAME)$$
 
+.PHONY: dryrun-tests
+dryrun-tests: static-dryrun
+	@./scripts/dryrun-tests.sh
+
 .PHONY: build-ttl.sh
 build-ttl.sh:
 	$(MAKE) -C local-artifact-mirror build-ttl.sh \
diff --git a/cmd/embedded-cluster/main.go b/cmd/embedded-cluster/main.go
index bf6038305..f6e70a11f 100644
--- a/cmd/embedded-cluster/main.go
+++ b/cmd/embedded-cluster/main.go
@@ -2,15 +2,14 @@ package main
 
 import (
 	"context"
-	"fmt"
 	"os"
 	"os/signal"
 	"path"
 	"syscall"
 
 	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli/v2"
 
+	"github.com/replicatedhq/embedded-cluster/pkg/cmd"
 	"github.com/replicatedhq/embedded-cluster/pkg/logging"
 )
 
@@ -22,25 +21,9 @@ func main() {
 	)
 	defer cancel()
 	logging.SetupLogging()
+
 	name := path.Base(os.Args[0])
-	var app = &cli.App{
-		Name:    name,
-		Usage:   fmt.Sprintf("Install and manage %s", name),
-		Suggest: true,
-		Commands: []*cli.Command{
-			installCommand(),
-			shellCommand(),
-			nodeCommands,
-			versionCommand,
-			joinCommand,
-			resetCommand(),
-			materializeCommand(),
-			updateCommand(),
-			restoreCommand(),
-			adminConsoleCommand(),
-			supportBundleCommand(),
-		},
-	}
+	app := cmd.NewApp(name)
 	if err := app.RunContext(ctx, os.Args); err != nil {
 		logrus.Fatal(err)
 	}
diff --git a/e2e/cluster/docker/cluster.go b/e2e/cluster/docker/cluster.go
index 08c48bac7..8b8251eb1 100644
--- a/e2e/cluster/docker/cluster.go
+++ b/e2e/cluster/docker/cluster.go
@@ -84,7 +84,10 @@ func (c *Cluster) WaitForReady() {
 func (c *Cluster) Cleanup(envs ...map[string]string) {
 	c.generateSupportBundle(envs...)
 	c.copyPlaywrightReport()
+	c.Destroy()
+}
 
+func (c *Cluster) Destroy() {
 	for _, node := range c.Nodes {
 		node.Destroy()
 	}
diff --git a/pkg/addons/embeddedclusteroperator/embeddedclusteroperator.go b/pkg/addons/embeddedclusteroperator/embeddedclusteroperator.go
index 3a93d94f9..3a59a82ca 100644
--- a/pkg/addons/embeddedclusteroperator/embeddedclusteroperator.go
+++ b/pkg/addons/embeddedclusteroperator/embeddedclusteroperator.go
@@ -6,7 +6,6 @@ import (
 	"context"
 	_ "embed"
 	"encoding/json"
-	"errors"
 	"fmt"
 	"strings"
 	"time"
@@ -167,6 +166,10 @@ func (e *EmbeddedClusterOperator) createVersionMetadataConfigmap(ctx context.Con
 	// the result as a suffix for the config map name.
 	slugver := slug.Make(strings.TrimPrefix(versions.Version, "v"))
 	configmap := &corev1.ConfigMap{
+		TypeMeta: metav1.TypeMeta{
+			APIVersion: "v1",
+			Kind:       "ConfigMap",
+		},
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      fmt.Sprintf("version-metadata-%s", slugver),
 			Namespace: e.namespace,
@@ -256,6 +259,10 @@ func (e *EmbeddedClusterOperator) Outro(ctx context.Context, provider *defaults.
 	}
 
 	installation := ecv1beta1.Installation{
+		TypeMeta: metav1.TypeMeta{
+			APIVersion: ecv1beta1.GroupVersion.String(),
+			Kind:       "Installation",
+		},
 		ObjectMeta: metav1.ObjectMeta{
 			Name: time.Now().Format("20060102150405"),
 			Labels: map[string]string{
@@ -282,13 +289,8 @@ func (e *EmbeddedClusterOperator) Outro(ctx context.Context, provider *defaults.
 	}
 
 	// we wait for the installation to exist here because items do not show up in the apiserver instantaneously after being created
-	gotInstallation, err := waitForInstallationToExist(ctx, cli, installation.Name)
-	if err != nil {
-		return fmt.Errorf("unable to wait for installation to exist: %w", err)
-	}
-	gotInstallation.Status.State = ecv1beta1.InstallationStateKubernetesInstalled
-	if err := cli.Status().Update(ctx, gotInstallation); err != nil {
-		return fmt.Errorf("unable to update installation status: %w", err)
+	if err := kubeutils.WaitAndMarkInstallation(ctx, cli, installation.Name, ecv1beta1.InstallationStateKubernetesInstalled); err != nil {
+		return fmt.Errorf("unable to wait and mark installation: %w", err)
 	}
 
 	return nil
@@ -343,18 +345,3 @@ func k0sConfigToNetworkSpec(k0sCfg *k0sv1beta1.ClusterConfig) *ecv1beta1.Network
 
 	return network
 }
-
-func waitForInstallationToExist(ctx context.Context, cli client.Client, name string) (*ecv1beta1.Installation, error) {
-	for i := 0; i < 20; i++ {
-		in, err := kubeutils.GetInstallation(ctx, cli, name)
-		if err != nil {
-			if !errors.Is(err, kubeutils.ErrNoInstallations{}) {
-				return nil, fmt.Errorf("unable to get installation: %w", err)
-			}
-		} else {
-			return in, nil
-		}
-		time.Sleep(time.Second)
-	}
-	return nil, fmt.Errorf("installation %s not found after 20 seconds", name)
-}
diff --git a/cmd/embedded-cluster/admin_console.go b/pkg/cmd/admin_console.go
similarity index 99%
rename from cmd/embedded-cluster/admin_console.go
rename to pkg/cmd/admin_console.go
index 81c5548b1..0592caf8e 100644
--- a/cmd/embedded-cluster/admin_console.go
+++ b/pkg/cmd/admin_console.go
@@ -1,4 +1,4 @@
-package main
+package cmd
 
 import (
 	"fmt"
diff --git a/pkg/cmd/app.go b/pkg/cmd/app.go
new file mode 100644
index 000000000..498301ca0
--- /dev/null
+++ b/pkg/cmd/app.go
@@ -0,0 +1,28 @@
+package cmd
+
+import (
+	"fmt"
+
+	"github.com/urfave/cli/v2"
+)
+
+func NewApp(name string) *cli.App {
+	return &cli.App{
+		Name:    name,
+		Usage:   fmt.Sprintf("Install and manage %s", name),
+		Suggest: true,
+		Commands: []*cli.Command{
+			installCommand(),
+			shellCommand(),
+			nodeCommands,
+			versionCommand,
+			joinCommand,
+			resetCommand(),
+			materializeCommand(),
+			updateCommand(),
+			restoreCommand(),
+			adminConsoleCommand(),
+			supportBundleCommand(),
+		},
+	}
+}
diff --git a/cmd/embedded-cluster/assets/resource-modifiers.yaml b/pkg/cmd/assets/resource-modifiers.yaml
similarity index 100%
rename from cmd/embedded-cluster/assets/resource-modifiers.yaml
rename to pkg/cmd/assets/resource-modifiers.yaml
diff --git a/cmd/embedded-cluster/flags.go b/pkg/cmd/flags.go
similarity index 99%
rename from cmd/embedded-cluster/flags.go
rename to pkg/cmd/flags.go
index 481b48461..6f9d19c32 100644
--- a/cmd/embedded-cluster/flags.go
+++ b/pkg/cmd/flags.go
@@ -1,4 +1,4 @@
-package main
+package cmd
 
 import (
 	"fmt"
diff --git a/cmd/embedded-cluster/install.go b/pkg/cmd/install.go
similarity index 96%
rename from cmd/embedded-cluster/install.go
rename to pkg/cmd/install.go
index 1694482e9..5e44084bd 100644
--- a/cmd/embedded-cluster/install.go
+++ b/pkg/cmd/install.go
@@ -1,4 +1,4 @@
-package main
+package cmd
 
 import (
 	"fmt"
@@ -20,6 +20,7 @@ import (
 	"github.com/replicatedhq/embedded-cluster/pkg/config"
 	"github.com/replicatedhq/embedded-cluster/pkg/configutils"
 	"github.com/replicatedhq/embedded-cluster/pkg/defaults"
+	"github.com/replicatedhq/embedded-cluster/pkg/dryrun"
 	"github.com/replicatedhq/embedded-cluster/pkg/goods"
 	"github.com/replicatedhq/embedded-cluster/pkg/helpers"
 	"github.com/replicatedhq/embedded-cluster/pkg/metrics"
@@ -188,6 +189,11 @@ func RunHostPreflights(c *cli.Context, provider *defaults.Provider, applier *add
 		hpf.Analyzers = append(hpf.Analyzers, h.Spec.Analyzers...)
 	}
 
+	if dryrun.Enabled() {
+		dryrun.RecordHostPreflightSpec(hpf)
+		return nil
+	}
+
 	return runHostPreflights(c, provider, hpf, proxy)
 }
 
@@ -454,15 +460,9 @@ func ensureK0sConfig(c *cli.Context, provider *defaults.Provider, applier *addon
 	if err != nil {
 		return nil, fmt.Errorf("unable to marshal config: %w", err)
 	}
-	fp, err := os.OpenFile(cfgpath, os.O_RDWR|os.O_CREATE, 0600)
-	if err != nil {
-		return nil, fmt.Errorf("unable to create config file: %w", err)
-	}
-	defer fp.Close()
-	if _, err := fp.Write(data); err != nil {
+	if err := os.WriteFile(cfgpath, data, 0600); err != nil {
 		return nil, fmt.Errorf("unable to write config file: %w", err)
 	}
-
 	return cfg, nil
 }
 
@@ -529,18 +529,20 @@ func installK0s(c *cli.Context, provider *defaults.Provider) error {
 // waitForK0s waits for the k0s API to be available. We wait for the k0s socket to
 // appear in the system and until the k0s status command to finish.
 func waitForK0s() error {
-	var success bool
-	for i := 0; i < 30; i++ {
-		time.Sleep(2 * time.Second)
-		spath := defaults.PathToK0sStatusSocket()
-		if _, err := os.Stat(spath); err != nil {
-			continue
+	if !dryrun.Enabled() {
+		var success bool
+		for i := 0; i < 30; i++ {
+			time.Sleep(2 * time.Second)
+			spath := defaults.PathToK0sStatusSocket()
+			if _, err := os.Stat(spath); err != nil {
+				continue
+			}
+			success = true
+			break
+		}
+		if !success {
+			return fmt.Errorf("timeout waiting for %s", defaults.BinaryName())
 		}
-		success = true
-		break
-	}
-	if !success {
-		return fmt.Errorf("timeout waiting for %s", defaults.BinaryName())
 	}
 
 	for i := 1; ; i++ {
@@ -678,6 +680,18 @@ func installCommand() *cli.Command {
 			if c.String("airgap-bundle") != "" {
 				metrics.DisableMetrics()
 			}
+			if drFile := c.String("dry-run"); drFile != "" {
+				dryrun.Init(drFile)
+				dryrun.RecordFlags(c)
+			}
+			return nil
+		},
+		After: func(c *cli.Context) error {
+			if c.String("dry-run") != "" {
+				if err := dryrun.Dump(); err != nil {
+					return fmt.Errorf("unable to dump dry run info: %w", err)
+				}
+			}
 			return nil
 		},
 		Flags: withProxyFlags(withSubnetCIDRFlags(
@@ -693,6 +707,12 @@ func installCommand() *cli.Command {
 					Usage: "Path to the air gap bundle. If set, the installation will complete without internet access.",
 				},
 				getDataDirFlagWithDefault(runtimeConfig),
+				&cli.StringFlag{
+					Name:   "dry-run",
+					Usage:  "If set, dry run the installation and output the results to the provided file",
+					Value:  "",
+					Hidden: true,
+				},
 				&cli.StringFlag{
 					Name:    "license",
 					Aliases: []string{"l"},
diff --git a/cmd/embedded-cluster/install_test.go b/pkg/cmd/install_test.go
similarity index 99%
rename from cmd/embedded-cluster/install_test.go
rename to pkg/cmd/install_test.go
index a58bfd7a7..aa1e422dd 100644
--- a/cmd/embedded-cluster/install_test.go
+++ b/pkg/cmd/install_test.go
@@ -1,4 +1,4 @@
-package main
+package cmd
 
 import (
 	"flag"
diff --git a/cmd/embedded-cluster/join.go b/pkg/cmd/join.go
similarity index 98%
rename from cmd/embedded-cluster/join.go
rename to pkg/cmd/join.go
index 52325d6ec..80a26490f 100644
--- a/cmd/embedded-cluster/join.go
+++ b/pkg/cmd/join.go
@@ -1,4 +1,4 @@
-package main
+package cmd
 
 import (
 	"context"
@@ -443,11 +443,6 @@ func patchK0sConfig(path string, patch string) error {
 		}
 		finalcfg.Spec.Storage = result.Spec.Storage
 	}
-	out, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0600)
-	if err != nil {
-		return fmt.Errorf("unable to open node config file for writing: %w", err)
-	}
-	defer out.Close()
 	// This is necessary to install the previous version of k0s in e2e tests
 	// TODO: remove this once the previous version is > 1.29
 	unstructured, err := helpers.K0sClusterConfigTo129Compat(&finalcfg)
@@ -458,8 +453,8 @@ func patchK0sConfig(path string, patch string) error {
 	if err != nil {
 		return fmt.Errorf("unable to marshal node config: %w", err)
 	}
-	if _, err := out.Write(data); err != nil {
-		return fmt.Errorf("unable to write node config: %w", err)
+	if err := os.WriteFile(path, data, 0600); err != nil {
+		return fmt.Errorf("unable to write node config file: %w", err)
 	}
 	return nil
 }
diff --git a/cmd/embedded-cluster/join_test.go b/pkg/cmd/join_test.go
similarity index 99%
rename from cmd/embedded-cluster/join_test.go
rename to pkg/cmd/join_test.go
index e2fb45324..bff83c4df 100644
--- a/cmd/embedded-cluster/join_test.go
+++ b/pkg/cmd/join_test.go
@@ -1,4 +1,4 @@
-package main
+package cmd
 
 import (
 	"embed"
diff --git a/cmd/embedded-cluster/k0s.go b/pkg/cmd/k0s.go
similarity index 98%
rename from cmd/embedded-cluster/k0s.go
rename to pkg/cmd/k0s.go
index fd2a35e28..751d779bd 100644
--- a/cmd/embedded-cluster/k0s.go
+++ b/pkg/cmd/k0s.go
@@ -1,4 +1,4 @@
-package main
+package cmd
 
 import (
 	"context"
diff --git a/cmd/embedded-cluster/list_images.go b/pkg/cmd/list_images.go
similarity index 98%
rename from cmd/embedded-cluster/list_images.go
rename to pkg/cmd/list_images.go
index 7d9188d10..d6e83f03d 100644
--- a/cmd/embedded-cluster/list_images.go
+++ b/pkg/cmd/list_images.go
@@ -1,4 +1,4 @@
-package main
+package cmd
 
 import (
 	"fmt"
diff --git a/cmd/embedded-cluster/materialize.go b/pkg/cmd/materialize.go
similarity index 98%
rename from cmd/embedded-cluster/materialize.go
rename to pkg/cmd/materialize.go
index e6239e87b..e0bbc6f82 100644
--- a/cmd/embedded-cluster/materialize.go
+++ b/pkg/cmd/materialize.go
@@ -1,4 +1,4 @@
-package main
+package cmd
 
 import (
 	"fmt"
diff --git a/cmd/embedded-cluster/metadata.go b/pkg/cmd/metadata.go
similarity index 99%
rename from cmd/embedded-cluster/metadata.go
rename to pkg/cmd/metadata.go
index 74135793c..575257005 100644
--- a/cmd/embedded-cluster/metadata.go
+++ b/pkg/cmd/metadata.go
@@ -1,4 +1,4 @@
-package main
+package cmd
 
 import (
 	"encoding/json"
diff --git a/cmd/embedded-cluster/network.go b/pkg/cmd/network.go
similarity index 99%
rename from cmd/embedded-cluster/network.go
rename to pkg/cmd/network.go
index b7e56eb31..4c2db0594 100644
--- a/cmd/embedded-cluster/network.go
+++ b/pkg/cmd/network.go
@@ -1,4 +1,4 @@
-package main
+package cmd
 
 import (
 	"fmt"
diff --git a/cmd/embedded-cluster/network_test.go b/pkg/cmd/network_test.go
similarity index 99%
rename from cmd/embedded-cluster/network_test.go
rename to pkg/cmd/network_test.go
index f0d93fc34..7b7bc0ef9 100644
--- a/cmd/embedded-cluster/network_test.go
+++ b/pkg/cmd/network_test.go
@@ -1,4 +1,4 @@
-package main
+package cmd
 
 import (
 	"flag"
diff --git a/cmd/embedded-cluster/node.go b/pkg/cmd/node.go
similarity index 95%
rename from cmd/embedded-cluster/node.go
rename to pkg/cmd/node.go
index 69d97f026..6358de050 100644
--- a/cmd/embedded-cluster/node.go
+++ b/pkg/cmd/node.go
@@ -1,4 +1,4 @@
-package main
+package cmd
 
 import (
 	"github.com/urfave/cli/v2"
diff --git a/cmd/embedded-cluster/preflights.go b/pkg/cmd/preflights.go
similarity index 99%
rename from cmd/embedded-cluster/preflights.go
rename to pkg/cmd/preflights.go
index c7df015de..693bd9015 100644
--- a/cmd/embedded-cluster/preflights.go
+++ b/pkg/cmd/preflights.go
@@ -1,4 +1,4 @@
-package main
+package cmd
 
 import (
 	"fmt"
diff --git a/cmd/embedded-cluster/provider.go b/pkg/cmd/provider.go
similarity index 99%
rename from cmd/embedded-cluster/provider.go
rename to pkg/cmd/provider.go
index 124d883ba..c0d8b9799 100644
--- a/cmd/embedded-cluster/provider.go
+++ b/pkg/cmd/provider.go
@@ -1,4 +1,4 @@
-package main
+package cmd
 
 import (
 	"context"
diff --git a/cmd/embedded-cluster/proxy.go b/pkg/cmd/proxy.go
similarity index 99%
rename from cmd/embedded-cluster/proxy.go
rename to pkg/cmd/proxy.go
index 07ff7f20f..dab02b6f3 100644
--- a/cmd/embedded-cluster/proxy.go
+++ b/pkg/cmd/proxy.go
@@ -1,4 +1,4 @@
-package main
+package cmd
 
 import (
 	"fmt"
diff --git a/cmd/embedded-cluster/proxy_test.go b/pkg/cmd/proxy_test.go
similarity index 99%
rename from cmd/embedded-cluster/proxy_test.go
rename to pkg/cmd/proxy_test.go
index dc8e8be54..bae2731dd 100644
--- a/cmd/embedded-cluster/proxy_test.go
+++ b/pkg/cmd/proxy_test.go
@@ -1,4 +1,4 @@
-package main
+package cmd
 
 import (
 	"flag"
diff --git a/cmd/embedded-cluster/reset.go b/pkg/cmd/reset.go
similarity index 95%
rename from cmd/embedded-cluster/reset.go
rename to pkg/cmd/reset.go
index 490ba3480..8123c4d0f 100644
--- a/cmd/embedded-cluster/reset.go
+++ b/pkg/cmd/reset.go
@@ -1,4 +1,4 @@
-package main
+package cmd
 
 import (
 	"context"
@@ -6,7 +6,6 @@ import (
 	"errors"
 	"fmt"
 	"os"
-	"os/exec"
 	"regexp"
 	"time"
 
@@ -103,9 +102,9 @@ func (h *hostInfo) drainNode() error {
 		"--timeout", "60s",
 		h.Hostname,
 	}
-	out, err := exec.Command(k0s, drainArgList...).CombinedOutput()
+	out, err := helpers.RunCommand(k0s, drainArgList...)
 	if err != nil {
-		if notFoundRegex.Match(out) {
+		if notFoundRegex.Match([]byte(out + err.Error())) {
 			return nil
 		}
 		return fmt.Errorf("could not drain node: %w, %s", err, out)
@@ -208,12 +207,12 @@ func (h *hostInfo) checkResetSafety(c *cli.Context) (bool, string, error) {
 func (h *hostInfo) leaveEtcdcluster() error {
 
 	// if we're the only etcd member we don't need to leave the cluster
-	out, err := exec.Command(k0s, "etcd", "member-list").Output()
+	out, err := helpers.RunCommand(k0s, "etcd", "member-list")
 	if err != nil {
 		return err
 	}
 	memberlist := etcdMembers{}
-	err = json.Unmarshal(out, &memberlist)
+	err = json.Unmarshal([]byte(out), &memberlist)
 	if err != nil {
 		return err
 	}
@@ -221,22 +220,22 @@ func (h *hostInfo) leaveEtcdcluster() error {
 		return nil
 	}
 
-	out, err = exec.Command(k0s, "etcd", "leave").CombinedOutput()
+	out, err = helpers.RunCommand(k0s, "etcd", "leave")
 	if err != nil {
-		return fmt.Errorf("unable to leave etcd cluster: %w, %s", err, string(out))
+		return fmt.Errorf("unable to leave etcd cluster: %w, %s", err, out)
 	}
 	return nil
 }
 
 // stopK0s attempts to stop the k0s service
 func stopAndResetK0s(dataDir string) error {
-	out, err := exec.Command(k0s, "stop").CombinedOutput()
+	out, err := helpers.RunCommand(k0s, "stop")
 	if err != nil {
-		return fmt.Errorf("could not stop k0s service: %w, %s", err, string(out))
+		return fmt.Errorf("could not stop k0s service: %w, %s", err, out)
 	}
-	out, err = exec.Command(k0s, "reset", "--data-dir", dataDir).CombinedOutput()
+	out, err = helpers.RunCommand(k0s, "reset", "--data-dir", dataDir)
 	if err != nil {
-		return fmt.Errorf("could not reset k0s: %w, %s", err, string(out))
+		return fmt.Errorf("could not reset k0s: %w, %s", err, out)
 	}
 	return nil
 }
@@ -497,7 +496,7 @@ func resetCommand() *cli.Command {
 				return fmt.Errorf("failed to remove embedded cluster data config: %w", err)
 			}
 
-			if _, err := exec.Command("reboot").Output(); err != nil {
+			if _, err := helpers.RunCommand("reboot"); err != nil {
 				return err
 			}
 
diff --git a/cmd/embedded-cluster/restore.go b/pkg/cmd/restore.go
similarity index 99%
rename from cmd/embedded-cluster/restore.go
rename to pkg/cmd/restore.go
index 4322dde47..7dc0a623b 100644
--- a/cmd/embedded-cluster/restore.go
+++ b/pkg/cmd/restore.go
@@ -1,4 +1,4 @@
-package main
+package cmd
 
 import (
 	"context"
@@ -363,12 +363,7 @@ func ensureK0sConfigForRestore(c *cli.Context, provider *defaults.Provider, appl
 	if err != nil {
 		return nil, fmt.Errorf("unable to marshal config: %w", err)
 	}
-	fp, err := os.OpenFile(cfgpath, os.O_RDWR|os.O_CREATE, 0600)
-	if err != nil {
-		return nil, fmt.Errorf("unable to create config file: %w", err)
-	}
-	defer fp.Close()
-	if _, err := fp.Write(data); err != nil {
+	if err := os.WriteFile(cfgpath, data, 0600); err != nil {
 		return nil, fmt.Errorf("unable to write config file: %w", err)
 	}
 	return cfg, nil
diff --git a/cmd/embedded-cluster/shell.go b/pkg/cmd/shell.go
similarity index 99%
rename from cmd/embedded-cluster/shell.go
rename to pkg/cmd/shell.go
index 8ec21b807..0bfc3881c 100644
--- a/cmd/embedded-cluster/shell.go
+++ b/pkg/cmd/shell.go
@@ -1,4 +1,4 @@
-package main
+package cmd
 
 import (
 	"fmt"
diff --git a/cmd/embedded-cluster/support_bundle.go b/pkg/cmd/support_bundle.go
similarity index 97%
rename from cmd/embedded-cluster/support_bundle.go
rename to pkg/cmd/support_bundle.go
index abe090953..561cd1ff4 100644
--- a/cmd/embedded-cluster/support_bundle.go
+++ b/pkg/cmd/support_bundle.go
@@ -1,4 +1,4 @@
-package main
+package cmd
 
 import (
 	"bytes"
@@ -70,8 +70,8 @@ func supportBundleCommand() *cli.Command {
 			stderr := bytes.NewBuffer(nil)
 			if err := helpers.RunCommandWithOptions(
 				helpers.RunCommandOptions{
-					Writer:       stdout,
-					ErrWriter:    stderr,
+					Stdout:       stdout,
+					Stderr:       stderr,
 					LogOnSuccess: true,
 				},
 				supportBundle,
diff --git a/cmd/embedded-cluster/testdata/join-command-response-empty-overrides.yaml b/pkg/cmd/testdata/join-command-response-empty-overrides.yaml
similarity index 100%
rename from cmd/embedded-cluster/testdata/join-command-response-empty-overrides.yaml
rename to pkg/cmd/testdata/join-command-response-empty-overrides.yaml
diff --git a/cmd/embedded-cluster/testdata/join-command-response-multiple-port-overrides.yaml b/pkg/cmd/testdata/join-command-response-multiple-port-overrides.yaml
similarity index 100%
rename from cmd/embedded-cluster/testdata/join-command-response-multiple-port-overrides.yaml
rename to pkg/cmd/testdata/join-command-response-multiple-port-overrides.yaml
diff --git a/cmd/embedded-cluster/testdata/join-command-response-multiple-unused-overlays.yaml b/pkg/cmd/testdata/join-command-response-multiple-unused-overlays.yaml
similarity index 100%
rename from cmd/embedded-cluster/testdata/join-command-response-multiple-unused-overlays.yaml
rename to pkg/cmd/testdata/join-command-response-multiple-unused-overlays.yaml
diff --git a/cmd/embedded-cluster/testdata/join-command-response-no-overrides.yaml b/pkg/cmd/testdata/join-command-response-no-overrides.yaml
similarity index 100%
rename from cmd/embedded-cluster/testdata/join-command-response-no-overrides.yaml
rename to pkg/cmd/testdata/join-command-response-no-overrides.yaml
diff --git a/cmd/embedded-cluster/testdata/join-command-response-override-ports.yaml b/pkg/cmd/testdata/join-command-response-override-ports.yaml
similarity index 100%
rename from cmd/embedded-cluster/testdata/join-command-response-override-ports.yaml
rename to pkg/cmd/testdata/join-command-response-override-ports.yaml
diff --git a/cmd/embedded-cluster/testdata/join-command-response-storage-and-port-overrides.yaml b/pkg/cmd/testdata/join-command-response-storage-and-port-overrides.yaml
similarity index 100%
rename from cmd/embedded-cluster/testdata/join-command-response-storage-and-port-overrides.yaml
rename to pkg/cmd/testdata/join-command-response-storage-and-port-overrides.yaml
diff --git a/cmd/embedded-cluster/testdata/patch-k0s-config-change-external-address.yaml b/pkg/cmd/testdata/patch-k0s-config-change-external-address.yaml
similarity index 100%
rename from cmd/embedded-cluster/testdata/patch-k0s-config-change-external-address.yaml
rename to pkg/cmd/testdata/patch-k0s-config-change-external-address.yaml
diff --git a/cmd/embedded-cluster/testdata/patch-k0s-config-change-sans.yaml b/pkg/cmd/testdata/patch-k0s-config-change-sans.yaml
similarity index 100%
rename from cmd/embedded-cluster/testdata/patch-k0s-config-change-sans.yaml
rename to pkg/cmd/testdata/patch-k0s-config-change-sans.yaml
diff --git a/cmd/embedded-cluster/testdata/patch-k0s-config-extra-args.yaml b/pkg/cmd/testdata/patch-k0s-config-extra-args.yaml
similarity index 100%
rename from cmd/embedded-cluster/testdata/patch-k0s-config-extra-args.yaml
rename to pkg/cmd/testdata/patch-k0s-config-extra-args.yaml
diff --git a/cmd/embedded-cluster/testdata/patch-k0s-config-k0s-storage.yaml b/pkg/cmd/testdata/patch-k0s-config-k0s-storage.yaml
similarity index 100%
rename from cmd/embedded-cluster/testdata/patch-k0s-config-k0s-storage.yaml
rename to pkg/cmd/testdata/patch-k0s-config-k0s-storage.yaml
diff --git a/cmd/embedded-cluster/testdata/patch-k0s-config-no-embedded-cluster-config.yaml b/pkg/cmd/testdata/patch-k0s-config-no-embedded-cluster-config.yaml
similarity index 100%
rename from cmd/embedded-cluster/testdata/patch-k0s-config-no-embedded-cluster-config.yaml
rename to pkg/cmd/testdata/patch-k0s-config-no-embedded-cluster-config.yaml
diff --git a/cmd/embedded-cluster/testdata/patch-k0s-config-no-overrides.yaml b/pkg/cmd/testdata/patch-k0s-config-no-overrides.yaml
similarity index 100%
rename from cmd/embedded-cluster/testdata/patch-k0s-config-no-overrides.yaml
rename to pkg/cmd/testdata/patch-k0s-config-no-overrides.yaml
diff --git a/cmd/embedded-cluster/update.go b/pkg/cmd/update.go
similarity index 99%
rename from cmd/embedded-cluster/update.go
rename to pkg/cmd/update.go
index 6a33f0272..87cb219ca 100644
--- a/cmd/embedded-cluster/update.go
+++ b/pkg/cmd/update.go
@@ -1,4 +1,4 @@
-package main
+package cmd
 
 import (
 	"fmt"
diff --git a/cmd/embedded-cluster/util.go b/pkg/cmd/util.go
similarity index 83%
rename from cmd/embedded-cluster/util.go
rename to pkg/cmd/util.go
index 3edaa3801..f3ab8479a 100644
--- a/cmd/embedded-cluster/util.go
+++ b/pkg/cmd/util.go
@@ -1,4 +1,4 @@
-package main
+package cmd
 
 import (
 	"fmt"
@@ -51,20 +51,15 @@ func ensureProxyConfig(servicePath string, httpProxy string, httpsProxy string,
 		return fmt.Errorf("unable to create directory: %w", err)
 	}
 
-	// create the file
-	fp, err := os.OpenFile(filepath.Join(servicePath, "http-proxy.conf"), os.O_RDWR|os.O_CREATE, 0644)
-	if err != nil {
-		return fmt.Errorf("unable to create proxy file: %w", err)
-	}
-	defer fp.Close()
-
-	// write the file
-	if _, err := fp.WriteString(fmt.Sprintf(`[Service]
+	// create and write the file
+	content := fmt.Sprintf(`[Service]
 Environment="HTTP_PROXY=%s"
 Environment="HTTPS_PROXY=%s"
-Environment="NO_PROXY=%s"`,
-		httpProxy, httpsProxy, noProxy)); err != nil {
-		return fmt.Errorf("unable to write proxy file: %w", err)
+Environment="NO_PROXY=%s"`, httpProxy, httpsProxy, noProxy)
+
+	err := os.WriteFile(filepath.Join(servicePath, "http-proxy.conf"), []byte(content), 0644)
+	if err != nil {
+		return fmt.Errorf("unable to create and write proxy file: %w", err)
 	}
 
 	return nil
diff --git a/cmd/embedded-cluster/version.go b/pkg/cmd/version.go
similarity index 99%
rename from cmd/embedded-cluster/version.go
rename to pkg/cmd/version.go
index 4ec8421ae..9b5504455 100644
--- a/cmd/embedded-cluster/version.go
+++ b/pkg/cmd/version.go
@@ -1,4 +1,4 @@
-package main
+package cmd
 
 import (
 	"encoding/json"
diff --git a/pkg/dryrun/dryrun.go b/pkg/dryrun/dryrun.go
new file mode 100644
index 000000000..5466f763f
--- /dev/null
+++ b/pkg/dryrun/dryrun.go
@@ -0,0 +1,101 @@
+package dryrun
+
+import (
+	"fmt"
+	"os"
+	"strings"
+	"sync"
+
+	"github.com/replicatedhq/embedded-cluster/pkg/dryrun/types"
+	"github.com/replicatedhq/embedded-cluster/pkg/helpers"
+	"github.com/replicatedhq/embedded-cluster/pkg/kubeutils"
+	"github.com/replicatedhq/embedded-cluster/pkg/metrics"
+	troubleshootv1beta2 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta2"
+	"github.com/urfave/cli/v2"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/yaml"
+)
+
+var (
+	dr     *types.DryRun
+	drFile string
+	mu     sync.Mutex
+)
+
+func Init(outputFile string) {
+	dr = &types.DryRun{
+		Flags:             map[string]interface{}{},
+		Commands:          []types.Command{},
+		Metrics:           []types.Metric{},
+		HostPreflightSpec: &troubleshootv1beta2.HostPreflightSpec{},
+	}
+	drFile = outputFile
+	kubeutils.Set(&KubeUtils{})
+	helpers.Set(&Helpers{})
+	metrics.Set(&Sender{})
+}
+
+func Dump() error {
+	mu.Lock()
+	defer mu.Unlock()
+
+	output, err := yaml.Marshal(dr)
+	if err != nil {
+		return fmt.Errorf("marshal dry run info: %w", err)
+	}
+	if err := os.WriteFile(drFile, output, 0644); err != nil {
+		return fmt.Errorf("write dry run info to file: %w", err)
+	}
+	return nil
+}
+
+func RecordFlags(c *cli.Context) {
+	mu.Lock()
+	defer mu.Unlock()
+
+	for _, flag := range c.Command.Flags {
+		for _, name := range flag.Names() {
+			dr.Flags[name] = c.Value(name)
+		}
+	}
+}
+
+func RecordCommand(cmd string, args []string, env map[string]string) {
+	mu.Lock()
+	defer mu.Unlock()
+
+	fullCmd := cmd
+	if len(args) > 0 {
+		fullCmd += " " + strings.Join(args, " ")
+	}
+	dr.Commands = append(dr.Commands, types.Command{
+		Cmd: fullCmd,
+		Env: env,
+	})
+}
+
+func RecordMetric(title string, url string, payload []byte) {
+	mu.Lock()
+	defer mu.Unlock()
+
+	dr.Metrics = append(dr.Metrics, types.Metric{
+		Title:   title,
+		URL:     url,
+		Payload: string(payload),
+	})
+}
+
+func RecordHostPreflightSpec(hpf *troubleshootv1beta2.HostPreflightSpec) {
+	mu.Lock()
+	defer mu.Unlock()
+
+	dr.HostPreflightSpec = hpf
+}
+
+func KubeClient() (client.Client, error) {
+	return dr.KubeClient()
+}
+
+func Enabled() bool {
+	return dr != nil
+}
diff --git a/pkg/dryrun/helpers.go b/pkg/dryrun/helpers.go
new file mode 100644
index 000000000..74d2f0cf3
--- /dev/null
+++ b/pkg/dryrun/helpers.go
@@ -0,0 +1,29 @@
+package dryrun
+
+import (
+	"bytes"
+	"context"
+
+	"github.com/replicatedhq/embedded-cluster/pkg/helpers"
+)
+
+type Helpers struct{}
+
+var _ helpers.HelpersInterface = (*Helpers)(nil)
+
+func (h *Helpers) RunCommandWithOptions(opts helpers.RunCommandOptions, bin string, args ...string) error {
+	RecordCommand(bin, args, opts.Env)
+	return nil
+}
+
+func (h *Helpers) RunCommand(bin string, args ...string) (string, error) {
+	stdout := bytes.NewBuffer(nil)
+	if err := h.RunCommandWithOptions(helpers.RunCommandOptions{Stdout: stdout}, bin, args...); err != nil {
+		return "", err
+	}
+	return stdout.String(), nil
+}
+
+func (h *Helpers) IsSystemdServiceActive(ctx context.Context, svcname string) (bool, error) {
+	return false, nil
+}
diff --git a/pkg/dryrun/kubeutils.go b/pkg/dryrun/kubeutils.go
new file mode 100644
index 000000000..1963359e8
--- /dev/null
+++ b/pkg/dryrun/kubeutils.go
@@ -0,0 +1,81 @@
+package dryrun
+
+import (
+	"context"
+
+	"github.com/replicatedhq/embedded-cluster/pkg/kubeutils"
+	"github.com/replicatedhq/embedded-cluster/pkg/spinner"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+type KubeUtils struct{}
+
+var _ kubeutils.KubeUtilsInterface = (*KubeUtils)(nil)
+
+func (k *KubeUtils) WaitForNamespace(ctx context.Context, cli client.Client, ns string) error {
+	return nil
+}
+
+func (k *KubeUtils) WaitForDeployment(ctx context.Context, cli client.Client, ns, name string) error {
+	return nil
+}
+
+func (k *KubeUtils) WaitForDaemonset(ctx context.Context, cli client.Client, ns, name string) error {
+	return nil
+}
+
+func (k *KubeUtils) WaitForService(ctx context.Context, cli client.Client, ns, name string) error {
+	return nil
+}
+
+func (k *KubeUtils) WaitForInstallation(ctx context.Context, cli client.Client, writer *spinner.MessageWriter) error {
+	return nil
+}
+
+func (k *KubeUtils) WaitForHAInstallation(ctx context.Context, cli client.Client) error {
+	return nil
+}
+
+func (k *KubeUtils) WaitForNodes(ctx context.Context, cli client.Client) error {
+	return nil
+}
+
+func (k *KubeUtils) WaitForControllerNode(ctx context.Context, kcli client.Client, name string) error {
+	return nil
+}
+
+func (k *KubeUtils) WaitForJob(ctx context.Context, cli client.Client, ns, name string, maxSteps int, completions int32) error {
+	return nil
+}
+
+func (k *KubeUtils) IsNamespaceReady(ctx context.Context, cli client.Client, ns string) (bool, error) {
+	return true, nil
+}
+
+func (k *KubeUtils) IsDeploymentReady(ctx context.Context, cli client.Client, ns, name string) (bool, error) {
+	return true, nil
+}
+
+func (k *KubeUtils) IsStatefulSetReady(ctx context.Context, cli client.Client, ns, name string) (bool, error) {
+	return true, nil
+}
+
+func (k *KubeUtils) IsDaemonsetReady(ctx context.Context, cli client.Client, ns, name string) (bool, error) {
+	return true, nil
+}
+
+func (k *KubeUtils) IsJobComplete(ctx context.Context, cli client.Client, ns, name string, completions int32) (bool, error) {
+	return true, nil
+}
+
+func (k *KubeUtils) WaitForKubernetes(ctx context.Context, cli client.Client) <-chan error {
+	return nil
+}
+
+func (k *KubeUtils) WaitAndMarkInstallation(ctx context.Context, cli client.Client, name string, state string) error {
+	return nil
+}
+
+func (k *KubeUtils) KubeClient() (client.Client, error) {
+	return KubeClient()
+}
diff --git a/pkg/dryrun/metrics.go b/pkg/dryrun/metrics.go
new file mode 100644
index 000000000..3e3d09c5f
--- /dev/null
+++ b/pkg/dryrun/metrics.go
@@ -0,0 +1,23 @@
+package dryrun
+
+import (
+	"context"
+
+	"github.com/replicatedhq/embedded-cluster/pkg/metrics"
+	"github.com/replicatedhq/embedded-cluster/pkg/metrics/types"
+	"github.com/sirupsen/logrus"
+)
+
+type Sender struct{}
+
+var _ metrics.SenderInterface = (*Sender)(nil)
+
+func (s *Sender) Send(ctx context.Context, baseURL string, ev types.Event) {
+	url := metrics.EventURL(baseURL, ev)
+	payload, err := metrics.EventPayload(ev)
+	if err != nil {
+		logrus.Debugf("unable to get payload for event %s: %s", ev.Title(), err)
+		return
+	}
+	RecordMetric(ev.Title(), url, payload)
+}
diff --git a/pkg/dryrun/types/types.go b/pkg/dryrun/types/types.go
new file mode 100644
index 000000000..97fea8e56
--- /dev/null
+++ b/pkg/dryrun/types/types.go
@@ -0,0 +1,224 @@
+package types
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"os"
+	"strings"
+
+	ecv1beta1 "github.com/replicatedhq/embedded-cluster/kinds/apis/v1beta1"
+	troubleshootv1beta2 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta2"
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	rbacv1 "k8s.io/api/rbac/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime"
+	k8scheme "k8s.io/client-go/kubernetes/scheme"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+	"sigs.k8s.io/yaml"
+)
+
+type DryRun struct {
+	Flags             map[string]interface{}                 `json:"flags"`
+	Commands          []Command                              `json:"commands"`
+	Metrics           []Metric                               `json:"metrics"`
+	HostPreflightSpec *troubleshootv1beta2.HostPreflightSpec `json:"hostPreflightSpec"`
+
+	// These fields are set on marshal
+	OSEnv      map[string]string `json:"osEnv"`
+	K8sObjects []string          `json:"k8sObjects"`
+
+	// These fields are used as mocks
+	kcli client.Client `json:"-"`
+}
+
+type Metric struct {
+	Title   string `json:"title"`
+	URL     string `json:"url"`
+	Payload string `json:"payload"`
+}
+
+type Command struct {
+	Cmd string            `json:"cmd"`
+	Env map[string]string `json:"env,omitempty"`
+}
+
+func (d *DryRun) MarshalJSON() ([]byte, error) {
+	k8sObjects, err := d.K8sObjectsFromClient()
+	if err != nil {
+		return nil, fmt.Errorf("get k8s objects: %w", err)
+	}
+	alias := *d
+	alias.OSEnv = getOSEnv()
+	alias.K8sObjects = k8sObjects
+	return json.Marshal(alias)
+}
+
+func (d *DryRun) K8sObjectsFromClient() ([]string, error) {
+	kcli, err := d.KubeClient()
+	if err != nil {
+		return nil, fmt.Errorf("get kube client: %w", err)
+	}
+
+	ctx := context.Background()
+	result := []string{}
+
+	addToResult := func(o runtime.Object) error {
+		data, err := yaml.Marshal(o)
+		if err != nil {
+			return fmt.Errorf("marshal object: %w", err)
+		}
+		result = append(result, string(data))
+		return nil
+	}
+
+	// Services
+	var services corev1.ServiceList
+	if err := kcli.List(ctx, &services); err != nil {
+		return nil, fmt.Errorf("list services: %w", err)
+	}
+	for _, svc := range services.Items {
+		if err := addToResult(&svc); err != nil {
+			return nil, fmt.Errorf("add to result: %w", err)
+		}
+	}
+
+	// Deployments
+	var deployments appsv1.DeploymentList
+	if err := kcli.List(ctx, &deployments); err != nil {
+		return nil, fmt.Errorf("list deployments: %w", err)
+	}
+	for _, dpl := range deployments.Items {
+		if err := addToResult(&dpl); err != nil {
+			return nil, fmt.Errorf("add to result: %w", err)
+		}
+	}
+
+	// StatefulSets
+	var statefulSets appsv1.StatefulSetList
+	if err := kcli.List(ctx, &statefulSets); err != nil {
+		return nil, fmt.Errorf("list statefulsets: %w", err)
+	}
+	for _, sts := range statefulSets.Items {
+		if err := addToResult(&sts); err != nil {
+			return nil, fmt.Errorf("add to result: %w", err)
+		}
+	}
+
+	// DaemonSets
+	var daemonSets appsv1.DaemonSetList
+	if err := kcli.List(ctx, &daemonSets); err != nil {
+		return nil, fmt.Errorf("list daemonsets: %w", err)
+	}
+	for _, ds := range daemonSets.Items {
+		if err := addToResult(&ds); err != nil {
+			return nil, fmt.Errorf("add to result: %w", err)
+		}
+	}
+
+	// Nodes
+	var nodes corev1.NodeList
+	if err := kcli.List(ctx, &nodes); err != nil {
+		return nil, fmt.Errorf("list nodes: %w", err)
+	}
+	for _, node := range nodes.Items {
+		if err := addToResult(&node); err != nil {
+			return nil, fmt.Errorf("add to result: %w", err)
+		}
+	}
+
+	// ConfigMaps
+	var configMaps corev1.ConfigMapList
+	if err := kcli.List(ctx, &configMaps); err != nil {
+		return nil, fmt.Errorf("list configmaps: %w", err)
+	}
+	for _, cm := range configMaps.Items {
+		if err := addToResult(&cm); err != nil {
+			return nil, fmt.Errorf("add to result: %w", err)
+		}
+	}
+
+	// Secrets
+	var secrets corev1.SecretList
+	if err := kcli.List(ctx, &secrets); err != nil {
+		return nil, fmt.Errorf("list secrets: %w", err)
+	}
+	for _, secret := range secrets.Items {
+		if err := addToResult(&secret); err != nil {
+			return nil, fmt.Errorf("add to result: %w", err)
+		}
+	}
+
+	// Roles
+	var roles rbacv1.RoleList
+	if err := kcli.List(ctx, &roles); err != nil {
+		return nil, fmt.Errorf("list roles: %w", err)
+	}
+	for _, role := range roles.Items {
+		if err := addToResult(&role); err != nil {
+			return nil, fmt.Errorf("add to result: %w", err)
+		}
+	}
+
+	// RoleBindings
+	var roleBindings rbacv1.RoleBindingList
+	if err := kcli.List(ctx, &roleBindings); err != nil {
+		return nil, fmt.Errorf("list rolebindings: %w", err)
+	}
+	for _, rb := range roleBindings.Items {
+		if err := addToResult(&rb); err != nil {
+			return nil, fmt.Errorf("add to result: %w", err)
+		}
+	}
+
+	// Installation CRs
+	var installations ecv1beta1.InstallationList
+	if err := kcli.List(ctx, &installations); err != nil {
+		return nil, fmt.Errorf("list installations: %w", err)
+	}
+	for _, install := range installations.Items {
+		if err := addToResult(&install); err != nil {
+			return nil, fmt.Errorf("add to result: %w", err)
+		}
+	}
+
+	return result, nil
+}
+
+func (d *DryRun) KubeClient() (client.Client, error) {
+	if d.kcli == nil {
+		scheme := runtime.NewScheme()
+		if err := k8scheme.AddToScheme(scheme); err != nil {
+			return nil, fmt.Errorf("add k8s scheme: %w", err)
+		}
+		if err := ecv1beta1.AddToScheme(scheme); err != nil {
+			return nil, fmt.Errorf("add ec v1beta1 scheme: %w", err)
+		}
+		clientObjs := []client.Object{}
+		for _, o := range d.K8sObjects {
+			var m map[string]interface{}
+			if err := yaml.Unmarshal([]byte(o), &m); err != nil {
+				return nil, fmt.Errorf("unmarshal: %w", err)
+			}
+			clientObjs = append(clientObjs, &unstructured.Unstructured{Object: m})
+		}
+		d.kcli = fake.NewClientBuilder().
+			WithScheme(scheme).
+			WithObjects(clientObjs...).
+			Build()
+	}
+	return d.kcli, nil
+}
+
+func getOSEnv() map[string]string {
+	osEnv := make(map[string]string)
+	for _, env := range os.Environ() {
+		parts := strings.SplitN(env, "=", 2)
+		if len(parts) == 2 {
+			osEnv[parts[0]] = parts[1]
+		}
+	}
+	return osEnv
+}
diff --git a/pkg/helm/values.go b/pkg/helm/values.go
index ea40895d1..1fb73a3f0 100644
--- a/pkg/helm/values.go
+++ b/pkg/helm/values.go
@@ -42,3 +42,15 @@ func SetValue(values map[string]interface{}, path string, newValue interface{})
 
 	return newValuesMap, nil
 }
+
+func GetValue(values map[string]interface{}, path string) (interface{}, error) {
+	x, err := jp.ParseString(path)
+	if err != nil {
+		return nil, fmt.Errorf("parse json path %q: %w", path, err)
+	}
+	v := x.Get(values)
+	if len(v) == 0 {
+		return nil, fmt.Errorf("value not found in path %q", path)
+	}
+	return v[0], nil
+}
diff --git a/pkg/helm/values_test.go b/pkg/helm/values_test.go
index 0e22cbc64..12aec8ada 100644
--- a/pkg/helm/values_test.go
+++ b/pkg/helm/values_test.go
@@ -79,3 +79,111 @@ func TestSetValue(t *testing.T) {
 		})
 	}
 }
+
+func TestGetValue(t *testing.T) {
+	type args struct {
+		values map[string]interface{}
+		path   string
+	}
+	tests := []struct {
+		name    string
+		args    args
+		want    interface{}
+		wantErr bool
+	}{
+		{
+			name: "get value",
+			args: args{
+				values: map[string]interface{}{
+					"foo": "bar",
+				},
+				path: "foo",
+			},
+			want: "bar",
+		},
+		{
+			name: "get value from array",
+			args: args{
+				values: map[string]interface{}{
+					"foo": []interface{}{"bar", "baz"},
+				},
+				path: "foo[0]",
+			},
+			want: "bar",
+		},
+		{
+			name: "get value from nested map",
+			args: args{
+				values: map[string]interface{}{
+					"foo": map[string]interface{}{
+						"bar": "baz",
+					},
+				},
+				path: "foo.bar",
+			},
+			want: "baz",
+		},
+		{
+			name: "get value from nested array",
+			args: args{
+				values: map[string]interface{}{
+					"foo": []interface{}{
+						map[string]interface{}{
+							"bar": []interface{}{
+								"baz",
+							},
+						},
+					},
+				},
+				path: "foo[0].bar[0]",
+			},
+			want: "baz",
+		},
+		{
+			name: "get value from missing map",
+			args: args{
+				values: map[string]interface{}{
+					"foo": map[string]interface{}{
+						"bar": "baz",
+					},
+				},
+				path: "foo.bar.baz",
+			},
+			wantErr: true,
+		},
+		{
+			name: "get value for key with hyphen",
+			args: args{
+				values: map[string]interface{}{
+					"foo-bar": "baz",
+				},
+				path: "['foo-bar']",
+			},
+			want: "baz",
+		},
+		{
+			name: "get value for nested key with hyphen",
+			args: args{
+				values: map[string]interface{}{
+					"foo": map[string]interface{}{
+						"bar-baz": "baz",
+					},
+				},
+				path: "foo['bar-baz']",
+			},
+			want: "baz",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := GetValue(tt.args.values, tt.args.path)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("GetValue() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("GetValue() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
diff --git a/pkg/helpers/command.go b/pkg/helpers/command.go
index 191179e39..b4b00d25b 100644
--- a/pkg/helpers/command.go
+++ b/pkg/helpers/command.go
@@ -9,21 +9,8 @@ import (
 	"github.com/sirupsen/logrus"
 )
 
-type RunCommandOptions struct {
-	// Writer is an additional io.Writer to write the stdout of the command to.
-	Writer io.Writer
-	// ErrWriter is an additional io.Writer to write the stderr of the command to.
-	ErrWriter io.Writer
-	// Env is a map of additional environment variables to set for the command.
-	Env map[string]string
-	// Stdin is the standard input to be used when running the command.
-	Stdin io.Reader
-	// LogOnSuccess makes the command output to be logged even when it succeeds.
-	LogOnSuccess bool
-}
-
 // RunCommandWithOptions runs a the provided command with the options specified.
-func RunCommandWithOptions(opts RunCommandOptions, bin string, args ...string) error {
+func (h *Helpers) RunCommandWithOptions(opts RunCommandOptions, bin string, args ...string) error {
 	fullcmd := append([]string{bin}, args...)
 	logrus.Debugf("running command: %v", fullcmd)
 
@@ -31,15 +18,15 @@ func RunCommandWithOptions(opts RunCommandOptions, bin string, args ...string) e
 	stdout := bytes.NewBuffer(nil)
 	cmd := exec.Command(bin, args...)
 	cmd.Stdout = stdout
-	if opts.Writer != nil {
-		cmd.Stdout = io.MultiWriter(opts.Writer, stdout)
+	if opts.Stdout != nil {
+		cmd.Stdout = io.MultiWriter(opts.Stdout, stdout)
 	}
 	if opts.Stdin != nil {
 		cmd.Stdin = opts.Stdin
 	}
 	cmd.Stderr = stderr
-	if opts.ErrWriter != nil {
-		cmd.Stderr = io.MultiWriter(opts.ErrWriter, stderr)
+	if opts.Stderr != nil {
+		cmd.Stderr = io.MultiWriter(opts.Stderr, stderr)
 	}
 	cmdEnv := cmd.Environ()
 	for k, v := range opts.Env {
@@ -68,9 +55,9 @@ func RunCommandWithOptions(opts RunCommandOptions, bin string, args ...string) e
 
 // RunCommand spawns a command and capture its output. Outputs are logged using the
 // logrus package and stdout is returned as a string.
-func RunCommand(bin string, args ...string) (string, error) {
+func (h *Helpers) RunCommand(bin string, args ...string) (string, error) {
 	stdout := bytes.NewBuffer(nil)
-	if err := RunCommandWithOptions(RunCommandOptions{Writer: stdout}, bin, args...); err != nil {
+	if err := h.RunCommandWithOptions(RunCommandOptions{Stdout: stdout}, bin, args...); err != nil {
 		return "", err
 	}
 	return stdout.String(), nil
diff --git a/pkg/helpers/interface.go b/pkg/helpers/interface.go
new file mode 100644
index 000000000..6cb00fed6
--- /dev/null
+++ b/pkg/helpers/interface.go
@@ -0,0 +1,54 @@
+package helpers
+
+import (
+	"context"
+	"io"
+)
+
+var h HelpersInterface
+
+type Helpers struct{}
+
+var _ HelpersInterface = (*Helpers)(nil)
+
+func init() {
+	Set(&Helpers{})
+}
+
+func Set(_h HelpersInterface) {
+	h = _h
+}
+
+// HelpersInterface is an interface that wraps the RunCommand function.
+type HelpersInterface interface {
+	RunCommandWithOptions(opts RunCommandOptions, bin string, args ...string) error
+	RunCommand(bin string, args ...string) (string, error)
+	IsSystemdServiceActive(ctx context.Context, svcname string) (bool, error)
+}
+
+type RunCommandOptions struct {
+	// Stdout is an additional io.Stdout to write the stdout of the command to.
+	Stdout io.Writer
+	// Stderr is an additional io.Stderr to write the stderr of the command to.
+	Stderr io.Writer
+	// Env is a map of additional environment variables to set for the command.
+	Env map[string]string
+	// Stdin is the standard input to be used when running the command.
+	Stdin io.Reader
+	// LogOnSuccess makes the command output to be logged even when it succeeds.
+	LogOnSuccess bool
+}
+
+// Convenience functions
+
+func RunCommandWithOptions(opts RunCommandOptions, bin string, args ...string) error {
+	return h.RunCommandWithOptions(opts, bin, args...)
+}
+
+func RunCommand(bin string, args ...string) (string, error) {
+	return h.RunCommand(bin, args...)
+}
+
+func IsSystemdServiceActive(ctx context.Context, svcname string) (bool, error) {
+	return h.IsSystemdServiceActive(ctx, svcname)
+}
diff --git a/pkg/helpers/systemd.go b/pkg/helpers/systemd.go
index df1ae507f..ede7db490 100644
--- a/pkg/helpers/systemd.go
+++ b/pkg/helpers/systemd.go
@@ -9,7 +9,7 @@ import (
 )
 
 // IsSystemdServiceActive checks if a systemd service is active or not.
-func IsSystemdServiceActive(ctx context.Context, svcname string) (bool, error) {
+func (h *Helpers) IsSystemdServiceActive(ctx context.Context, svcname string) (bool, error) {
 	conn, err := dbus.NewSystemConnectionContext(ctx)
 	if err != nil {
 		return false, fmt.Errorf("unable to establish connection to systemd: %w", err)
diff --git a/pkg/kotscli/kotscli.go b/pkg/kotscli/kotscli.go
index 8ee921aea..c5359770d 100644
--- a/pkg/kotscli/kotscli.go
+++ b/pkg/kotscli/kotscli.go
@@ -83,7 +83,7 @@ func Install(provider *defaults.Provider, opts InstallOptions, msg *spinner.Mess
 	defer msg.SetLineBreaker(nil)
 
 	runCommandOptions := helpers.RunCommandOptions{
-		Writer: msg,
+		Stdout: msg,
 		Env: map[string]string{
 			"EMBEDDED_CLUSTER_ID": metrics.ClusterID().String(),
 		},
@@ -144,7 +144,7 @@ func AirgapUpdate(provider *defaults.Provider, opts AirgapUpdateOptions) error {
 
 	loading := spinner.Start(spinner.WithMask(maskfn), spinner.WithLineBreaker(lbreakfn))
 	runCommandOptions := helpers.RunCommandOptions{
-		Writer: loading,
+		Stdout: loading,
 		Env: map[string]string{
 			"EMBEDDED_CLUSTER_ID": metrics.ClusterID().String(),
 		},
diff --git a/pkg/kubeutils/client.go b/pkg/kubeutils/client.go
deleted file mode 100644
index 3baae436b..000000000
--- a/pkg/kubeutils/client.go
+++ /dev/null
@@ -1,24 +0,0 @@
-package kubeutils
-
-import (
-	"fmt"
-	"io"
-
-	"sigs.k8s.io/controller-runtime/pkg/client"
-	"sigs.k8s.io/controller-runtime/pkg/client/config"
-	"sigs.k8s.io/controller-runtime/pkg/log"
-	"sigs.k8s.io/controller-runtime/pkg/log/zap"
-)
-
-// KubeClient returns a new kubernetes client.
-func KubeClient() (client.Client, error) {
-	k8slogger := zap.New(func(o *zap.Options) {
-		o.DestWriter = io.Discard
-	})
-	log.SetLogger(k8slogger)
-	cfg, err := config.GetConfig()
-	if err != nil {
-		return nil, fmt.Errorf("unable to process kubernetes config: %w", err)
-	}
-	return client.New(cfg, client.Options{})
-}
diff --git a/pkg/kubeutils/interface.go b/pkg/kubeutils/interface.go
new file mode 100644
index 000000000..bd683b7e9
--- /dev/null
+++ b/pkg/kubeutils/interface.go
@@ -0,0 +1,108 @@
+package kubeutils
+
+import (
+	"context"
+
+	"github.com/replicatedhq/embedded-cluster/pkg/spinner"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+var kb KubeUtilsInterface
+
+func init() {
+	Set(&KubeUtils{})
+}
+
+func Set(_kb KubeUtilsInterface) {
+	kb = _kb
+}
+
+type KubeUtilsInterface interface {
+	WaitForNamespace(ctx context.Context, cli client.Client, ns string) error
+	WaitForDeployment(ctx context.Context, cli client.Client, ns, name string) error
+	WaitForDaemonset(ctx context.Context, cli client.Client, ns, name string) error
+	WaitForService(ctx context.Context, cli client.Client, ns, name string) error
+	WaitForInstallation(ctx context.Context, cli client.Client, writer *spinner.MessageWriter) error
+	WaitForHAInstallation(ctx context.Context, cli client.Client) error
+	WaitForNodes(ctx context.Context, cli client.Client) error
+	WaitForControllerNode(ctx context.Context, kcli client.Client, name string) error
+	WaitForJob(ctx context.Context, cli client.Client, ns, name string, maxSteps int, completions int32) error
+	IsNamespaceReady(ctx context.Context, cli client.Client, ns string) (bool, error)
+	IsDeploymentReady(ctx context.Context, cli client.Client, ns, name string) (bool, error)
+	IsStatefulSetReady(ctx context.Context, cli client.Client, ns, name string) (bool, error)
+	IsDaemonsetReady(ctx context.Context, cli client.Client, ns, name string) (bool, error)
+	IsJobComplete(ctx context.Context, cli client.Client, ns, name string, completions int32) (bool, error)
+	WaitForKubernetes(ctx context.Context, cli client.Client) <-chan error
+	WaitAndMarkInstallation(ctx context.Context, cli client.Client, name string, state string) error
+	KubeClient() (client.Client, error)
+}
+
+// Convenience functions
+
+func WaitForNamespace(ctx context.Context, cli client.Client, ns string) error {
+	return kb.WaitForNamespace(ctx, cli, ns)
+}
+
+func WaitForDeployment(ctx context.Context, cli client.Client, ns, name string) error {
+	return kb.WaitForDeployment(ctx, cli, ns, name)
+}
+
+func WaitForDaemonset(ctx context.Context, cli client.Client, ns, name string) error {
+	return kb.WaitForDaemonset(ctx, cli, ns, name)
+}
+
+func WaitForService(ctx context.Context, cli client.Client, ns, name string) error {
+	return kb.WaitForService(ctx, cli, ns, name)
+}
+
+func WaitForInstallation(ctx context.Context, cli client.Client, writer *spinner.MessageWriter) error {
+	return kb.WaitForInstallation(ctx, cli, writer)
+}
+
+func WaitForHAInstallation(ctx context.Context, cli client.Client) error {
+	return kb.WaitForHAInstallation(ctx, cli)
+}
+
+func WaitForNodes(ctx context.Context, cli client.Client) error {
+	return kb.WaitForNodes(ctx, cli)
+}
+
+func WaitForControllerNode(ctx context.Context, kcli client.Client, name string) error {
+	return kb.WaitForControllerNode(ctx, kcli, name)
+}
+
+func WaitForJob(ctx context.Context, cli client.Client, ns, name string, maxSteps int, completions int32) error {
+	return kb.WaitForJob(ctx, cli, ns, name, maxSteps, completions)
+}
+
+func IsNamespaceReady(ctx context.Context, cli client.Client, ns string) (bool, error) {
+	return kb.IsNamespaceReady(ctx, cli, ns)
+}
+
+func IsDeploymentReady(ctx context.Context, cli client.Client, ns, name string) (bool, error) {
+	return kb.IsDeploymentReady(ctx, cli, ns, name)
+}
+
+func IsStatefulSetReady(ctx context.Context, cli client.Client, ns, name string) (bool, error) {
+	return kb.IsStatefulSetReady(ctx, cli, ns, name)
+}
+
+func IsDaemonsetReady(ctx context.Context, cli client.Client, ns, name string) (bool, error) {
+	return kb.IsDaemonsetReady(ctx, cli, ns, name)
+}
+
+func IsJobComplete(ctx context.Context, cli client.Client, ns, name string, completions int32) (bool, error) {
+	return kb.IsJobComplete(ctx, cli, ns, name, completions)
+}
+
+func WaitForKubernetes(ctx context.Context, cli client.Client) <-chan error {
+	return kb.WaitForKubernetes(ctx, cli)
+}
+
+func WaitAndMarkInstallation(ctx context.Context, cli client.Client, name string, state string) error {
+	return kb.WaitAndMarkInstallation(ctx, cli, name, state)
+}
+
+func KubeClient() (client.Client, error) {
+	return kb.KubeClient()
+}
diff --git a/pkg/kubeutils/kubeutils.go b/pkg/kubeutils/kubeutils.go
index 52c22e5a1..1ef9930bb 100644
--- a/pkg/kubeutils/kubeutils.go
+++ b/pkg/kubeutils/kubeutils.go
@@ -2,7 +2,9 @@ package kubeutils
 
 import (
 	"context"
+	"errors"
 	"fmt"
+	"io"
 	"regexp"
 	"sort"
 	"time"
@@ -19,8 +21,15 @@ import (
 	"k8s.io/apimachinery/pkg/util/wait"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/client/config"
+	"sigs.k8s.io/controller-runtime/pkg/log"
+	"sigs.k8s.io/controller-runtime/pkg/log/zap"
 )
 
+type KubeUtils struct{}
+
+var _ KubeUtilsInterface = (*KubeUtils)(nil)
+
 type ErrNoInstallations struct{}
 
 func (e ErrNoInstallations) Error() string {
@@ -44,12 +53,12 @@ func BackOffToDuration(backoff wait.Backoff) time.Duration {
 	return total
 }
 
-func WaitForNamespace(ctx context.Context, cli client.Client, ns string) error {
+func (k *KubeUtils) WaitForNamespace(ctx context.Context, cli client.Client, ns string) error {
 	backoff := wait.Backoff{Steps: 60, Duration: 5 * time.Second, Factor: 1.0, Jitter: 0.1}
 	var lasterr error
 	if err := wait.ExponentialBackoffWithContext(
 		ctx, backoff, func(ctx context.Context) (bool, error) {
-			ready, err := IsNamespaceReady(ctx, cli, ns)
+			ready, err := k.IsNamespaceReady(ctx, cli, ns)
 			if err != nil {
 				lasterr = fmt.Errorf("unable to get namespace %s status: %v", ns, err)
 				return false, nil
@@ -67,12 +76,12 @@ func WaitForNamespace(ctx context.Context, cli client.Client, ns string) error {
 }
 
 // WaitForDeployment waits for the provided deployment to be ready.
-func WaitForDeployment(ctx context.Context, cli client.Client, ns, name string) error {
+func (k *KubeUtils) WaitForDeployment(ctx context.Context, cli client.Client, ns, name string) error {
 	backoff := wait.Backoff{Steps: 60, Duration: 5 * time.Second, Factor: 1.0, Jitter: 0.1}
 	var lasterr error
 	if err := wait.ExponentialBackoffWithContext(
 		ctx, backoff, func(ctx context.Context) (bool, error) {
-			ready, err := IsDeploymentReady(ctx, cli, ns, name)
+			ready, err := k.IsDeploymentReady(ctx, cli, ns, name)
 			if err != nil {
 				lasterr = fmt.Errorf("unable to get deploy %s status: %v", name, err)
 				return false, nil
@@ -90,12 +99,12 @@ func WaitForDeployment(ctx context.Context, cli client.Client, ns, name string)
 }
 
 // WaitForDaemonset waits for the provided daemonset to be ready.
-func WaitForDaemonset(ctx context.Context, cli client.Client, ns, name string) error {
+func (k *KubeUtils) WaitForDaemonset(ctx context.Context, cli client.Client, ns, name string) error {
 	backoff := wait.Backoff{Steps: 60, Duration: 5 * time.Second, Factor: 1.0, Jitter: 0.1}
 	var lasterr error
 	if err := wait.ExponentialBackoffWithContext(
 		ctx, backoff, func(ctx context.Context) (bool, error) {
-			ready, err := IsDaemonsetReady(ctx, cli, ns, name)
+			ready, err := k.IsDaemonsetReady(ctx, cli, ns, name)
 			if err != nil {
 				lasterr = fmt.Errorf("unable to get daemonset %s status: %v", name, err)
 				return false, nil
@@ -112,7 +121,7 @@ func WaitForDaemonset(ctx context.Context, cli client.Client, ns, name string) e
 	return nil
 }
 
-func WaitForService(ctx context.Context, cli client.Client, ns, name string) error {
+func (k *KubeUtils) WaitForService(ctx context.Context, cli client.Client, ns, name string) error {
 	backoff := wait.Backoff{Steps: 60, Duration: 5 * time.Second, Factor: 1.0, Jitter: 0.1}
 	var lasterr error
 	if err := wait.ExponentialBackoffWithContext(
@@ -135,7 +144,7 @@ func WaitForService(ctx context.Context, cli client.Client, ns, name string) err
 	return nil
 }
 
-func WaitForInstallation(ctx context.Context, cli client.Client, writer *spinner.MessageWriter) error {
+func (k *KubeUtils) WaitForInstallation(ctx context.Context, cli client.Client, writer *spinner.MessageWriter) error {
 	backoff := wait.Backoff{Steps: 60 * 5, Duration: time.Second, Factor: 1.0, Jitter: 0.1}
 	var lasterr error
 
@@ -340,7 +349,7 @@ func writeStatusMessage(writer *spinner.MessageWriter, install *embeddedclusterv
 	}
 }
 
-func WaitForHAInstallation(ctx context.Context, cli client.Client) error {
+func (k *KubeUtils) WaitForHAInstallation(ctx context.Context, cli client.Client) error {
 	for {
 		select {
 		case <-ctx.Done():
@@ -369,7 +378,7 @@ func CheckConditionStatus(inStat embeddedclusterv1beta1.InstallationStatus, cond
 	return ""
 }
 
-func WaitForNodes(ctx context.Context, cli client.Client) error {
+func (k *KubeUtils) WaitForNodes(ctx context.Context, cli client.Client) error {
 	backoff := wait.Backoff{Steps: 60, Duration: 5 * time.Second, Factor: 1.0, Jitter: 0.1}
 	var lasterr error
 	if err := wait.ExponentialBackoffWithContext(
@@ -400,7 +409,7 @@ func WaitForNodes(ctx context.Context, cli client.Client) error {
 }
 
 // WaitForControllerNode waits for a specific controller node to be registered with the cluster.
-func WaitForControllerNode(ctx context.Context, kcli client.Client, name string) error {
+func (k *KubeUtils) WaitForControllerNode(ctx context.Context, kcli client.Client, name string) error {
 	backoff := wait.Backoff{Steps: 60, Duration: 5 * time.Second, Factor: 1.0, Jitter: 0.1}
 	var lasterr error
 	if err := wait.ExponentialBackoffWithContext(
@@ -433,12 +442,12 @@ func WaitForControllerNode(ctx context.Context, kcli client.Client, name string)
 }
 
 // WaitForJob waits for a job to have a certain number of completions.
-func WaitForJob(ctx context.Context, cli client.Client, ns, name string, maxSteps int, completions int32) error {
+func (k *KubeUtils) WaitForJob(ctx context.Context, cli client.Client, ns, name string, maxSteps int, completions int32) error {
 	backoff := wait.Backoff{Steps: maxSteps, Duration: 5 * time.Second, Factor: 1.0, Jitter: 0.1}
 	var lasterr error
 	if err := wait.ExponentialBackoffWithContext(
 		ctx, backoff, func(ctx context.Context) (bool, error) {
-			ready, err := IsJobComplete(ctx, cli, ns, name, completions)
+			ready, err := k.IsJobComplete(ctx, cli, ns, name, completions)
 			if err != nil {
 				lasterr = fmt.Errorf("unable to get job status: %w", err)
 				return false, nil
@@ -455,7 +464,7 @@ func WaitForJob(ctx context.Context, cli client.Client, ns, name string, maxStep
 	return nil
 }
 
-func IsNamespaceReady(ctx context.Context, cli client.Client, ns string) (bool, error) {
+func (k *KubeUtils) IsNamespaceReady(ctx context.Context, cli client.Client, ns string) (bool, error) {
 	var namespace corev1.Namespace
 	if err := cli.Get(ctx, types.NamespacedName{Name: ns}, &namespace); err != nil {
 		return false, err
@@ -464,7 +473,7 @@ func IsNamespaceReady(ctx context.Context, cli client.Client, ns string) (bool,
 }
 
 // IsDeploymentReady returns true if the deployment is ready.
-func IsDeploymentReady(ctx context.Context, cli client.Client, ns, name string) (bool, error) {
+func (k *KubeUtils) IsDeploymentReady(ctx context.Context, cli client.Client, ns, name string) (bool, error) {
 	var deploy appsv1.Deployment
 	nsn := types.NamespacedName{Namespace: ns, Name: name}
 	if err := cli.Get(ctx, nsn, &deploy); err != nil {
@@ -477,7 +486,7 @@ func IsDeploymentReady(ctx context.Context, cli client.Client, ns, name string)
 }
 
 // IsStatefulSetReady returns true if the statefulset is ready.
-func IsStatefulSetReady(ctx context.Context, cli client.Client, ns, name string) (bool, error) {
+func (k *KubeUtils) IsStatefulSetReady(ctx context.Context, cli client.Client, ns, name string) (bool, error) {
 	var statefulset appsv1.StatefulSet
 	nsn := types.NamespacedName{Namespace: ns, Name: name}
 	if err := cli.Get(ctx, nsn, &statefulset); err != nil {
@@ -490,7 +499,7 @@ func IsStatefulSetReady(ctx context.Context, cli client.Client, ns, name string)
 }
 
 // IsDaemonsetReady returns true if the daemonset is ready.
-func IsDaemonsetReady(ctx context.Context, cli client.Client, ns, name string) (bool, error) {
+func (k *KubeUtils) IsDaemonsetReady(ctx context.Context, cli client.Client, ns, name string) (bool, error) {
 	var daemonset appsv1.DaemonSet
 	nsn := types.NamespacedName{Namespace: ns, Name: name}
 	if err := cli.Get(ctx, nsn, &daemonset); err != nil {
@@ -503,7 +512,7 @@ func IsDaemonsetReady(ctx context.Context, cli client.Client, ns, name string) (
 }
 
 // IsJobComplete returns true if the job has been completed successfully.
-func IsJobComplete(ctx context.Context, cli client.Client, ns, name string, completions int32) (bool, error) {
+func (k *KubeUtils) IsJobComplete(ctx context.Context, cli client.Client, ns, name string, completions int32) (bool, error) {
 	var job batchv1.Job
 	nsn := types.NamespacedName{Namespace: ns, Name: name}
 	if err := cli.Get(ctx, nsn, &job); err != nil {
@@ -517,7 +526,7 @@ func IsJobComplete(ctx context.Context, cli client.Client, ns, name string, comp
 
 // WaitForKubernetes waits for all deployments to be ready in kube-system, and returns an error channel.
 // if either of them fails to become healthy, an error is returned via the channel.
-func WaitForKubernetes(ctx context.Context, cli client.Client) <-chan error {
+func (k *KubeUtils) WaitForKubernetes(ctx context.Context, cli client.Client) <-chan error {
 	errch := make(chan error, 1)
 
 	// wait until there is at least one deployment in kube-system
@@ -538,7 +547,7 @@ func WaitForKubernetes(ctx context.Context, cli client.Client) <-chan error {
 
 	for _, dep := range deps.Items {
 		go func(depName string) {
-			err := WaitForDeployment(ctx, cli, "kube-system", depName)
+			err := k.WaitForDeployment(ctx, cli, "kube-system", depName)
 			if err != nil {
 				errch <- fmt.Errorf("%s failed to become healthy: %w", depName, err)
 			}
@@ -560,3 +569,35 @@ func NumOfControlPlaneNodes(ctx context.Context, cli client.Client) (int, error)
 	}
 	return len(nodes.Items), nil
 }
+
+func (k *KubeUtils) WaitAndMarkInstallation(ctx context.Context, cli client.Client, name string, state string) error {
+	for i := 0; i < 20; i++ {
+		in, err := GetInstallation(ctx, cli, name)
+		if err != nil {
+			if !errors.Is(err, ErrNoInstallations{}) {
+				return fmt.Errorf("unable to get installation: %w", err)
+			}
+		} else {
+			in.Status.State = state
+			if err := cli.Status().Update(ctx, in); err != nil {
+				return fmt.Errorf("unable to update installation status: %w", err)
+			}
+			return nil
+		}
+		time.Sleep(time.Second)
+	}
+	return fmt.Errorf("installation %s not found after 20 seconds", name)
+}
+
+// KubeClient returns a new kubernetes client.
+func (k *KubeUtils) KubeClient() (client.Client, error) {
+	k8slogger := zap.New(func(o *zap.Options) {
+		o.DestWriter = io.Discard
+	})
+	log.SetLogger(k8slogger)
+	cfg, err := config.GetConfig()
+	if err != nil {
+		return nil, fmt.Errorf("unable to process kubernetes config: %w", err)
+	}
+	return client.New(cfg, client.Options{})
+}
diff --git a/pkg/metrics/interface.go b/pkg/metrics/interface.go
new file mode 100644
index 000000000..08f40c1bd
--- /dev/null
+++ b/pkg/metrics/interface.go
@@ -0,0 +1,41 @@
+package metrics
+
+import (
+	"context"
+
+	"github.com/replicatedhq/embedded-cluster/pkg/metrics/types"
+	"github.com/sirupsen/logrus"
+)
+
+var s SenderInterface
+
+// Sender sends events to the metrics endpoint.
+type Sender struct{}
+
+var _ SenderInterface = (*Sender)(nil)
+
+func init() {
+	Set(&Sender{})
+}
+
+func Set(_s SenderInterface) {
+	s = _s
+}
+
+type SenderInterface interface {
+	Send(ctx context.Context, baseURL string, ev types.Event)
+}
+
+// Convenience functions
+
+// Send is a helper function that sends an event to the metrics endpoint.
+// Metrics endpoint can be overwritten by the license.spec.endpoint field
+// or by the EMBEDDED_CLUSTER_METRICS_BASEURL environment variable, the latter has
+// precedence over the former.
+func Send(ctx context.Context, baseURL string, ev types.Event) {
+	if metricsDisabled {
+		logrus.Debugf("metrics are disabled, not sending event %s", ev.Title())
+		return
+	}
+	s.Send(ctx, baseURL, ev)
+}
diff --git a/pkg/metrics/reporter.go b/pkg/metrics/reporter.go
index 9ea1bae15..558c8b912 100644
--- a/pkg/metrics/reporter.go
+++ b/pkg/metrics/reporter.go
@@ -13,6 +13,7 @@ import (
 
 	"github.com/replicatedhq/embedded-cluster/pkg/defaults"
 	"github.com/replicatedhq/embedded-cluster/pkg/helpers"
+	"github.com/replicatedhq/embedded-cluster/pkg/metrics/types"
 	"github.com/replicatedhq/embedded-cluster/pkg/release"
 	"github.com/replicatedhq/embedded-cluster/pkg/versions"
 	kotsv1beta1 "github.com/replicatedhq/kotskinds/apis/kots/v1beta1"
@@ -73,7 +74,7 @@ func ReportInstallationStarted(ctx context.Context, license *kotsv1beta1.License
 		appVersion = rel.VersionLabel
 	}
 
-	Send(ctx, BaseURL(license), InstallationStarted{
+	Send(ctx, BaseURL(license), types.InstallationStarted{
 		ClusterID:    ClusterID(),
 		Version:      versions.Version,
 		Flags:        strings.Join(os.Args[1:], " "),
@@ -87,12 +88,16 @@ func ReportInstallationStarted(ctx context.Context, license *kotsv1beta1.License
 
 // ReportInstallationSucceeded reports that the installation has succeeded.
 func ReportInstallationSucceeded(ctx context.Context, license *kotsv1beta1.License) {
-	Send(ctx, BaseURL(license), InstallationSucceeded{ClusterID: ClusterID(), Version: versions.Version})
+	Send(ctx, BaseURL(license), types.InstallationSucceeded{ClusterID: ClusterID(), Version: versions.Version})
 }
 
 // ReportInstallationFailed reports that the installation has failed.
 func ReportInstallationFailed(ctx context.Context, license *kotsv1beta1.License, err error) {
-	Send(ctx, BaseURL(license), InstallationFailed{ClusterID(), versions.Version, err.Error()})
+	Send(ctx, BaseURL(license), types.InstallationFailed{
+		ClusterID: ClusterID(),
+		Version:   versions.Version,
+		Reason:    err.Error(),
+	})
 }
 
 // ReportJoinStarted reports that a join has started.
@@ -102,7 +107,11 @@ func ReportJoinStarted(ctx context.Context, baseURL string, clusterID uuid.UUID)
 		logrus.Warnf("unable to get hostname: %s", err)
 		hostname = "unknown"
 	}
-	Send(ctx, baseURL, JoinStarted{clusterID, versions.Version, hostname})
+	Send(ctx, baseURL, types.JoinStarted{
+		ClusterID: clusterID,
+		Version:   versions.Version,
+		NodeName:  hostname,
+	})
 }
 
 // ReportJoinSucceeded reports that a join has finished successfully.
@@ -112,7 +121,11 @@ func ReportJoinSucceeded(ctx context.Context, baseURL string, clusterID uuid.UUI
 		logrus.Warnf("unable to get hostname: %s", err)
 		hostname = "unknown"
 	}
-	Send(ctx, baseURL, JoinSucceeded{clusterID, versions.Version, hostname})
+	Send(ctx, baseURL, types.JoinSucceeded{
+		ClusterID: clusterID,
+		Version:   versions.Version,
+		NodeName:  hostname,
+	})
 }
 
 // ReportJoinFailed reports that a join has failed.
@@ -122,7 +135,12 @@ func ReportJoinFailed(ctx context.Context, baseURL string, clusterID uuid.UUID,
 		logrus.Warnf("unable to get hostname: %s", err)
 		hostname = "unknown"
 	}
-	Send(ctx, baseURL, JoinFailed{clusterID, versions.Version, hostname, exterr.Error()})
+	Send(ctx, baseURL, types.JoinFailed{
+		ClusterID: clusterID,
+		Version:   versions.Version,
+		NodeName:  hostname,
+		Reason:    exterr.Error(),
+	})
 }
 
 // ReportApplyStarted reports an InstallationStarted event.
diff --git a/pkg/metrics/sender.go b/pkg/metrics/sender.go
index fe1af08a5..0172d1246 100644
--- a/pkg/metrics/sender.go
+++ b/pkg/metrics/sender.go
@@ -3,37 +3,16 @@ package metrics
 import (
 	"bytes"
 	"context"
-	"encoding/json"
-	"fmt"
 	"net/http"
 
-	"github.com/replicatedhq/embedded-cluster/pkg/versions"
+	"github.com/replicatedhq/embedded-cluster/pkg/metrics/types"
 	"github.com/sirupsen/logrus"
 )
 
-// Send is a helper function that sends an event to the metrics endpoint.
-// Metrics endpoint can be overwritten by the license.spec.endpoint field
-// or by the EMBEDDED_CLUSTER_METRICS_BASEURL environment variable, the latter has
-// precedence over the former.
-func Send(ctx context.Context, baseURL string, ev Event) {
-	sender := Sender{baseURL}
-	sender.Send(ctx, ev)
-}
-
-// Sender sends events to the metrics endpoint.
-type Sender struct {
-	baseURL string
-}
-
 // Send sends an event to the metrics endpoint.
-func (s *Sender) Send(ctx context.Context, ev Event) {
-	if metricsDisabled {
-		logrus.Debugf("metrics are disabled, not sending event %s", ev.Title())
-		return
-	}
-
-	url := fmt.Sprintf("%s/embedded_cluster_metrics/%s", s.baseURL, ev.Title())
-	payload, err := s.payload(ev)
+func (s *Sender) Send(ctx context.Context, baseURL string, ev types.Event) {
+	url := EventURL(baseURL, ev)
+	payload, err := EventPayload(ev)
 	if err != nil {
 		logrus.Debugf("unable to get payload for event %s: %s", ev.Title(), err)
 		return
@@ -54,13 +33,3 @@ func (s *Sender) Send(ctx context.Context, ev Event) {
 		logrus.Debugf("unable to confirm event %s: %d", ev.Title(), response.StatusCode)
 	}
 }
-
-// payload returns the payload to be sent to the metrics endpoint.
-func (s *Sender) payload(ev Event) ([]byte, error) {
-	vmap := map[string]string{
-		"EmbeddedCluster": versions.Version,
-		"Kubernetes":      versions.K0sVersion,
-	}
-	payload := map[string]interface{}{"event": ev, "versions": vmap}
-	return json.Marshal(payload)
-}
diff --git a/pkg/metrics/sender_test.go b/pkg/metrics/sender_test.go
index 6f750e7c9..a1f2bbe14 100644
--- a/pkg/metrics/sender_test.go
+++ b/pkg/metrics/sender_test.go
@@ -11,17 +11,18 @@ import (
 	"testing"
 
 	"github.com/google/uuid"
+	"github.com/replicatedhq/embedded-cluster/pkg/metrics/types"
 	"github.com/stretchr/testify/assert"
 )
 
 func TestSend(t *testing.T) {
 	for _, tt := range []struct {
 		name  string
-		event Event
+		event types.Event
 	}{
 		{
 			name: "InstallationStarted",
-			event: InstallationStarted{
+			event: types.InstallationStarted{
 				ClusterID:  uuid.New(),
 				Version:    "1.2.3",
 				Flags:      "foo",
@@ -32,34 +33,34 @@ func TestSend(t *testing.T) {
 		},
 		{
 			name: "InstallationSucceeded",
-			event: InstallationSucceeded{
+			event: types.InstallationSucceeded{
 				ClusterID: uuid.New(),
 			},
 		},
 		{
 			name: "InstallationFailed",
-			event: InstallationFailed{
+			event: types.InstallationFailed{
 				ClusterID: uuid.New(),
 				Reason:    "foo",
 			},
 		},
 		{
 			name: "JoinStarted",
-			event: JoinStarted{
+			event: types.JoinStarted{
 				ClusterID: uuid.New(),
 				NodeName:  "foo",
 			},
 		},
 		{
 			name: "JoinSucceeded",
-			event: JoinSucceeded{
+			event: types.JoinSucceeded{
 				ClusterID: uuid.New(),
 				NodeName:  "foo",
 			},
 		},
 		{
 			name: "JoinFailed",
-			event: JoinFailed{
+			event: types.JoinFailed{
 				ClusterID: uuid.New(),
 				NodeName:  "foo",
 				Reason:    "bar",
@@ -89,8 +90,7 @@ func TestSend(t *testing.T) {
 				),
 			)
 			defer server.Close()
-			sender := Sender{baseURL: server.URL}
-			sender.Send(context.Background(), tt.event)
+			Send(context.Background(), server.URL, tt.event)
 		})
 	}
 }
diff --git a/pkg/metrics/events.go b/pkg/metrics/types/types.go
similarity index 99%
rename from pkg/metrics/events.go
rename to pkg/metrics/types/types.go
index ec5fe2483..751b38b2e 100644
--- a/pkg/metrics/events.go
+++ b/pkg/metrics/types/types.go
@@ -1,4 +1,4 @@
-package metrics
+package types
 
 import (
 	"github.com/google/uuid"
diff --git a/pkg/metrics/util.go b/pkg/metrics/util.go
new file mode 100644
index 000000000..e99096c66
--- /dev/null
+++ b/pkg/metrics/util.go
@@ -0,0 +1,24 @@
+package metrics
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"github.com/replicatedhq/embedded-cluster/pkg/metrics/types"
+	"github.com/replicatedhq/embedded-cluster/pkg/versions"
+)
+
+// EventURL returns the URL to be used when sending an event to the metrics endpoint.
+func EventURL(baseURL string, ev types.Event) string {
+	return fmt.Sprintf("%s/embedded_cluster_metrics/%s", baseURL, ev.Title())
+}
+
+// EventPayload returns the payload to be sent to the metrics endpoint.
+func EventPayload(ev types.Event) ([]byte, error) {
+	vmap := map[string]string{
+		"EmbeddedCluster": versions.Version,
+		"Kubernetes":      versions.K0sVersion,
+	}
+	payload := map[string]interface{}{"event": ev, "versions": vmap}
+	return json.Marshal(payload)
+}
diff --git a/scripts/dryrun-tests.sh b/scripts/dryrun-tests.sh
new file mode 100755
index 000000000..d3839ba82
--- /dev/null
+++ b/scripts/dryrun-tests.sh
@@ -0,0 +1,56 @@
+#!/bin/bash
+
+set -eo pipefail
+
+function plog() {
+    local type="$1"
+    local message="$2"
+    local emoji="$3"
+    echo -e "[$(date +'%Y-%m-%d %H:%M:%S')] $emoji [$type] $message"
+}
+
+# Build the test container
+plog "INFO" "Building test container..." "🔨"
+docker build -q -t ec-dryrun ./tests/dryrun > /dev/null
+
+# Get all test functions
+tests=$(grep -o 'func Test[^ (]*' ./tests/dryrun/*.go | awk '{print $2}')
+
+# Run tests in separate containers
+for test in $tests; do
+    plog "INFO" "Starting test: $test" "🚀"
+    docker rm -f --volumes "$test" > /dev/null 2>&1 || true
+    docker run -d \
+        -v "$(pwd)":/ec \
+        -w /ec \
+        -e GOCACHE=/ec/dev/.gocache \
+        -e GOMODCACHE=/ec/dev/.gomodcache \
+        --name "$test" \
+        ec-dryrun \
+        go test -timeout 1m -v ./tests/dryrun/... -run "^$test$" > /dev/null
+done
+
+plog "INFO" "Waiting for tests to complete..." "⏳"
+
+# Check test results
+failed_tests=()
+for test in $tests; do
+    exit_code=$(docker wait "$test")
+    if [ "$exit_code" -ne 0 ]; then
+        failed_tests+=("$test")
+        plog "ERROR" "$test failed" "❌"
+        docker logs "$test"
+    else
+        plog "INFO" "$test passed" "✅"
+        docker rm -f --volumes "$test" > /dev/null
+    fi
+done
+
+# Display final summary
+if [ ${#failed_tests[@]} -eq 0 ]; then
+    plog "SUCCESS" "All tests passed successfully!" "🎉"
+    exit 0
+else
+    plog "FAILURE" "Some tests failed: ${failed_tests[*]}" "🚨"
+    exit 1
+fi
diff --git a/tests/dryrun/Dockerfile b/tests/dryrun/Dockerfile
new file mode 100644
index 000000000..95f031da7
--- /dev/null
+++ b/tests/dryrun/Dockerfile
@@ -0,0 +1,5 @@
+FROM golang:1.23-alpine AS build
+
+RUN apk add --no-cache ca-certificates curl git make bash
+
+RUN mkdir -p /etc/systemd/system
diff --git a/tests/dryrun/assets/install-license.yaml b/tests/dryrun/assets/install-license.yaml
new file mode 100644
index 000000000..df850ab91
--- /dev/null
+++ b/tests/dryrun/assets/install-license.yaml
@@ -0,0 +1,36 @@
+apiVersion: kots.io/v1beta1
+kind: License
+metadata:
+  name: dryrun-install
+spec:
+  appSlug: fake-app-slug
+  channelID: fake-channel-id
+  channelName: fake-channel-name
+  channels:
+  - channelID: fake-channel-id
+    channelName: fake-channel-name
+    channelSlug: fake-channel-slug
+    endpoint: https://fake-endpoint.com
+    isDefault: true
+    replicatedProxyDomain: fake-replicated-proxy.test.net
+  customerEmail: salah@replicated.com
+  customerName: Salah EC Dev
+  endpoint: https://fake-endpoint.com
+  entitlements:
+    expires_at:
+      description: License Expiration
+      signature: {}
+      title: Expiration
+      value: ""
+      valueType: String
+  isDisasterRecoverySupported: true
+  isEmbeddedClusterDownloadEnabled: true
+  isKotsInstallEnabled: true
+  isNewKotsUiEnabled: true
+  isSnapshotSupported: true
+  isSupportBundleUploadSupported: true
+  licenseID: fake-license-id
+  licenseSequence: 4
+  licenseType: dev
+  replicatedProxyDomain: fake-replicated-proxy.test.net
+  signature: ZmFrZS1zaWduYXR1cmU=
diff --git a/tests/dryrun/assets/install-release.yaml b/tests/dryrun/assets/install-release.yaml
new file mode 100644
index 000000000..c887b4acd
--- /dev/null
+++ b/tests/dryrun/assets/install-release.yaml
@@ -0,0 +1,5 @@
+# channel release object
+channelID: "fake-channel-id"
+channelSlug: "fake-channel-slug"
+appSlug: "fake-app-slug"
+versionLabel: "fake-version-label"
diff --git a/tests/dryrun/install_test.go b/tests/dryrun/install_test.go
new file mode 100644
index 000000000..d6291dc58
--- /dev/null
+++ b/tests/dryrun/install_test.go
@@ -0,0 +1,242 @@
+package dryrun
+
+import (
+	"context"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/replicatedhq/embedded-cluster/pkg/kubeutils"
+	troubleshootv1beta2 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta2"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestDefaultInstallation(t *testing.T) {
+	dr := dryrunInstall(t)
+
+	// --- validate os env --- //
+	assertEnv(t, dr.OSEnv, map[string]string{
+		"TMPDIR":     "/var/lib/embedded-cluster/tmp",
+		"KUBECONFIG": "/var/lib/embedded-cluster/k0s/pki/admin.conf",
+	})
+
+	// --- validate commands --- //
+	for _, c := range dr.Commands {
+		if strings.Contains(c.Cmd, "k0s install controller") {
+			assert.Contains(t, c.Cmd, "--data-dir /var/lib/embedded-cluster/k0s")
+		}
+	}
+
+	// --- validate host preflight spec --- //
+	assertCollectors(t, dr.HostPreflightSpec.Collectors, map[string]struct {
+		match    func(*troubleshootv1beta2.HostCollect) bool
+		validate func(*troubleshootv1beta2.HostCollect)
+	}{
+		"FilesystemPerformance": {
+			match: func(hc *troubleshootv1beta2.HostCollect) bool {
+				return hc.FilesystemPerformance != nil
+			},
+			validate: func(hc *troubleshootv1beta2.HostCollect) {
+				assert.Equal(t, "/var/lib/embedded-cluster/k0s/etcd", hc.FilesystemPerformance.Directory)
+			},
+		},
+		"LAM TCPPortStatus": {
+			match: func(hc *troubleshootv1beta2.HostCollect) bool {
+				return hc.TCPPortStatus != nil && hc.TCPPortStatus.CollectorName == "Local Artifact Mirror Port"
+			},
+			validate: func(hc *troubleshootv1beta2.HostCollect) {
+				assert.Equal(t, 50000, hc.TCPPortStatus.Port)
+			},
+		},
+		"Kotsadm TCPPortStatus": {
+			match: func(hc *troubleshootv1beta2.HostCollect) bool {
+				return hc.TCPPortStatus != nil && hc.TCPPortStatus.CollectorName == "Kotsadm Node Port"
+			},
+			validate: func(hc *troubleshootv1beta2.HostCollect) {
+				assert.Equal(t, 30000, hc.TCPPortStatus.Port)
+			},
+		},
+	})
+
+	// --- validate metrics --- //
+	assertMetrics(t, dr.Metrics, []struct {
+		title    string
+		validate func(string)
+	}{
+		{
+			title:    "InstallationStarted",
+			validate: func(payload string) {},
+		},
+		{
+			title:    "InstallationSucceeded",
+			validate: func(payload string) {},
+		},
+	})
+
+	// --- validate cluster resources --- //
+	kcli, err := dr.KubeClient()
+	if err != nil {
+		t.Fatalf("failed to create kube client: %v", err)
+	}
+
+	assertConfigMapExists(t, kcli, "embedded-cluster-host-support-bundle", "kotsadm")
+	assertSecretExists(t, kcli, "kotsadm-password", "kotsadm")
+	assertSecretExists(t, kcli, "cloud-credentials", "velero")
+
+	// --- validate installation object --- //
+	in, err := kubeutils.GetLatestInstallation(context.TODO(), kcli)
+	if err != nil {
+		t.Fatalf("failed to get latest installation: %v", err)
+	}
+
+	assert.Equal(t, "80-32767", in.Spec.Network.NodePortRange)
+	assert.Equal(t, "10.244.0.0/16", dr.Flags["cidr"])
+	assert.Equal(t, "10.244.0.0/17", in.Spec.Network.PodCIDR)
+	assert.Equal(t, "10.244.128.0/17", in.Spec.Network.ServiceCIDR)
+	assert.Equal(t, 30000, in.Spec.RuntimeConfig.AdminConsole.Port)
+	assert.Equal(t, "/var/lib/embedded-cluster", in.Spec.RuntimeConfig.DataDir)
+	assert.Equal(t, 50000, in.Spec.RuntimeConfig.LocalArtifactMirror.Port)
+	assert.Equal(t, "ec-install", in.ObjectMeta.Labels["replicated.com/disaster-recovery"])
+
+	// --- validate k0s cluster config --- //
+	k0sConfig := readK0sConfig(t)
+
+	assert.Equal(t, "10.244.0.0/17", k0sConfig.Spec.Network.PodCIDR)
+	assert.Equal(t, "10.244.128.0/17", k0sConfig.Spec.Network.ServiceCIDR)
+
+	assertHelmValues(t, k0sConfig, "openebs", map[string]interface{}{
+		"['localpv-provisioner'].localpv.basePath": "/var/lib/embedded-cluster/openebs-local",
+	})
+	assertHelmValues(t, k0sConfig, "velero", map[string]interface{}{
+		"nodeAgent.podVolumePath": "/var/lib/embedded-cluster/k0s/kubelet/pods",
+	})
+
+	t.Logf("%s: test complete", time.Now().Format(time.RFC3339))
+}
+
+func TestCustomDataDir(t *testing.T) {
+	dr := dryrunInstall(t,
+		"--data-dir", "/custom/data/dir",
+	)
+
+	// --- validate os env --- //
+	assertEnv(t, dr.OSEnv, map[string]string{
+		"TMPDIR":     "/custom/data/dir/tmp",
+		"KUBECONFIG": "/custom/data/dir/k0s/pki/admin.conf",
+	})
+
+	// --- validate commands --- //
+	for _, c := range dr.Commands {
+		if strings.Contains(c.Cmd, "k0s install controller") {
+			assert.Contains(t, c.Cmd, "--data-dir /custom/data/dir/k0s")
+		}
+	}
+
+	// --- validate host preflight spec --- //
+	assertCollectors(t, dr.HostPreflightSpec.Collectors, map[string]struct {
+		match    func(*troubleshootv1beta2.HostCollect) bool
+		validate func(*troubleshootv1beta2.HostCollect)
+	}{
+		"FilesystemPerformance": {
+			match: func(hc *troubleshootv1beta2.HostCollect) bool {
+				return hc.FilesystemPerformance != nil
+			},
+			validate: func(hc *troubleshootv1beta2.HostCollect) {
+				assert.Equal(t, "/custom/data/dir/k0s/etcd", hc.FilesystemPerformance.Directory)
+			},
+		},
+	})
+
+	// --- validate installation object --- //
+	kcli, err := dr.KubeClient()
+	if err != nil {
+		t.Fatalf("failed to create kube client: %v", err)
+	}
+	in, err := kubeutils.GetLatestInstallation(context.TODO(), kcli)
+	if err != nil {
+		t.Fatalf("failed to get latest installation: %v", err)
+	}
+	assert.Equal(t, "/custom/data/dir", in.Spec.RuntimeConfig.DataDir)
+
+	// --- validate k0s cluster config --- //
+	k0sConfig := readK0sConfig(t)
+
+	assertHelmValues(t, k0sConfig, "openebs", map[string]interface{}{
+		"['localpv-provisioner'].localpv.basePath": "/custom/data/dir/openebs-local",
+	})
+	assertHelmValues(t, k0sConfig, "velero", map[string]interface{}{
+		"nodeAgent.podVolumePath": "/custom/data/dir/k0s/kubelet/pods",
+	})
+
+	t.Logf("%s: test complete", time.Now().Format(time.RFC3339))
+}
+
+func TestCustomPortsInstallation(t *testing.T) {
+	dr := dryrunInstall(t,
+		"--local-artifact-mirror-port", "50001",
+		"--admin-console-port", "30002",
+	)
+
+	// --- validate host preflight spec --- //
+	assertCollectors(t, dr.HostPreflightSpec.Collectors, map[string]struct {
+		match    func(*troubleshootv1beta2.HostCollect) bool
+		validate func(*troubleshootv1beta2.HostCollect)
+	}{
+		"LAM TCPPortStatus": {
+			match: func(hc *troubleshootv1beta2.HostCollect) bool {
+				return hc.TCPPortStatus != nil && hc.TCPPortStatus.CollectorName == "Local Artifact Mirror Port"
+			},
+			validate: func(hc *troubleshootv1beta2.HostCollect) {
+				assert.Equal(t, 50001, hc.TCPPortStatus.Port)
+			},
+		},
+		"Kotsadm TCPPortStatus": {
+			match: func(hc *troubleshootv1beta2.HostCollect) bool {
+				return hc.TCPPortStatus != nil && hc.TCPPortStatus.CollectorName == "Kotsadm Node Port"
+			},
+			validate: func(hc *troubleshootv1beta2.HostCollect) {
+				assert.Equal(t, 30002, hc.TCPPortStatus.Port)
+			},
+		},
+	})
+
+	// --- validate metrics --- //
+	assertMetrics(t, dr.Metrics, []struct {
+		title    string
+		validate func(string)
+	}{
+		{
+			title: "InstallationStarted",
+			validate: func(payload string) {
+				assert.Contains(t, payload, "--local-artifact-mirror-port 50001")
+				assert.Contains(t, payload, "--admin-console-port 30002")
+			},
+		},
+		{
+			title:    "InstallationSucceeded",
+			validate: func(payload string) {},
+		},
+	})
+
+	// --- validate installation object --- //
+	kcli, err := dr.KubeClient()
+	if err != nil {
+		t.Fatalf("failed to create kube client: %v", err)
+	}
+	in, err := kubeutils.GetLatestInstallation(context.TODO(), kcli)
+	if err != nil {
+		t.Fatalf("failed to get latest installation: %v", err)
+	}
+
+	assert.Equal(t, 30002, in.Spec.RuntimeConfig.AdminConsole.Port)
+	assert.Equal(t, 50001, in.Spec.RuntimeConfig.LocalArtifactMirror.Port)
+
+	// --- validate k0s cluster config --- //
+	k0sConfig := readK0sConfig(t)
+
+	assertHelmValues(t, k0sConfig, "admin-console", map[string]interface{}{
+		"kurlProxy.nodePort": float64(30002),
+	})
+
+	t.Logf("%s: test complete", time.Now().Format(time.RFC3339))
+}
diff --git a/tests/dryrun/util.go b/tests/dryrun/util.go
new file mode 100644
index 000000000..63b10f272
--- /dev/null
+++ b/tests/dryrun/util.go
@@ -0,0 +1,178 @@
+package dryrun
+
+import (
+	"context"
+	_ "embed"
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"testing"
+
+	k0sv1beta1 "github.com/k0sproject/k0s/pkg/apis/k0s/v1beta1"
+	"github.com/replicatedhq/embedded-cluster/pkg/cmd"
+	"github.com/replicatedhq/embedded-cluster/pkg/defaults"
+	dryruntypes "github.com/replicatedhq/embedded-cluster/pkg/dryrun/types"
+	"github.com/replicatedhq/embedded-cluster/pkg/helm"
+	"github.com/replicatedhq/embedded-cluster/pkg/release"
+	troubleshootv1beta2 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta2"
+	"github.com/stretchr/testify/assert"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/yaml"
+)
+
+//go:embed assets/install-release.yaml
+var releaseData string
+
+func dryrunInstall(t *testing.T, args ...string) dryruntypes.DryRun {
+	if err := embedReleaseData(); err != nil {
+		t.Fatalf("fail to embed release data: %v", err)
+	}
+
+	drFile := filepath.Join(t.TempDir(), "ec-dryrun.yaml")
+	defer os.Remove(drFile)
+
+	if err := runEmbeddedClusterCmd(
+		append([]string{
+			"install",
+			"--dry-run", drFile,
+			"--no-prompt",
+			"--license", "./assets/install-license.yaml",
+		}, args...)...,
+	); err != nil {
+		t.Fatalf("fail to dryrun install embedded-cluster: %v", err)
+	}
+
+	stdout, err := exec.Command("cat", drFile).Output()
+	if err != nil {
+		t.Fatalf("fail to get dryrun output: %v", err)
+	}
+
+	dr := dryruntypes.DryRun{}
+	if err := yaml.Unmarshal([]byte(stdout), &dr); err != nil {
+		t.Fatalf("fail to unmarshal dryrun output: %v", err)
+	}
+	return dr
+}
+
+func embedReleaseData() error {
+	if err := release.SetReleaseDataForTests(map[string][]byte{
+		"release.yaml": []byte(releaseData),
+	}); err != nil {
+		return fmt.Errorf("set release data: %v", err)
+	}
+	return nil
+}
+
+func runEmbeddedClusterCmd(args ...string) error {
+	fullArgs := append([]string{"embedded-cluster"}, args...)
+	os.Args = fullArgs // for reporting
+	return cmd.NewApp("embedded-cluster").Run(fullArgs)
+}
+
+func readK0sConfig(t *testing.T) k0sv1beta1.ClusterConfig {
+	stdout, err := exec.Command("cat", defaults.PathToK0sConfig()).Output()
+	if err != nil {
+		t.Fatalf("fail to get k0s config: %v", err)
+	}
+	k0sConfig := k0sv1beta1.ClusterConfig{}
+	if err := yaml.Unmarshal(stdout, &k0sConfig); err != nil {
+		t.Fatalf("fail to unmarshal k0s config: %v", err)
+	}
+	return k0sConfig
+}
+
+func assertCollectors(t *testing.T, actual []*troubleshootv1beta2.HostCollect, expected map[string]struct {
+	match    func(*troubleshootv1beta2.HostCollect) bool
+	validate func(*troubleshootv1beta2.HostCollect)
+}) {
+	found := make(map[string]bool)
+	for _, collector := range actual {
+		for name, assertion := range expected {
+			if assertion.match(collector) {
+				found[name] = true
+				assertion.validate(collector)
+			}
+		}
+	}
+	for name := range expected {
+		assert.True(t, found[name], fmt.Sprintf("%s collector not found", name))
+	}
+}
+
+func assertAnalyzers(t *testing.T, actual []*troubleshootv1beta2.HostAnalyze, expected map[string]struct {
+	match    func(*troubleshootv1beta2.HostAnalyze) bool
+	validate func(*troubleshootv1beta2.HostAnalyze)
+}) {
+	found := make(map[string]bool)
+	for _, collector := range actual {
+		for name, assertion := range expected {
+			if assertion.match(collector) {
+				found[name] = true
+				assertion.validate(collector)
+			}
+		}
+	}
+	for name := range expected {
+		assert.True(t, found[name], fmt.Sprintf("%s collector not found", name))
+	}
+}
+
+func assertMetrics(t *testing.T, actual []dryruntypes.Metric, expected []struct {
+	title    string
+	validate func(string)
+}) {
+	if len(actual) != len(expected) {
+		t.Errorf("expected %d metrics, got %d", len(expected), len(actual))
+		return
+	}
+	for i, exp := range expected {
+		m := actual[i]
+		if m.Title != exp.title {
+			t.Errorf("expected metric %s at position %d, got %s", exp.title, i, m.Title)
+			continue
+		}
+		exp.validate(m.Payload)
+	}
+}
+
+func assertEnv(t *testing.T, actual, expected map[string]string) {
+	for expectedKey, expectedValue := range expected {
+		assert.Equal(t, expectedValue, actual[expectedKey])
+	}
+}
+
+func assertConfigMapExists(t *testing.T, kcli client.Client, name string, namespace string) {
+	var cm corev1.ConfigMap
+	err := kcli.Get(context.TODO(), types.NamespacedName{Name: name, Namespace: namespace}, &cm)
+	assert.NoError(t, err, "failed to get configmap %s in namespace %s", name, namespace)
+}
+
+func assertSecretExists(t *testing.T, kcli client.Client, name string, namespace string) {
+	var secret corev1.Secret
+	err := kcli.Get(context.TODO(), types.NamespacedName{Name: name, Namespace: namespace}, &secret)
+	assert.NoError(t, err, "failed to get secret %s in namespace %s", name, namespace)
+}
+
+func assertHelmValues(
+	t *testing.T,
+	k0sConfig k0sv1beta1.ClusterConfig,
+	chartName string,
+	expectedValues map[string]interface{},
+) {
+	actualValues := map[string]interface{}{}
+	for _, ext := range k0sConfig.Spec.Extensions.Helm.Charts {
+		if ext.Name == chartName {
+			if err := yaml.Unmarshal([]byte(ext.Values), &actualValues); err != nil {
+				t.Fatalf("fail to unmarshal %s helm values: %v", chartName, err)
+			}
+		}
+	}
+	for expectedKey, expectedValue := range expectedValues {
+		actualValue, err := helm.GetValue(actualValues, expectedKey)
+		assert.NoError(t, err)
+		assert.Equal(t, expectedValue, actualValue)
+	}
+}