Implement EC install CLI command integration test (#1420)

* Implement EC install CLI command integration test

.github/workflows/ci.yaml

@@ -59,6 +59,21 @@ jobs:
         run: |
           make unit-tests
 
+  dryrun-tests:
+    name: Dryrun tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Setup go
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+          cache-dependency-path: "**/*.sum"
+      - name: Dryrun tests
+        run: |
+          make dryrun-tests
+
   check-operator-crds:
     name: Check operator CRDs
     runs-on: ubuntu-latest

Makefile

@@ -202,6 +202,16 @@ static: pkg/goods/bins/k0s \
 	pkg/goods/bins/fio \
 	pkg/goods/internal/bins/kubectl-kots
 
+.PHONY: static-dryrun
+static-dryrun:
+	@mkdir -p pkg/goods/bins pkg/goods/internal/bins
+	@touch pkg/goods/bins/k0s \
+		pkg/goods/bins/kubectl-preflight \
+		pkg/goods/bins/kubectl-support_bundle \
+		pkg/goods/bins/local-artifact-mirror \
+		pkg/goods/bins/fio \
+		pkg/goods/internal/bins/kubectl-kots
+
 .PHONY: embedded-cluster-linux-amd64
 embedded-cluster-linux-amd64: export OS = linux
 embedded-cluster-linux-amd64: export ARCH = amd64
@@ -250,6 +260,10 @@ e2e-tests: embedded-release
 e2e-test:
 	go test -timeout 60m -ldflags="$(LD_FLAGS)" -v ./e2e -run ^$(TEST_NAME)$$
 
+.PHONY: dryrun-tests
+dryrun-tests: static-dryrun
+	@./scripts/dryrun-tests.sh
+
 .PHONY: build-ttl.sh
 build-ttl.sh:
 	$(MAKE) -C local-artifact-mirror build-ttl.sh \

cmd/embedded-cluster/main.go

@@ -2,15 +2,14 @@ package main
 
 import (
 	"context"
-	"fmt"
 	"os"
 	"os/signal"
 	"path"
 	"syscall"
 
 	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli/v2"
 
+	"github.com/replicatedhq/embedded-cluster/pkg/cmd"
 	"github.com/replicatedhq/embedded-cluster/pkg/logging"
 )
 
@@ -22,25 +21,9 @@ func main() {
 	)
 	defer cancel()
 	logging.SetupLogging()
+
 	name := path.Base(os.Args[0])
-	var app = &cli.App{
-		Name:    name,
-		Usage:   fmt.Sprintf("Install and manage %s", name),
-		Suggest: true,
-		Commands: []*cli.Command{
-			installCommand(),
-			shellCommand(),
-			nodeCommands,
-			versionCommand,
-			joinCommand,
-			resetCommand(),
-			materializeCommand(),
-			updateCommand(),
-			restoreCommand(),
-			adminConsoleCommand(),
-			supportBundleCommand(),
-		},
-	}
+	app := cmd.NewApp(name)
 	if err := app.RunContext(ctx, os.Args); err != nil {
 		logrus.Fatal(err)
 	}

e2e/cluster/docker/cluster.go

@@ -84,7 +84,10 @@ func (c *Cluster) WaitForReady() {
 func (c *Cluster) Cleanup(envs ...map[string]string) {
 	c.generateSupportBundle(envs...)
 	c.copyPlaywrightReport()
+	c.Destroy()
+}
 
+func (c *Cluster) Destroy() {
 	for _, node := range c.Nodes {
 		node.Destroy()
 	}

pkg/addons/embeddedclusteroperator/embeddedclusteroperator.go

@@ -6,7 +6,6 @@ import (
 	"context"
 	_ "embed"
 	"encoding/json"
-	"errors"
 	"fmt"
 	"strings"
 	"time"
@@ -167,6 +166,10 @@ func (e *EmbeddedClusterOperator) createVersionMetadataConfigmap(ctx context.Con
 	// the result as a suffix for the config map name.
 	slugver := slug.Make(strings.TrimPrefix(versions.Version, "v"))
 	configmap := &corev1.ConfigMap{
+		TypeMeta: metav1.TypeMeta{
+			APIVersion: "v1",
+			Kind:       "ConfigMap",
+		},
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      fmt.Sprintf("version-metadata-%s", slugver),
 			Namespace: e.namespace,
@@ -256,6 +259,10 @@ func (e *EmbeddedClusterOperator) Outro(ctx context.Context, provider *defaults.
 	}
 
 	installation := ecv1beta1.Installation{
+		TypeMeta: metav1.TypeMeta{
+			APIVersion: ecv1beta1.GroupVersion.String(),
+			Kind:       "Installation",
+		},
 		ObjectMeta: metav1.ObjectMeta{
 			Name: time.Now().Format("20060102150405"),
 			Labels: map[string]string{
@@ -282,13 +289,8 @@ func (e *EmbeddedClusterOperator) Outro(ctx context.Context, provider *defaults.
 	}
 
 	// we wait for the installation to exist here because items do not show up in the apiserver instantaneously after being created
-	gotInstallation, err := waitForInstallationToExist(ctx, cli, installation.Name)
-	if err != nil {
-		return fmt.Errorf("unable to wait for installation to exist: %w", err)
-	}
-	gotInstallation.Status.State = ecv1beta1.InstallationStateKubernetesInstalled
-	if err := cli.Status().Update(ctx, gotInstallation); err != nil {
-		return fmt.Errorf("unable to update installation status: %w", err)
+	if err := kubeutils.WaitAndMarkInstallation(ctx, cli, installation.Name, ecv1beta1.InstallationStateKubernetesInstalled); err != nil {
+		return fmt.Errorf("unable to wait and mark installation: %w", err)
 	}
 
 	return nil
@@ -343,18 +345,3 @@ func k0sConfigToNetworkSpec(k0sCfg *k0sv1beta1.ClusterConfig) *ecv1beta1.Network
 
 	return network
 }
-
-func waitForInstallationToExist(ctx context.Context, cli client.Client, name string) (*ecv1beta1.Installation, error) {
-	for i := 0; i < 20; i++ {
-		in, err := kubeutils.GetInstallation(ctx, cli, name)
-		if err != nil {
-			if !errors.Is(err, kubeutils.ErrNoInstallations{}) {
-				return nil, fmt.Errorf("unable to get installation: %w", err)
-			}
-		} else {
-			return in, nil
-		}
-		time.Sleep(time.Second)
-	}
-	return nil, fmt.Errorf("installation %s not found after 20 seconds", name)
-}

cmd/embedded-cluster/admin_console.go → pkg/cmd/admin_console.go

@@ -1,4 +1,4 @@
-package main
+package cmd
 
 import (
 	"fmt"

pkg/cmd/app.go

@@ -0,0 +1,28 @@
+package cmd
+
+import (
+	"fmt"
+
+	"github.com/urfave/cli/v2"
+)
+
+func NewApp(name string) *cli.App {
+	return &cli.App{
+		Name:    name,
+		Usage:   fmt.Sprintf("Install and manage %s", name),
+		Suggest: true,
+		Commands: []*cli.Command{
+			installCommand(),
+			shellCommand(),
+			nodeCommands,
+			versionCommand,
+			joinCommand,
+			resetCommand(),
+			materializeCommand(),
+			updateCommand(),
+			restoreCommand(),
+			adminConsoleCommand(),
+			supportBundleCommand(),
+		},
+	}
+}

cmd/embedded-cluster/flags.go → pkg/cmd/flags.go

@@ -1,4 +1,4 @@
-package main
+package cmd
 
 import (
 	"fmt"

cmd/embedded-cluster/install.go → pkg/cmd/install.go

@@ -1,4 +1,4 @@
-package main
+package cmd
 
 import (
 	"fmt"
@@ -20,6 +20,7 @@ import (
 	"github.com/replicatedhq/embedded-cluster/pkg/config"
 	"github.com/replicatedhq/embedded-cluster/pkg/configutils"
 	"github.com/replicatedhq/embedded-cluster/pkg/defaults"
+	"github.com/replicatedhq/embedded-cluster/pkg/dryrun"
 	"github.com/replicatedhq/embedded-cluster/pkg/goods"
 	"github.com/replicatedhq/embedded-cluster/pkg/helpers"
 	"github.com/replicatedhq/embedded-cluster/pkg/metrics"
@@ -188,6 +189,11 @@ func RunHostPreflights(c *cli.Context, provider *defaults.Provider, applier *add
 		hpf.Analyzers = append(hpf.Analyzers, h.Spec.Analyzers...)
 	}
 
+	if dryrun.Enabled() {
+		dryrun.RecordHostPreflightSpec(hpf)
+		return nil
+	}
+
 	return runHostPreflights(c, provider, hpf, proxy)
 }
 
@@ -454,15 +460,9 @@ func ensureK0sConfig(c *cli.Context, provider *defaults.Provider, applier *addon
 	if err != nil {
 		return nil, fmt.Errorf("unable to marshal config: %w", err)
 	}
-	fp, err := os.OpenFile(cfgpath, os.O_RDWR|os.O_CREATE, 0600)
-	if err != nil {
-		return nil, fmt.Errorf("unable to create config file: %w", err)
-	}
-	defer fp.Close()
-	if _, err := fp.Write(data); err != nil {
+	if err := os.WriteFile(cfgpath, data, 0600); err != nil {
 		return nil, fmt.Errorf("unable to write config file: %w", err)
 	}
-
 	return cfg, nil
 }
 
@@ -529,18 +529,20 @@ func installK0s(c *cli.Context, provider *defaults.Provider) error {
 // waitForK0s waits for the k0s API to be available. We wait for the k0s socket to
 // appear in the system and until the k0s status command to finish.
 func waitForK0s() error {
-	var success bool
-	for i := 0; i < 30; i++ {
-		time.Sleep(2 * time.Second)
-		spath := defaults.PathToK0sStatusSocket()
-		if _, err := os.Stat(spath); err != nil {
-			continue
+	if !dryrun.Enabled() {
+		var success bool
+		for i := 0; i < 30; i++ {
+			time.Sleep(2 * time.Second)
+			spath := defaults.PathToK0sStatusSocket()
+			if _, err := os.Stat(spath); err != nil {
+				continue
+			}
+			success = true
+			break
+		}
+		if !success {
+			return fmt.Errorf("timeout waiting for %s", defaults.BinaryName())
 		}
-		success = true
-		break
-	}
-	if !success {
-		return fmt.Errorf("timeout waiting for %s", defaults.BinaryName())
 	}
 
 	for i := 1; ; i++ {
@@ -678,6 +680,18 @@ func installCommand() *cli.Command {
 			if c.String("airgap-bundle") != "" {
 				metrics.DisableMetrics()
 			}
+			if drFile := c.String("dry-run"); drFile != "" {
+				dryrun.Init(drFile)
+				dryrun.RecordFlags(c)
+			}
+			return nil
+		},
+		After: func(c *cli.Context) error {
+			if c.String("dry-run") != "" {
+				if err := dryrun.Dump(); err != nil {
+					return fmt.Errorf("unable to dump dry run info: %w", err)
+				}
+			}
 			return nil
 		},
 		Flags: withProxyFlags(withSubnetCIDRFlags(
@@ -693,6 +707,12 @@ func installCommand() *cli.Command {
 					Usage: "Path to the air gap bundle. If set, the installation will complete without internet access.",
 				},
 				getDataDirFlagWithDefault(runtimeConfig),
+				&cli.StringFlag{
+					Name:   "dry-run",
+					Usage:  "If set, dry run the installation and output the results to the provided file",
+					Value:  "",
+					Hidden: true,
+				},
 				&cli.StringFlag{
 					Name:    "license",
 					Aliases: []string{"l"},

cmd/embedded-cluster/install_test.go → pkg/cmd/install_test.go

@@ -1,4 +1,4 @@
-package main
+package cmd
 
 import (
 	"flag"

cmd/embedded-cluster/join.go → pkg/cmd/join.go

@@ -1,4 +1,4 @@
-package main
+package cmd
 
 import (
 	"context"
@@ -443,11 +443,6 @@ func patchK0sConfig(path string, patch string) error {
 		}
 		finalcfg.Spec.Storage = result.Spec.Storage
 	}
-	out, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0600)
-	if err != nil {
-		return fmt.Errorf("unable to open node config file for writing: %w", err)
-	}
-	defer out.Close()
 	// This is necessary to install the previous version of k0s in e2e tests
 	// TODO: remove this once the previous version is > 1.29
 	unstructured, err := helpers.K0sClusterConfigTo129Compat(&finalcfg)
@@ -458,8 +453,8 @@ func patchK0sConfig(path string, patch string) error {
 	if err != nil {
 		return fmt.Errorf("unable to marshal node config: %w", err)
 	}
-	if _, err := out.Write(data); err != nil {
-		return fmt.Errorf("unable to write node config: %w", err)
+	if err := os.WriteFile(path, data, 0600); err != nil {
+		return fmt.Errorf("unable to write node config file: %w", err)
 	}
 	return nil
 }

cmd/embedded-cluster/join_test.go → pkg/cmd/join_test.go

@@ -1,4 +1,4 @@
-package main
+package cmd
 
 import (
 	"embed"

cmd/embedded-cluster/k0s.go → pkg/cmd/k0s.go

@@ -1,4 +1,4 @@
-package main
+package cmd
 
 import (
 	"context"

cmd/embedded-cluster/list_images.go → pkg/cmd/list_images.go

@@ -1,4 +1,4 @@
-package main
+package cmd
 
 import (
 	"fmt"

cmd/embedded-cluster/materialize.go → pkg/cmd/materialize.go

@@ -1,4 +1,4 @@
-package main
+package cmd
 
 import (
 	"fmt"

cmd/embedded-cluster/metadata.go → pkg/cmd/metadata.go

@@ -1,4 +1,4 @@
-package main
+package cmd
 
 import (
 	"encoding/json"

cmd/embedded-cluster/network.go → pkg/cmd/network.go

@@ -1,4 +1,4 @@
-package main
+package cmd
 
 import (
 	"fmt"

cmd/embedded-cluster/network_test.go → pkg/cmd/network_test.go

@@ -1,4 +1,4 @@
-package main
+package cmd
 
 import (
 	"flag"