Skip to content

Commit

Permalink
Merge pull request #2240 from Nordix/mquhuy/use-kustomize-to-install-…
Browse files Browse the repository at this point in the history 
…ironic-bmo-pivot-test

🌱 Only use Kustomize to install Ironic and BMO in e2e tests
  • Loading branch information
@metal3-io-bot
metal3-io-bot authored Jan 21, 2025
2 parents c9d6f72 + f1db6fc commit 6d21061
Show file tree
Hide file tree
Showing 3 changed files with 87 additions and 161 deletions.
8 changes: 0 additions & 8 deletions test/e2e/common.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@ import (
"path"
"path/filepath"
"sort"
"strconv"
"strings"
"text/tabwriter"
"time"
Expand Down Expand Up @@ -83,13 +82,6 @@ func LogFromFile(logFile string) {
Logf(string(data))
}

// return only the boolean value from ParseBool.
func getBool(s string) bool {
b, err := strconv.ParseBool(s)
Expect(err).ToNot(HaveOccurred())
return b
}

// logTable print a formatted table into the e2e logs.
func logTable(title string, rows [][]string) {
getRowFormatted := func(row []string) string {
Expand Down
36 changes: 31 additions & 5 deletions test/e2e/logcollector.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (
"os/exec"
"path"
"path/filepath"
"time"

. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
Expand Down Expand Up @@ -116,14 +117,39 @@ func FetchClusterLogs(clusterProxy framework.ClusterProxy, outputPath string) er
// Print the Pods' information to file
// This does the same thing as:
// kubectl --kubeconfig="${KUBECONFIG_WORKLOAD}" get pods -A
out, err := exec.Command("kubectl", "--kubeconfig", clusterProxy.GetKubeconfigPath(), "get", "pods", "-A", "-o", "wide").Output() // #nosec G204:gosec
outputFile := filepath.Join(baseDir, "pods.log")
file, err := os.Create(outputFile)
if err != nil {
return fmt.Errorf("couldn't get pods: %v", err)
return fmt.Errorf("failed to create output file: %v", err)
}
file := filepath.Join(baseDir, "pods.log")
err = os.WriteFile(file, out, 0600)
defer file.Close()

// List pods across all namespaces
pods, err := clientset.CoreV1().Pods("").List(context.Background(), metav1.ListOptions{})
if err != nil {
return fmt.Errorf("couldn't write to file: %v", err)
return fmt.Errorf("failed to list pods: %v", err)
}

// Print header to file
header := fmt.Sprintf("%-32s %-16s %-12s %-24s %-8s\n", "NAMESPACE", "NAME", "STATUS", "NODE", "AGE")
if _, err = file.WriteString(header); err != nil {
return fmt.Errorf("error writing to file: %v", err)
}

// Iterate through pods and print information
for _, pod := range pods.Items {
age := time.Since(pod.CreationTimestamp.Time).Round(time.Second)
podInfo := fmt.Sprintf("%-32s %-16s %-12s %-24s %-8s\n",
pod.Namespace,
pod.Name,
string(pod.Status.Phase),
pod.Spec.NodeName,
age,
)

if _, err = file.WriteString(podInfo); err != nil {
return fmt.Errorf("error writing to file: %v", err)
}
}

// Get all namespaces
Expand Down
204 changes: 56 additions & 148 deletions test/e2e/pivoting.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@ import (
"os"
"os/exec"
"path/filepath"
"strconv"

containerTypes "github.com/docker/docker/api/types/container"
docker "github.com/docker/docker/client"
Expand Down Expand Up @@ -155,42 +154,36 @@ func pivoting(ctx context.Context, inputGetter func() PivotingInput) {

By("Install Ironic in the target cluster")
// TODO(dtantsur): support ironic-standalone-operator
installIronicBMO(ctx, func() installIronicBMOInput {
return installIronicBMOInput{
ManagementCluster: input.TargetCluster,
BMOPath: input.E2EConfig.GetVariable(bmoPath),
deployIronic: true,
deployBMO: false,
deployIronicTLSSetup: getBool(input.E2EConfig.GetVariable(ironicTLSSetup)),
deployIronicBasicAuth: getBool(input.E2EConfig.GetVariable(ironicBasicAuth)),
deployIronicKeepalived: getBool(input.E2EConfig.GetVariable(ironicKeepalived)),
deployIronicMariadb: getBool(input.E2EConfig.GetVariable(ironicMariadb)),
Namespace: ironicNamespaceObj.Name,
NamePrefix: input.E2EConfig.GetVariable(NamePrefix),
RestartContainerCertUpdate: getBool(input.E2EConfig.GetVariable(restartContainerCertUpdate)),
E2EConfig: input.E2EConfig,
SpecName: input.SpecName,
}
ironicDeployLogFolder := filepath.Join(os.TempDir(), "target_cluster_logs", "ironic-deploy-logs", input.TargetCluster.GetName())
ironicKustomization := input.E2EConfig.GetVariable("IRONIC_RELEASE_LATEST")
By(fmt.Sprintf("Installing Ironic from kustomization %s on the target cluster", ironicKustomization))
err = BuildAndApplyKustomization(ctx, &BuildAndApplyKustomizationInput{
Kustomization: ironicKustomization,
ClusterProxy: input.TargetCluster,
WaitForDeployment: true,
WatchDeploymentLogs: true,
LogPath: ironicDeployLogFolder,
DeploymentName: "baremetal-operator-ironic",
DeploymentNamespace: ironicNamespaceObj.Name,
WaitIntervals: input.E2EConfig.GetIntervals("default", "wait-deployment"),
})

By("Install BMO")
installIronicBMO(ctx, func() installIronicBMOInput {
return installIronicBMOInput{
ManagementCluster: input.TargetCluster,
BMOPath: input.E2EConfig.GetVariable(bmoPath),
deployIronic: false,
deployBMO: true,
deployIronicTLSSetup: getBool(input.E2EConfig.GetVariable(ironicTLSSetup)),
deployIronicBasicAuth: getBool(input.E2EConfig.GetVariable(ironicBasicAuth)),
deployIronicKeepalived: getBool(input.E2EConfig.GetVariable(ironicKeepalived)),
deployIronicMariadb: getBool(input.E2EConfig.GetVariable(ironicMariadb)),
Namespace: ironicNamespaceObj.Name,
NamePrefix: input.E2EConfig.GetVariable(NamePrefix),
RestartContainerCertUpdate: getBool(input.E2EConfig.GetVariable(restartContainerCertUpdate)),
E2EConfig: input.E2EConfig,
SpecName: input.SpecName,
}
Expect(err).NotTo(HaveOccurred())

By("Install BMO in the target cluster")
bmoDeployLogFolder := filepath.Join(os.TempDir(), "target_cluster_logs", "bmo-deploy-logs", input.TargetCluster.GetName())
bmoKustomization := input.E2EConfig.GetVariable("BMO_RELEASE_LATEST")
By(fmt.Sprintf("Installing BMO from kustomization %s on the target cluster", bmoKustomization))
err = BuildAndApplyKustomization(ctx, &BuildAndApplyKustomizationInput{
Kustomization: bmoKustomization,
ClusterProxy: input.TargetCluster,
WaitForDeployment: true,
WatchDeploymentLogs: true,
LogPath: bmoDeployLogFolder,
DeploymentName: "baremetal-operator-controller-manager",
DeploymentNamespace: ironicNamespaceObj.Name,
WaitIntervals: input.E2EConfig.GetIntervals("default", "wait-deployment"),
})
Expect(err).NotTo(HaveOccurred())

By("Add labels to BMO CRDs in the target cluster")
labelBMOCRDs(ctx, input.TargetCluster)
Expand Down Expand Up @@ -227,7 +220,6 @@ func pivoting(ctx context.Context, inputGetter func() PivotingInput) {
Name: input.E2EConfig.GetVariable(NamePrefix) + "-controller-manager",
}
})

pivotingCluster := framework.DiscoveryAndWaitForCluster(ctx, framework.DiscoveryAndWaitForClusterInput{
Getter: input.TargetCluster.GetClient(),
Namespace: input.Namespace,
Expand Down Expand Up @@ -268,82 +260,6 @@ func pivoting(ctx context.Context, inputGetter func() PivotingInput) {
By("PIVOTING TESTS PASSED!")
}

type installIronicBMOInput struct {
ManagementCluster framework.ClusterProxy
BMOPath string
deployIronic bool
deployBMO bool
deployIronicTLSSetup bool
deployIronicBasicAuth bool
deployIronicKeepalived bool
deployIronicMariadb bool
Namespace string
NamePrefix string
RestartContainerCertUpdate bool
E2EConfig *clusterctl.E2EConfig
SpecName string
}

func installIronicBMO(ctx context.Context, inputGetter func() installIronicBMOInput) {
input := inputGetter()

ironicHost := os.Getenv("CLUSTER_BARE_METAL_PROVISIONER_IP")
path := fmt.Sprintf("%s/tools/", input.BMOPath)

args := []string{}
if input.deployBMO {
args = append(args, "-b")
}
if input.deployIronic {
args = append(args, "-i")
}
if input.deployIronicTLSSetup {
args = append(args, "-t")
}
if !input.deployIronicBasicAuth {
args = append(args, "-n")
}
if input.deployIronicKeepalived {
args = append(args, "-k")
}
if input.deployIronicMariadb {
args = append(args, "-m")
}

env := []string{
fmt.Sprintf("IRONIC_HOST=%s", ironicHost),
fmt.Sprintf("IRONIC_HOST_IP=%s", ironicHost),
fmt.Sprintf("KUBECTL_ARGS=--kubeconfig=%s", input.ManagementCluster.GetKubeconfigPath()),
fmt.Sprintf("NAMEPREFIX=%s", input.NamePrefix),
fmt.Sprintf("RESTART_CONTAINER_CERTIFICATE_UPDATED=%s", strconv.FormatBool(input.RestartContainerCertUpdate)),
"USER=ubuntu",
}
cmd := exec.Command("./deploy.sh", args...) // #nosec G204:gosec
cmd.Dir = path
cmd.Env = append(env, os.Environ()...)

stdoutStderr, er := cmd.CombinedOutput()
Logf("%s\n", stdoutStderr)
Expect(er).ToNot(HaveOccurred(), "Failed to deploy Ironic")
deploymentNameList := []string{}
if input.deployIronic {
deploymentNameList = append(deploymentNameList, ironicSuffix)
}
if input.deployBMO {
deploymentNameList = append(deploymentNameList, "-controller-manager")
}
// Wait for the deployments to become available
clientSet := input.ManagementCluster.GetClientSet()
for _, name := range deploymentNameList {
deployment, err := clientSet.AppsV1().Deployments(input.Namespace).Get(ctx, input.NamePrefix+name, metav1.GetOptions{})
Expect(err).ToNot(HaveOccurred(), "Unable to get the deployment %s in namespace %s \n error message: %s", input.NamePrefix+name, input.Namespace, err)
framework.WaitForDeploymentsAvailable(ctx, framework.WaitForDeploymentsAvailableInput{
Getter: input.ManagementCluster.GetClient(),
Deployment: deployment,
}, input.E2EConfig.GetIntervals(input.SpecName, "wait-deployment")...)
}
}

type IronicDeploymentType string

const (
Expand Down Expand Up @@ -476,27 +392,7 @@ func rePivoting(ctx context.Context, inputGetter func() RePivotingInput) {
}
})

By("Reinstate BMO in Source cluster")
installIronicBMO(ctx, func() installIronicBMOInput {
return installIronicBMOInput{
ManagementCluster: input.BootstrapClusterProxy,
BMOPath: input.E2EConfig.GetVariable(bmoPath),
deployIronic: false,
deployBMO: true,
deployIronicTLSSetup: getBool(input.E2EConfig.GetVariable(ironicTLSSetup)),
deployIronicBasicAuth: getBool(input.E2EConfig.GetVariable(ironicBasicAuth)),
deployIronicKeepalived: getBool(input.E2EConfig.GetVariable(ironicKeepalived)),
deployIronicMariadb: getBool(input.E2EConfig.GetVariable(ironicMariadb)),
Namespace: input.E2EConfig.GetVariable(ironicNamespace),
NamePrefix: input.E2EConfig.GetVariable(NamePrefix),
RestartContainerCertUpdate: getBool(input.E2EConfig.GetVariable(restartContainerCertUpdate)),
E2EConfig: input.E2EConfig,
SpecName: input.SpecName,
}
})

By("Reinstate Ironic containers and BMH")
// TODO(mboukhalfa): add this local ironic deployment case to installIronicBMO function
ephemeralCluster := os.Getenv("EPHEMERAL_CLUSTER")
if ephemeralCluster == Kind {
bmoPath := input.E2EConfig.GetVariable("BMOPATH")
Expand All @@ -508,25 +404,37 @@ func rePivoting(ctx context.Context, inputGetter func() RePivotingInput) {
Expect(err).ToNot(HaveOccurred(), "Cannot run local ironic")
} else {
By("Install Ironic in the bootstrap cluster")
installIronicBMO(ctx, func() installIronicBMOInput {
return installIronicBMOInput{
ManagementCluster: input.BootstrapClusterProxy,
BMOPath: input.E2EConfig.GetVariable(bmoPath),
deployIronic: true,
deployBMO: false,
deployIronicTLSSetup: getBool(input.E2EConfig.GetVariable(ironicTLSSetup)),
deployIronicBasicAuth: getBool(input.E2EConfig.GetVariable(ironicBasicAuth)),
deployIronicKeepalived: getBool(input.E2EConfig.GetVariable(ironicKeepalived)),
deployIronicMariadb: getBool(input.E2EConfig.GetVariable(ironicMariadb)),
Namespace: input.E2EConfig.GetVariable(ironicNamespace),
NamePrefix: input.E2EConfig.GetVariable(NamePrefix),
RestartContainerCertUpdate: getBool(input.E2EConfig.GetVariable(restartContainerCertUpdate)),
E2EConfig: input.E2EConfig,
SpecName: input.SpecName,
}
ironicKustomization := input.E2EConfig.GetVariable("IRONIC_RELEASE_LATEST")
ironicDeployLogFolder := filepath.Join(os.TempDir(), "source_cluster_logs", "ironic-deploy-logs", input.TargetCluster.GetName())
err = BuildAndApplyKustomization(ctx, &BuildAndApplyKustomizationInput{
Kustomization: ironicKustomization,
ClusterProxy: input.BootstrapClusterProxy,
WaitForDeployment: true,
WatchDeploymentLogs: true,
LogPath: ironicDeployLogFolder,
DeploymentName: "baremetal-operator-ironic",
DeploymentNamespace: input.E2EConfig.GetVariable(ironicNamespace),
WaitIntervals: input.E2EConfig.GetIntervals("default", "wait-deployment"),
})
Expect(err).NotTo(HaveOccurred())
}

By("Reinstate BMO in Source cluster")
bmoKustomization := input.E2EConfig.GetVariable("BMO_RELEASE_LATEST")
bmoDeployLogFolder := filepath.Join(os.TempDir(), "source_cluster_logs", "bmo-deploy-logs", input.TargetCluster.GetName())
By(fmt.Sprintf("Installing BMO from kustomization %s on the source cluster", bmoKustomization))
err = BuildAndApplyKustomization(ctx, &BuildAndApplyKustomizationInput{
Kustomization: bmoKustomization,
ClusterProxy: input.BootstrapClusterProxy,
WaitForDeployment: true,
WatchDeploymentLogs: true,
LogPath: bmoDeployLogFolder,
DeploymentName: "baremetal-operator-controller-manager",
DeploymentNamespace: input.E2EConfig.GetVariable(ironicNamespace),
WaitIntervals: input.E2EConfig.GetIntervals("default", "wait-deployment"),
})
Expect(err).NotTo(HaveOccurred())

By("Ensure API servers are stable before doing move")
// Nb. This check was introduced to prevent doing move to self-hosted in an aggressive way and thus avoid flakes.
// More specifically, it was observed that the test was failing to get objects from the API server during move, so now
Expand Down

0 comments on commit 6d21061

Please sign in to comment.