Skip to content

Commit

Permalink
Simplifying clusterctl upgrade tests
Browse files Browse the repository at this point in the history 
Signed-off-by: muhammad adil ghaffar <[email protected]>
  • Loading branch information
@adilGhaffarDev
adilGhaffarDev committed Feb 27, 2024
1 parent c9a0a3b commit ebd8797
Show file tree
Hide file tree
Showing 2 changed files with 138 additions and 117 deletions.
91 changes: 51 additions & 40 deletions test/e2e/config/e2e_conf.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,85 +17,96 @@ providers:
- name: cluster-api
type: CoreProvider
versions:
- name: ${CAPI_FROM_RELEASE}
value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/${CAPI_FROM_RELEASE}/core-components.yaml"
- name: v1.6.1
value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.6.1/core-components.yaml"
type: "url"
contract: ${CONTRACT_FROM}
contract: v1beta1
replacements:
- old: --metrics-addr=127.0.0.1:8080
new: --metrics-addr=:8080
files:
- sourcePath: "../data/shared/${CAPI_FROM_RELEASE:0:4}/metadata.yaml"
- name: ${CAPI_TO_RELEASE}
# Use manifest from source files
value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/${CAPI_TO_RELEASE}/core-components.yaml"
- sourcePath: "../data/shared/v1.6/metadata.yaml"
- name: v1.5.5
value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.5.5/core-components.yaml"
type: "url"
contract: ${CONTRACT_TO}
files:
- sourcePath: "../data/shared/${CAPI_TO_RELEASE:0:4}/metadata.yaml"
contract: v1beta1
replacements:
- old: "--leader-elect"
new: "--leader-elect=false"
- old: --metrics-addr=127.0.0.1:8080
new: --metrics-addr=:8080
files:
- sourcePath: "../data/shared/v1.5/metadata.yaml"
- name: kubeadm
type: BootstrapProvider
versions:
- name: ${CAPI_FROM_RELEASE} # latest published release in the v1alpha4 series; this is used for v1alpha4 --> v1beta1 clusterctl upgrades test only.
value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/${CAPI_FROM_RELEASE}/bootstrap-components.yaml"
- name: v1.6.1 # latest published release in the v1alpha4 series; this is used for v1alpha4 --> v1beta1 clusterctl upgrades test only.
value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.6.1/bootstrap-components.yaml"
type: "url"
contract: ${CONTRACT_FROM}
contract: v1beta1
replacements:
- old: --metrics-addr=127.0.0.1:8080
new: --metrics-addr=:8080
files:
- sourcePath: "../data/shared/${CAPI_FROM_RELEASE:0:4}/metadata.yaml"
- name: ${CAPI_TO_RELEASE}
# Use manifest from source files
value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/${CAPI_TO_RELEASE}/bootstrap-components.yaml"
- sourcePath: "../data/shared/v1.6/metadata.yaml"
- name: v1.5.5 # latest published release in the v1alpha4 series; this is used for v1alpha4 --> v1beta1 clusterctl upgrades test only.
value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.5.5/bootstrap-components.yaml"
type: "url"
contract: ${CONTRACT_TO}
files:
- sourcePath: "../data/shared/${CAPI_TO_RELEASE:0:4}/metadata.yaml"
contract: v1beta1
replacements:
- old: "--leader-elect"
new: "--leader-elect=false"
- old: --metrics-addr=127.0.0.1:8080
new: --metrics-addr=:8080
files:
- sourcePath: "../data/shared/v1.5/metadata.yaml"
- name: kubeadm
type: ControlPlaneProvider
versions:
- name: ${CAPI_FROM_RELEASE} # latest published release in the v1alpha4 series; this is used for v1alpha4 --> v1beta1 clusterctl upgrades test only.
value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/${CAPI_FROM_RELEASE}/control-plane-components.yaml"
- name: v1.6.1 # latest published release in the v1alpha4 series; this is used for v1alpha4 --> v1beta1 clusterctl upgrades test only.
value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.6.1/control-plane-components.yaml"
type: "url"
contract: ${CONTRACT_FROM}
contract: v1beta1
replacements:
- old: --metrics-addr=127.0.0.1:8080
new: --metrics-addr=:8080
files:
- sourcePath: "../data/shared/${CAPI_FROM_RELEASE:0:4}/metadata.yaml"
- name: ${CAPI_TO_RELEASE}
# Use manifest from source files
value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/${CAPI_TO_RELEASE}/control-plane-components.yaml"
- sourcePath: "../data/shared/v1.6/metadata.yaml"
- name: v1.5.5 # latest published release in the v1alpha4 series; this is used for v1alpha4 --> v1beta1 clusterctl upgrades test only.
value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.5.5/control-plane-components.yaml"
type: "url"
contract: ${CONTRACT_TO}
files:
- sourcePath: "../data/shared/${CAPI_TO_RELEASE:0:4}/metadata.yaml"
contract: v1beta1
replacements:
- old: "--leader-elect"
new: "--leader-elect=false"
- old: --metrics-addr=127.0.0.1:8080
new: --metrics-addr=:8080
files:
- sourcePath: "../data/shared/v1.5/metadata.yaml"
- name: metal3
type: InfrastructureProvider
versions:
- name: ${CAPM3_FROM_RELEASE}
value: "https://github.com/metal3-io/cluster-api-provider-metal3/releases/download/${CAPM3_FROM_RELEASE}/infrastructure-components.yaml"
- name: v1.6.0
value: "https://github.com/metal3-io/cluster-api-provider-metal3/releases/download/v1.6.0/infrastructure-components.yaml"
type: "url"
contract: v1beta1
files:
- sourcePath: "../data/infrastructure-metal3/v1.6/metadata.yaml"
targetName: "metadata.yaml"
- sourcePath: "../_out/cluster-template-ubuntu.yaml"
targetName: "cluster-template-ubuntu.yaml"
- sourcePath: "../_out/cluster-template-upgrade-workload.yaml"
targetName: "cluster-template-upgrade-workload.yaml"
- name: v1.5.1
value: "https://github.com/metal3-io/cluster-api-provider-metal3/releases/download/v1.5.1/infrastructure-components.yaml"
type: "url"
contract: ${CONTRACT_FROM}
contract: v1beta1
files:
- sourcePath: "../data/infrastructure-metal3/${CAPM3_FROM_RELEASE:0:4}/metadata.yaml"
- sourcePath: "../data/infrastructure-metal3/v1.5/metadata.yaml"
targetName: "metadata.yaml"
- sourcePath: "../_out/cluster-template-ubuntu.yaml"
targetName: "cluster-template-ubuntu.yaml"
- sourcePath: "../_out/cluster-template-upgrade-workload.yaml"
targetName: "cluster-template-upgrade-workload.yaml"
- name: v1.6.99
value: "${PWD}/config/default"
replacements:
- old: "--leader-elect"
new: "--leader-elect\n - --logging-format=json"
files:
- sourcePath: "../data/infrastructure-metal3/main/metadata.yaml"
targetName: "metadata.yaml"
Expand Down
164 changes: 87 additions & 77 deletions test/e2e/upgrade_clusterctl_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,95 +18,105 @@ import (

const workDir = "/opt/metal3-dev-env/"

var _ = Describe(fmt.Sprintf("When testing cluster upgrade from releases %s > current [clusterctl-upgrade]", os.Getenv("CAPM3_FROM_RELEASE")), func() {
var _ = Describe("When testing cluster upgrade from releases (v1.6=>current) [clusterctl-upgrade]", func() {
BeforeEach(func() {
osType := strings.ToLower(os.Getenv("OS"))
Expect(osType).ToNot(Equal(""))
validateGlobals(specName)
imageURL, imageChecksum := EnsureImage(e2eConfig.GetVariable("INIT_WITH_KUBERNETES_VERSION"))
imageURL, imageChecksum := EnsureImage("v1.29.0")
os.Setenv("IMAGE_RAW_CHECKSUM", imageChecksum)
os.Setenv("IMAGE_RAW_URL", imageURL)
// We need to override clusterctl apply log folder to avoid getting our credentials exposed.
clusterctlLogFolder = filepath.Join(os.TempDir(), "clusters", bootstrapClusterProxy.GetName())
})
Releasev1 := strings.Contains(os.Getenv("CAPM3_FROM_RELEASE"), "v1")
if Releasev1 {
capi_e2e.ClusterctlUpgradeSpec(ctx, func() capi_e2e.ClusterctlUpgradeSpecInput {
return capi_e2e.ClusterctlUpgradeSpecInput{
E2EConfig: e2eConfig,
ClusterctlConfigPath: clusterctlConfigPath,
BootstrapClusterProxy: bootstrapClusterProxy,
ArtifactFolder: artifactFolder,
SkipCleanup: skipCleanup,
InitWithCoreProvider: fmt.Sprintf("cluster-api:%s", os.Getenv("CAPI_FROM_RELEASE")),
InitWithBootstrapProviders: []string{fmt.Sprintf("kubeadm:%s", os.Getenv("CAPI_FROM_RELEASE"))},
InitWithControlPlaneProviders: []string{fmt.Sprintf("kubeadm:%s", os.Getenv("CAPI_FROM_RELEASE"))},
InitWithInfrastructureProviders: []string{fmt.Sprintf("metal3:%s", os.Getenv("CAPM3_FROM_RELEASE"))},
InitWithKubernetesVersion: e2eConfig.GetVariable("INIT_WITH_KUBERNETES_VERSION"),
WorkloadKubernetesVersion: e2eConfig.GetVariable("INIT_WITH_KUBERNETES_VERSION"),
InitWithBinary: e2eConfig.GetVariable("INIT_WITH_BINARY"),
PreInit: func(clusterProxy framework.ClusterProxy) {
preInitFunc(clusterProxy)
// Override capi/capm3 versions exported in preInit
os.Setenv("CAPI_VERSION", "v1beta1")
os.Setenv("CAPM3_VERSION", "v1beta1")
os.Setenv("KUBECONFIG_BOOTSTRAP", bootstrapClusterProxy.GetKubeconfigPath())
},
PreWaitForCluster: preWaitForCluster,
PreUpgrade: preUpgrade,
PreCleanupManagementCluster: preCleanupManagementCluster,
MgmtFlavor: osType,
WorkloadFlavor: osType,
}
capi_e2e.ClusterctlUpgradeSpec(ctx, func() capi_e2e.ClusterctlUpgradeSpecInput {
return capi_e2e.ClusterctlUpgradeSpecInput{
E2EConfig: e2eConfig,
ClusterctlConfigPath: clusterctlConfigPath,
BootstrapClusterProxy: bootstrapClusterProxy,
ArtifactFolder: artifactFolder,
SkipCleanup: skipCleanup,
InitWithCoreProvider: "cluster-api:v1.6.1",
InitWithBootstrapProviders: []string{"kubeadm:v1.6.1"},
InitWithControlPlaneProviders: []string{"kubeadm:v1.6.1"},
InitWithInfrastructureProviders: []string{"metal3:v1.6.0"},
InitWithKubernetesVersion: "v1.29.0",
WorkloadKubernetesVersion: "v1.29.0",
InitWithBinary: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.6.1/clusterctl-{OS}-{ARCH}",
PreInit: func(clusterProxy framework.ClusterProxy) {
preInitFunc(clusterProxy)
// Override capi/capm3 versions exported in preInit
os.Setenv("CAPI_VERSION", "v1beta1")
os.Setenv("CAPM3_VERSION", "v1beta1")
os.Setenv("KUBECONFIG_BOOTSTRAP", bootstrapClusterProxy.GetKubeconfigPath())
},
PreWaitForCluster: preWaitForCluster,
PreUpgrade: preUpgrade,
PreCleanupManagementCluster: preCleanupManagementCluster,
MgmtFlavor: osType,
WorkloadFlavor: osType,
}
})
AfterEach(func() {
// Recreate bmh that was used in capi namespace in metal3
//#nosec G204 -- We need to pass in the file name here.
cmd := exec.Command("bash", "-c", "kubectl apply -f bmhosts_crs.yaml -n metal3")
cmd.Dir = workDir
output, err := cmd.CombinedOutput()
Logf("Applying bmh to metal3 namespace : \n %v", string(output))
Expect(err).ToNot(HaveOccurred())
// wait for all bmh to become available
bootstrapClient := bootstrapClusterProxy.GetClient()
ListBareMetalHosts(ctx, bootstrapClient, client.InNamespace(namespace))
WaitForNumBmhInState(ctx, bmov1alpha1.StateAvailable, WaitForNumInput{
Client: bootstrapClient,
Options: []client.ListOption{client.InNamespace(namespace)},
Replicas: 5,
Intervals: e2eConfig.GetIntervals(specName, "wait-bmh-available"),
})
} else {

isPreRelease := strings.Contains(os.Getenv("CAPI_TO_RELEASE"), "-")
ListBareMetalHosts(ctx, bootstrapClient, client.InNamespace(namespace))
})
})

if isPreRelease {
capi_e2e.ClusterctlUpgradeSpec(ctx, func() capi_e2e.ClusterctlUpgradeSpecInput {
return capi_e2e.ClusterctlUpgradeSpecInput{
E2EConfig: e2eConfig,
ClusterctlConfigPath: clusterctlConfigPath,
BootstrapClusterProxy: bootstrapClusterProxy,
ArtifactFolder: artifactFolder,
SkipCleanup: skipCleanup,
InitWithProvidersContract: "v1alpha4",
InitWithCoreProvider: fmt.Sprintf("capi-system/cluster-api:%s", os.Getenv("CAPI_TO_RELEASE")),
InitWithBootstrapProviders: []string{fmt.Sprintf("capi-kubeadm-bootstrap-system/kubeadm:%s", os.Getenv("CAPI_TO_RELEASE"))},
InitWithControlPlaneProviders: []string{fmt.Sprintf("capi-kubeadm-control-plane-system/kubeadm:%s", os.Getenv("CAPI_TO_RELEASE"))},
InitWithInfrastructureProviders: []string{fmt.Sprintf("capm3-system/metal3:%s", os.Getenv("CAPM3_TO_RELEASE"))},
InitWithBinary: e2eConfig.GetVariable("INIT_WITH_BINARY"),
InitWithKubernetesVersion: e2eConfig.GetVariable("INIT_WITH_KUBERNETES_VERSION"),
WorkloadKubernetesVersion: e2eConfig.GetVariable("INIT_WITH_KUBERNETES_VERSION"),
PreInit: preInitFunc,
PreWaitForCluster: preWaitForCluster,
PreUpgrade: preUpgrade,
PreCleanupManagementCluster: preCleanupManagementCluster,
MgmtFlavor: osType,
WorkloadFlavor: osType,
}
})
} else {
capi_e2e.ClusterctlUpgradeSpec(ctx, func() capi_e2e.ClusterctlUpgradeSpecInput {
return capi_e2e.ClusterctlUpgradeSpecInput{
E2EConfig: e2eConfig,
ClusterctlConfigPath: clusterctlConfigPath,
BootstrapClusterProxy: bootstrapClusterProxy,
ArtifactFolder: artifactFolder,
SkipCleanup: skipCleanup,
InitWithProvidersContract: "v1alpha4",
InitWithBinary: e2eConfig.GetVariable("INIT_WITH_BINARY"),
PreInit: preInitFunc,
PreWaitForCluster: preWaitForCluster,
PreUpgrade: preUpgrade,
PreCleanupManagementCluster: preCleanupManagementCluster,
MgmtFlavor: osType,
WorkloadFlavor: osType,
}
})
var _ = Describe("When testing cluster upgrade from releases (v1.5=>current) [clusterctl-upgrade]", func() {
BeforeEach(func() {
osType := strings.ToLower(os.Getenv("OS"))
Expect(osType).ToNot(Equal(""))
validateGlobals(specName)
imageURL, imageChecksum := EnsureImage("v1.28.0")
os.Setenv("IMAGE_RAW_CHECKSUM", imageChecksum)
os.Setenv("IMAGE_RAW_URL", imageURL)
// We need to override clusterctl apply log folder to avoid getting our credentials exposed.
clusterctlLogFolder = filepath.Join(os.TempDir(), "clusters", bootstrapClusterProxy.GetName())
})
capi_e2e.ClusterctlUpgradeSpec(ctx, func() capi_e2e.ClusterctlUpgradeSpecInput {
return capi_e2e.ClusterctlUpgradeSpecInput{
E2EConfig: e2eConfig,
ClusterctlConfigPath: clusterctlConfigPath,
BootstrapClusterProxy: bootstrapClusterProxy,
ArtifactFolder: artifactFolder,
SkipCleanup: skipCleanup,
InitWithCoreProvider: "cluster-api:v1.5.5",
InitWithBootstrapProviders: []string{"kubeadm:v1.5.5"},
InitWithControlPlaneProviders: []string{"kubeadm:v1.5.5"},
InitWithInfrastructureProviders: []string{"metal3:v1.5.1"},
InitWithKubernetesVersion: "v1.28.0",
WorkloadKubernetesVersion: "v1.28.0",
InitWithBinary: "https://github.com/kubernetes-sigs/cluster-api/releases/download/vv1.5.5/clusterctl-{OS}-{ARCH}",
PreInit: func(clusterProxy framework.ClusterProxy) {
preInitFunc(clusterProxy)
// Override capi/capm3 versions exported in preInit
os.Setenv("CAPI_VERSION", "v1beta1")
os.Setenv("CAPM3_VERSION", "v1beta1")
os.Setenv("KUBECONFIG_BOOTSTRAP", bootstrapClusterProxy.GetKubeconfigPath())
},
PreWaitForCluster: preWaitForCluster,
PreUpgrade: preUpgrade,
PreCleanupManagementCluster: preCleanupManagementCluster,
MgmtFlavor: osType,
WorkloadFlavor: osType,
}
}
})
AfterEach(func() {
// Recreate bmh that was used in capi namespace in metal3
//#nosec G204 -- We need to pass in the file name here.
Expand Down

0 comments on commit ebd8797

Please sign in to comment.