From 74fb3034a49f66c0b396b3bf2b97cae74c402645 Mon Sep 17 00:00:00 2001 From: Enrique Llorente Pastora Date: Wed, 22 Jul 2020 09:45:44 +0200 Subject: [PATCH] Remove OKD/OCP cluster provision (#413) * Remove OKD/OCP cluster provision As second part of ocp/okd kubevirtci cleanup this commits remove the provision part the gocli and the pack8s from it. Signed-off-by: Quique Llorente * Update images.sh with gocli sha Signed-off-by: Quique Llorente --- OKD.md | 85 ---- cluster-provision/README.md | 61 --- cluster-provision/gocli/cmd/okd/provision.go | 287 ------------- cluster-provision/gocli/cmd/okd/run.go | 330 --------------- cluster-provision/gocli/cmd/provision.go | 5 - cluster-provision/gocli/cmd/run.go | 4 - .../manifests/okd/install-config.yaml | 26 -- .../manifests/okd/local-storage-cr.yaml | 12 - .../manifests/okd/local-storage.yaml | 32 -- .../manifests/okd/registries.conf | 7 - .../manifests/okd/registries.yaml | 32 -- cluster-provision/ocp/4.3/provision.sh | 39 -- cluster-provision/ocp/4.3/publish.sh | 5 - cluster-provision/ocp/4.3/run.sh | 10 - cluster-provision/ocp/4.4/provision.sh | 42 -- cluster-provision/ocp/4.4/publish.sh | 7 - cluster-provision/ocp/4.4/run.sh | 10 - cluster-provision/ocp/README.md | 11 - cluster-provision/okd/4.1/provision.sh | 29 -- cluster-provision/okd/4.1/publish.sh | 4 - cluster-provision/okd/4.1/run.sh | 16 - cluster-provision/okd/4.2/provision.sh | 29 -- cluster-provision/okd/4.2/publish.sh | 4 - cluster-provision/okd/4.2/run.sh | 16 - cluster-provision/okd/4.3/provision.sh | 33 -- cluster-provision/okd/4.3/publish.sh | 4 - cluster-provision/okd/4.3/run.sh | 16 - cluster-provision/okd/README.md | 13 - cluster-provision/okd/base/Dockerfile | 47 --- cluster-provision/okd/base/build.sh | 3 - cluster-provision/okd/base/entrypoint.sh | 25 -- cluster-provision/okd/base/haproxy.cfg | 90 ----- cluster-provision/okd/base/publish.sh | 4 - cluster-provision/okd/base/vagrant.key | 27 -- cluster-provision/okd/hacks/release-4.1 | 209 ---------- cluster-provision/okd/hacks/release-4.2 | 209 ---------- cluster-provision/okd/hacks/release-4.3 | 210 ---------- cluster-provision/okd/hacks/release-4.4 | 201 ---------- .../okd/scripts/create-local-disks.sh | 15 - cluster-provision/okd/scripts/provision.sh | 376 ------------------ cluster-provision/okd/scripts/run.sh | 225 ----------- cluster-up/cluster/images.sh | 2 +- pack8s/STATUS.md | 2 - pack8s/cmd/okd/run.go | 203 ---------- pack8s/cmd/run.go | 5 - 45 files changed, 1 insertion(+), 3021 deletions(-) delete mode 100644 OKD.md delete mode 100644 cluster-provision/gocli/cmd/okd/provision.go delete mode 100644 cluster-provision/gocli/cmd/okd/run.go delete mode 100644 cluster-provision/manifests/okd/install-config.yaml delete mode 100644 cluster-provision/manifests/okd/local-storage-cr.yaml delete mode 100644 cluster-provision/manifests/okd/local-storage.yaml delete mode 100644 cluster-provision/manifests/okd/registries.conf delete mode 100644 cluster-provision/manifests/okd/registries.yaml delete mode 100755 cluster-provision/ocp/4.3/provision.sh delete mode 100755 cluster-provision/ocp/4.3/publish.sh delete mode 100755 cluster-provision/ocp/4.3/run.sh delete mode 100755 cluster-provision/ocp/4.4/provision.sh delete mode 100755 cluster-provision/ocp/4.4/publish.sh delete mode 100755 cluster-provision/ocp/4.4/run.sh delete mode 100644 cluster-provision/ocp/README.md delete mode 100755 cluster-provision/okd/4.1/provision.sh delete mode 100755 cluster-provision/okd/4.1/publish.sh delete mode 100755 cluster-provision/okd/4.1/run.sh delete mode 100755 cluster-provision/okd/4.2/provision.sh delete mode 100755 cluster-provision/okd/4.2/publish.sh delete mode 100755 cluster-provision/okd/4.2/run.sh delete mode 100755 cluster-provision/okd/4.3/provision.sh delete mode 100755 cluster-provision/okd/4.3/publish.sh delete mode 100755 cluster-provision/okd/4.3/run.sh delete mode 100644 cluster-provision/okd/README.md delete mode 100644 cluster-provision/okd/base/Dockerfile delete mode 100755 cluster-provision/okd/base/build.sh delete mode 100755 cluster-provision/okd/base/entrypoint.sh delete mode 100644 cluster-provision/okd/base/haproxy.cfg delete mode 100755 cluster-provision/okd/base/publish.sh delete mode 100644 cluster-provision/okd/base/vagrant.key delete mode 100644 cluster-provision/okd/hacks/release-4.1 delete mode 100644 cluster-provision/okd/hacks/release-4.2 delete mode 100644 cluster-provision/okd/hacks/release-4.3 delete mode 100644 cluster-provision/okd/hacks/release-4.4 delete mode 100755 cluster-provision/okd/scripts/create-local-disks.sh delete mode 100755 cluster-provision/okd/scripts/provision.sh delete mode 100755 cluster-provision/okd/scripts/run.sh delete mode 100644 pack8s/cmd/okd/run.go diff --git a/OKD.md b/OKD.md deleted file mode 100644 index fb564d57f1..0000000000 --- a/OKD.md +++ /dev/null @@ -1,85 +0,0 @@ -# [kubevirtci](README.md): Getting Started with multi-node OKD Provider - -Download this repo -``` -git clone https://github.com/kubevirt/kubevirtci.git -cd kubevirtci -``` - -Start okd cluster (pre-configured with a master and worker node) -``` -export KUBEVIRT_PROVIDER=okd-4.1 -# export OKD_CONSOLE_PORT=443 # Uncomment to access OKD console -make cluster-up -``` - -Stop okd cluster -``` -make cluster-down -``` - -Use provider's OC client with oc.sh wrapper script -``` -cluster-up/oc.sh get nodes -cluster-up/oc.sh get pods --all-namespaces -``` - -Use your own OC client by defining the KUBECONFIG environment variable -``` -export KUBECONFIG=$(cluster-up/kubeconfig.sh) - -oc get nodes -oc apply -f -``` - -SSH into master -``` -cluster-up/ssh.sh master-0 -``` - -SSH into worker -``` -cluster-up/ssh.sh worker-0 -``` - -Connect to the container (with KUBECONFIG exported) -``` -make connect -``` - -In order to check newly created provider run, -this will point to the local created provider upon cluster-up -``` -export KUBEVIRTCI_PROVISION_CHECK=1 -``` - -## OKD Console -To access the OKD UI from the host running `docker`, remember to export `OKD_CONSOLE_PORT=443` before `make cluster-up`. -You should find out the IP address of the OKD docker container -``` -clusterip=$(docker inspect $(docker ps | grep "kubevirtci/$KUBEVIRT_PROVIDER" | awk '{print $1}') | jq -r '.[0].NetworkSettings.IPAddress' ) -``` -and make it known in `/etc/hosts` via -``` -cat << EOF >> /etc/hosts -$clusterip console-openshift-console.apps.test-1.tt.testing -$clusterip oauth-openshift.apps.test-1.tt.testing -EOF -``` -Now you can browse to https://console-openshift-console.apps.test-1.tt.testing -and log in by picking the `htpasswd_provider` option. The credentials are `admin/admin`. - -To access the OKD UI from a remote client, forward incoming port 433 into the OKD cluster -on the host running kubevirtci: -``` -$ nic=em1 # the interface facing your remote client -$ sudo iptables -t nat -A PREROUTING -p tcp -i $nic --dport 443 -j DNAT --to-destination $clusterip -``` -On your remote client host, point the cluster fqdn to the host running kubevirtci -``` -kubevirtci_ip=a.b.c.d # put here the ip address of the host running kubevirtci -cat << EOF >> /etc/hosts -$kubevirtci_ip console-openshift-console.apps.test-1.tt.testing -$kubevirtci_ip oauth-openshift.apps.test-1.tt.testing -EOF -``` diff --git a/cluster-provision/README.md b/cluster-provision/README.md index 1c9eca3842..31e452c6ec 100644 --- a/cluster-provision/README.md +++ b/cluster-provision/README.md @@ -22,19 +22,6 @@ * `kubevirtci/k8s-1.15.1`: `sha256:14d7b1806f24e527167d2913deafd910ea46e69b830bf0b094dde35ba961b159` * `kubevirtci/k8s-1.16.2`: `sha256:5bae6a5f3b996952c5ceb4ba12ac635146425909801df89d34a592f3d3502b0c` -# OKD clusters in the container with libvirt - -* `okd-base` contains all needed packages to provision and run OKD cluster on top of the libvirt provider -* `okd-4.1` okd-4.1 cluster provisioned with OpenShift installer on top of the libvirt provider, this image contains custom libvirt image that includes fixes to deploy new nodes without need to apply any W/A -* `okd-4.3` okd-4.3 cluster provisioned with OpenShift installer on top of the libvirt provider, this image contains custom libvirt image that includes fixes to deploy new nodes without need to apply any W/A - -## Versions to use - -* `kubevirtci/okd-base`: `sha256:73ede51ce464546a82b81956b7f58cf98662a4c5fded9c659b57746bc131e047` -* `kubevirtci/okd-4.1`: `sha256:e7e3a03bb144eb8c0be4dcd700592934856fb623d51a2b53871d69267ca51c86` -* `kubevirtci/okd-4.2`: `sha256:a830064ca7bf5c5c2f15df180f816534e669a9a038fef4919116d61eb33e84c5` -* `kubevirtci/okd-4.3`: `sha256:63abc3884002a615712dfac5f42785be864ea62006892bf8a086ccdbca8b3d38` - ## Using gocli `gocli` is a tiny go binary which helps managing the containerized clusters. It @@ -46,54 +33,6 @@ alias gocli="docker run --net=host --privileged --rm -it -v /var/run/docker.sock gocli help ``` -### How to provision OKD cluster - -First you will need to create installer pull token file with the content: -``` -{"auths":{...}} -``` - -and after you should run `gocli` command: -```bash -gocli provision okd \ ---prefix okd-4.1 \ ---dir-scripts /scripts \ ---dir-hacks /hacks \ ---master-memory 10240 \ ---installer-pull-secret-file \ ---installer-repo-tag release-4.1 \ ---installer-release-image quay.io/openshift-release-dev/ocp-release:4.1 \ -kubevirtci/okd-base@sha256:73ede51ce464546a82b81956b7f58cf98662a4c5fded9c659b57746bc131e047 -``` - -*** -NOTE: you can get the pull secret [here](https://cloud.redhat.com/openshift). -*** - -*** -NOTE: OpenShift cluster consumes a lot of resources, you should have at least 18Gb of the memory on the machine where do you run the container. -*** - -### How to run OKD cluster - -You should run `gocli` command: -```bash -gocli run okd --prefix okd-4.1 --ocp-console-port 443 --installer-pull-secret-file --background kubevirtci/okd-4.1@sha256:e7e3a03bb144eb8c0be4dcd700592934856fb623d51a2b53871d69267ca51c86 -``` - -### How to connect to the OKD console - -To connect the OKD console you should add once hosts to the `/etc/hosts` - -```bash -127.0.0.1 console-openshift-console.apps.test-1.tt.testing -127.0.0.1 oauth-openshift.apps.test-1.tt.testing -``` - -and specify the `--ocp-console-port` under the `gocli` run command to `443`. - -After you can connect to the https://console-openshift-console.apps.test-1.tt.testing and login via `htpasswd_provider` provider with `admin` user and password. - ## Quickstart Kubernetes ### Start the cluster diff --git a/cluster-provision/gocli/cmd/okd/provision.go b/cluster-provision/gocli/cmd/okd/provision.go deleted file mode 100644 index 65dbd3e2c9..0000000000 --- a/cluster-provision/gocli/cmd/okd/provision.go +++ /dev/null @@ -1,287 +0,0 @@ -package okd - -import ( - "fmt" - "os" - "os/signal" - - "github.com/docker/docker/api/types" - "github.com/docker/docker/api/types/container" - "github.com/docker/docker/api/types/mount" - "github.com/docker/docker/client" - "github.com/docker/docker/pkg/archive" - "github.com/spf13/cobra" - - "golang.org/x/net/context" - - "kubevirt.io/kubevirtci/cluster-provision/gocli/docker" -) - -type copyConfig struct { - srcPath string - dstPath string - container string -} - -// NewProvisionCommand provision the OKD cluster with one master and one worker -func NewProvisionCommand() *cobra.Command { - provision := &cobra.Command{ - Use: "okd", - Short: "provision okd command will provision new OKD cluster", - RunE: provision, - Args: cobra.ExactArgs(1), - } - - provision.Flags().Bool("skip-cnao", false, "skip installing cluster network addons operator") - provision.Flags().String("networking-type", "OpenShiftSDN", "networking type: OpenShiftSDN, OVNKubernetes") - provision.Flags().String("dir-hacks", "", "directory with installer hack that should be copied to the container") - provision.Flags().String("dir-manifests", "", "directory with additional manifests that should be installed") - provision.Flags().String("dir-scripts", "", "directory with scripts that should be copied to the container") - provision.Flags().String("master-memory", "8192", "amount of RAM in MB on the master") - provision.Flags().String("master-cpu", "4", "number of CPU cores on the master") - provision.Flags().String("workers-memory", "4096", "amount of RAM in MB per worker") - provision.Flags().String("workers-cpu", "2", "number of CPU per worker") - provision.Flags().String("installer-pull-secret-file", "", "file that contains the installer pull secret") - provision.Flags().String("installer-repo-tag", "", "installer repository tag that you want to compile from") - provision.Flags().String("installer-release-image", "", "the OKD release image that you want to use") - - return provision -} - -func provision(cmd *cobra.Command, args []string) error { - prefix, err := cmd.Flags().GetString("prefix") - if err != nil { - return err - } - - dirHacks, err := cmd.Flags().GetString("dir-hacks") - if err != nil { - return err - } - - dirManifests, err := cmd.Flags().GetString("dir-manifests") - if err != nil { - return err - } - - dirScripts, err := cmd.Flags().GetString("dir-scripts") - if err != nil { - return err - } - - if dirScripts == "" { - return fmt.Errorf("you should provide the directory with scripts") - } - - envs := []string{} - - skipCnao, err := cmd.Flags().GetBool("skip-cnao") - if err != nil { - return err - } - envs = append(envs, fmt.Sprintf("CNAO=%t", !skipCnao)) - - networkingType, err := cmd.Flags().GetString("networking-type") - if err != nil { - return err - } - envs = append(envs, fmt.Sprintf("NETWORKING_TYPE=%s", networkingType)) - - masterMemory, err := cmd.Flags().GetString("master-memory") - if err != nil { - return err - } - envs = append(envs, fmt.Sprintf("MASTER_MEMORY=%s", masterMemory)) - - masterCPU, err := cmd.Flags().GetString("master-cpu") - if err != nil { - return err - } - envs = append(envs, fmt.Sprintf("MASTER_CPU=%s", masterCPU)) - - workersMemory, err := cmd.Flags().GetString("workers-memory") - if err != nil { - return err - } - envs = append(envs, fmt.Sprintf("WORKERS_MEMORY=%s", workersMemory)) - - workersCPU, err := cmd.Flags().GetString("workers-cpu") - if err != nil { - return err - } - envs = append(envs, fmt.Sprintf("WORKERS_CPU=%s", workersCPU)) - - installerTag, err := cmd.Flags().GetString("installer-repo-tag") - if err != nil { - return err - } - - if installerTag == "" { - return fmt.Errorf("you should provide the installer tag") - } - envs = append(envs, fmt.Sprintf("INSTALLER_TAG=%s", installerTag)) - - installerReleaseImage, err := cmd.Flags().GetString("installer-release-image") - if err != nil { - return err - } - - if installerReleaseImage != "" { - envs = append(envs, fmt.Sprintf("INSTALLER_RELEASE_IMAGE=%s", installerReleaseImage)) - } - - pullSecretFile, err := cmd.Flags().GetString("installer-pull-secret-file") - if err != nil { - return err - } - - if pullSecretFile == "" { - return fmt.Errorf("you should provide the installer secret token") - } - - base := args[0] - - cli, err := client.NewEnvClient() - if err != nil { - return err - } - - b := context.Background() - ctx, cancel := context.WithCancel(b) - - containers, _, done := docker.NewCleanupHandler(cli, cmd.OutOrStderr()) - - defer func() { - done <- err - }() - - go func() { - interrupt := make(chan os.Signal, 1) - signal.Notify(interrupt, os.Interrupt) - <-interrupt - cancel() - done <- fmt.Errorf("Interrupt received, clean up") - }() - - // Pull the base image - baseImage := "docker.io/" + base - fmt.Printf("Download the image %s\n", baseImage) - reader, err := cli.ImagePull(ctx, baseImage, types.ImagePullOptions{}) - if err != nil { - panic(err) - } - docker.PrintProgress(reader, os.Stdout) - - clusterContainerName := prefix + "-cluster" - // Start cluster container - cluster, err := cli.ContainerCreate(ctx, &container.Config{ - Image: base, - Env: envs, - }, &container.HostConfig{ - Mounts: []mount.Mount{ - { - Type: mount.TypeBind, - Source: pullSecretFile, - Target: "/etc/installer/token", - }, - }, - Privileged: true, - }, nil, clusterContainerName) - if err != nil { - return err - } - containers <- cluster.ID - - fmt.Printf("Start the container %s\n", clusterContainerName) - if err := cli.ContainerStart(ctx, cluster.ID, types.ContainerStartOptions{}); err != nil { - return err - } - - // Copy hacks directory to the container - if dirHacks != "" { - fmt.Printf("Copy hacks directory to the container %s\n", clusterContainerName) - config := ©Config{ - srcPath: dirHacks, - dstPath: "/", - container: cluster.ID, - } - err = copyToContainer(ctx, cli, config) - if err != nil { - return err - } - } - - // Copy manifests directory to the container - if dirManifests != "" { - fmt.Printf("Copy manifests directory to the container %s\n", clusterContainerName) - config := ©Config{ - srcPath: dirManifests, - dstPath: "/", - container: cluster.ID, - } - err = copyToContainer(ctx, cli, config) - if err != nil { - return err - } - } - - // Copy scripts directory to the container - fmt.Printf("Copy scripts directory to the container %s\n", clusterContainerName) - config := ©Config{ - srcPath: dirScripts, - dstPath: "/", - container: cluster.ID, - } - err = copyToContainer(ctx, cli, config) - if err != nil { - return err - } - - // Run provision script - fmt.Printf("Run provision script\n") - success, err := docker.Exec(cli, clusterContainerName, []string{"/bin/bash", "-c", "/scripts/provision.sh"}, os.Stdout) - if err != nil { - return err - } - - if !success { - return fmt.Errorf("failed to provision OKD cluster under the container %s", clusterContainerName) - } - - fmt.Printf("Commit the container %s\n", clusterContainerName) - _, err = cli.ContainerCommit(ctx, clusterContainerName, types.ContainerCommitOptions{Reference: "kubevirtci/" + prefix}) - if err != nil { - return fmt.Errorf("failed to commit the provisioned container %s: %v", clusterContainerName, err) - } - - done <- fmt.Errorf("Done. Cleanup") - - return nil -} - -func copyToContainer(ctx context.Context, cli *client.Client, config *copyConfig) error { - dstInfo := archive.CopyInfo{ - Exists: true, - IsDir: true, - Path: config.dstPath, - } - - srcInfo, err := archive.CopyInfoSourcePath(config.srcPath, true) - if err != nil { - return err - } - - srcArchive, err := archive.TarResource(srcInfo) - if err != nil { - return err - } - defer srcArchive.Close() - - dstDir, preparedArchive, err := archive.PrepareArchiveCopy(srcArchive, srcInfo, dstInfo) - if err != nil { - return err - } - defer preparedArchive.Close() - - return cli.CopyToContainer(ctx, config.container, dstDir, preparedArchive, types.CopyToContainerOptions{}) -} diff --git a/cluster-provision/gocli/cmd/okd/run.go b/cluster-provision/gocli/cmd/okd/run.go deleted file mode 100644 index a16e4f5022..0000000000 --- a/cluster-provision/gocli/cmd/okd/run.go +++ /dev/null @@ -1,330 +0,0 @@ -package okd - -import ( - "encoding/base64" - "encoding/json" - "fmt" - "os" - "os/signal" - "path/filepath" - - "github.com/docker/docker/api/types" - "github.com/docker/docker/api/types/container" - "github.com/docker/docker/api/types/mount" - "github.com/docker/docker/api/types/volume" - "github.com/docker/docker/client" - "github.com/docker/go-connections/nat" - "github.com/spf13/cobra" - "golang.org/x/net/context" - - "kubevirt.io/kubevirtci/cluster-provision/gocli/cmd/utils" - "kubevirt.io/kubevirtci/cluster-provision/gocli/docker" -) - -// NewRunCommand returns command that runs OKD cluster -func NewRunCommand() *cobra.Command { - run := &cobra.Command{ - Use: "okd", - Short: "run OKD cluster", - RunE: run, - Args: cobra.ExactArgs(1), - } - run.Flags().String("master-memory", "12288", "amount of RAM in MB on the master") - run.Flags().String("master-cpu", "4", "number of CPU cores on the master") - run.Flags().String("workers", "1", "number of cluster worker nodes to start") - run.Flags().String("workers-memory", "6144", "amount of RAM in MB per worker") - run.Flags().String("workers-cpu", "2", "number of CPU per worker") - run.Flags().UintP("secondary-nics", "", 0, "number of secondary nics to add") - run.Flags().String("registry-volume", "", "cache docker registry content in the specified volume") - run.Flags().String("nfs-data", "", "path to data which should be exposed via nfs to the nodes") - run.Flags().Uint("registry-port", 0, "port on localhost for the docker registry") - run.Flags().Uint("ocp-console-port", 0, "port on localhost for the ocp console") - run.Flags().Uint("k8s-port", 0, "port on localhost for the k8s cluster") - run.Flags().Uint("ssh-master-port", 0, "port on localhost to ssh to master node") - run.Flags().Uint("ssh-worker-port", 0, "port on localhost to ssh to worker node") - run.Flags().Bool("background", false, "go to background after nodes are up") - run.Flags().Bool("random-ports", true, "expose all ports on random localhost ports") - run.Flags().String("container-registry", "docker.io", "the registry to pull cluster container from") - run.Flags().String("container-registry-user", "", "the user to pull cluster container from") - run.Flags().String("container-registry-password", "", "the password to pull cluster container from") - run.Flags().String("installer-pull-secret-file", "", "file that contains the installer pull secret") - return run -} - -func run(cmd *cobra.Command, args []string) (err error) { - - prefix, err := cmd.Flags().GetString("prefix") - if err != nil { - return err - } - - envs := []string{} - workers, err := cmd.Flags().GetString("workers") - if err != nil { - return err - } - envs = append(envs, fmt.Sprintf("WORKERS=%s", workers)) - - masterMemory, err := cmd.Flags().GetString("master-memory") - if err != nil { - return err - } - envs = append(envs, fmt.Sprintf("MASTER_MEMORY=%s", masterMemory)) - - masterCPU, err := cmd.Flags().GetString("master-cpu") - if err != nil { - return err - } - envs = append(envs, fmt.Sprintf("MASTER_CPU=%s", masterCPU)) - - workersMemory, err := cmd.Flags().GetString("workers-memory") - if err != nil { - return err - } - envs = append(envs, fmt.Sprintf("WORKERS_MEMORY=%s", workersMemory)) - - workersCPU, err := cmd.Flags().GetString("workers-cpu") - if err != nil { - return err - } - envs = append(envs, fmt.Sprintf("WORKERS_CPU=%s", workersCPU)) - - secondaryNics, err := cmd.Flags().GetUint("secondary-nics") - if err != nil { - return err - } - envs = append(envs, fmt.Sprintf("NUM_SECONDARY_NICS=%d", secondaryNics)) - - pullSecretFile, err := cmd.Flags().GetString("installer-pull-secret-file") - if err != nil { - return err - } - - mounts := []mount.Mount{} - if pullSecretFile == "" { - fmt.Println("you should provide the installer pull secret file, if you want to install additional machines") - } else { - mounts = append(mounts, mount.Mount{ - Type: mount.TypeBind, - Source: pullSecretFile, - Target: "/etc/installer/token", - }) - } - - randomPorts, err := cmd.Flags().GetBool("random-ports") - if err != nil { - return err - } - - portMap := nat.PortMap{} - - containerRegistry, err := cmd.Flags().GetString("container-registry") - if err != nil { - return err - } - containerRegistryUser, err := cmd.Flags().GetString("container-registry-user") - if err != nil { - return err - } - containerRegistryPassword, err := cmd.Flags().GetString("container-registry-password") - if err != nil { - return err - } - if containerRegistryUser != "" && containerRegistryPassword == "" { - return fmt.Errorf("Missing --container-registry-password") - } - if containerRegistryPassword != "" && containerRegistryUser == "" { - return fmt.Errorf("Missing --container-registry-user") - } - - utils.AppendIfExplicit(portMap, utils.PortSSH, cmd.Flags(), "ssh-master-port") - utils.AppendIfExplicit(portMap, utils.PortSSHWorker, cmd.Flags(), "ssh-worker-port") - utils.AppendIfExplicit(portMap, utils.PortAPI, cmd.Flags(), "k8s-port") - utils.AppendIfExplicit(portMap, utils.PortOCPConsole, cmd.Flags(), "ocp-console-port") - utils.AppendIfExplicit(portMap, utils.PortRegistry, cmd.Flags(), "registry-port") - - registryVol, err := cmd.Flags().GetString("registry-volume") - if err != nil { - return err - } - - nfsData, err := cmd.Flags().GetString("nfs-data") - if err != nil { - return err - } - - cluster := args[0] - - background, err := cmd.Flags().GetBool("background") - if err != nil { - return err - } - - cli, err := client.NewEnvClient() - if err != nil { - return err - } - - b := context.Background() - ctx, cancel := context.WithCancel(b) - - containers, _, done := docker.NewCleanupHandler(cli, cmd.OutOrStderr()) - - defer func() { - done <- err - }() - - go func() { - interrupt := make(chan os.Signal, 1) - signal.Notify(interrupt, os.Interrupt) - <-interrupt - cancel() - done <- fmt.Errorf("Interrupt received, clean up") - }() - - if len(containerRegistry) > 0 { - cluster = containerRegistry + "/" + cluster - // Pull the cluster image - imagePullOptions := types.ImagePullOptions{} - if containerRegistryUser != "" { - authConfig := types.AuthConfig{ - Username: containerRegistryUser, - Password: containerRegistryPassword, - } - encodedAuthConfig, err := json.Marshal(authConfig) - if err != nil { - panic(err) - } - imagePullOptions.RegistryAuth = base64.URLEncoding.EncodeToString(encodedAuthConfig) - } - - fmt.Printf("Download the image %s\n", cluster) - err = docker.ImagePull(cli, ctx, cluster, imagePullOptions) - if err != nil { - panic(err) - } - } - - clusterContainerName := prefix + "-cluster" - // Start cluster container - clusterContainer, err := cli.ContainerCreate(ctx, &container.Config{ - Image: cluster, - Env: envs, - ExposedPorts: nat.PortSet{ - utils.TCPPortOrDie(utils.PortSSH): {}, - utils.TCPPortOrDie(utils.PortSSHWorker): {}, - utils.TCPPortOrDie(utils.PortRegistry): {}, - utils.TCPPortOrDie(utils.PortOCPConsole): {}, - utils.TCPPortOrDie(utils.PortAPI): {}, - }, - }, &container.HostConfig{ - Mounts: mounts, - Privileged: true, - PublishAllPorts: randomPorts, - PortBindings: portMap, - }, nil, clusterContainerName) - if err != nil { - return err - } - containers <- clusterContainer.ID - fmt.Printf("Start the container %s\n", clusterContainerName) - if err := cli.ContainerStart(ctx, clusterContainer.ID, types.ContainerStartOptions{}); err != nil { - return err - } - - // Pull the registry image - fmt.Printf("Download the image %s\n", utils.DockerRegistryImage) - err = docker.ImagePull(cli, ctx, utils.DockerRegistryImage, types.ImagePullOptions{}) - if err != nil { - panic(err) - } - - // Create registry volume - var registryMounts []mount.Mount - if registryVol != "" { - vol, err := cli.VolumeCreate(ctx, volume.VolumesCreateBody{ - Name: fmt.Sprintf("%s-%s", prefix, "registry"), - }) - if err != nil { - return err - } - registryMounts = []mount.Mount{ - { - Type: mount.TypeVolume, - Source: vol.Name, - Target: "/var/lib/registry", - }, - } - } - - // Start registry - registry, err := cli.ContainerCreate(ctx, &container.Config{ - Image: utils.DockerRegistryImage, - }, &container.HostConfig{ - Mounts: registryMounts, - Privileged: true, // fixme we just need proper selinux volume labeling - NetworkMode: container.NetworkMode("container:" + clusterContainer.ID), - }, nil, prefix+"-registry") - if err != nil { - return err - } - containers <- registry.ID - fmt.Printf("Start the container %s\n", prefix+"-registry") - if err := cli.ContainerStart(ctx, registry.ID, types.ContainerStartOptions{}); err != nil { - return err - } - - if nfsData != "" { - nfsData, err := filepath.Abs(nfsData) - if err != nil { - return err - } - // Pull the ganesha image - fmt.Printf("Download the image %s\n", utils.NFSGaneshaImage) - err = docker.ImagePull(cli, ctx, utils.NFSGaneshaImage, types.ImagePullOptions{}) - if err != nil { - panic(err) - } - - // Start the ganesha image - nfsServer, err := cli.ContainerCreate(ctx, &container.Config{ - Image: utils.NFSGaneshaImage, - }, &container.HostConfig{ - Mounts: []mount.Mount{ - { - Type: mount.TypeBind, - Source: nfsData, - Target: "/data/nfs", - }, - }, - Privileged: true, - NetworkMode: container.NetworkMode("container:" + clusterContainer.ID), - }, nil, prefix+"-nfs-ganesha") - if err != nil { - return err - } - containers <- nfsServer.ID - fmt.Printf("Start the container %s\n", prefix+"-nfs-ganesha") - if err := cli.ContainerStart(ctx, nfsServer.ID, types.ContainerStartOptions{}); err != nil { - return err - } - } - - // Run the cluster - fmt.Printf("Run the cluster\n") - success, err := docker.Exec(cli, clusterContainerName, []string{"/bin/bash", "-c", "/scripts/run.sh"}, os.Stdout) - if err != nil { - return err - } - - if !success { - return fmt.Errorf("failed to run the OKD cluster under the container %s", clusterContainerName) - } - - // If background flag was specified, we don't want to clean up if we reach that state - if !background { - done <- fmt.Errorf("Done. please clean up") - } - - return nil -} diff --git a/cluster-provision/gocli/cmd/provision.go b/cluster-provision/gocli/cmd/provision.go index 3cbe43a2d0..8cfb6ff4cb 100644 --- a/cluster-provision/gocli/cmd/provision.go +++ b/cluster-provision/gocli/cmd/provision.go @@ -15,7 +15,6 @@ import ( "github.com/spf13/cobra" "golang.org/x/net/context" - "kubevirt.io/kubevirtci/cluster-provision/gocli/cmd/okd" "kubevirt.io/kubevirtci/cluster-provision/gocli/cmd/utils" "kubevirt.io/kubevirtci/cluster-provision/gocli/docker" ) @@ -37,10 +36,6 @@ func NewProvisionCommand() *cobra.Command { provision.Flags().Uint("vnc-port", 0, "port on localhost for vnc") provision.Flags().Uint("ssh-port", 0, "port on localhost for ssh server") - provision.AddCommand( - okd.NewProvisionCommand(), - ) - return provision } diff --git a/cluster-provision/gocli/cmd/run.go b/cluster-provision/gocli/cmd/run.go index ba0509ea07..14db7e02c8 100644 --- a/cluster-provision/gocli/cmd/run.go +++ b/cluster-provision/gocli/cmd/run.go @@ -21,7 +21,6 @@ import ( "github.com/spf13/cobra" "golang.org/x/net/context" - "kubevirt.io/kubevirtci/cluster-provision/gocli/cmd/okd" "kubevirt.io/kubevirtci/cluster-provision/gocli/cmd/utils" "kubevirt.io/kubevirtci/cluster-provision/gocli/docker" ) @@ -74,9 +73,6 @@ func NewRunCommand() *cobra.Command { run.Flags().String("docker-proxy", "", "sets network proxy for docker daemon") run.Flags().String("container-registry", "docker.io", "the registry to pull cluster container from") - run.AddCommand( - okd.NewRunCommand(), - ) return run } diff --git a/cluster-provision/manifests/okd/install-config.yaml b/cluster-provision/manifests/okd/install-config.yaml deleted file mode 100644 index f71003b5c6..0000000000 --- a/cluster-provision/manifests/okd/install-config.yaml +++ /dev/null @@ -1,26 +0,0 @@ -apiVersion: v1beta3 -baseDomain: tt.testing -compute: -- name: worker - platform: {} - replicas: 2 -controlPlane: - name: master - platform: {} - replicas: 1 -metadata: - name: test-1 -networking: - clusterNetworks: - - cidr: 10.128.0.0/14 - hostSubnetLength: 9 - machineCIDR: 192.168.126.0/24 - serviceCIDR: 172.30.0.0/16 - type: ${NETWORKING_TYPE} -platform: - libvirt: - URI: qemu+tcp://192.168.122.1/system - network: - if: tt0 -pullSecret: '${PULL_SECRET}' -sshKey: '${SSH_PUBLIC_KEY}' diff --git a/cluster-provision/manifests/okd/local-storage-cr.yaml b/cluster-provision/manifests/okd/local-storage-cr.yaml deleted file mode 100644 index 86b1439720..0000000000 --- a/cluster-provision/manifests/okd/local-storage-cr.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: local.storage.openshift.io/v1 -kind: LocalVolume -metadata: - name: local-disks - namespace: local-storage -spec: - storageClassDevices: - - storageClassName: local - volumeMode: Filesystem - fsType: ext4 - devicePaths: - - /mnt/local-storage/local diff --git a/cluster-provision/manifests/okd/local-storage.yaml b/cluster-provision/manifests/okd/local-storage.yaml deleted file mode 100644 index edb01f2269..0000000000 --- a/cluster-provision/manifests/okd/local-storage.yaml +++ /dev/null @@ -1,32 +0,0 @@ ---- -apiVersion: operators.coreos.com/v1 -kind: OperatorGroup -metadata: - name: local-operator-group - namespace: local-storage - spec: - targetNamespaces: - - local-storage ---- -apiVersion: operators.coreos.com/v1alpha1 -kind: CatalogSource -metadata: - name: local-storage-manifests - namespace: local-storage -spec: - sourceType: grpc - image: quay.io/gnufied/local-registry:v4.2.0 - displayName: Local Storage Operator - publisher: Red Hat - description: An operator to manage local volumes ---- -apiVersion: operators.coreos.com/v1alpha1 -kind: Subscription -metadata: - name: local-storage-subscription - namespace: local-storage -spec: - channel: stable - name: local-storage-operator - source: local-storage-manifests - sourceNamespace: local-storage diff --git a/cluster-provision/manifests/okd/registries.conf b/cluster-provision/manifests/okd/registries.conf deleted file mode 100644 index 53720300de..0000000000 --- a/cluster-provision/manifests/okd/registries.conf +++ /dev/null @@ -1,7 +0,0 @@ -[registries] - [registries.search] - registries = ["registry.access.redhat.com", "docker.io"] - [registries.insecure] - registries = ["brew-pulp-docker01.web.prod.ext.phx2.redhat.com:8888", "registry:5000", "registry-proxy.engineering.redhat.com"] - [registries.block] - registries = [] diff --git a/cluster-provision/manifests/okd/registries.yaml b/cluster-provision/manifests/okd/registries.yaml deleted file mode 100644 index f348daed4e..0000000000 --- a/cluster-provision/manifests/okd/registries.yaml +++ /dev/null @@ -1,32 +0,0 @@ -spec: - config: - ignition: - config: {} - security: - tls: {} - timeouts: {} - version: 2.2.0 - networkd: {} - passwd: {} - storage: { - "files": [ - { - "path": "/etc/containers/registries.conf", - "filesystem": "root", - "mode": 420, - "contents": { - "source": "data:;base64,${REGISTRIES_CONF}" - } - } - ] - } - systemd: { - "units": [ - { - "contents": "[Unit]\nDescription=Update system CA\nAfter=syslog.target network.target\n\n[Service]\nType=oneshot\nExecStart=/usr/bin/update-ca-trust\nRemainAfterExit=true\n\n[Install]\nWantedBy=multi-user.target\n", - "enabled": true, - "name": "update-ca.service" - } - ] - } - osImageURL: "" diff --git a/cluster-provision/ocp/4.3/provision.sh b/cluster-provision/ocp/4.3/provision.sh deleted file mode 100755 index 03b44262ef..0000000000 --- a/cluster-provision/ocp/4.3/provision.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash - -set -x - -PARENT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../.. && pwd )" -KUBEVIRTCI_DIR="$( cd ${PARENT_DIR}/../kubevirtci && pwd)" - -okd_base_hash="sha256:73ede51ce464546a82b81956b7f58cf98662a4c5fded9c659b57746bc131e047" -gocli_image_hash="sha256:220f55f6b1bcb3975d535948d335bd0e6b6297149a3eba1a4c14cad9ac80f80d" - -gocli="docker run \ ---privileged \ ---net=host \ ---rm -t \ --v /var/run/docker.sock:/var/run/docker.sock \ --v ${PARENT_DIR}:${PARENT_DIR} \ -docker.io/kubevirtci/gocli@${gocli_image_hash}" - -provisioner_container_id=$(docker ps --filter name=ocp-4.3-provision-cluster --format {{.ID}}) -docker kill $provisioner_container_id -docker container rm $provisioner_container_id - -${gocli} provision okd \ ---prefix ocp-4.3-provision \ ---dir-scripts ${PARENT_DIR}/okd/scripts \ ---dir-manifests ${PARENT_DIR}/manifests \ ---dir-hacks ${PARENT_DIR}/okd/hacks \ ---skip-cnao \ ---workers-memory 8192 \ ---workers-cpu 4 \ ---installer-pull-secret-file ${INSTALLER_PULL_SECRET} \ ---installer-repo-tag release-4.3 \ ---installer-release-image registry.svc.ci.openshift.org/ocp/release:4.3 \ -"kubevirtci/okd-base@${okd_base_hash}" -rc=$? - - - -exit $rc diff --git a/cluster-provision/ocp/4.3/publish.sh b/cluster-provision/ocp/4.3/publish.sh deleted file mode 100755 index 63ab8d0436..0000000000 --- a/cluster-provision/ocp/4.3/publish.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash -tag=$(git log -1 --pretty=%h)-$(date +%s) -destination="quay.io/kubevirtci/ocp-4.3:$tag" -docker tag kubevirtci/ocp-4.3-provision:latest $destination -docker push $destination diff --git a/cluster-provision/ocp/4.3/run.sh b/cluster-provision/ocp/4.3/run.sh deleted file mode 100755 index 4152ab035d..0000000000 --- a/cluster-provision/ocp/4.3/run.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash - -set -x - -ocp_image_hash="sha256:16a70403141142aae387a50feb2fd039a745c6916aa3f61e1a5d5a74efb6be39" -gocli_image_hash="sha256:a7880757e2d2755c6a784c1b64c64b096769ed3ccfac9d8e535df481731c2144" - -gocli="docker run --privileged --net=host --rm -t -v /var/run/docker.sock:/var/run/docker.sock docker.io/kubevirtci/gocli@${gocli_image_hash}" - -${gocli} run ocp --random-ports --background --prefix ocp-4.3 --registry-volume ocp-4.3-registry "kubevirtci/ocp-4.3@${ocp_image_hash}" diff --git a/cluster-provision/ocp/4.4/provision.sh b/cluster-provision/ocp/4.4/provision.sh deleted file mode 100755 index 33484c4319..0000000000 --- a/cluster-provision/ocp/4.4/provision.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/bash - -set -x - -PARENT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../.. && pwd )" -KUBEVIRTCI_DIR="$( cd ${PARENT_DIR}/../kubevirtci && pwd)" - -okd_base_hash="sha256:73ede51ce464546a82b81956b7f58cf98662a4c5fded9c659b57746bc131e047" -gocli_image_hash="sha256:220f55f6b1bcb3975d535948d335bd0e6b6297149a3eba1a4c14cad9ac80f80d" - -gocli="docker run \ ---privileged \ ---net=host \ ---rm -t \ --v /var/run/docker.sock:/var/run/docker.sock \ --v ${PARENT_DIR}:${PARENT_DIR} \ -docker.io/kubevirtci/gocli@${gocli_image_hash}" - -provisioner_container_id=$(docker ps --filter name=ocp-4.4-provision-cluster --format {{.ID}}) -docker kill $provisioner_container_id -docker container rm $provisioner_container_id - -# For ocp-4.4 we want OVNKubernetes -${gocli} provision okd \ ---prefix ocp-4.4-provision \ ---dir-scripts ${PARENT_DIR}/okd/scripts \ ---dir-manifests ${PARENT_DIR}/manifests \ ---dir-hacks ${PARENT_DIR}/okd/hacks \ ---skip-cnao \ ---master-memory 10240 \ ---workers-memory 8192 \ ---workers-cpu 4 \ ---networking-type OVNKubernetes \ ---installer-pull-secret-file ${INSTALLER_PULL_SECRET} \ ---installer-repo-tag release-4.4 \ ---installer-release-image registry.svc.ci.openshift.org/ocp/release:4.4 \ -"kubevirtci/okd-base@${okd_base_hash}" -rc=$? - - - -exit $rc diff --git a/cluster-provision/ocp/4.4/publish.sh b/cluster-provision/ocp/4.4/publish.sh deleted file mode 100755 index 00b875dec3..0000000000 --- a/cluster-provision/ocp/4.4/publish.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash - -tag=$(git log -1 --pretty=%h)-$(date +%s) -destination="quay.io/kubevirtci/ocp-4.4:$tag" - -docker tag kubevirtci/ocp-4.4-provision:latest $destination -docker push $destination diff --git a/cluster-provision/ocp/4.4/run.sh b/cluster-provision/ocp/4.4/run.sh deleted file mode 100755 index a556246bc6..0000000000 --- a/cluster-provision/ocp/4.4/run.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash - -set -x - -ocp_image_hash="sha256:16a70403141142aae387a50feb2fd039a745c6916aa3f61e1a5d5a74efb6be39" -gocli_image_hash="sha256:a7880757e2d2755c6a784c1b64c64b096769ed3ccfac9d8e535df481731c2144" - -gocli="docker run --privileged --net=host --rm -t -v /var/run/docker.sock:/var/run/docker.sock docker.io/kubevirtci/gocli@${gocli_image_hash}" - -${gocli} run ocp --random-ports --background --prefix ocp-4.4 --registry-volume ocp-4.4-registry "kubevirtci/ocp-4.4@${ocp_image_hash}" diff --git a/cluster-provision/ocp/README.md b/cluster-provision/ocp/README.md deleted file mode 100644 index 4560a3a40a..0000000000 --- a/cluster-provision/ocp/README.md +++ /dev/null @@ -1,11 +0,0 @@ -# How to create new OCP release - -1. Get pull secrets from https://cloud.redhat.com/openshift/install/metal/user-provisioned - -2. Export location INSTALLER_PULL_SECRET=pull-secret.txt - -2. Provision ocp-4.3 provider ./cluster-provision/ocp/4.3/provision.sh - -3. Log into quay.io container registry make sure you have push permissiong for openshift-cnv organization - -4. Push the ocp-4.3 provider container with ./cluster-provision/ocp/4.3/publish.sh diff --git a/cluster-provision/okd/4.1/provision.sh b/cluster-provision/okd/4.1/provision.sh deleted file mode 100755 index 09da131879..0000000000 --- a/cluster-provision/okd/4.1/provision.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash - -set -x - -PARENT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../.. && pwd )" - -okd_base_hash="sha256:73ede51ce464546a82b81956b7f58cf98662a4c5fded9c659b57746bc131e047" -gocli_image_hash="sha256:220f55f6b1bcb3975d535948d335bd0e6b6297149a3eba1a4c14cad9ac80f80d" - -gocli="docker run \ ---privileged \ ---net=host \ ---rm -t \ --v /var/run/docker.sock:/var/run/docker.sock \ --v ${PARENT_DIR}:${PARENT_DIR} \ -docker.io/kubevirtci/gocli@${gocli_image_hash}" - -${gocli} provision okd \ ---prefix okd-4.1-provision \ ---dir-scripts ${PARENT_DIR}/okd/scripts \ ---dir-manifests ${PARENT_DIR}/manifests \ ---dir-hacks ${PARENT_DIR}/okd/hacks \ ---master-memory 10240 \ ---workers-cpu 4 \ ---workers-memory 6144 \ ---installer-pull-secret-file ${INSTALLER_PULL_SECRET} \ ---installer-repo-tag release-4.1 \ ---installer-release-image docker.io/kubevirtci/ocp-release:4.1.24 \ -"kubevirtci/okd-base@${okd_base_hash}" diff --git a/cluster-provision/okd/4.1/publish.sh b/cluster-provision/okd/4.1/publish.sh deleted file mode 100755 index 50b5ea748f..0000000000 --- a/cluster-provision/okd/4.1/publish.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash - -docker tag kubevirtci/okd-4.1-provision:latest docker.io/kubevirtci/okd-4.1:latest -docker push docker.io/kubevirtci/okd-4.1:latest diff --git a/cluster-provision/okd/4.1/run.sh b/cluster-provision/okd/4.1/run.sh deleted file mode 100755 index a9af5f5961..0000000000 --- a/cluster-provision/okd/4.1/run.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash - -set -x - -okd_image_hash="sha256:e7e3a03bb144eb8c0be4dcd700592934856fb623d51a2b53871d69267ca51c86" -gocli_image_hash="sha256:220f55f6b1bcb3975d535948d335bd0e6b6297149a3eba1a4c14cad9ac80f80d" - -gocli="docker run --privileged --net=host --rm -t -v /var/run/docker.sock:/var/run/docker.sock docker.io/kubevirtci/gocli@${gocli_image_hash}" - -${gocli} run okd \ ---random-ports \ ---background \ ---prefix okd-4.1 \ ---registry-volume okd-4.1-registry \ ---installer-pull-secret-file ${INSTALLER_PULL_SECRET} \ -"kubevirtci/okd-4.1@${okd_image_hash}" diff --git a/cluster-provision/okd/4.2/provision.sh b/cluster-provision/okd/4.2/provision.sh deleted file mode 100755 index 3eae7ced54..0000000000 --- a/cluster-provision/okd/4.2/provision.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash - -set -x - -PARENT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../.. && pwd )" - -okd_base_hash="sha256:73ede51ce464546a82b81956b7f58cf98662a4c5fded9c659b57746bc131e047" -gocli_image_hash="sha256:220f55f6b1bcb3975d535948d335bd0e6b6297149a3eba1a4c14cad9ac80f80d" - -gocli="docker run \ ---privileged \ ---net=host \ ---rm -t \ --v /var/run/docker.sock:/var/run/docker.sock \ --v ${PARENT_DIR}:${PARENT_DIR} \ -docker.io/kubevirtci/gocli@${gocli_image_hash}" - -${gocli} provision okd \ ---prefix okd-4.2-provision \ ---dir-scripts ${PARENT_DIR}/okd/scripts \ ---dir-manifests ${PARENT_DIR}/manifests \ ---dir-hacks ${PARENT_DIR}/okd/hacks \ ---master-memory 10240 \ ---workers-cpu 4 \ ---workers-memory 6144 \ ---installer-pull-secret-file ${INSTALLER_PULL_SECRET} \ ---installer-repo-tag release-4.2 \ ---installer-release-image docker.io/kubevirtci/ocp-release:4.2.5 \ -"kubevirtci/okd-base@${okd_base_hash}" diff --git a/cluster-provision/okd/4.2/publish.sh b/cluster-provision/okd/4.2/publish.sh deleted file mode 100755 index 8e3aba5d41..0000000000 --- a/cluster-provision/okd/4.2/publish.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash - -docker tag kubevirtci/okd-4.2-provision:latest docker.io/kubevirtci/okd-4.2:latest -docker push docker.io/kubevirtci/okd-4.2:latest diff --git a/cluster-provision/okd/4.2/run.sh b/cluster-provision/okd/4.2/run.sh deleted file mode 100755 index fcbee924fb..0000000000 --- a/cluster-provision/okd/4.2/run.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash - -set -x - -okd_image_hash="sha256:a830064ca7bf5c5c2f15df180f816534e669a9a038fef4919116d61eb33e84c5" -gocli_image_hash="sha256:220f55f6b1bcb3975d535948d335bd0e6b6297149a3eba1a4c14cad9ac80f80d" - -gocli="docker run --privileged --net=host --rm -t -v /var/run/docker.sock:/var/run/docker.sock docker.io/kubevirtci/gocli@${gocli_image_hash}" - -${gocli} run okd \ ---random-ports \ ---background \ ---prefix okd-4.2 \ ---registry-volume okd-4.2-registry \ ---installer-pull-secret-file ${INSTALLER_PULL_SECRET} \ -"kubevirtci/okd-4.2@${okd_image_hash}" diff --git a/cluster-provision/okd/4.3/provision.sh b/cluster-provision/okd/4.3/provision.sh deleted file mode 100755 index d810c43bfe..0000000000 --- a/cluster-provision/okd/4.3/provision.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/bash - -set -x - -PARENT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../.. && pwd )" - -okd_base_hash="sha256:73ede51ce464546a82b81956b7f58cf98662a4c5fded9c659b57746bc131e047" -gocli_image_hash="sha256:220f55f6b1bcb3975d535948d335bd0e6b6297149a3eba1a4c14cad9ac80f80d" - -gocli="docker run \ ---privileged \ ---net=host \ ---rm -t \ --v /var/run/docker.sock:/var/run/docker.sock \ --v ${PARENT_DIR}:${PARENT_DIR} \ -docker.io/kubevirtci/gocli@${gocli_image_hash}" - -provisioner_container_id=$(docker ps --filter name=okd-4.3-provision-cluster --format {{.ID}}) -docker kill $provisioner_container_id -docker container rm $provisioner_container_id - -${gocli} provision okd \ ---prefix okd-4.3-provision \ ---dir-scripts ${PARENT_DIR}/okd/scripts \ ---dir-manifests ${PARENT_DIR}/manifests \ ---dir-hacks ${PARENT_DIR}/okd/hacks \ ---workers-memory 8192 \ ---workers-cpu 4 \ ---workers-memory 6144 \ ---installer-pull-secret-file ${INSTALLER_PULL_SECRET} \ ---installer-repo-tag release-4.3 \ -"kubevirtci/okd-base@${okd_base_hash}" -exit $? diff --git a/cluster-provision/okd/4.3/publish.sh b/cluster-provision/okd/4.3/publish.sh deleted file mode 100755 index 9b915fca41..0000000000 --- a/cluster-provision/okd/4.3/publish.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash - -docker tag kubevirtci/okd-4.3-provision:latest docker.io/kubevirtci/okd-4.3:latest -docker push docker.io/kubevirtci/okd-4.3:latest diff --git a/cluster-provision/okd/4.3/run.sh b/cluster-provision/okd/4.3/run.sh deleted file mode 100755 index 7113097e94..0000000000 --- a/cluster-provision/okd/4.3/run.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash - -set -x - -okd_image_hash="sha256:63abc3884002a615712dfac5f42785be864ea62006892bf8a086ccdbca8b3d38" -gocli_image_hash="sha256:220f55f6b1bcb3975d535948d335bd0e6b6297149a3eba1a4c14cad9ac80f80d" - -gocli="docker run --privileged --net=host --rm -t -v /var/run/docker.sock:/var/run/docker.sock docker.io/kubevirtci/gocli@${gocli_image_hash}" - -${gocli} run okd \ ---random-ports \ ---background \ ---prefix okd-4.3 \ ---registry-volume okd-4.3-registry \ ---installer-pull-secret-file ${INSTALLER_PULL_SECRET} \ -"kubevirtci/okd-4.3@${okd_image_hash}" diff --git a/cluster-provision/okd/README.md b/cluster-provision/okd/README.md deleted file mode 100644 index 6cb0293af0..0000000000 --- a/cluster-provision/okd/README.md +++ /dev/null @@ -1,13 +0,0 @@ -# How to create new OKD release - -Possible the situatution when the specific OKD componenet need have some bug fix, that does not exist under the release image, in this case you can build new component image with the fix and create a new release image that will use this component image. - -1. You will need to download [`oc`](https://mirror.openshift.com/pub/openshift-v4/clients/ocp/latest/) binary and extract it. - -2. After you can check all images via `oc adm release info quay.io/openshift-release-dev/ocp-release:4.1.18` - -3. And create a new image that will use a new component image `oc adm release new --to-image docker.io/kubevirtci/ocp-release:4.1.18 --from-release quay.io/openshift-release-dev/ocp-release:4.1.18 libvirt-machine-controllers=docker.io/kubevirtci/origin-libvirt-machine-controllers@sha256:090d4035c6558cdc956d5fed70b0646998c9c4058ed1791370d76d8553130244` - -*** -Note: Be sure that you have `pull` permissions for the image repository, in the case of above example, you should have permissions for `docker.io/kubevirtci` and you will need read permissions for `quay.io/openshift-release-dev` repository. -*** diff --git a/cluster-provision/okd/base/Dockerfile b/cluster-provision/okd/base/Dockerfile deleted file mode 100644 index 576a42d496..0000000000 --- a/cluster-provision/okd/base/Dockerfile +++ /dev/null @@ -1,47 +0,0 @@ -FROM fedora@sha256:a66c6fa97957087176fede47846e503aeffc0441050dd7d6d2ed9e2fae50ea8e - -RUN dnf install -y \ -libvirt \ -libvirt-devel \ -libvirt-daemon-kvm \ -libvirt-client \ -qemu-kvm \ -openssh-clients \ -haproxy \ -jq \ -virt-install \ -socat \ -selinux-policy \ -selinux-policy-targeted \ -httpd-tools \ -python2-pip && \ -dnf clean all - -RUN pip2 install yq - -# install golang -ENV GO_VERSION 1.12.12 -RUN curl https://dl.google.com/go/go${GO_VERSION}.linux-amd64.tar.gz -o go.tar.gz && \ -tar -xvzf go.tar.gz -C /usr/local/ && \ -rm -rf go.tar.gz - -# Install oc client -ENV OC_VERSION 4.3 -RUN curl https://mirror.openshift.com/pub/openshift-v4/clients/oc/${OC_VERSION}/linux/oc.tar.gz -o oc.tar.gz && \ -tar -xvzf oc.tar.gz -C /usr/local/bin/ && \ -rm -rf oc.tar.gz - -# configure libvirt -RUN echo 'listen_tls = 0' >> /etc/libvirt/libvirtd.conf; \ -echo 'listen_tcp = 1' >> /etc/libvirt/libvirtd.conf; \ -echo 'auth_tcp="none"' >> /etc/libvirt/libvirtd.conf; \ -echo 'tcp_port = "16509"' >> /etc/libvirt/libvirtd.conf; \ -echo 'cgroup_controllers = [ ]' >> /etc/libvirt/qemu.conf - -COPY vagrant.key / -RUN chmod 600 /vagrant.key - -COPY haproxy.cfg /etc/haproxy - -COPY entrypoint.sh /entrypoint.sh -ENTRYPOINT [ "/entrypoint.sh" ] diff --git a/cluster-provision/okd/base/build.sh b/cluster-provision/okd/base/build.sh deleted file mode 100755 index 70952461ab..0000000000 --- a/cluster-provision/okd/base/build.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -docker build -t kubevirtci/okd-base . diff --git a/cluster-provision/okd/base/entrypoint.sh b/cluster-provision/okd/base/entrypoint.sh deleted file mode 100755 index 8dd4c93f47..0000000000 --- a/cluster-provision/okd/base/entrypoint.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/bash - -# set KVM device permissions -chown root:kvm /dev/kvm -chmod 660 /dev/kvm - -# configure iptables -iptables \ - -I INPUT \ - -p tcp \ - -s 192.168.126.0/24 \ - -d 192.168.124.1 \ - --dport 16509 \ - -j ACCEPT \ - -m comment \ - --comment "Allow insecure libvirt clients" - -# add go ENV variables -echo 'export GOROOT=/usr/local/go' >> /root/.bashrc -echo 'export GOPATH=/root/go/' >> /root/.bashrc -echo 'export PATH=$GOPATH/bin:$GOROOT/bin:$PATH' >> /root/.bashrc - -# start libvirt -/usr/sbin/virtlogd --daemon -/usr/sbin/libvirtd --listen diff --git a/cluster-provision/okd/base/haproxy.cfg b/cluster-provision/okd/base/haproxy.cfg deleted file mode 100644 index 8309d56cda..0000000000 --- a/cluster-provision/okd/base/haproxy.cfg +++ /dev/null @@ -1,90 +0,0 @@ -#--------------------------------------------------------------------- -# Example configuration for a possible web application. See the -# full configuration options online. -# -# https://www.haproxy.org/download/1.8/doc/configuration.txt -# -#--------------------------------------------------------------------- - -#--------------------------------------------------------------------- -# Global settings -#--------------------------------------------------------------------- -global - # to have these messages end up in /var/log/haproxy.log you will - # need to: - # - # 1) configure syslog to accept network log events. This is done - # by adding the '-r' option to the SYSLOGD_OPTIONS in - # /etc/sysconfig/syslog - # - # 2) configure local2 events to go to the /var/log/haproxy.log - # file. A line like the following can be added to - # /etc/sysconfig/syslog - # - # local2.* /var/log/haproxy.log - # - log 127.0.0.1 local2 - - chroot /var/lib/haproxy - pidfile /var/run/haproxy.pid - maxconn 4000 - user haproxy - group haproxy - daemon - - # turn on stats unix socket - stats socket /var/lib/haproxy/stats - - # utilize system-wide crypto-policies - ssl-default-bind-ciphers PROFILE=SYSTEM - ssl-default-server-ciphers PROFILE=SYSTEM - -#--------------------------------------------------------------------- -# common defaults that all the 'listen' and 'backend' sections will -# use if not designated in their block -#--------------------------------------------------------------------- - -resolvers libvirt-dns - nameserver dns1 192.168.126.1:53 - resolve_retries 3 - timeout retry 1s - hold other 30s - hold refused 30s - hold nx 30s - hold timeout 30s - hold valid 10s - -defaults - mode tcp - log global - option tcplog - option dontlognull - option http-server-close - option redispatch - retries 3 - timeout http-request 10s - timeout queue 1m - timeout connect 10s - timeout client 1m - timeout server 1m - timeout http-keep-alive 10s - timeout check 10s - maxconn 3000 - default-server init-addr none - -listen console - bind *:443 - server worker-0 console-openshift-console.apps.test-1.tt.testing:443 resolvers libvirt-dns check - -listen api - bind *:6443 - server master-0 api.test-1.tt.testing:6443 resolvers libvirt-dns check - -# TODO: this hardcode values should be filled on runtime -listen ssh-master - bind *:2201 - server master-0 192.168.126.11:22 check - -listen ssh-worker - bind *:2202 - server worker-0 192.168.126.51:22 check diff --git a/cluster-provision/okd/base/publish.sh b/cluster-provision/okd/base/publish.sh deleted file mode 100755 index ba9c596b12..0000000000 --- a/cluster-provision/okd/base/publish.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash - -docker tag kubevirtci/okd-base docker.io/kubevirtci/okd-base -docker push docker.io/kubevirtci/okd-base diff --git a/cluster-provision/okd/base/vagrant.key b/cluster-provision/okd/base/vagrant.key deleted file mode 100644 index 7d6a083909..0000000000 --- a/cluster-provision/okd/base/vagrant.key +++ /dev/null @@ -1,27 +0,0 @@ ------BEGIN RSA PRIVATE KEY----- -MIIEogIBAAKCAQEA6NF8iallvQVp22WDkTkyrtvp9eWW6A8YVr+kz4TjGYe7gHzI -w+niNltGEFHzD8+v1I2YJ6oXevct1YeS0o9HZyN1Q9qgCgzUFtdOKLv6IedplqoP -kcmF0aYet2PkEDo3MlTBckFXPITAMzF8dJSIFo9D8HfdOV0IAdx4O7PtixWKn5y2 -hMNG0zQPyUecp4pzC6kivAIhyfHilFR61RGL+GPXQ2MWZWFYbAGjyiYJnAmCP3NO -Td0jMZEnDkbUvxhMmBYSdETk1rRgm+R4LOzFUGaHqHDLKLX+FIPKcF96hrucXzcW -yLbIbEgE98OHlnVYCzRdK8jlqm8tehUc9c9WhQIBIwKCAQEA4iqWPJXtzZA68mKd -ELs4jJsdyky+ewdZeNds5tjcnHU5zUYE25K+ffJED9qUWICcLZDc81TGWjHyAqD1 -Bw7XpgUwFgeUJwUlzQurAv+/ySnxiwuaGJfhFM1CaQHzfXphgVml+fZUvnJUTvzf -TK2Lg6EdbUE9TarUlBf/xPfuEhMSlIE5keb/Zz3/LUlRg8yDqz5w+QWVJ4utnKnK -iqwZN0mwpwU7YSyJhlT4YV1F3n4YjLswM5wJs2oqm0jssQu/BT0tyEXNDYBLEF4A -sClaWuSJ2kjq7KhrrYXzagqhnSei9ODYFShJu8UWVec3Ihb5ZXlzO6vdNQ1J9Xsf -4m+2ywKBgQD6qFxx/Rv9CNN96l/4rb14HKirC2o/orApiHmHDsURs5rUKDx0f9iP -cXN7S1uePXuJRK/5hsubaOCx3Owd2u9gD6Oq0CsMkE4CUSiJcYrMANtx54cGH7Rk -EjFZxK8xAv1ldELEyxrFqkbE4BKd8QOt414qjvTGyAK+OLD3M2QdCQKBgQDtx8pN -CAxR7yhHbIWT1AH66+XWN8bXq7l3RO/ukeaci98JfkbkxURZhtxV/HHuvUhnPLdX -3TwygPBYZFNo4pzVEhzWoTtnEtrFueKxyc3+LjZpuo+mBlQ6ORtfgkr9gBVphXZG -YEzkCD3lVdl8L4cw9BVpKrJCs1c5taGjDgdInQKBgHm/fVvv96bJxc9x1tffXAcj -3OVdUN0UgXNCSaf/3A/phbeBQe9xS+3mpc4r6qvx+iy69mNBeNZ0xOitIjpjBo2+ -dBEjSBwLk5q5tJqHmy/jKMJL4n9ROlx93XS+njxgibTvU6Fp9w+NOFD/HvxB3Tcz -6+jJF85D5BNAG3DBMKBjAoGBAOAxZvgsKN+JuENXsST7F89Tck2iTcQIT8g5rwWC -P9Vt74yboe2kDT531w8+egz7nAmRBKNM751U/95P9t88EDacDI/Z2OwnuFQHCPDF -llYOUI+SpLJ6/vURRbHSnnn8a/XG+nzedGH5JGqEJNQsz+xT2axM0/W/CRknmGaJ -kda/AoGANWrLCz708y7VYgAtW2Uf1DPOIYMdvo6fxIB5i9ZfISgcJ/bbCUkFrhoH -+vq/5CIWxCPp0f85R4qxxQ5ihxJ0YDQT9Jpx4TMss4PSavPaBH3RXow5Ohe+bYoQ -NE5OgEXk2wVfZczCZpigBKbKZHNYcelXtTt/nP3rsCuGcM4h53s= ------END RSA PRIVATE KEY----- diff --git a/cluster-provision/okd/hacks/release-4.1 b/cluster-provision/okd/hacks/release-4.1 deleted file mode 100644 index 4c4b6cb450..0000000000 --- a/cluster-provision/okd/hacks/release-4.1 +++ /dev/null @@ -1,209 +0,0 @@ -diff --git a/cmd/openshift-install/create.go b/cmd/openshift-install/create.go -index 2e3d442c6..f7c52a3a5 100644 ---- a/cmd/openshift-install/create.go -+++ b/cmd/openshift-install/create.go -@@ -238,7 +238,7 @@ func waitForBootstrapComplete(ctx context.Context, config *rest.Config, director - - discovery := client.Discovery() - -- apiTimeout := 30 * time.Minute -+ apiTimeout := 120 * time.Minute - logrus.Infof("Waiting up to %v for the Kubernetes API at %s...", apiTimeout, config.Host) - apiContext, cancel := context.WithTimeout(ctx, apiTimeout) - defer cancel() -@@ -279,7 +279,7 @@ func waitForBootstrapComplete(ctx context.Context, config *rest.Config, director - // and waits for the bootstrap configmap to report that bootstrapping has - // completed. - func waitForBootstrapConfigMap(ctx context.Context, client *kubernetes.Clientset) error { -- timeout := 30 * time.Minute -+ timeout := 120 * time.Minute - logrus.Infof("Waiting up to %v for bootstrapping to complete...", timeout) - - waitCtx, cancel := context.WithTimeout(ctx, timeout) -@@ -317,7 +317,7 @@ func waitForBootstrapConfigMap(ctx context.Context, client *kubernetes.Clientset - // waitForInitializedCluster watches the ClusterVersion waiting for confirmation - // that the cluster has been initialized. - func waitForInitializedCluster(ctx context.Context, config *rest.Config) error { -- timeout := 30 * time.Minute -+ timeout := 120 * time.Minute - logrus.Infof("Waiting up to %v for the cluster at %s to initialize...", timeout, config.Host) - cc, err := configclient.NewForConfig(config) - if err != nil { -diff --git a/data/data/libvirt/main.tf b/data/data/libvirt/main.tf -index 4013ed0ec..cb9be72fa 100644 ---- a/data/data/libvirt/main.tf -+++ b/data/data/libvirt/main.tf -@@ -23,6 +23,7 @@ resource "libvirt_volume" "master" { - count = "${var.master_count}" - name = "${var.cluster_id}-master-${count.index}" - base_volume_id = "${module.volume.coreos_base_volume_id}" -+ size = 32212254720 - } - - resource "libvirt_ignition" "master" { -@@ -55,6 +56,8 @@ resource "libvirt_network" "net" { - data.libvirt_network_dns_host_template.masters.*.rendered, - data.libvirt_network_dns_host_template.masters_int.*.rendered, - data.libvirt_network_dns_host_template.etcds.*.rendered, -+ data.libvirt_network_dns_host_template.console.*.rendered, -+ data.libvirt_network_dns_host_template.auth.*.rendered, - ))}"] - }] - -@@ -121,6 +124,18 @@ data "libvirt_network_dns_host_template" "etcds" { - hostname = "etcd-${count.index}.${var.cluster_domain}" - } - -+data "libvirt_network_dns_host_template" "auth" { -+ count = "${var.master_count}" -+ ip = "${var.libvirt_auth_ip}" -+ hostname = "oauth-openshift.apps.${var.cluster_domain}" -+} -+ -+data "libvirt_network_dns_host_template" "console" { -+ count = "${var.master_count}" -+ ip = "${var.libvirt_auth_ip}" -+ hostname = "console-openshift-console.apps.${var.cluster_domain}" -+} -+ - data "libvirt_network_dns_srv_template" "etcd_cluster" { - count = "${var.master_count}" - service = "etcd-server-ssl" -diff --git a/data/data/libvirt/variables-libvirt.tf b/data/data/libvirt/variables-libvirt.tf -index cea25a520..a1e3722fc 100644 ---- a/data/data/libvirt/variables-libvirt.tf -+++ b/data/data/libvirt/variables-libvirt.tf -@@ -28,6 +28,11 @@ variable "libvirt_master_ips" { - description = "the list of desired master ips. Must match master_count" - } - -+variable "libvirt_auth_ip" { -+ type = "string" -+ description = "node with authentication server ip" -+} -+ - # It's definitely recommended to bump this if you can. - variable "libvirt_master_memory" { - type = "string" -diff --git a/pkg/asset/tls/aggregator.go b/pkg/asset/tls/aggregator.go -index 9ec6432da..6dac0b736 100644 ---- a/pkg/asset/tls/aggregator.go -+++ b/pkg/asset/tls/aggregator.go -@@ -27,7 +27,7 @@ func (a *AggregatorCA) Generate(dependencies asset.Parents) error { - cfg := &CertCfg{ - Subject: pkix.Name{CommonName: "aggregator", OrganizationalUnit: []string{"bootkube"}}, - KeyUsages: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, -- Validity: ValidityOneDay, -+ Validity: ValidityOneYear, - IsCA: true, - } - -@@ -65,7 +65,7 @@ func (a *APIServerProxyCertKey) Generate(dependencies asset.Parents) error { - Subject: pkix.Name{CommonName: "system:kube-apiserver-proxy", Organization: []string{"kube-master"}}, - KeyUsages: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, - ExtKeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, -- Validity: ValidityOneDay, -+ Validity: ValidityOneYear, - } - - return a.SignedCertKey.Generate(cfg, aggregatorCA, "apiserver-proxy", DoNotAppendParent) -@@ -93,7 +93,7 @@ func (c *AggregatorSignerCertKey) Generate(parents asset.Parents) error { - cfg := &CertCfg{ - Subject: pkix.Name{CommonName: "aggregator-signer", OrganizationalUnit: []string{"openshift"}}, - KeyUsages: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, -- Validity: ValidityOneDay, -+ Validity: ValidityOneYear, - IsCA: true, - } - -@@ -158,7 +158,7 @@ func (a *AggregatorClientCertKey) Generate(dependencies asset.Parents) error { - Subject: pkix.Name{CommonName: "system:kube-apiserver-proxy", Organization: []string{"kube-master"}}, - KeyUsages: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, - ExtKeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, -- Validity: ValidityOneDay, -+ Validity: ValidityOneYear, - } - - return a.SignedCertKey.Generate(cfg, ca, "aggregator-client", DoNotAppendParent) -diff --git a/pkg/asset/tls/apiserver.go b/pkg/asset/tls/apiserver.go -index a50bee836..cd63ff13c 100644 ---- a/pkg/asset/tls/apiserver.go -+++ b/pkg/asset/tls/apiserver.go -@@ -185,7 +185,7 @@ func (a *KubeAPIServerLocalhostServerCertKey) Generate(dependencies asset.Parent - Subject: pkix.Name{CommonName: "system:kube-apiserver", Organization: []string{"kube-master"}}, - KeyUsages: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, - ExtKeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, -- Validity: ValidityOneDay, -+ Validity: ValidityOneYear, - DNSNames: []string{ - "localhost", - }, -@@ -288,7 +288,7 @@ func (a *KubeAPIServerServiceNetworkServerCertKey) Generate(dependencies asset.P - Subject: pkix.Name{CommonName: "system:kube-apiserver", Organization: []string{"kube-master"}}, - KeyUsages: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, - ExtKeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, -- Validity: ValidityOneDay, -+ Validity: ValidityOneYear, - DNSNames: []string{ - "kubernetes", "kubernetes.default", - "kubernetes.default.svc", -@@ -392,7 +392,7 @@ func (a *KubeAPIServerExternalLBServerCertKey) Generate(dependencies asset.Paren - Subject: pkix.Name{CommonName: "system:kube-apiserver", Organization: []string{"kube-master"}}, - KeyUsages: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, - ExtKeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, -- Validity: ValidityOneDay, -+ Validity: ValidityOneYear, - DNSNames: []string{ - apiAddress(installConfig.Config), - }, -@@ -431,7 +431,7 @@ func (a *KubeAPIServerInternalLBServerCertKey) Generate(dependencies asset.Paren - Subject: pkix.Name{CommonName: "system:kube-apiserver", Organization: []string{"kube-master"}}, - KeyUsages: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, - ExtKeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, -- Validity: ValidityOneDay, -+ Validity: ValidityOneYear, - DNSNames: []string{ - internalAPIAddress(installConfig.Config), - }, -diff --git a/pkg/asset/tls/kubelet.go b/pkg/asset/tls/kubelet.go -index 01264e898..32cc8059d 100644 ---- a/pkg/asset/tls/kubelet.go -+++ b/pkg/asset/tls/kubelet.go -@@ -24,7 +24,7 @@ func (c *KubeletCSRSignerCertKey) Generate(parents asset.Parents) error { - cfg := &CertCfg{ - Subject: pkix.Name{CommonName: "kubelet-signer", OrganizationalUnit: []string{"openshift"}}, - KeyUsages: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, -- Validity: ValidityOneDay, -+ Validity: ValidityOneYear, - IsCA: true, - } - -@@ -181,7 +181,7 @@ func (a *KubeletClientCertKey) Generate(dependencies asset.Parents) error { - Subject: pkix.Name{CommonName: "system:serviceaccount:openshift-machine-config-operator:node-bootstrapper", Organization: []string{"system:serviceaccounts:openshift-machine-config-operator"}}, - KeyUsages: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, - ExtKeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, -- Validity: ValidityOneDay, -+ Validity: ValidityOneYear, - } - - return a.SignedCertKey.Generate(cfg, ca, "kubelet-client", DoNotAppendParent) -diff --git a/pkg/tfvars/libvirt/libvirt.go b/pkg/tfvars/libvirt/libvirt.go -index dc9710fa1..88603f4f7 100644 ---- a/pkg/tfvars/libvirt/libvirt.go -+++ b/pkg/tfvars/libvirt/libvirt.go -@@ -17,6 +17,7 @@ type config struct { - IfName string `json:"libvirt_network_if"` - MasterIPs []string `json:"libvirt_master_ips,omitempty"` - BootstrapIP string `json:"libvirt_bootstrap_ip,omitempty"` -+ AuthNodeIP string `json:"libvirt_auth_ip,omitempty"` - } - - // TFVars generates libvirt-specific Terraform variables. -@@ -42,6 +43,7 @@ func TFVars(masterConfig *v1alpha1.LibvirtMachineProviderConfig, osImage string, - IfName: bridge, - BootstrapIP: bootstrapIP.String(), - MasterIPs: masterIPs, -+ AuthNodeIP: "192.168.126.51", - } - - return json.MarshalIndent(cfg, "", " ") diff --git a/cluster-provision/okd/hacks/release-4.2 b/cluster-provision/okd/hacks/release-4.2 deleted file mode 100644 index 4084185883..0000000000 --- a/cluster-provision/okd/hacks/release-4.2 +++ /dev/null @@ -1,209 +0,0 @@ -diff --git a/cmd/openshift-install/create.go b/cmd/openshift-install/create.go -index 9021025b6..a88d973ed 100644 ---- a/cmd/openshift-install/create.go -+++ b/cmd/openshift-install/create.go -@@ -238,7 +238,7 @@ func waitForBootstrapComplete(ctx context.Context, config *rest.Config, director - - discovery := client.Discovery() - -- apiTimeout := 30 * time.Minute -+ apiTimeout := 120 * time.Minute - logrus.Infof("Waiting up to %v for the Kubernetes API at %s...", apiTimeout, config.Host) - apiContext, cancel := context.WithTimeout(ctx, apiTimeout) - defer cancel() -@@ -279,7 +279,7 @@ func waitForBootstrapComplete(ctx context.Context, config *rest.Config, director - // and waits for the bootstrap configmap to report that bootstrapping has - // completed. - func waitForBootstrapConfigMap(ctx context.Context, client *kubernetes.Clientset) error { -- timeout := 30 * time.Minute -+ timeout := 120 * time.Minute - logrus.Infof("Waiting up to %v for bootstrapping to complete...", timeout) - - waitCtx, cancel := context.WithTimeout(ctx, timeout) -@@ -317,7 +317,7 @@ func waitForBootstrapConfigMap(ctx context.Context, client *kubernetes.Clientset - // waitForInitializedCluster watches the ClusterVersion waiting for confirmation - // that the cluster has been initialized. - func waitForInitializedCluster(ctx context.Context, config *rest.Config) error { -- timeout := 30 * time.Minute -+ timeout := 120 * time.Minute - logrus.Infof("Waiting up to %v for the cluster at %s to initialize...", timeout, config.Host) - cc, err := configclient.NewForConfig(config) - if err != nil { -diff --git a/data/data/libvirt/main.tf b/data/data/libvirt/main.tf -index 9ba88c9cf..0b899734d 100644 ---- a/data/data/libvirt/main.tf -+++ b/data/data/libvirt/main.tf -@@ -33,6 +33,7 @@ resource "libvirt_volume" "master" { - name = "${var.cluster_id}-master-${count.index}" - base_volume_id = module.volume.coreos_base_volume_id - pool = libvirt_pool.storage_pool.name -+ size = 32212254720 - } - - resource "libvirt_ignition" "master" { -@@ -73,6 +74,8 @@ resource "libvirt_network" "net" { - data.libvirt_network_dns_host_template.masters.*.rendered, - data.libvirt_network_dns_host_template.masters_int.*.rendered, - data.libvirt_network_dns_host_template.etcds.*.rendered, -+ data.libvirt_network_dns_host_template.console.*.rendered, -+ data.libvirt_network_dns_host_template.auth.*.rendered, - ) - content { - hostname = hosts.value.hostname -@@ -114,6 +117,18 @@ resource "libvirt_domain" "master" { - } - } - -+data "libvirt_network_dns_host_template" "auth" { -+ count = "${var.master_count}" -+ ip = "${var.libvirt_auth_ip}" -+ hostname = "oauth-openshift.apps.${var.cluster_domain}" -+} -+ -+data "libvirt_network_dns_host_template" "console" { -+ count = "${var.master_count}" -+ ip = "${var.libvirt_auth_ip}" -+ hostname = "console-openshift-console.apps.${var.cluster_domain}" -+} -+ - data "libvirt_network_dns_host_template" "bootstrap" { - count = var.bootstrap_dns ? 1 : 0 - ip = var.libvirt_bootstrap_ip -diff --git a/data/data/libvirt/variables-libvirt.tf b/data/data/libvirt/variables-libvirt.tf -index 53cf68bae..3c5f7f905 100644 ---- a/data/data/libvirt/variables-libvirt.tf -+++ b/data/data/libvirt/variables-libvirt.tf -@@ -28,6 +28,11 @@ variable "libvirt_master_ips" { - description = "the list of desired master ips. Must match master_count" - } - -+variable "libvirt_auth_ip" { -+ type = "string" -+ description = "node with authentication server ip" -+} -+ - # It's definitely recommended to bump this if you can. - variable "libvirt_master_memory" { - type = string -diff --git a/pkg/asset/tls/aggregator.go b/pkg/asset/tls/aggregator.go -index 9ec6432da..6dac0b736 100644 ---- a/pkg/asset/tls/aggregator.go -+++ b/pkg/asset/tls/aggregator.go -@@ -27,7 +27,7 @@ func (a *AggregatorCA) Generate(dependencies asset.Parents) error { - cfg := &CertCfg{ - Subject: pkix.Name{CommonName: "aggregator", OrganizationalUnit: []string{"bootkube"}}, - KeyUsages: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, -- Validity: ValidityOneDay, -+ Validity: ValidityOneYear, - IsCA: true, - } - -@@ -65,7 +65,7 @@ func (a *APIServerProxyCertKey) Generate(dependencies asset.Parents) error { - Subject: pkix.Name{CommonName: "system:kube-apiserver-proxy", Organization: []string{"kube-master"}}, - KeyUsages: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, - ExtKeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, -- Validity: ValidityOneDay, -+ Validity: ValidityOneYear, - } - - return a.SignedCertKey.Generate(cfg, aggregatorCA, "apiserver-proxy", DoNotAppendParent) -@@ -93,7 +93,7 @@ func (c *AggregatorSignerCertKey) Generate(parents asset.Parents) error { - cfg := &CertCfg{ - Subject: pkix.Name{CommonName: "aggregator-signer", OrganizationalUnit: []string{"openshift"}}, - KeyUsages: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, -- Validity: ValidityOneDay, -+ Validity: ValidityOneYear, - IsCA: true, - } - -@@ -158,7 +158,7 @@ func (a *AggregatorClientCertKey) Generate(dependencies asset.Parents) error { - Subject: pkix.Name{CommonName: "system:kube-apiserver-proxy", Organization: []string{"kube-master"}}, - KeyUsages: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, - ExtKeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, -- Validity: ValidityOneDay, -+ Validity: ValidityOneYear, - } - - return a.SignedCertKey.Generate(cfg, ca, "aggregator-client", DoNotAppendParent) -diff --git a/pkg/asset/tls/apiserver.go b/pkg/asset/tls/apiserver.go -index a50bee836..cd63ff13c 100644 ---- a/pkg/asset/tls/apiserver.go -+++ b/pkg/asset/tls/apiserver.go -@@ -185,7 +185,7 @@ func (a *KubeAPIServerLocalhostServerCertKey) Generate(dependencies asset.Parent - Subject: pkix.Name{CommonName: "system:kube-apiserver", Organization: []string{"kube-master"}}, - KeyUsages: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, - ExtKeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, -- Validity: ValidityOneDay, -+ Validity: ValidityOneYear, - DNSNames: []string{ - "localhost", - }, -@@ -288,7 +288,7 @@ func (a *KubeAPIServerServiceNetworkServerCertKey) Generate(dependencies asset.P - Subject: pkix.Name{CommonName: "system:kube-apiserver", Organization: []string{"kube-master"}}, - KeyUsages: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, - ExtKeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, -- Validity: ValidityOneDay, -+ Validity: ValidityOneYear, - DNSNames: []string{ - "kubernetes", "kubernetes.default", - "kubernetes.default.svc", -@@ -392,7 +392,7 @@ func (a *KubeAPIServerExternalLBServerCertKey) Generate(dependencies asset.Paren - Subject: pkix.Name{CommonName: "system:kube-apiserver", Organization: []string{"kube-master"}}, - KeyUsages: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, - ExtKeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, -- Validity: ValidityOneDay, -+ Validity: ValidityOneYear, - DNSNames: []string{ - apiAddress(installConfig.Config), - }, -@@ -431,7 +431,7 @@ func (a *KubeAPIServerInternalLBServerCertKey) Generate(dependencies asset.Paren - Subject: pkix.Name{CommonName: "system:kube-apiserver", Organization: []string{"kube-master"}}, - KeyUsages: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, - ExtKeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, -- Validity: ValidityOneDay, -+ Validity: ValidityOneYear, - DNSNames: []string{ - internalAPIAddress(installConfig.Config), - }, -diff --git a/pkg/asset/tls/kubelet.go b/pkg/asset/tls/kubelet.go -index 01264e898..32cc8059d 100644 ---- a/pkg/asset/tls/kubelet.go -+++ b/pkg/asset/tls/kubelet.go -@@ -24,7 +24,7 @@ func (c *KubeletCSRSignerCertKey) Generate(parents asset.Parents) error { - cfg := &CertCfg{ - Subject: pkix.Name{CommonName: "kubelet-signer", OrganizationalUnit: []string{"openshift"}}, - KeyUsages: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, -- Validity: ValidityOneDay, -+ Validity: ValidityOneYear, - IsCA: true, - } - -@@ -181,7 +181,7 @@ func (a *KubeletClientCertKey) Generate(dependencies asset.Parents) error { - Subject: pkix.Name{CommonName: "system:serviceaccount:openshift-machine-config-operator:node-bootstrapper", Organization: []string{"system:serviceaccounts:openshift-machine-config-operator"}}, - KeyUsages: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, - ExtKeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, -- Validity: ValidityOneDay, -+ Validity: ValidityOneYear, - } - - return a.SignedCertKey.Generate(cfg, ca, "kubelet-client", DoNotAppendParent) -diff --git a/pkg/tfvars/libvirt/libvirt.go b/pkg/tfvars/libvirt/libvirt.go -index 4b7b7f50b..056bc2de1 100644 ---- a/pkg/tfvars/libvirt/libvirt.go -+++ b/pkg/tfvars/libvirt/libvirt.go -@@ -17,6 +17,7 @@ type config struct { - IfName string `json:"libvirt_network_if"` - MasterIPs []string `json:"libvirt_master_ips,omitempty"` - BootstrapIP string `json:"libvirt_bootstrap_ip,omitempty"` -+ AuthNodeIP string `json:"libvirt_auth_ip,omitempty"` - } - - // TFVars generates libvirt-specific Terraform variables. -@@ -42,6 +43,7 @@ func TFVars(masterConfig *v1beta1.LibvirtMachineProviderConfig, osImage string, - IfName: bridge, - BootstrapIP: bootstrapIP.String(), - MasterIPs: masterIPs, -+ AuthNodeIP: "192.168.126.51", - } - - return json.MarshalIndent(cfg, "", " ") diff --git a/cluster-provision/okd/hacks/release-4.3 b/cluster-provision/okd/hacks/release-4.3 deleted file mode 100644 index d2dab565dc..0000000000 --- a/cluster-provision/okd/hacks/release-4.3 +++ /dev/null @@ -1,210 +0,0 @@ -diff --git a/cmd/openshift-install/create.go b/cmd/openshift-install/create.go -index c5c035e72..e44a4d03e 100644 ---- a/cmd/openshift-install/create.go -+++ b/cmd/openshift-install/create.go -@@ -252,7 +252,7 @@ func waitForBootstrapComplete(ctx context.Context, config *rest.Config, director - - discovery := client.Discovery() - -- apiTimeout := 30 * time.Minute -+ apiTimeout := 120 * time.Minute - logrus.Infof("Waiting up to %v for the Kubernetes API at %s...", apiTimeout, config.Host) - apiContext, cancel := context.WithTimeout(ctx, apiTimeout) - defer cancel() -@@ -293,7 +293,7 @@ func waitForBootstrapComplete(ctx context.Context, config *rest.Config, director - // and waits for the bootstrap configmap to report that bootstrapping has - // completed. - func waitForBootstrapConfigMap(ctx context.Context, client *kubernetes.Clientset) error { -- timeout := 30 * time.Minute -+ timeout := 120 * time.Minute - logrus.Infof("Waiting up to %v for bootstrapping to complete...", timeout) - - waitCtx, cancel := context.WithTimeout(ctx, timeout) -@@ -331,7 +331,7 @@ func waitForBootstrapConfigMap(ctx context.Context, client *kubernetes.Clientset - // waitForInitializedCluster watches the ClusterVersion waiting for confirmation - // that the cluster has been initialized. - func waitForInitializedCluster(ctx context.Context, config *rest.Config) error { -- timeout := 30 * time.Minute -+ timeout := 120 * time.Minute - - // Wait longer for baremetal, due to length of time it takes to boot - if assetStore, err := assetstore.NewStore(rootOpts.dir); err == nil { -diff --git a/data/data/libvirt/main.tf b/data/data/libvirt/main.tf -index 9ba88c9cf..09f6500bf 100644 ---- a/data/data/libvirt/main.tf -+++ b/data/data/libvirt/main.tf -@@ -33,6 +33,7 @@ resource "libvirt_volume" "master" { - name = "${var.cluster_id}-master-${count.index}" - base_volume_id = module.volume.coreos_base_volume_id - pool = libvirt_pool.storage_pool.name -+ size = 32212254720 - } - - resource "libvirt_ignition" "master" { -@@ -73,6 +74,8 @@ resource "libvirt_network" "net" { - data.libvirt_network_dns_host_template.masters.*.rendered, - data.libvirt_network_dns_host_template.masters_int.*.rendered, - data.libvirt_network_dns_host_template.etcds.*.rendered, -+ data.libvirt_network_dns_host_template.console.*.rendered, -+ data.libvirt_network_dns_host_template.auth.*.rendered, - ) - content { - hostname = hosts.value.hostname -@@ -114,6 +117,19 @@ resource "libvirt_domain" "master" { - } - } - -+data "libvirt_network_dns_host_template" "auth" { -+ count = "${var.master_count}" -+ ip = "${var.libvirt_auth_ip}" -+ hostname = "oauth-openshift.apps.${var.cluster_domain}" -+} -+ -+data "libvirt_network_dns_host_template" "console" { -+ count = "${var.master_count}" -+ ip = "${var.libvirt_auth_ip}" -+ hostname = "console-openshift-console.apps.${var.cluster_domain}" -+} -+ -+ - data "libvirt_network_dns_host_template" "bootstrap" { - count = var.bootstrap_dns ? 1 : 0 - ip = var.libvirt_bootstrap_ip -diff --git a/data/data/libvirt/variables-libvirt.tf b/data/data/libvirt/variables-libvirt.tf -index 53cf68bae..3c5f7f905 100644 ---- a/data/data/libvirt/variables-libvirt.tf -+++ b/data/data/libvirt/variables-libvirt.tf -@@ -28,6 +28,11 @@ variable "libvirt_master_ips" { - description = "the list of desired master ips. Must match master_count" - } - -+variable "libvirt_auth_ip" { -+ type = "string" -+ description = "node with authentication server ip" -+} -+ - # It's definitely recommended to bump this if you can. - variable "libvirt_master_memory" { - type = string -diff --git a/pkg/asset/tls/aggregator.go b/pkg/asset/tls/aggregator.go -index 9ec6432da..6dac0b736 100644 ---- a/pkg/asset/tls/aggregator.go -+++ b/pkg/asset/tls/aggregator.go -@@ -27,7 +27,7 @@ func (a *AggregatorCA) Generate(dependencies asset.Parents) error { - cfg := &CertCfg{ - Subject: pkix.Name{CommonName: "aggregator", OrganizationalUnit: []string{"bootkube"}}, - KeyUsages: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, -- Validity: ValidityOneDay, -+ Validity: ValidityOneYear, - IsCA: true, - } - -@@ -65,7 +65,7 @@ func (a *APIServerProxyCertKey) Generate(dependencies asset.Parents) error { - Subject: pkix.Name{CommonName: "system:kube-apiserver-proxy", Organization: []string{"kube-master"}}, - KeyUsages: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, - ExtKeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, -- Validity: ValidityOneDay, -+ Validity: ValidityOneYear, - } - - return a.SignedCertKey.Generate(cfg, aggregatorCA, "apiserver-proxy", DoNotAppendParent) -@@ -93,7 +93,7 @@ func (c *AggregatorSignerCertKey) Generate(parents asset.Parents) error { - cfg := &CertCfg{ - Subject: pkix.Name{CommonName: "aggregator-signer", OrganizationalUnit: []string{"openshift"}}, - KeyUsages: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, -- Validity: ValidityOneDay, -+ Validity: ValidityOneYear, - IsCA: true, - } - -@@ -158,7 +158,7 @@ func (a *AggregatorClientCertKey) Generate(dependencies asset.Parents) error { - Subject: pkix.Name{CommonName: "system:kube-apiserver-proxy", Organization: []string{"kube-master"}}, - KeyUsages: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, - ExtKeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, -- Validity: ValidityOneDay, -+ Validity: ValidityOneYear, - } - - return a.SignedCertKey.Generate(cfg, ca, "aggregator-client", DoNotAppendParent) -diff --git a/pkg/asset/tls/apiserver.go b/pkg/asset/tls/apiserver.go -index a50bee836..cd63ff13c 100644 ---- a/pkg/asset/tls/apiserver.go -+++ b/pkg/asset/tls/apiserver.go -@@ -185,7 +185,7 @@ func (a *KubeAPIServerLocalhostServerCertKey) Generate(dependencies asset.Parent - Subject: pkix.Name{CommonName: "system:kube-apiserver", Organization: []string{"kube-master"}}, - KeyUsages: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, - ExtKeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, -- Validity: ValidityOneDay, -+ Validity: ValidityOneYear, - DNSNames: []string{ - "localhost", - }, -@@ -288,7 +288,7 @@ func (a *KubeAPIServerServiceNetworkServerCertKey) Generate(dependencies asset.P - Subject: pkix.Name{CommonName: "system:kube-apiserver", Organization: []string{"kube-master"}}, - KeyUsages: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, - ExtKeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, -- Validity: ValidityOneDay, -+ Validity: ValidityOneYear, - DNSNames: []string{ - "kubernetes", "kubernetes.default", - "kubernetes.default.svc", -@@ -392,7 +392,7 @@ func (a *KubeAPIServerExternalLBServerCertKey) Generate(dependencies asset.Paren - Subject: pkix.Name{CommonName: "system:kube-apiserver", Organization: []string{"kube-master"}}, - KeyUsages: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, - ExtKeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, -- Validity: ValidityOneDay, -+ Validity: ValidityOneYear, - DNSNames: []string{ - apiAddress(installConfig.Config), - }, -@@ -431,7 +431,7 @@ func (a *KubeAPIServerInternalLBServerCertKey) Generate(dependencies asset.Paren - Subject: pkix.Name{CommonName: "system:kube-apiserver", Organization: []string{"kube-master"}}, - KeyUsages: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, - ExtKeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, -- Validity: ValidityOneDay, -+ Validity: ValidityOneYear, - DNSNames: []string{ - internalAPIAddress(installConfig.Config), - }, -diff --git a/pkg/asset/tls/kubelet.go b/pkg/asset/tls/kubelet.go -index 01264e898..32cc8059d 100644 ---- a/pkg/asset/tls/kubelet.go -+++ b/pkg/asset/tls/kubelet.go -@@ -24,7 +24,7 @@ func (c *KubeletCSRSignerCertKey) Generate(parents asset.Parents) error { - cfg := &CertCfg{ - Subject: pkix.Name{CommonName: "kubelet-signer", OrganizationalUnit: []string{"openshift"}}, - KeyUsages: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, -- Validity: ValidityOneDay, -+ Validity: ValidityOneYear, - IsCA: true, - } - -@@ -181,7 +181,7 @@ func (a *KubeletClientCertKey) Generate(dependencies asset.Parents) error { - Subject: pkix.Name{CommonName: "system:serviceaccount:openshift-machine-config-operator:node-bootstrapper", Organization: []string{"system:serviceaccounts:openshift-machine-config-operator"}}, - KeyUsages: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, - ExtKeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, -- Validity: ValidityOneDay, -+ Validity: ValidityOneYear, - } - - return a.SignedCertKey.Generate(cfg, ca, "kubelet-client", DoNotAppendParent) -diff --git a/pkg/tfvars/libvirt/libvirt.go b/pkg/tfvars/libvirt/libvirt.go -index e4e5e4927..43bcdc8a2 100644 ---- a/pkg/tfvars/libvirt/libvirt.go -+++ b/pkg/tfvars/libvirt/libvirt.go -@@ -21,6 +21,7 @@ type config struct { - BootstrapIP string `json:"libvirt_bootstrap_ip,omitempty"` - MasterMemory string `json:"libvirt_master_memory,omitempty"` - MasterVcpu string `json:"libvirt_master_vcpu,omitempty"` -+ AuthNodeIP string `json:"libvirt_auth_ip,omitempty"` - } - - // TFVars generates libvirt-specific Terraform variables. -@@ -46,6 +47,7 @@ func TFVars(masterConfig *v1beta1.LibvirtMachineProviderConfig, osImage string, - IfName: bridge, - BootstrapIP: bootstrapIP.String(), - MasterIPs: masterIPs, -+ AuthNodeIP: "192.168.126.51", - MasterMemory: strconv.Itoa(masterConfig.DomainMemory), - MasterVcpu: strconv.Itoa(masterConfig.DomainVcpu), - } diff --git a/cluster-provision/okd/hacks/release-4.4 b/cluster-provision/okd/hacks/release-4.4 deleted file mode 100644 index 1e52a15cf8..0000000000 --- a/cluster-provision/okd/hacks/release-4.4 +++ /dev/null @@ -1,201 +0,0 @@ -diff --git a/cmd/openshift-install/create.go b/cmd/openshift-install/create.go -index 70a8201a8..e44a4d03e 100644 ---- a/cmd/openshift-install/create.go -+++ b/cmd/openshift-install/create.go -@@ -252,7 +252,7 @@ func waitForBootstrapComplete(ctx context.Context, config *rest.Config, director - - discovery := client.Discovery() - -- apiTimeout := 20 * time.Minute -+ apiTimeout := 120 * time.Minute - logrus.Infof("Waiting up to %v for the Kubernetes API at %s...", apiTimeout, config.Host) - apiContext, cancel := context.WithTimeout(ctx, apiTimeout) - defer cancel() -@@ -293,7 +293,7 @@ func waitForBootstrapComplete(ctx context.Context, config *rest.Config, director - // and waits for the bootstrap configmap to report that bootstrapping has - // completed. - func waitForBootstrapConfigMap(ctx context.Context, client *kubernetes.Clientset) error { -- timeout := 40 * time.Minute -+ timeout := 120 * time.Minute - logrus.Infof("Waiting up to %v for bootstrapping to complete...", timeout) - - waitCtx, cancel := context.WithTimeout(ctx, timeout) -@@ -331,7 +331,7 @@ func waitForBootstrapConfigMap(ctx context.Context, client *kubernetes.Clientset - // waitForInitializedCluster watches the ClusterVersion waiting for confirmation - // that the cluster has been initialized. - func waitForInitializedCluster(ctx context.Context, config *rest.Config) error { -- timeout := 30 * time.Minute -+ timeout := 120 * time.Minute - - // Wait longer for baremetal, due to length of time it takes to boot - if assetStore, err := assetstore.NewStore(rootOpts.dir); err == nil { -diff --git a/data/data/libvirt/main.tf b/data/data/libvirt/main.tf -index 8e96fa27e..0984b0410 100644 ---- a/data/data/libvirt/main.tf -+++ b/data/data/libvirt/main.tf -@@ -33,6 +33,7 @@ resource "libvirt_volume" "master" { - name = "${var.cluster_id}-master-${count.index}" - base_volume_id = module.volume.coreos_base_volume_id - pool = libvirt_pool.storage_pool.name -+ size = 32212254720 - } - - resource "libvirt_ignition" "master" { -@@ -60,6 +61,8 @@ resource "libvirt_network" "net" { - data.libvirt_network_dns_host_template.bootstrap_int.*.rendered, - data.libvirt_network_dns_host_template.masters.*.rendered, - data.libvirt_network_dns_host_template.masters_int.*.rendered, -+ data.libvirt_network_dns_host_template.console.*.rendered, -+ data.libvirt_network_dns_host_template.auth.*.rendered, - ) - content { - hostname = hosts.value.hostname -@@ -101,6 +104,19 @@ resource "libvirt_domain" "master" { - } - } - -+data "libvirt_network_dns_host_template" "auth" { -+ count = "${var.master_count}" -+ ip = "${var.libvirt_auth_ip}" -+ hostname = "oauth-openshift.apps.${var.cluster_domain}" -+} -+ -+data "libvirt_network_dns_host_template" "console" { -+ count = "${var.master_count}" -+ ip = "${var.libvirt_auth_ip}" -+ hostname = "console-openshift-console.apps.${var.cluster_domain}" -+} -+ -+ - data "libvirt_network_dns_host_template" "bootstrap" { - count = var.bootstrap_dns ? 1 : 0 - ip = var.libvirt_bootstrap_ip -diff --git a/data/data/libvirt/variables-libvirt.tf b/data/data/libvirt/variables-libvirt.tf -index 53cf68bae..3c5f7f905 100644 ---- a/data/data/libvirt/variables-libvirt.tf -+++ b/data/data/libvirt/variables-libvirt.tf -@@ -28,6 +28,11 @@ variable "libvirt_master_ips" { - description = "the list of desired master ips. Must match master_count" - } - -+variable "libvirt_auth_ip" { -+ type = "string" -+ description = "node with authentication server ip" -+} -+ - # It's definitely recommended to bump this if you can. - variable "libvirt_master_memory" { - type = string -diff --git a/pkg/asset/tls/aggregator.go b/pkg/asset/tls/aggregator.go -index 9ec6432da..6dac0b736 100644 ---- a/pkg/asset/tls/aggregator.go -+++ b/pkg/asset/tls/aggregator.go -@@ -27,7 +27,7 @@ func (a *AggregatorCA) Generate(dependencies asset.Parents) error { - cfg := &CertCfg{ - Subject: pkix.Name{CommonName: "aggregator", OrganizationalUnit: []string{"bootkube"}}, - KeyUsages: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, -- Validity: ValidityOneDay, -+ Validity: ValidityOneYear, - IsCA: true, - } - -@@ -65,7 +65,7 @@ func (a *APIServerProxyCertKey) Generate(dependencies asset.Parents) error { - Subject: pkix.Name{CommonName: "system:kube-apiserver-proxy", Organization: []string{"kube-master"}}, - KeyUsages: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, - ExtKeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, -- Validity: ValidityOneDay, -+ Validity: ValidityOneYear, - } - - return a.SignedCertKey.Generate(cfg, aggregatorCA, "apiserver-proxy", DoNotAppendParent) -@@ -93,7 +93,7 @@ func (c *AggregatorSignerCertKey) Generate(parents asset.Parents) error { - cfg := &CertCfg{ - Subject: pkix.Name{CommonName: "aggregator-signer", OrganizationalUnit: []string{"openshift"}}, - KeyUsages: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, -- Validity: ValidityOneDay, -+ Validity: ValidityOneYear, - IsCA: true, - } - -@@ -158,7 +158,7 @@ func (a *AggregatorClientCertKey) Generate(dependencies asset.Parents) error { - Subject: pkix.Name{CommonName: "system:kube-apiserver-proxy", Organization: []string{"kube-master"}}, - KeyUsages: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, - ExtKeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, -- Validity: ValidityOneDay, -+ Validity: ValidityOneYear, - } - - return a.SignedCertKey.Generate(cfg, ca, "aggregator-client", DoNotAppendParent) -diff --git a/pkg/asset/tls/apiserver.go b/pkg/asset/tls/apiserver.go -index a50bee836..cd63ff13c 100644 ---- a/pkg/asset/tls/apiserver.go -+++ b/pkg/asset/tls/apiserver.go -@@ -185,7 +185,7 @@ func (a *KubeAPIServerLocalhostServerCertKey) Generate(dependencies asset.Parent - Subject: pkix.Name{CommonName: "system:kube-apiserver", Organization: []string{"kube-master"}}, - KeyUsages: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, - ExtKeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, -- Validity: ValidityOneDay, -+ Validity: ValidityOneYear, - DNSNames: []string{ - "localhost", - }, -@@ -288,7 +288,7 @@ func (a *KubeAPIServerServiceNetworkServerCertKey) Generate(dependencies asset.P - Subject: pkix.Name{CommonName: "system:kube-apiserver", Organization: []string{"kube-master"}}, - KeyUsages: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, - ExtKeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, -- Validity: ValidityOneDay, -+ Validity: ValidityOneYear, - DNSNames: []string{ - "kubernetes", "kubernetes.default", - "kubernetes.default.svc", -@@ -392,7 +392,7 @@ func (a *KubeAPIServerExternalLBServerCertKey) Generate(dependencies asset.Paren - Subject: pkix.Name{CommonName: "system:kube-apiserver", Organization: []string{"kube-master"}}, - KeyUsages: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, - ExtKeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, -- Validity: ValidityOneDay, -+ Validity: ValidityOneYear, - DNSNames: []string{ - apiAddress(installConfig.Config), - }, -@@ -431,7 +431,7 @@ func (a *KubeAPIServerInternalLBServerCertKey) Generate(dependencies asset.Paren - Subject: pkix.Name{CommonName: "system:kube-apiserver", Organization: []string{"kube-master"}}, - KeyUsages: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, - ExtKeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, -- Validity: ValidityOneDay, -+ Validity: ValidityOneYear, - DNSNames: []string{ - internalAPIAddress(installConfig.Config), - }, -diff --git a/pkg/asset/tls/kubelet.go b/pkg/asset/tls/kubelet.go -index 34049cba3..53178675c 100644 ---- a/pkg/asset/tls/kubelet.go -+++ b/pkg/asset/tls/kubelet.go -@@ -24,7 +24,7 @@ func (c *KubeletCSRSignerCertKey) Generate(parents asset.Parents) error { - cfg := &CertCfg{ - Subject: pkix.Name{CommonName: "kubelet-signer", OrganizationalUnit: []string{"openshift"}}, - KeyUsages: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, -- Validity: ValidityOneDay, -+ Validity: ValidityOneYear, - IsCA: true, - } - -diff --git a/pkg/tfvars/libvirt/libvirt.go b/pkg/tfvars/libvirt/libvirt.go -index e4e5e4927..43bcdc8a2 100644 ---- a/pkg/tfvars/libvirt/libvirt.go -+++ b/pkg/tfvars/libvirt/libvirt.go -@@ -21,6 +21,7 @@ type config struct { - BootstrapIP string `json:"libvirt_bootstrap_ip,omitempty"` - MasterMemory string `json:"libvirt_master_memory,omitempty"` - MasterVcpu string `json:"libvirt_master_vcpu,omitempty"` -+ AuthNodeIP string `json:"libvirt_auth_ip,omitempty"` - } - - // TFVars generates libvirt-specific Terraform variables. -@@ -46,6 +47,7 @@ func TFVars(masterConfig *v1beta1.LibvirtMachineProviderConfig, osImage string, - IfName: bridge, - BootstrapIP: bootstrapIP.String(), - MasterIPs: masterIPs, -+ AuthNodeIP: "192.168.126.51", - MasterMemory: strconv.Itoa(masterConfig.DomainMemory), - MasterVcpu: strconv.Itoa(masterConfig.DomainVcpu), - } diff --git a/cluster-provision/okd/scripts/create-local-disks.sh b/cluster-provision/okd/scripts/create-local-disks.sh deleted file mode 100755 index b321b37c21..0000000000 --- a/cluster-provision/okd/scripts/create-local-disks.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env bash - -set -e - -if [ ! -e /mnt/local-storage/local/disk1 ]; then - # Create local-volume directories - for i in {1..10}; do - sudo mkdir -p /var/local/kubevirt-storage/local-volume/disk${i} - sudo mkdir -p /mnt/local-storage/local/disk${i} - sudo mount --bind /var/local/kubevirt-storage/local-volume/disk${i} /mnt/local-storage/local/disk${i} - done - sudo chmod -R 777 /var/local/kubevirt-storage/local-volume - # Setup selinux permissions to local volume directories. - sudo chcon -R unconfined_u:object_r:svirt_sandbox_file_t:s0 /mnt/local-storage/ -fi diff --git a/cluster-provision/okd/scripts/provision.sh b/cluster-provision/okd/scripts/provision.sh deleted file mode 100755 index 773113fe77..0000000000 --- a/cluster-provision/okd/scripts/provision.sh +++ /dev/null @@ -1,376 +0,0 @@ -#!/bin/bash - -set -xe - -if [ ! -f "/etc/installer/token" ]; then - echo "You need to provide installer pull secret file to the container" - exit 1 -fi - -if [ ! -z $INSTALLER_RELEASE_IMAGE ]; then - until export INSTALLER_COMMIT=$(oc adm release info -a /etc/installer/token $INSTALLER_RELEASE_IMAGE --commits | grep installer | awk '{print $3}' | head -n 1); do - sleep 1 - done -fi - -MASTERS=1 -if [[ $INSTALLER_TAG == "release-4.4" ]]; then - MASTERS=3 -fi - -function oc_retry { - until oc $@; do - sleep 1 - done -} - -compile_installer () { - # install build dependencies - local build_pkgs="git gcc-c++" - dnf install -y ${build_pkgs} - - # get installer code - local installer_dir="/root/go/src/github.com/openshift/installer" - mkdir -p ${installer_dir} - cd ${installer_dir} - git clone https://github.com/openshift/installer.git ${installer_dir} - - if [ ! -z $INSTALLER_COMMIT ]; then - git checkout $INSTALLER_COMMIT - else - git checkout $INSTALLER_TAG - fi - - # compile the installer - if [ -d "/hacks" ]; then - git apply /hacks/$INSTALLER_TAG - fi - - GOROOT=/usr/local/go - GOPATH=/root/go/ - PATH=$GOPATH/bin:$GOROOT/bin:$PATH - TAGS=libvirt ./hack/build.sh - cp bin/openshift-install / - - # clean after the compilation - cd / - rm -rf ${installer_dir} ${GOROOT} - dnf erase -y ${build_pkgs} && dnf clean all -} - -compile_installer - -until virsh list -do - sleep 5 -done - -# create libvirt storage pool -virsh pool-define /dev/stdin < - default - - /var/lib/libvirt/images - - -EOF -virsh pool-start default -virsh pool-autostart default - -# dnsmasq configuration -original_dnss=$(cat /etc/resolv.conf | egrep "^nameserver" | awk '{print $2}') -echo "nameserver 127.0.0.1" > /etc/resolv.conf - -mkdir -p /etc/dnsmasq.d -echo "server=/tt.testing/192.168.126.1" >> /etc/dnsmasq.d/openshift.conf -for dns in $original_dnss; do - echo "server=/#/$dns" >> /etc/dnsmasq.d/openshift.conf -done - -/usr/sbin/dnsmasq \ ---no-resolv \ ---keep-in-foreground \ ---no-hosts \ ---bind-interfaces \ ---pid-file=/var/run/dnsmasq.pid \ ---listen-address=127.0.0.1 \ ---cache-size=400 \ ---clear-on-reload \ ---conf-file=/dev/null \ ---proxy-dnssec \ ---strict-order \ ---conf-file=/etc/dnsmasq.d/openshift.conf & - -# wait until dnsmasq will start -sleep 10 - -export CLUSTER_DIR=/root/install -INSTALL_CONFIG_FILE=$CLUSTER_DIR/install-config.yaml - -function yq_inline { - local expression="$1" - local file="$2" - if [ ! -f "$file" ]; then - echo "$file is not a file!" - return 1 - fi - tmp_file=$(mktemp /tmp/output.XXXXXXXXXX) - yq -y "$expression" "$file" > "$tmp_file" - if [ $? -ne 0 ]; then - return $? - fi - mv "$file" "$file.tmp" - mv "$tmp_file" "$file" -} - -mkdir -p $CLUSTER_DIR - -# fill registries.yaml with registries from the conf -export REGISTRIES_CONF=$(base64 -w0 /manifests/okd/registries.conf) -envsubst < /manifests/okd/registries.yaml > /registries.yaml - -# inject PULL_SECRET and SSH_PUBLIC_KEY into install-config -set +x -export PULL_SECRET=$(cat /etc/installer/token) -export SSH_PUBLIC_KEY="ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA6NF8iallvQVp22WDkTkyrtvp9eWW6A8YVr+kz4TjGYe7gHzIw+niNltGEFHzD8+v1I2YJ6oXevct1YeS0o9HZyN1Q9qgCgzUFtdOKLv6IedplqoPkcmF0aYet2PkEDo3MlTBckFXPITAMzF8dJSIFo9D8HfdOV0IAdx4O7PtixWKn5y2hMNG0zQPyUecp4pzC6kivAIhyfHilFR61RGL+GPXQ2MWZWFYbAGjyiYJnAmCP3NOTd0jMZEnDkbUvxhMmBYSdETk1rRgm+R4LOzFUGaHqHDLKLX+FIPKcF96hrucXzcWyLbIbEgE98OHlnVYCzRdK8jlqm8tehUc9c9WhQ== vagrant insecure public key" - -envsubst < /manifests/okd/install-config.yaml > ${INSTALL_CONFIG_FILE} -unset PULL_SECRET -set -x - -if [ ! -z $INSTALLER_RELEASE_IMAGE ]; then - export OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE=$INSTALLER_RELEASE_IMAGE -fi - -# update number of masters for okd-4.4, because 1 master isnt supported atm to to a bug of etcd -sed -i "s/replicas: 1/replicas: ${MASTERS}/" ${INSTALL_CONFIG_FILE} - -# Generate manifests -/openshift-install create manifests --dir=$CLUSTER_DIR - -# change master memory and vcpu -yq_inline '.spec.providerSpec.value.domainMemory = '"$MASTER_MEMORY"' | .spec.providerSpec.value.domainVcpu = '"$MASTER_CPU" \ - $CLUSTER_DIR/openshift/99_openshift-cluster-api_master-machines-0.yaml - -# change workers memory and vcpu -yq_inline '.spec.template.spec.providerSpec.value.domainMemory = '"$WORKERS_MEMORY"' | .spec.template.spec.providerSpec.value.domainVcpu = '"$WORKERS_CPU" \ - $CLUSTER_DIR/openshift/99_openshift-cluster-api_worker-machineset-0.yaml - -cat > "${CLUSTER_DIR}/openshift/99-master-registries.yaml" << __EOF__ ---- -apiVersion: machineconfiguration.openshift.io/v1 -kind: MachineConfig -metadata: - labels: - machineconfiguration.openshift.io/role: master - name: 99-master-registries -$(cat /registries.yaml) -__EOF__ - -cat > "${CLUSTER_DIR}/openshift/99-worker-registries.yaml" << __EOF__ ---- -apiVersion: machineconfiguration.openshift.io/v1 -kind: MachineConfig -metadata: - labels: - machineconfiguration.openshift.io/role: worker - name: 99-worker-registries -$(cat /registries.yaml) -__EOF__ - -# for debug -cp "${CLUSTER_DIR}/openshift/99-master-registries.yaml" ./ -cp "${CLUSTER_DIR}/openshift/99-worker-registries.yaml" ./ - -# Generate ignition configs -/openshift-install --dir "${CLUSTER_DIR}" create ignition-configs - -# Excecute installer -export TF_VAR_libvirt_master_memory=$MASTER_MEMORY -export TF_VAR_libvirt_master_vcpu=$MASTER_CPU -/openshift-install create cluster --dir "$CLUSTER_DIR" --log-level debug - -export KUBECONFIG=$CLUSTER_DIR/auth/kubeconfig - -oc wait --for=condition=Ready $(oc get node -o name) --timeout=900s - -# Create htpasswd with user admin -htpasswd -c -B -b /root/htpasswd admin admin - -# Create OpenShift HTPasswd provider with user and password admin -oc_retry create secret generic htpass-secret --from-file=htpasswd=/root/htpasswd -n openshift-config -oc_retry apply -f - < - ceph - nfs - registry -" --live --config - -domain_number=1 -for domain in $(virsh list --name --all); do - - # Add secondary nics - if [ "$NUM_SECONDARY_NICS" -gt 0 ]; then - domain_idx=$(printf "%02d" $domain_number) - for nic_idx in $(seq -f "%02g" 1 ${NUM_SECONDARY_NICS}); do - secondary_nic_mac=52:54:00:4b:$domain_idx:$nic_idx - virsh attach-interface --config --model virtio --domain $domain --type network --mac $secondary_nic_mac --source $cluster_network - done - fi - - domain_number=$(expr $domain_number + 1) - # Update master nodes memory - virt-xml --edit --memory ${MASTER_MEMORY} $domain - - # Update VM's CPU mode to passthroug - virt-xml --edit --cpu host-passthrough $domain - - # Update master nodes CPU - [[ $domain =~ master ]] && virt-xml --edit --vcpu ${MASTER_CPU} $domain - - # Update worker nodes memory and CPU - [[ $domain =~ worker ]] && virt-xml --edit --memory ${WORKERS_MEMORY} $domain && virt-xml --edit --vcpu ${WORKERS_CPU} $domain - - virsh start $domain -done - -while [[ "$(virsh list --name --all)" != "$(virsh list --name)" ]]; do - sleep 1 -done - -export KUBECONFIG=/root/install/auth/kubeconfig - -# update bashrc to make life easier -echo "" >> /root/.bashrc -echo 'export KUBECONFIG=/root/install/auth/kubeconfig' >> /root/.bashrc -echo "alias podc=\"oc get pods -A | grep -ivE 'run|comp'\"" >> /root/.bashrc -echo "alias pods=\"oc get pods -A\"" >> /root/.bashrc -echo "alias nodes=\"oc get nodes\"" >> /root/.bashrc -echo "alias podcw=\"oc get pods -A -owide | grep -ivE 'run|comp'\"" >> /root/.bashrc - -oc_retry config set-cluster test-1 --server=https://127.0.0.1:6443 -oc_retry config set-cluster test-1 --insecure-skip-tls-verify=true - -# Wait for API server to be up -oc_retry get nodes - -# wait half minute, just to be sure that we do not get old cluster state -sleep 30 - -# wait for the router pod to start on the worker -until [[ $(oc -n openshift-ingress get pods -o custom-columns=NAME:.metadata.name,HOST_IP:.status.hostIP,PHASE:.status.phase | grep route | grep Running | head -n 1 | awk '{print $2}') != "" ]]; do - sleep 5 -done - -# get_value fetches command output, with retry and timeout -# syntax nodes=$(get_value 10 oc get nodes) -# first parameter is the number of iterations to try, each has 6 seconds delay -function get_value() -{ - local val="" - timeout="$1" - shift - - n=0 - val=$("$@") - until [[ ${val} != "" ]]; do - sleep 6 - n=$[$n+1] - - if [ "$n" -ge "$timeout" ]; then - break - fi - - val=$("$@") - done - - echo "$val" -} - -worker_node_ip=$(get_value 50 oc -n openshift-ingress get pods -o custom-columns=NAME:.metadata.name,HOST_IP:.status.hostIP,PHASE:.status.phase | grep route | grep Running | head -n 1 | awk '{print $2}') -if [[ ${worker_node_ip} == "" ]]; then - echo "Failed to get worker_node_ip, exiting" - exit 1 -fi - -if [[ ${worker_node_ip} != "192.168.126.51" ]]; then - virsh net-update $cluster_network delete dns-host \ -" - console-openshift-console.apps.test-1.tt.testing - oauth-openshift.apps.test-1.tt.testing -" --live --config - - virsh net-update $cluster_network add dns-host \ -" - console-openshift-console.apps.test-1.tt.testing - oauth-openshift.apps.test-1.tt.testing -" --live --config - - sed -i "s/192.168.126.51/${worker_node_ip}/" /etc/haproxy/haproxy.cfg - pkill haproxy - haproxy -f /etc/haproxy/haproxy.cfg -fi - -set +xe -n=0 -# Following while should iterate as long as more than 3 pods arent Ready. -# we use /tmp/num_pods.txt because we need to check NUM_PODS just in case the -# oc command itself succeeded (else value will be a fake zero). -# /tmp/timeout.inject is just optional and can be used to shrink or extend the timeout. -while true; do - # get number of pods, when all but 3 pods are ready, continue - oc get pods --all-namespaces --no-headers > /tmp/num_pods.txt - if [ $? -eq 0 ]; then - NUM_PODS=$(cat /tmp/num_pods.txt | grep -v revision-pruner | grep -v Running | grep -v Completed | wc -l) - if [ $NUM_PODS -le 3 ] && [ $n -ge 20 ]; then - echo $NUM_PODS "pods are not Ready, continuing cluster-up" - break - fi - fi - - echo "Num of not ready pods" $NUM_PODS", waiting for pods to come up, cycle" $n - sleep 10 - - # allow to override timeout by echo timeout to timeout.inject in the container - TIMEOUT_FILE=/tmp/timeout.inject - timeout=90 - RE='^[0-9]+$' - if [ -f "$TIMEOUT_FILE" ]; then - input=$(cat $TIMEOUT_FILE) - if [[ $input =~ $RE ]]; then - timeout=$input - echo "$TIMEOUT_FILE exist, overriding timeout to $timeout" - fi - fi - - # check if loop timeout occured - n=$[$n+1] - if [ "$n" -gt "$timeout" ]; then - echo "Warning: timeout waiting for pods to come up" - break - fi -done -set -xe - -# update the pull-secret from the file -if [ -s "/etc/installer/token" ]; then - set +x - pull_secret=$(cat /etc/installer/token | base64 -w0) - until oc -n openshift-config patch secret pull-secret --type merge --patch "{\"data\": {\".dockerconfigjson\": \"${pull_secret}\"}}"; do - sleep 5 - done - set -x -fi - -# update worker machine set with desired number of CPU and memory -worker_machine_set=$(oc -n openshift-machine-api get machineset --no-headers | grep worker | awk '{print $1}') -until oc -n openshift-machine-api patch machineset ${worker_machine_set} --type merge --patch "{\"spec\": {\"template\": {\"spec\": {\"providerSpec\": {\"value\": {\"domainMemory\": ${WORKERS_MEMORY}, \"domainVcpu\": ${WORKERS_CPU}}}}}}}"; do - worker_machine_set=$(oc -n openshift-machine-api get machineset --no-headers | grep worker | awk '{print $1}') - sleep 5 -done - -# update number of workers -until oc -n openshift-machine-api scale --replicas=${WORKERS} machineset ${worker_machine_set}; do - sleep 5 -done - -echo "wait until all worker nodes will be ready" -until [[ $(oc get nodes | grep worker | grep -w Ready | wc -l) == ${WORKERS} ]]; do - sleep 5 -done - -echo "wait until all master nodes will be ready" -until [[ $(oc get nodes | grep master | grep -w Ready | wc -l) == $(oc get nodes | grep master | wc -l) ]]; do - sleep 5 -done - -# create local disks under all nodes, possible that we configured different number of nodes on the runtime -network_name=$(virsh net-list | grep test | awk '{print $1}') -vms=$(virsh list --name) -for vm in ${vms}; do - vm_ip=$(virsh net-dhcp-leases ${network_name} | grep ${vm} | awk '{print $5}' | tr "/" "\t" | awk '{print $1}') - ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -q -lcore -i vagrant.key ${vm_ip} < /scripts/create-local-disks.sh -done - -set +xe -echo "cluster non ready pods:" -timeout 1m bash -c "until oc get pods -A | grep -ivE 'run|comp'; do sleep 1; done" -echo "cluster nodes status:" -timeout 1m bash -c "until oc get nodes; do sleep 1; done" -echo "NOTE: check pods state, in case it doesnt converge in reasonable time, try to restart nodes / kubelets" diff --git a/cluster-up/cluster/images.sh b/cluster-up/cluster/images.sh index a402719324..bda87aadb2 100644 --- a/cluster-up/cluster/images.sh +++ b/cluster-up/cluster/images.sh @@ -3,7 +3,7 @@ set -e declare -A IMAGES -IMAGES[gocli]="gocli@sha256:2ce75db868bbf2811395dfe4f73a68160d2ad700b1e08aeee6956b4943ceb237" +IMAGES[gocli]="gocli@sha256:0cec3163e5f8b050f129fa795b76c20cb1eb27c91564c63663160b095064a1f9" if [ -z $KUBEVIRTCI_PROVISION_CHECK ]; then IMAGES[k8s-fedora-1.17.0]="k8s-fedora-1.17.0@sha256:aebf67b8b1b499c721f4d98a7ab9542c680553a14cbc144d1fa701fe611f3c0d" IMAGES[k8s-1.18]="k8s-1.18@sha256:fcf196a188cdb600d35e594b26b39804106e4ad263deea012a8a9a863232c4e4" diff --git a/pack8s/STATUS.md b/pack8s/STATUS.md index ce8b2ed7f2..33c4505636 100644 --- a/pack8s/STATUS.md +++ b/pack8s/STATUS.md @@ -10,8 +10,6 @@ kubevirtci `56f69bb5867db7517f70a0787b32570a861e124a` | k8s-1.15.1 | Yes | Planned | | | k8s-1.16.2 | Yes | Planned | | | k8s-multus-1.13.3 | Yes | N/A | | -| okd-4.1 | Yes | N/A | | -| okd-4.3 | Yes | Planned | | | os-3.11.0 | Yes | N/A | | | os-3.11.0-crio | Yes | N/A | | | os-3.11.0-multus | Yes | N/A | | diff --git a/pack8s/cmd/okd/run.go b/pack8s/cmd/okd/run.go deleted file mode 100644 index ed1565d63b..0000000000 --- a/pack8s/cmd/okd/run.go +++ /dev/null @@ -1,203 +0,0 @@ -package okd - -import ( - "context" - "fmt" - "os" - "os/signal" - - "github.com/spf13/cobra" - - "github.com/fromanirh/pack8s/iopodman" - - "github.com/fromanirh/pack8s/internal/pkg/ledger" - "github.com/fromanirh/pack8s/internal/pkg/podman" - "github.com/fromanirh/pack8s/internal/pkg/ports" - - "github.com/fromanirh/pack8s/cmd/cmdutil" -) - -type okdRunOptions struct { - privileged bool - masterMemory string - masterCpu string - workers string - workersMemory string - workersCpu string - secondaryNics uint - registryVolume string - nfsData string - registryPort uint - ocpConsolePort uint - k8sPort uint - sshMasterPort uint - sshWorkerPort uint - background bool - randomPorts bool - volume string - downloadOnly bool -} - -func (ro okdRunOptions) WantsNFS() bool { - return ro.nfsData != "" -} - -func (ro okdRunOptions) WantsCeph() bool { - return false -} - -func (ro okdRunOptions) WantsFluentd() bool { - return false -} - -var okdRunOpts okdRunOptions - -// NewRunCommand returns command that runs OKD cluster -func NewRunCommand() *cobra.Command { - run := &cobra.Command{ - Use: "okd", - Short: "run OKD cluster", - RunE: run, - Args: cobra.ExactArgs(1), - } - - okdRunOpts.privileged = true // always - run.Flags().StringVar(&okdRunOpts.masterMemory, "master-memory", "12288", "amount of RAM in MB on the master") - run.Flags().StringVar(&okdRunOpts.masterCpu, "master-cpu", "4", "number of CPU cores on the master") - run.Flags().StringVar(&okdRunOpts.workers, "workers", "1", "number of cluster worker nodes to start") - run.Flags().StringVar(&okdRunOpts.workersMemory, "workers-memory", "6144", "amount of RAM in MB per worker") - run.Flags().StringVar(&okdRunOpts.workersCpu, "workers-cpu", "2", "number of CPU per worker") - run.Flags().UintVar(&okdRunOpts.secondaryNics, "secondary-nics", 0, "number of secondary nics to add") - run.Flags().StringVar(&okdRunOpts.registryVolume, "registry-volume", "", "cache docker registry content in the specified volume") - run.Flags().StringVar(&okdRunOpts.nfsData, "nfs-data", "", "path to data which should be exposed via nfs to the nodes") - run.Flags().UintVar(&okdRunOpts.registryPort, "registry-port", 0, "port on localhost for the docker registry") - run.Flags().UintVar(&okdRunOpts.ocpConsolePort, "ocp-console-port", 0, "port on localhost for the ocp console") - run.Flags().UintVar(&okdRunOpts.k8sPort, "k8s-port", 0, "port on localhost for the k8s cluster") - run.Flags().UintVar(&okdRunOpts.sshMasterPort, "ssh-master-port", 0, "port on localhost to ssh to master node") - run.Flags().UintVar(&okdRunOpts.sshWorkerPort, "ssh-worker-port", 0, "port on localhost to ssh to worker node") - run.Flags().BoolVar(&okdRunOpts.background, "background", false, "go to background after nodes are up") - run.Flags().BoolVar(&okdRunOpts.randomPorts, "random-ports", true, "expose all ports on random localhost ports") - run.Flags().StringVar(&okdRunOpts.volume, "volume", "", "Bind mount a volume into the container") - return run -} - -func run(cmd *cobra.Command, args []string) (err error) { - cOpts, err := cmdutil.GetCommonOpts(cmd) - if err != nil { - return err - } - - envs := []string{} - envs = append(envs, fmt.Sprintf("WORKERS=%s", okdRunOpts.workers)) - envs = append(envs, fmt.Sprintf("MASTER_MEMORY=%s", okdRunOpts.masterMemory)) - envs = append(envs, fmt.Sprintf("MASTER_CPU=%s", okdRunOpts.masterCpu)) - envs = append(envs, fmt.Sprintf("WORKERS_MEMORY=%s", okdRunOpts.workersMemory)) - envs = append(envs, fmt.Sprintf("WORKERS_CPU=%s", okdRunOpts.workersCpu)) - envs = append(envs, fmt.Sprintf("NUM_SECONDARY_NICS=%d", okdRunOpts.secondaryNics)) - - portMap, err := ports.NewMappingFromFlags(cmd.Flags(), []ports.PortInfo{ - ports.PortInfo{ - ExposedPort: ports.PortSSH, - Name: "ssh-master-port", - }, - ports.PortInfo{ - ExposedPort: ports.PortSSHWorker, - Name: "ssh-worker-port", - }, - ports.PortInfo{ - ExposedPort: ports.PortAPI, - Name: "k8s-port", - }, - ports.PortInfo{ - ExposedPort: ports.PortOCPConsole, - Name: "ocp-console-port", - }, - ports.PortInfo{ - ExposedPort: ports.PortRegistry, - Name: "registry-port", - }, - }) - if err != nil { - return err - } - - cluster := args[0] - - ctx, cancel := context.WithCancel(context.Background()) - - log := cOpts.GetLogger() - hnd, err := podman.NewHandle(ctx, cOpts.PodmanSocket, log) - if err != nil { - return err - } - - log.Noticef("downloading all the images needed for %s (from %s)", cluster, cOpts.Registry) - err = hnd.PullClusterImages(okdRunOpts, cOpts.Registry, cluster) - if err != nil || okdRunOpts.downloadOnly { - return err - } - log.Infof("downloaded all the images needed for %s, bringing cluster up", cluster) - - ldgr := ledger.NewLedger(hnd, cmd.OutOrStderr(), log) - - defer func() { - ldgr.Done <- err - }() - - go func() { - interrupt := make(chan os.Signal, 1) - signal.Notify(interrupt, os.Interrupt) - <-interrupt - cancel() - ldgr.Done <- fmt.Errorf("Interrupt received, clean up") - }() - - clusterContainerName := cOpts.Prefix + "-cluster" - clusterExpose := ports.ToStrings( - ports.PortSSH, ports.PortSSHWorker, ports.PortRegistry, - ports.PortOCPConsole, ports.PortAPI, - ) - clusterPorts := portMap.ToStrings() - clusterLabels := []string{fmt.Sprintf("%s=000", podman.LabelGeneration)} - clusterID, err := ldgr.RunContainer(iopodman.Create{ - Args: []string{cluster}, - Env: &envs, - Expose: &clusterExpose, - Label: &clusterLabels, - Name: &clusterContainerName, - Privileged: &okdRunOpts.privileged, - Publish: &clusterPorts, - PublishAll: &okdRunOpts.randomPorts, - Volume: &[]string{okdRunOpts.volume}, - }) - if err != nil { - return err - } - - clusterNetwork := fmt.Sprintf("container:%s", clusterID) - - err = cmdutil.SetupRegistry(ldgr, cOpts.Prefix, clusterNetwork, okdRunOpts.registryVolume, okdRunOpts.privileged) - if err != nil { - return err - } - if okdRunOpts.nfsData != "" { - err = cmdutil.SetupNFS(ldgr, cOpts.Prefix, clusterNetwork, okdRunOpts.nfsData, okdRunOpts.privileged) - if err != nil { - return err - } - } - - // Run the cluster - fmt.Printf("Run the cluster\n") - err = hnd.Exec(clusterContainerName, []string{"/bin/bash", "-c", "/scripts/run.sh"}, os.Stdout) - if err != nil { - return fmt.Errorf("failed to run the OKD cluster under the container %s: %s", clusterContainerName, err) - } - - // If background flag was specified, we don't want to clean up if we reach that state - if !okdRunOpts.background { - ldgr.Done <- fmt.Errorf("Done. please clean up") - } - - return nil -} diff --git a/pack8s/cmd/run.go b/pack8s/cmd/run.go index 2d434be300..ce55f42a10 100644 --- a/pack8s/cmd/run.go +++ b/pack8s/cmd/run.go @@ -22,8 +22,6 @@ import ( "github.com/fromanirh/pack8s/internal/pkg/ports" "github.com/fromanirh/pack8s/cmd/cmdutil" - - "github.com/fromanirh/pack8s/cmd/okd" ) type runOptions struct { @@ -92,9 +90,6 @@ func NewRunCommand() *cobra.Command { run.Flags().BoolVar(&runOpts.enableCeph, "enable-ceph", false, "enables dynamic storage provisioning using Ceph") run.Flags().BoolVar(&runOpts.downloadOnly, "download-only", false, "download cluster images and exith") - run.AddCommand( - okd.NewRunCommand(), - ) return run }