diff --git a/README.adoc b/README.adoc index 40325f6..1537ec6 100644 --- a/README.adoc +++ b/README.adoc @@ -187,6 +187,7 @@ ansible-playbook -i staging -e cluster=[cluster_name] restricted_static_ips_ova. == Post Install (Hybrid clusters) +=== Iso Generation for Additional Nodes In the event that you need to add nodes to a hybrid cluster post install, there is a `new_worker_iso.yml` that can generate additional ISOs for new nodes. The requirements to this playbook are the same as the other playbooks here with 1 exception, you need to create a new `{{ clusters_folder }}/{{ cluster }}_additional_nodes.yaml` file. The format of that file is as follows: @@ -207,6 +208,25 @@ NOTE: The role that we use for this playbook is a shared role and is used by the ansible-playbook -i staging -e "cluster=ocp-example" new_worker_isos.yml ---- +=== OVA template for Additional Nodes +In the event that you need to add nodes to a hybrid cluster post install, there is a `new_worker_ova.yml` that can reuse the existing OVA template for new nodes. The requirements to this playbook are the same as the other playbooks here with 1 exception, you need to create a new `{{ clusters_folder }}/{{ cluster }}_additional_nodes_ova.yaml` file. +The format of that file is as follows: + +.Additional node file +[source,yaml] +==== +include::clusters/ocp-example_additional_nodes_ova.yaml[] +==== + +By using this file, the playbook will create only the nodes registered in this file. As per now, this playbook is only adapted to create new worker/infra nodes in an existing cluster. + +NOTE: This role uses a new role called _vsphere_vm_new_nodes_. + +.Example run +---- +ansible-playbook -i staging -e "cluster=ocp-example" new_worker_ova.yml +---- + == Final Check: If everything goes well you should be able validate the cluster using the included `validateCluster.yml` playbook. diff --git a/clusters/ocp-example.yaml b/clusters/ocp-example.yaml index 4e52074..3a86b05 100644 --- a/clusters/ocp-example.yaml +++ b/clusters/ocp-example.yaml @@ -28,6 +28,9 @@ download: govc: "https://github.com/vmware/govmomi/releases/download/v0.30.0" clients_url: "https://mirror.openshift.com/pub/openshift-v4/x86_64/clients/ocp/{{ config.cluster_version }}" nodes: +# Sample node definition: +# - By adding isinfra, the node will be labeled as infra node +# - { name: "worker0", type: "worker", ipaddr: "192.168.66.11", cpu: 8, ram: 16384, size_gb: 120, isinfra: ""} - { name: "bootstrap", type: "bootstrap", macaddr: "00:50:56:be:ce:8e", ipaddr: "192.168.1.62", cpu: 4, ram: 16384, size_gb: 120, cores_per_socket: 2} - { name: "master0", type: "master", macaddr: "00:50:56:be:1a:89", ipaddr: "192.168.1.43", cpu: 8, ram: 16384, size_gb: 120, cores_per_socket: 2} - { name: "master1", type: "master", macaddr: "00:50:56:be:4c:49", ipaddr: "192.168.1.44", cpu: 8, ram: 16384, size_gb: 120, cores_per_socket: 2} diff --git a/clusters/ocp-example_additional_nodes_ova.yaml b/clusters/ocp-example_additional_nodes_ova.yaml new file mode 100644 index 0000000..ae2116b --- /dev/null +++ b/clusters/ocp-example_additional_nodes_ova.yaml @@ -0,0 +1,16 @@ +# ova: The ova template created in the cluster folder in vSphere +ova: "rhcos-vmware-4.14.15.ova" +# additional_nodes: The list of additional nodes to be created +# -name: The name of the node +# -macaddr: The mac address of the node +# -ipaddr: The ip address of the node +# -cpu: The number of cpus for the node +# -ram: The amount of ram for the node +# -size_gb: The size of the disk for the node +# -cores_per_socket: The number of cores per socket for the node +# -isinfra: The flag to indicate if the node is an infra node. The node will be labeled as infra. +# -isodf: The flag to indicate if the node is an odf node. The node will be labeled and tainted for ODF usage. +additional_nodes: + - { name: "worker3", macaddr: "00:50:56:be:d3:cb", ipaddr: "192.168.1.51", cpu: 4, ram: 16384, size_gb: 120, cores_per_socket: 2, isinfra: "", isodf: ""} + - { name: "worker4", macaddr: "00:50:56:be:0f:22", ipaddr: "192.168.1.52", cpu: 4, ram: 16384, size_gb: 120, cores_per_socket: 2, isinfra: "", isodf: ""} + - { name: "worker5", macaddr: "00:50:56:be:42:8a", ipaddr: "192.168.1.53", cpu: 4, ram: 16384, size_gb: 120, cores_per_socket: 2, isinfra: "", isodf: ""} \ No newline at end of file diff --git a/new_worker_ova.yml b/new_worker_ova.yml new file mode 100644 index 0000000..0da24fe --- /dev/null +++ b/new_worker_ova.yml @@ -0,0 +1,48 @@ +--- +- name: Initial Environment setup + hosts: localhost + gather_facts: True + environment: + PATH: "{{ playbook_dir }}/bin:{{ ansible_env.PATH }}" + vars_files: + - "{{ clusters_folder }}/{{ cluster }}_additional_nodes_ova.yaml" + tasks: + - name: Check for additional nodes file + stat: + path: "{{ clusters_folder }}/{{ cluster }}_additional_nodes_ova.yaml" + register: nodes_file + + - fail: + msg: "Bailing out: nodes file is not found in the '{{ clusters_folder }}' folder" + when: not nodes_file.stat.exists + + - name: Initial setup and checks + import_role: + name: setup + tags: + - always + + - name: Fetch the content of the sha256sum.txt from the dependencies downloads page + uri: + url: "{{ download.dependencies_url }}/sha256sum.txt" + return_content: yes + register: dependencies_content + + +- name: Deploy VMs and finish cluster install + hosts: localhost + gather_facts: True + environment: + PATH: "{{ playbook_dir }}/bin:{{ ansible_env.PATH }}" + vars_files: + - "{{ clusters_folder }}/{{ cluster }}_additional_nodes_ova.yaml" + tasks: + - name: Create, Configure and boot the VMs + environment: + GOVC_USERNAME: "{{ vcenter.username }}" + GOVC_PASSWORD: "{{ vcenter.password }}" + GOVC_URL: "https://{{ vcenter.ip }}" + GOVC_DATACENTER: "{{ vcenter.datacenter }}" + GOVC_INSECURE: 1 + import_role: + name: vsphere_vm_new_nodes \ No newline at end of file diff --git a/roles/setup/tasks/main.yml b/roles/setup/tasks/main.yml index ecba933..7f2005d 100644 --- a/roles/setup/tasks/main.yml +++ b/roles/setup/tasks/main.yml @@ -81,6 +81,28 @@ loop_control: label: "{{ item.name }}" +- name: Set additional nodes list + set_fact: + worker_additional_nodes: "{{ worker_additional_nodes|default([]) + [item] }}" + when: nodes_file is defined and item.type == 'worker' + loop: "{{ additional_nodes }}" + loop_control: + label: "{{ item.name }}" + +- name: Set count of worker nodes to 0 by default + set_fact: + worker_count: "0" + +- name: Set count of worker nodes + set_fact: + worker_count: "{{ worker_vms | length }}" + when: worker_vms is defined + +- name: Update count of worker nodes if control planes are schedulable + set_fact: + worker_count: "{{ (worker_count | int) + (master_vms | length) }}" + when: config.master_schedulable + - name: Display the absolute folder path of the vCenter folder debug: var: vcenter.folder_absolute_path diff --git a/roles/vsphere_vm_new_nodes/tasks/dhcp.yml b/roles/vsphere_vm_new_nodes/tasks/dhcp.yml new file mode 100644 index 0000000..c0566ef --- /dev/null +++ b/roles/vsphere_vm_new_nodes/tasks/dhcp.yml @@ -0,0 +1,52 @@ +- name: Create worker VMs from the template + vmware_guest: + hostname: "{{ vcenter.ip }}" + username: "{{ vcenter.username }}" + password: "{{ vcenter.password }}" + datacenter: "{{ vcenter.datacenter }}" + guest_id: "rhel8_64Guest" + validate_certs: no + folder: "{{ vcenter.folder_absolute_path }}" + name: "{{ item.name }}.{{config.cluster_name}}.{{config.base_domain}}" + state: poweredoff + template: "{{ vcenter.folder_absolute_path }}/{{ vcenter.template_name }}" + disk: + - size_gb: "{{ item.size_gb | default(120) }}" + type: thin + datastore: "{{ vcenter.datastore }}" + hardware: + memory_mb: "{{ item.ram | default(16384) }}" + num_cpus: "{{ item.cpu | default(4) }}" + num_cpu_cores_per_socket: "{{ vm_mods.worker_cores_per_socket | default(omit) }}" + memory_reservation_lock: True + version: "{{ vcenter.hw_version }}" + hotadd_cpu: "{{ vm_mods.hotadd_cpu | default(omit) }}" + hotremove_cpu: "{{ vm_mods.hotremove_cpu | default(omit) }}" + hotadd_memory: "{{ vm_mods.hotadd_memory | default(omit) }}" + wait_for_ip_address: no + advanced_settings: + - key: guestinfo.ignition.config.data + value: "{{ workerContent }}" + - key: guestinfo.ignition.config.data.encoding + value: base64 + - key: disk.EnableUUID + value: TRUE + loop: "{{ worker_additional_nodes }}" + loop_control: + label: "{{ item.name }}" + +- name: Configure network for worker VMs + vmware_guest_network: + hostname: "{{ vcenter.ip }}" + username: "{{ vcenter.username }}" + password: "{{ vcenter.password }}" + datacenter: "{{ vcenter.datacenter }}" + validate_certs: no + name: "{{ item.name }}.{{config.cluster_name}}.{{config.base_domain}}" + network_name: "{{ vcenter.network }}" + mac_address: "{{ item.macaddr | default(omit) }}" + state: present + start_connected: true + loop: "{{ worker_additional_nodes | default([]) }}" + loop_control: + label: "{{ item.name }}" diff --git a/roles/vsphere_vm_new_nodes/tasks/finish_install.yml b/roles/vsphere_vm_new_nodes/tasks/finish_install.yml new file mode 100644 index 0000000..a7b9ddd --- /dev/null +++ b/roles/vsphere_vm_new_nodes/tasks/finish_install.yml @@ -0,0 +1,92 @@ +- name: Power-On the worker VMs + vmware_guest: + hostname: "{{ vcenter.ip }}" + username: "{{ vcenter.username }}" + password: "{{ vcenter.password }}" + datacenter: "{{ vcenter.datacenter }}" + validate_certs: no + folder: "{{ vcenter.folder_absolute_path }}" + name: "{{ item.name }}.{{config.cluster_name}}.{{config.base_domain}}" + state: poweredon + loop: "{{ worker_additional_nodes | default([]) }}" + loop_control: + label: "Power On {{ item.name }}" + when: not config.hybrid | bool + +- name: Approve csrs for worker nodes + environment: + KUBECONFIG: "{{ playbook_dir }}/install-dir/auth/kubeconfig" + shell: | + oc get csr -o name | xargs oc adm certificate approve &> /dev/null + oc get nodes -l "node-role.kubernetes.io/worker=" -o name | grep {{ item.name }} | wc -l + register: result + until: (result.stdout | int > 0) + retries: 100 + delay: 10 + loop: "{{ worker_additional_nodes | default([]) }}" + loop_control: + label: "{{ item.name }}" + +- name: Create Infra List + set_fact: + infra_vms: "{{ worker_additional_nodes|select('search','isinfra') | list }}" + +- name: Check for infra nodes and label accordingly + environment: + KUBECONFIG: "{{ playbook_dir }}/install-dir/auth/kubeconfig" + k8s: + state: patched + merge_type: merge + definition: + apiVersion: v1 + kind: Node + metadata: + name: "{{ item.name }}.{{ cluster }}.{{ config.base_domain }}" + labels: + node-role.kubernetes.io/infra: "" + loop: "{{ infra_vms }}" + when: infra_vms | length > 0 + +- name: Create ODF node list + set_fact: + odf_vms: "{{ worker_additional_nodes|select('search','isodf') | list }}" + +- name: Check for ODF nodes and label them accordingly + environment: + KUBECONFIG: "{{ playbook_dir }}/install-dir/auth/kubeconfig" + k8s: + state: patched + merge_type: merge + definition: + apiVersion: v1 + kind: Node + metadata: + name: "{{ item.name }}.{{ cluster }}.{{ config.base_domain }}" + labels: + cluster.ocs.openshift.io/openshift-storage: "" + spec: + taints: + - effect: NoSchedule + key: node.ocs.openshift.io/storage + value: "true" + loop: "{{ odf_vms }}" + when: odf_vms | length > 0 + +- name: Change ingress controller node selector + environment: + KUBECONFIG: "{{ playbook_dir }}/install-dir/auth/kubeconfig" + k8s: + state: patched + merge_type: merge + definition: + apiVersion: "operator.openshift.io/v1" + kind: IngressController + metadata: + name: default + namespace: openshift-ingress-operator + spec: + nodePlacement: + nodeSelector: + matchLabels: + node-role.kubernetes.io/infra: "" + when: infra_vms | length > 0 \ No newline at end of file diff --git a/roles/vsphere_vm_new_nodes/tasks/main.yml b/roles/vsphere_vm_new_nodes/tasks/main.yml new file mode 100644 index 0000000..b22ea53 --- /dev/null +++ b/roles/vsphere_vm_new_nodes/tasks/main.yml @@ -0,0 +1,20 @@ +- name: Encode node ignition files + set_fact: + workerContent: "{{ lookup('file', '{{ playbook_dir }}/install-dir/worker.ign') | b64encode }}" + +- name: "Worker base64 " + debug: + msg: "{{ workerContent }}" + verbosity: 1 + +- name: Create VMs + vars: + ip_alloc: "{% if dhcp | default(false) %}dhcp{% else %}static{% endif %}" + medium: "{% if iso | default(false) | bool %}iso{% else %}ova{% endif %}" + name: "{% if ip_alloc == 'dhcp' %}{{ ip_alloc }}{% else %}{{ ip_alloc }}-{{ medium }}{% endif %}.yml" + include_tasks: + file: "{{ name }}" + +- name: Run steps to finish cluster install + include_tasks: + file: "finish_install.yml" \ No newline at end of file diff --git a/roles/vsphere_vm_new_nodes/tasks/static-ova.yml b/roles/vsphere_vm_new_nodes/tasks/static-ova.yml new file mode 100644 index 0000000..b71951c --- /dev/null +++ b/roles/vsphere_vm_new_nodes/tasks/static-ova.yml @@ -0,0 +1,53 @@ +- name: Create worker VMs from the template + vmware_guest: + hostname: "{{ vcenter.ip }}" + username: "{{ vcenter.username }}" + password: "{{ vcenter.password }}" + datacenter: "{{ vcenter.datacenter }}" + validate_certs: no + folder: "{{ vcenter.folder_absolute_path }}" + name: "{{ item.name }}.{{config.cluster_name}}.{{config.base_domain}}" + state: poweredoff + template: "{{ vcenter.folder_absolute_path }}/{{ ova }}" + disk: + - size_gb: "{{ item.size_gb | default(120) }}" + type: thin + datastore: "{{ vcenter.datastore }}" + hardware: + memory_mb: "{{ item.ram | default(16384) }}" + num_cpus: "{{ item.cpu | default(4) }}" + num_cpu_cores_per_socket: "{{ vm_mods.worker_cores_per_socket | default(omit) }}" + memory_reservation_lock: True + version: "{{ vcenter.hw_version }}" + hotadd_cpu: "{{ vm_mods.hotadd_cpu | default(omit) }}" + hotremove_cpu: "{{ vm_mods.hotremove_cpu | default(omit) }}" + hotadd_memory: "{{ vm_mods.hotadd_memory | default(omit) }}" + wait_for_ip_address: no + advanced_settings: + - key: guestinfo.ignition.config.data + value: "{{ workerContent }}" + - key: guestinfo.ignition.config.data.encoding + value: base64 + - key: disk.EnableUUID + value: TRUE + - key: guestinfo.afterburn.initrd.network-kargs + value: "ip={{ item.ipaddr }}::{{ static_ip.gateway }}:{{ static_ip.netmask }}:{{ item.name }}.{{config.cluster_name}}.{{config.base_domain}}:ens192:off:{{ static_ip.dns }}" + loop: "{{ worker_additional_nodes }}" + loop_control: + label: "{{ item.name }}" + +- name: Configure network for worker VMs + vmware_guest_network: + hostname: "{{ vcenter.ip }}" + username: "{{ vcenter.username }}" + password: "{{ vcenter.password }}" + datacenter: "{{ vcenter.datacenter }}" + validate_certs: no + name: "{{ item.name }}.{{config.cluster_name}}.{{config.base_domain}}" + network_name: "{{ vcenter.network }}" + mac_address: "{{ item.macaddr | default(omit) }}" + state: present + start_connected: true + loop: "{{ worker_additional_nodes | default([]) }}" + loop_control: + label: "{{ item.name }}" \ No newline at end of file diff --git a/roles/vsphere_vm_new_nodes/vars/main.yml b/roles/vsphere_vm_new_nodes/vars/main.yml new file mode 100644 index 0000000..f2d38f2 --- /dev/null +++ b/roles/vsphere_vm_new_nodes/vars/main.yml @@ -0,0 +1 @@ +workerContent: "{{ lookup('file', '{{ playbook_dir }}/install-dir/worker.ign') | b64encode }}" \ No newline at end of file