Feature/add new worker nodes by using ova

README.adoc

@@ -187,6 +187,7 @@ ansible-playbook -i staging -e cluster=[cluster_name] restricted_static_ips_ova.
 
 == Post Install (Hybrid clusters)
 
+=== Iso Generation for Additional Nodes
 In the event that you need to add nodes to a hybrid cluster post install, there is a `new_worker_iso.yml` that can generate additional ISOs for new nodes. The requirements to this playbook are the same as the other playbooks here with 1 exception, you need to create a new `{{ clusters_folder }}/{{ cluster }}_additional_nodes.yaml` file.
 The format of that file is as follows:
 
@@ -207,6 +208,25 @@ NOTE: The role that we use for this playbook is a shared role and is used by the
 ansible-playbook -i staging -e "cluster=ocp-example" new_worker_isos.yml
 ----
 
+=== OVA template for Additional Nodes
+In the event that you need to add nodes to a hybrid cluster post install, there is a `new_worker_ova.yml` that can reuse the existing OVA template for new nodes. The requirements to this playbook are the same as the other playbooks here with 1 exception, you need to create a new `{{ clusters_folder }}/{{ cluster }}_additional_nodes_ova.yaml` file.
+The format of that file is as follows:
+
+.Additional node file
+[source,yaml]
+====
+include::clusters/ocp-example_additional_nodes_ova.yaml[]
+====
+
+By using this file, the playbook will create only the nodes registered in this file. As per now, this playbook is only adapted to create new worker/infra nodes in an existing cluster.
+
+NOTE: This role uses a new role called _vsphere_vm_new_nodes_.
+
+.Example run
+----
+ansible-playbook -i staging -e "cluster=ocp-example" new_worker_ova.yml
+----
+
 == Final Check:
 
 If everything goes well you should be able validate the cluster using the included `validateCluster.yml` playbook.

clusters/ocp-example.yaml

@@ -28,6 +28,9 @@ download:
   govc: "https://github.com/vmware/govmomi/releases/download/v0.30.0"
   clients_url: "https://mirror.openshift.com/pub/openshift-v4/x86_64/clients/ocp/{{ config.cluster_version }}"
 nodes:
+# Sample node definition:
+#  - By adding isinfra, the node will be labeled as infra node
+# - { name: "worker0", type: "worker", ipaddr: "192.168.66.11", cpu: 8, ram: 16384, size_gb: 120, isinfra: ""}
   - { name: "bootstrap", type: "bootstrap", macaddr: "00:50:56:be:ce:8e", ipaddr: "192.168.1.62", cpu: 4, ram: 16384, size_gb: 120, cores_per_socket: 2}
   - { name: "master0", type: "master", macaddr: "00:50:56:be:1a:89", ipaddr: "192.168.1.43", cpu: 8, ram: 16384, size_gb: 120, cores_per_socket: 2}
   - { name: "master1", type: "master", macaddr: "00:50:56:be:4c:49", ipaddr: "192.168.1.44", cpu: 8, ram: 16384, size_gb: 120, cores_per_socket: 2}

clusters/ocp-example_additional_nodes_ova.yaml

@@ -0,0 +1,16 @@
+# ova: The ova template created in the cluster folder in vSphere
+ova: "rhcos-vmware-4.14.15.ova"
+# additional_nodes: The list of additional nodes to be created
+# -name: The name of the node
+# -macaddr: The mac address of the node
+# -ipaddr: The ip address of the node
+# -cpu: The number of cpus for the node
+# -ram: The amount of ram for the node
+# -size_gb: The size of the disk for the node
+# -cores_per_socket: The number of cores per socket for the node
+# -isinfra: The flag to indicate if the node is an infra node. The node will be labeled as infra.
+# -isodf: The flag to indicate if the node is an odf node. The node will be labeled and tainted for ODF usage.
+additional_nodes:
+  - { name: "worker3", macaddr: "00:50:56:be:d3:cb", ipaddr: "192.168.1.51", cpu: 4, ram: 16384, size_gb: 120, cores_per_socket: 2, isinfra: "", isodf: ""}
+  - { name: "worker4", macaddr: "00:50:56:be:0f:22", ipaddr: "192.168.1.52", cpu: 4, ram: 16384, size_gb: 120, cores_per_socket: 2, isinfra: "", isodf: ""}
+  - { name: "worker5", macaddr: "00:50:56:be:42:8a", ipaddr: "192.168.1.53", cpu: 4, ram: 16384, size_gb: 120, cores_per_socket: 2, isinfra: "", isodf: ""}

new_worker_ova.yml

@@ -0,0 +1,48 @@
+---
+- name: Initial Environment setup
+  hosts: localhost
+  gather_facts: True
+  environment:
+    PATH: "{{ playbook_dir }}/bin:{{ ansible_env.PATH }}"
+  vars_files:
+  - "{{ clusters_folder }}/{{ cluster }}_additional_nodes_ova.yaml"
+  tasks:
+    - name: Check for additional nodes file
+      stat:
+        path: "{{ clusters_folder }}/{{ cluster }}_additional_nodes_ova.yaml"
+      register: nodes_file
+
+    - fail:
+        msg: "Bailing out: nodes file is not found in the '{{ clusters_folder }}' folder"
+      when: not nodes_file.stat.exists
+
+    - name: Initial setup and checks
+      import_role:
+        name: setup
+      tags:
+      - always
+
+    - name: Fetch the content of the sha256sum.txt from the dependencies downloads page
+      uri:
+        url: "{{ download.dependencies_url }}/sha256sum.txt"
+        return_content: yes
+      register: dependencies_content
+
+
+- name: Deploy VMs and finish cluster install
+  hosts: localhost
+  gather_facts: True
+  environment:
+    PATH: "{{ playbook_dir }}/bin:{{ ansible_env.PATH }}"
+  vars_files:
+  - "{{ clusters_folder }}/{{ cluster }}_additional_nodes_ova.yaml"
+  tasks:
+    - name: Create, Configure and boot the VMs
+      environment:
+        GOVC_USERNAME: "{{ vcenter.username }}"
+        GOVC_PASSWORD: "{{ vcenter.password }}"
+        GOVC_URL: "https://{{ vcenter.ip }}"
+        GOVC_DATACENTER: "{{ vcenter.datacenter }}"
+        GOVC_INSECURE: 1
+      import_role:
+        name: vsphere_vm_new_nodes

roles/setup/tasks/main.yml

@@ -81,6 +81,28 @@
   loop_control:
     label: "{{ item.name }}"
 
+- name: Set additional nodes list
+  set_fact:
+    worker_additional_nodes: "{{ worker_additional_nodes|default([]) + [item] }}"
+  when: nodes_file is defined and item.type == 'worker'
+  loop: "{{ additional_nodes }}"
+  loop_control:
+    label: "{{ item.name }}"
+
+- name: Set count of worker nodes to 0 by default
+  set_fact:
+    worker_count: "0"
+
+- name: Set count of worker nodes
+  set_fact:
+    worker_count: "{{ worker_vms | length }}"
+  when: worker_vms is defined
+
+- name: Update count of worker nodes if control planes are schedulable
+  set_fact:
+    worker_count: "{{ (worker_count | int) + (master_vms | length) }}"
+  when: config.master_schedulable
+
 - name: Display the absolute folder path of the vCenter folder
   debug:
     var: vcenter.folder_absolute_path

roles/vsphere_vm_new_nodes/tasks/dhcp.yml

@@ -0,0 +1,52 @@
+- name: Create worker VMs from the template
+  vmware_guest:
+    hostname: "{{ vcenter.ip }}"
+    username: "{{ vcenter.username }}"
+    password: "{{ vcenter.password }}"
+    datacenter: "{{ vcenter.datacenter }}"
+    guest_id: "rhel8_64Guest"
+    validate_certs: no
+    folder: "{{ vcenter.folder_absolute_path }}"
+    name: "{{ item.name }}.{{config.cluster_name}}.{{config.base_domain}}"
+    state: poweredoff
+    template: "{{ vcenter.folder_absolute_path }}/{{ vcenter.template_name }}"
+    disk:
+    - size_gb: "{{ item.size_gb | default(120) }}"
+      type: thin
+      datastore: "{{ vcenter.datastore }}"
+    hardware:
+      memory_mb: "{{ item.ram | default(16384) }}"
+      num_cpus: "{{ item.cpu | default(4) }}"
+      num_cpu_cores_per_socket: "{{ vm_mods.worker_cores_per_socket | default(omit) }}"
+      memory_reservation_lock: True
+      version: "{{ vcenter.hw_version }}"
+      hotadd_cpu: "{{ vm_mods.hotadd_cpu | default(omit) }}"
+      hotremove_cpu: "{{ vm_mods.hotremove_cpu | default(omit) }}"
+      hotadd_memory: "{{ vm_mods.hotadd_memory | default(omit) }}"
+    wait_for_ip_address: no
+    advanced_settings:
+    - key: guestinfo.ignition.config.data
+      value: "{{ workerContent }}"
+    - key: guestinfo.ignition.config.data.encoding
+      value: base64
+    - key: disk.EnableUUID
+      value: TRUE
+  loop: "{{ worker_additional_nodes }}"
+  loop_control:
+    label: "{{ item.name }}"
+
+- name: Configure network for worker VMs
+  vmware_guest_network:
+    hostname: "{{ vcenter.ip }}"
+    username: "{{ vcenter.username }}"
+    password: "{{ vcenter.password }}"
+    datacenter: "{{ vcenter.datacenter }}"
+    validate_certs: no
+    name: "{{ item.name }}.{{config.cluster_name}}.{{config.base_domain}}"
+    network_name: "{{ vcenter.network }}"
+    mac_address: "{{ item.macaddr | default(omit) }}"
+    state: present
+    start_connected: true
+  loop: "{{ worker_additional_nodes | default([]) }}"
+  loop_control:
+    label: "{{ item.name }}"

roles/vsphere_vm_new_nodes/tasks/finish_install.yml

@@ -0,0 +1,92 @@
+- name: Power-On the worker VMs
+  vmware_guest:
+    hostname: "{{ vcenter.ip }}"
+    username: "{{ vcenter.username }}"
+    password: "{{ vcenter.password }}"
+    datacenter: "{{ vcenter.datacenter }}"
+    validate_certs: no
+    folder: "{{ vcenter.folder_absolute_path }}"
+    name: "{{ item.name }}.{{config.cluster_name}}.{{config.base_domain}}"
+    state: poweredon
+  loop: "{{ worker_additional_nodes | default([]) }}"
+  loop_control:
+    label: "Power On {{ item.name }}"
+  when: not config.hybrid | bool
+
+- name: Approve csrs for worker nodes
+  environment:
+    KUBECONFIG: "{{ playbook_dir }}/install-dir/auth/kubeconfig"
+  shell: |
+    oc get csr -o name | xargs oc adm certificate approve &> /dev/null
+    oc get nodes -l "node-role.kubernetes.io/worker=" -o name | grep {{ item.name }} | wc -l
+  register: result
+  until: (result.stdout | int > 0)
+  retries: 100
+  delay: 10
+  loop: "{{ worker_additional_nodes | default([]) }}"
+  loop_control:
+    label: "{{ item.name }}"
+
+- name: Create Infra List
+  set_fact:
+   infra_vms: "{{ worker_additional_nodes|select('search','isinfra') | list }}"
+
+- name: Check for infra nodes and label accordingly
+  environment:
+   KUBECONFIG: "{{ playbook_dir }}/install-dir/auth/kubeconfig"
+  k8s:
+   state: patched
+   merge_type: merge
+   definition:
+     apiVersion: v1
+     kind: Node
+     metadata:
+       name: "{{ item.name }}.{{ cluster }}.{{ config.base_domain }}"
+       labels:
+         node-role.kubernetes.io/infra: ""
+  loop: "{{ infra_vms }}"
+  when: infra_vms | length > 0
+
+- name: Create ODF node list
+  set_fact:
+   odf_vms: "{{ worker_additional_nodes|select('search','isodf') | list }}"
+
+- name: Check for ODF nodes and label them accordingly
+  environment:
+   KUBECONFIG: "{{ playbook_dir }}/install-dir/auth/kubeconfig"
+  k8s:
+    state: patched
+    merge_type: merge
+    definition:
+      apiVersion: v1
+      kind: Node
+      metadata:
+        name: "{{ item.name }}.{{ cluster }}.{{ config.base_domain }}"
+        labels:
+          cluster.ocs.openshift.io/openshift-storage: ""
+      spec:
+        taints:
+        - effect: NoSchedule
+          key: node.ocs.openshift.io/storage
+          value: "true"
+  loop: "{{ odf_vms }}"
+  when: odf_vms | length > 0
+
+- name: Change ingress controller node selector
+  environment:
+   KUBECONFIG: "{{ playbook_dir }}/install-dir/auth/kubeconfig"
+  k8s:
+   state: patched
+   merge_type: merge
+   definition:
+     apiVersion: "operator.openshift.io/v1"
+     kind: IngressController
+     metadata:
+       name: default
+       namespace: openshift-ingress-operator
+     spec:
+      nodePlacement:
+        nodeSelector:
+          matchLabels:
+            node-role.kubernetes.io/infra: ""
+  when: infra_vms | length > 0

roles/vsphere_vm_new_nodes/tasks/main.yml

@@ -0,0 +1,20 @@
+- name: Encode node ignition files
+  set_fact:
+    workerContent: "{{ lookup('file', '{{ playbook_dir }}/install-dir/worker.ign') | b64encode }}"
+
+- name: "Worker base64 "
+  debug:
+    msg: "{{ workerContent }}"
+    verbosity: 1
+
+- name: Create VMs
+  vars:
+    ip_alloc: "{% if dhcp | default(false) %}dhcp{% else %}static{% endif %}"
+    medium: "{% if iso | default(false) | bool %}iso{% else %}ova{% endif %}"
+    name: "{% if ip_alloc == 'dhcp' %}{{ ip_alloc }}{% else %}{{ ip_alloc }}-{{ medium }}{% endif %}.yml"
+  include_tasks:
+    file: "{{ name }}"
+
+- name: Run steps to finish cluster install
+  include_tasks:
+    file: "finish_install.yml"

roles/vsphere_vm_new_nodes/tasks/static-ova.yml

@@ -0,0 +1,53 @@
+- name: Create worker VMs from the template
+  vmware_guest:
+    hostname: "{{ vcenter.ip }}"
+    username: "{{ vcenter.username }}"
+    password: "{{ vcenter.password }}"
+    datacenter: "{{ vcenter.datacenter }}"
+    validate_certs: no
+    folder: "{{ vcenter.folder_absolute_path }}"
+    name: "{{ item.name }}.{{config.cluster_name}}.{{config.base_domain}}"
+    state: poweredoff
+    template: "{{ vcenter.folder_absolute_path }}/{{ ova }}"
+    disk:
+    - size_gb: "{{ item.size_gb | default(120) }}"
+      type: thin
+      datastore: "{{ vcenter.datastore }}"
+    hardware:
+      memory_mb: "{{ item.ram | default(16384) }}"
+      num_cpus: "{{ item.cpu | default(4) }}"
+      num_cpu_cores_per_socket: "{{ vm_mods.worker_cores_per_socket | default(omit) }}"
+      memory_reservation_lock: True
+      version: "{{ vcenter.hw_version }}"
+      hotadd_cpu: "{{ vm_mods.hotadd_cpu | default(omit) }}"
+      hotremove_cpu: "{{ vm_mods.hotremove_cpu | default(omit) }}"
+      hotadd_memory: "{{ vm_mods.hotadd_memory | default(omit) }}"
+    wait_for_ip_address: no
+    advanced_settings:
+      - key: guestinfo.ignition.config.data
+        value: "{{ workerContent }}"
+      - key: guestinfo.ignition.config.data.encoding
+        value: base64
+      - key: disk.EnableUUID
+        value: TRUE
+      - key: guestinfo.afterburn.initrd.network-kargs
+        value: "ip={{ item.ipaddr }}::{{ static_ip.gateway }}:{{ static_ip.netmask }}:{{ item.name }}.{{config.cluster_name}}.{{config.base_domain}}:ens192:off:{{ static_ip.dns }}"
+  loop: "{{ worker_additional_nodes }}"
+  loop_control:
+    label: "{{ item.name }}"
+
+- name: Configure network for worker VMs
+  vmware_guest_network:
+    hostname: "{{ vcenter.ip }}"
+    username: "{{ vcenter.username }}"
+    password: "{{ vcenter.password }}"
+    datacenter: "{{ vcenter.datacenter }}"
+    validate_certs: no
+    name: "{{ item.name }}.{{config.cluster_name}}.{{config.base_domain}}"
+    network_name: "{{ vcenter.network }}"
+    mac_address: "{{ item.macaddr | default(omit) }}"
+    state: present
+    start_connected: true
+  loop: "{{ worker_additional_nodes | default([]) }}"
+  loop_control:
+    label: "{{ item.name }}"

roles/vsphere_vm_new_nodes/vars/main.yml

@@ -0,0 +1 @@
+workerContent: "{{ lookup('file', '{{ playbook_dir }}/install-dir/worker.ign') | b64encode }}"