enable cilium CNI option

ansible/roles/cluster/defaults/main.yml

@@ -13,3 +13,4 @@ cluster_pod_subnet: ""
 # Default etcd values, change these if you experience "leader changed" issues when running on a SD card
 cluster_etcd_heartbeat_interval: 100
 cluster_etcd_election_timeout: 1000
+kube_proxy: enabled

ansible/roles/cluster/tasks/initialize.yml

@@ -25,10 +25,19 @@
     src: kubeadm-init.yml.j2
     dest: /etc/kubernetes/kubeadm-init.yml
 
-- name: initialize cluster
+- name: initialize cluster (with kube-proxy)
   command:
     cmd: kubeadm init --config /etc/kubernetes/kubeadm-init.yml --upload-certs
     creates: /etc/kubernetes/admin.conf
+  when:
+    - kube_proxy == "enabled"
+
+- name: initialize cluster (without kube-proxy)
+  command:
+    cmd: kubeadm init --skip-phases=addon/kube-proxy --config /etc/kubernetes/kubeadm-init.yml --upload-certs
+    creates: /etc/kubernetes/admin.conf
+  when:
+    - kube_proxy == "disabled"
 
 - name: get sha256 of ca certificate
   openssl_certificate_info:

ansible/roles/cni/defaults/main.yml

@@ -1,2 +1,8 @@
 ---
 cni_plugin: calico
+bgp_peer_address: 192.168.0.1
+bgp_peer_asn: 64512
+cilium_helm_version: 1.8.3
+cilium_image_version: v1.8.3
+k8s_service_host: "{{ hostvars[groups['masters'][0]]['ansible_host'] }}"
+k8s_service_port: 6443

ansible/roles/cni/tasks/cilium.yml

@@ -0,0 +1,61 @@
+---
+- name: Install Helm v3
+  shell: |
+    curl https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 | bash
+  args:
+    warn: false
+
+- name: Add Cilium Repo
+  command:
+    cmd: helm repo add cilium https://helm.cilium.io/
+
+- name: Deploy Cilium
+  shell: |
+    helm upgrade -i cilium cilium/cilium --version {{ cilium_helm_version }} \
+    --set global.registry="docker.io/cilium" \
+    --set global.tag="{{ cilium_image_version }}" \
+    --set global.tunnel="disabled" \
+    --set global.externalIPs.enabled="true" \
+    --set global.ipam.operator.clusterPoolIPv4PodCIDR="{{ cluster_pod_subnet }}" \
+    --set global.ipam.operator.clusterPoolIPv4MaskSize="24" \
+    --set global.endpointRoutes.enabled="true" \
+    --set global.hostServices.enabled="true" \
+    --set global.autoDirectNodeRoutes="true" \
+    --set global.nodePort.enabled="true" \
+    --set global.nodePort.mode="dsr" \
+    --set global.masquerade="false" \
+    --set global.hubble.enabled="true" \
+    --set global.hubble.ui.enabled="true" \
+    --set global.hubble.relay.enabled="true" \
+    --set global.hubble.metrics.enabled="{dns,drop,tcp,flow,port-distribution,icmp,http}" \
+    --set global.kubeProxyReplacement=strict \
+    --set global.k8sServiceHost={{ k8s_service_host }} \
+    --set global.k8sServicePort={{ k8s_service_port }} \
+    --set config.bpfMasquerade="false" \
+    --namespace kube-system
+
+- name: Create Manifests Directory
+  file:
+    path: /root/manifests
+    state: directory
+    mode: 0700
+
+- name: "Deploy manifests"
+  become: true
+  template:
+    src: "{{ item }}"
+    dest: "/root/manifests/{{ item | basename | replace('.j2','') }}"
+    mode: 0600
+  with_items:
+  - "generic-kuberouter-only-advertise-routes.yaml.j2"
+
+- name: Applying manifests
+  command:
+    cmd: "kubectl apply -f /root/manifests/{{ item }}"
+  with_items:
+  - "generic-kuberouter-only-advertise-routes.yaml"
+
+- name: Remove Manifests Directory
+  file:
+    path: /root/manifests
+    state: absent

ansible/roles/cni/tasks/pre_checks.yml

@@ -31,3 +31,11 @@
     fail_msg:
       - "Plugin 'flannel' requires pod subnet to be configured."
   when: cni_plugin == 'flannel'
+
+- name: 'validate pod subnet is used when using cilium'
+  assert:
+    that:
+      - cluster_pod_subnet | ipaddr
+    fail_msg:
+      - "Plugin 'cilium' requires pod subnet to be configured."
+  when: cni_plugin == 'cilium'

ansible/roles/cni/templates/generic-kuberouter-only-advertise-routes.yaml.j2

@@ -0,0 +1,129 @@
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  labels:
+    k8s-app: kube-router
+    tier: node
+  name: kube-router
+  namespace: kube-system
+spec:
+  selector:
+    matchLabels:
+      k8s-app: kube-router
+      tier: node
+  template:
+    metadata:
+      labels:
+        k8s-app: kube-router
+        tier: node
+    spec:
+      priorityClassName: system-node-critical
+      serviceAccountName: kube-router
+      containers:
+      - name: kube-router
+        image: "{{ kube_router_image }}"
+        imagePullPolicy: Always
+        args:
+        - "--run-router=true"
+        - "--run-firewall=false"
+        - "--run-service-proxy=false"
+        - "--bgp-graceful-restart=true"
+        - "--enable-cni=false"
+        - "--enable-pod-egress=false"
+        - "--enable-ibgp=true"
+        - "--enable-overlay=false"
+        - "--peer-router-ips={{ bgp_peer_address }}"
+        - "--peer-router-asns={{ bgp_peer_asn }}"
+        - "--cluster-asn={{ bgp_cluster_asn }}"
+        - "--advertise-cluster-ip=true"
+        - "--advertise-external-ip=true"
+        - "--advertise-loadbalancer-ip=true"
+        env:
+        - name: NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
+        livenessProbe:
+          httpGet:
+            path: /healthz
+            port: 20244
+          initialDelaySeconds: 10
+          periodSeconds: 3
+        resources:
+          requests:
+            cpu: 250m
+            memory: 250Mi
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - name: xtables-lock
+          mountPath: /run/xtables.lock
+          readOnly: false
+      hostNetwork: true
+      tolerations:
+      - effect: NoSchedule
+        operator: Exists
+      - key: CriticalAddonsOnly
+        operator: Exists
+      - effect: NoExecute
+        operator: Exists
+      volumes:
+      - name: xtables-lock
+        hostPath:
+          path: /run/xtables.lock
+          type: FileOrCreate
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: kube-router
+  namespace: kube-system
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+  name: kube-router
+  namespace: kube-system
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - namespaces
+  - pods
+  - services
+  - nodes
+  - endpoints
+  verbs:
+  - list
+  - get
+  - watch
+- apiGroups:
+  - "networking.k8s.io"
+  resources:
+  - networkpolicies
+  verbs:
+  - list
+  - get
+  - watch
+- apiGroups:
+  - extensions
+  resources:
+  - networkpolicies
+  verbs:
+  - get
+  - list
+  - watch
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+  name: kube-router
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: kube-router
+subjects:
+- kind: ServiceAccount
+  name: kube-router
+  namespace: kube-system

ansible/roles/cni/vars/main.yml

@@ -4,3 +4,4 @@ cni_supported_plugins:
   - calico
   - flannel
   - weave
+  - cilium