fast node slice initial implementation

.gitignore

@@ -10,6 +10,8 @@
 
 # Output of the go coverage tool, specifically when used with LiteIDE
 *.out
+.idea
+kind/
 
 bin/
 /github.com/

Dockerfile

@@ -9,5 +9,6 @@ FROM alpine:latest
 LABEL org.opencontainers.image.source https://github.com/k8snetworkplumbingwg/whereabouts
 COPY --from=0 /go/src/github.com/k8snetworkplumbingwg/whereabouts/bin/whereabouts .
 COPY --from=0 /go/src/github.com/k8snetworkplumbingwg/whereabouts/bin/ip-control-loop .
+COPY --from=0 /go/src/github.com/k8snetworkplumbingwg/whereabouts/bin/node-slice-controller .
 COPY script/install-cni.sh .
 CMD ["/install-cni.sh"]

cmd/nodeslicecontroller/node_slice_controller.go

@@ -0,0 +1,91 @@
+package main
+
+import (
+	"flag"
+	nadclient "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/clientset/versioned"
+	nadinformers "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions"
+	clientset "github.com/k8snetworkplumbingwg/whereabouts/pkg/client/clientset/versioned"
+	informers "github.com/k8snetworkplumbingwg/whereabouts/pkg/client/informers/externalversions"
+	node_controller "github.com/k8snetworkplumbingwg/whereabouts/pkg/node-controller"
+
+	"time"
+
+	kubeinformers "k8s.io/client-go/informers"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/tools/clientcmd"
+	"k8s.io/klog/v2"
+
+	"github.com/k8snetworkplumbingwg/whereabouts/pkg/node-controller/signals"
+)
+
+var (
+	masterURL  string
+	kubeconfig string
+)
+
+// TODO: leader election
+func main() {
+	klog.InitFlags(nil)
+	flag.Parse()
+
+	// set up signals so we handle the shutdown signal gracefully
+	ctx := signals.SetupSignalHandler()
+	logger := klog.FromContext(ctx)
+
+	cfg, err := clientcmd.BuildConfigFromFlags(masterURL, kubeconfig)
+	if err != nil {
+		logger.Error(err, "Error building kubeconfig")
+		klog.FlushAndExit(klog.ExitFlushTimeout, 1)
+	}
+
+	kubeClient, err := kubernetes.NewForConfig(cfg)
+	if err != nil {
+		logger.Error(err, "Error building kubernetes clientset")
+		klog.FlushAndExit(klog.ExitFlushTimeout, 1)
+	}
+
+	whereaboutsClient, err := clientset.NewForConfig(cfg)
+	if err != nil {
+		logger.Error(err, "Error building kubernetes clientset")
+		klog.FlushAndExit(klog.ExitFlushTimeout, 1)
+	}
+
+	nadClient, err := nadclient.NewForConfig(cfg)
+	if err != nil {
+		logger.Error(err, "Error building kubernetes clientset")
+		klog.FlushAndExit(klog.ExitFlushTimeout, 1)
+	}
+
+	kubeInformerFactory := kubeinformers.NewSharedInformerFactory(kubeClient, time.Second*30)
+	whereaboutsInformerFactory := informers.NewSharedInformerFactory(whereaboutsClient, time.Second*30)
+	nadInformerFactory := nadinformers.NewSharedInformerFactory(nadClient, time.Second*30)
+
+	controller := node_controller.NewController(
+		ctx,
+		kubeClient,
+		whereaboutsClient,
+		nadClient,
+		kubeInformerFactory.Core().V1().Nodes(),
+		whereaboutsInformerFactory.Whereabouts().V1alpha1().NodeSlicePools(),
+		whereaboutsInformerFactory.Whereabouts().V1alpha1().IPPools(),
+		nadInformerFactory.K8sCniCncfIo().V1().NetworkAttachmentDefinitions(),
+		false,
+	)
+
+	// notice that there is no need to run Start methods in a separate goroutine. (i.e. go kubeInformerFactory.Start(ctx.done())
+	// Start method is non-blocking and runs all registered informers in a dedicated goroutine.
+	kubeInformerFactory.Start(ctx.Done())
+	whereaboutsInformerFactory.Start(ctx.Done())
+	nadInformerFactory.Start(ctx.Done())
+
+	//TODO: make workers configurable via flag, what is a sane value here? How will concurrency work?
+	if err = controller.Run(ctx, 1); err != nil {
+		logger.Error(err, "Error running controller")
+		klog.FlushAndExit(klog.ExitFlushTimeout, 1)
+	}
+}
+
+func init() {
+	flag.StringVar(&kubeconfig, "kubeconfig", "", "Path to a kubeconfig. Only required if out-of-cluster.")
+	flag.StringVar(&masterURL, "master", "", "The address of the Kubernetes API server. Overrides any value in kubeconfig. Only required if out-of-cluster.")
+}

doc/crds/daemonset-install.yaml

@@ -28,6 +28,7 @@ rules:
   resources:
   - ippools
   - overlappingrangeipreservations
+  - nodeslicepools
   verbs:
   - get
   - list
@@ -48,11 +49,14 @@ rules:
   verbs:
   - list
   - watch
+  - get
 - apiGroups: [""]
   resources:
   - nodes
   verbs:
   - get
+  - list
+  - watch
 - apiGroups: ["k8s.cni.cncf.io"]
   resources:
     - network-attachment-definitions
@@ -118,6 +122,7 @@ spec:
             SLEEP=false /install-cni.sh &&
             /ip-control-loop -log-level debug
         image: ghcr.io/k8snetworkplumbingwg/whereabouts:latest
+        imagePullPolicy: IfNotPresent
         env:
         - name: NODENAME
           valueFrom:

doc/crds/node-slice-controller.yaml

@@ -0,0 +1,123 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: whereabouts-controller
+  namespace: kube-system
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: whereabouts-controller
+  template:
+    metadata:
+      labels:
+        app: whereabouts-controller
+    spec:
+      containers:
+        - command:
+            - /node-slice-controller
+          env:
+            - name: NODENAME
+              valueFrom:
+                fieldRef:
+                  apiVersion: v1
+                  fieldPath: spec.nodeName
+            - name: WHEREABOUTS_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  apiVersion: v1
+                  fieldPath: metadata.namespace
+          image: ghcr.io/k8snetworkplumbingwg/whereabouts:latest
+          imagePullPolicy: IfNotPresent
+          name: whereabouts
+          resources:
+            limits:
+              cpu: 100m
+              memory: 200Mi
+            requests:
+              cpu: 100m
+              memory: 100Mi
+          securityContext:
+            privileged: true
+          terminationMessagePath: /dev/termination-log
+          terminationMessagePolicy: File
+          volumeMounts:
+            - mountPath: /host/opt/cni/bin
+              name: cnibin
+            - mountPath: /host/etc/cni/net.d
+              name: cni-net-dir
+            - mountPath: /cron-schedule
+              name: cron-scheduler-configmap
+            - mountPath: /var/run/secrets/kubernetes.io/serviceaccount
+              name: kube-api-access-6kd6k
+              readOnly: true
+      dnsPolicy: ClusterFirst
+      enableServiceLinks: true
+      hostNetwork: true
+      nodeName: kind-worker3
+      preemptionPolicy: PreemptLowerPriority
+      priority: 0
+      restartPolicy: Always
+      schedulerName: default-scheduler
+      securityContext: {}
+      serviceAccount: whereabouts
+      serviceAccountName: whereabouts
+      terminationGracePeriodSeconds: 30
+      tolerations:
+        - effect: NoSchedule
+          operator: Exists
+        - effect: NoExecute
+          key: node.kubernetes.io/not-ready
+          operator: Exists
+        - effect: NoExecute
+          key: node.kubernetes.io/unreachable
+          operator: Exists
+        - effect: NoSchedule
+          key: node.kubernetes.io/disk-pressure
+          operator: Exists
+        - effect: NoSchedule
+          key: node.kubernetes.io/memory-pressure
+          operator: Exists
+        - effect: NoSchedule
+          key: node.kubernetes.io/pid-pressure
+          operator: Exists
+        - effect: NoSchedule
+          key: node.kubernetes.io/unschedulable
+          operator: Exists
+        - effect: NoSchedule
+          key: node.kubernetes.io/network-unavailable
+          operator: Exists
+      volumes:
+        - hostPath:
+            path: /opt/cni/bin
+            type: ""
+          name: cnibin
+        - hostPath:
+            path: /etc/cni/net.d
+            type: ""
+          name: cni-net-dir
+        - configMap:
+            defaultMode: 484
+            items:
+              - key: cron-expression
+                path: config
+            name: whereabouts-config
+          name: cron-scheduler-configmap
+        - name: kube-api-access-6kd6k
+          projected:
+            defaultMode: 420
+            sources:
+              - serviceAccountToken:
+                  expirationSeconds: 3607
+                  path: token
+              - configMap:
+                  items:
+                    - key: ca.crt
+                      path: ca.crt
+                  name: kube-root-ca.crt
+              - downwardAPI:
+                  items:
+                    - fieldRef:
+                        apiVersion: v1
+                        fieldPath: metadata.namespace
+                      path: namespace

doc/crds/whereabouts.cni.cncf.io_nodeslicepools.yaml

@@ -0,0 +1,76 @@
+
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.4.1
+  creationTimestamp: null
+  name: nodeslicepools.whereabouts.cni.cncf.io
+spec:
+  group: whereabouts.cni.cncf.io
+  names:
+    kind: NodeSlicePool
+    listKind: NodeSlicePoolList
+    plural: nodeslicepools
+    singular: nodeslicepool
+  scope: Namespaced
+  versions:
+  - name: v1alpha1
+    schema:
+      openAPIV3Schema:
+        description: NodeSlicePool is the Schema for the nodesliceippools API
+        properties:
+          apiVersion:
+            description: 'APIVersion defines the versioned schema of this representation
+              of an object. Servers should convert recognized schemas to the latest
+              internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            type: string
+          kind:
+            description: 'Kind is a string value representing the REST resource this
+              object represents. Servers may infer this from the endpoint the client
+              submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            type: string
+          metadata:
+            type: object
+          spec:
+            description: NodeSlicePoolSpec defines the desired state of NodeSlicePool
+            properties:
+              range:
+                description: Range is a RFC 4632/4291-style string that represents
+                  an IP address and prefix length in CIDR notation this refers to
+                  the entire range where the node is allocated a subset
+                type: string
+              sliceSize:
+                type: string
+            required:
+            - range
+            - sliceSize
+            type: object
+          status:
+            description: NodeSlicePoolStatus defines the desired state of NodeSlicePool
+            properties:
+              allocations:
+                items:
+                  properties:
+                    nodeName:
+                      type: string
+                    sliceRange:
+                      type: string
+                  required:
+                  - nodeName
+                  - sliceRange
+                  type: object
+                type: array
+            required:
+            - allocations
+            type: object
+        type: object
+    served: true
+    storage: true
+status:
+  acceptedNames:
+    kind: ""
+    plural: ""
+  conditions: []
+  storedVersions: []

e2e/client/ippool.go

@@ -7,6 +7,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"time"
 
 	kubeClient "github.com/k8snetworkplumbingwg/whereabouts/pkg/storage/kubernetes"
@@ -31,8 +32,39 @@ func isIPPoolAllocationsEmpty(ctx context.Context, k8sIPAM *kubeClient.Kubernete
 	}
 }
 
+func isIPPoolAllocationsEmptyForNodeSlices(k8sIPAM *kubeClient.KubernetesIPAM, ipPoolCIDR string, clientInfo *ClientInfo) wait.ConditionFunc {
+	return func() (bool, error) {
+		nodes, err := clientInfo.Client.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{})
+		if err != nil {
+			return false, err
+		}
+		for _, node := range nodes.Items {
+			ipPool, err := k8sIPAM.GetIPPool(context.Background(), kubeClient.PoolIdentifier{NodeName: node.Name, IpRange: ipPoolCIDR, NetworkName: k8sIPAM.Config.NetworkName})
+			//noPoolError := fmt.Errorf("k8s pool initialized")
+			if err != nil {
+				if err.Error() == "k8s pool initialized" {
+					continue
+				} else {
+					return false, err
+				}
+			}
+
+			if len(ipPool.Allocations()) != 0 {
+				return false, nil
+			}
+		}
+		return true, nil
+	}
+}
+
 // WaitForZeroIPPoolAllocations polls up to timeout seconds for IP pool allocations to be gone from the Kubernetes cluster.
 // Returns an error if any IP pool allocations remain after time limit, or if GETing IP pools causes an error.
 func WaitForZeroIPPoolAllocations(ctx context.Context, k8sIPAM *kubeClient.KubernetesIPAM, ipPoolCIDR string, timeout time.Duration) error {
 	return wait.PollUntilContextTimeout(ctx, time.Second, timeout, true, isIPPoolAllocationsEmpty(ctx, k8sIPAM, ipPoolCIDR))
 }
+
+// WaitForZeroIPPoolAllocationsAcrossNodeSlices polls up to timeout seconds for IP pool allocations to be gone from the Kubernetes cluster.
+// Returns an error if any IP pool allocations remain after time limit, or if GETing IP pools causes an error.
+func WaitForZeroIPPoolAllocationsAcrossNodeSlices(k8sIPAM *kubeClient.KubernetesIPAM, ipPoolCIDR string, timeout time.Duration, clientInfo *ClientInfo) error {
+	return wait.PollImmediate(time.Second, timeout, isIPPoolAllocationsEmptyForNodeSlices(k8sIPAM, ipPoolCIDR, clientInfo))
+}

e2e/client/node.go

@@ -0,0 +1 @@
+package client