[cinder-csi-plugin] ephemeral volume removal (#2602)

Remove openstack credits from node plugin

Signed-off-by: Serge Logvinov <[email protected]>

charts/cinder-csi-plugin/templates/controllerplugin-deployment.yaml

@@ -173,6 +173,7 @@ spec:
             - "--endpoint=$(CSI_ENDPOINT)"
             - "--cloud-config=$(CLOUD_CONFIG)"
             - "--cluster=$(CLUSTER_NAME)"
+            - "--provide-node-service=false"
             {{- if .Values.csi.plugin.httpEndpoint.enabled }}
             - "--http-endpoint=:{{ .Values.csi.plugin.httpEndpoint.port }}"
             {{- end }}

charts/cinder-csi-plugin/templates/nodeplugin-daemonset.yaml

@@ -91,7 +91,8 @@ spec:
             - /bin/cinder-csi-plugin
             - "-v={{ .Values.logVerbosityLevel }}"
             - "--endpoint=$(CSI_ENDPOINT)"
-            - "--cloud-config=$(CLOUD_CONFIG)"
+            - "--provide-controller-service=false"
+            # - "--cloud-config=$(CLOUD_CONFIG)"
             {{- if .Values.csi.plugin.extraArgs }}
             {{- with .Values.csi.plugin.extraArgs }}
             {{- tpl . $ | trim | nindent 12 }}
@@ -100,8 +101,8 @@ spec:
           env:
             - name: CSI_ENDPOINT
               value: unix://csi/csi.sock
-            - name: CLOUD_CONFIG
-              value: /etc/kubernetes/{{ .Values.secret.filename }}
+            # - name: CLOUD_CONFIG
+            #   value: /etc/kubernetes/{{ .Values.secret.filename }}
             {{- if .Values.csi.plugin.extraEnv }}
               {{- toYaml .Values.csi.plugin.extraEnv | nindent 12 }}
             {{- end }}

cmd/cinder-csi-plugin/main.go

@@ -31,13 +31,18 @@ import (
 )
 
 var (
-	endpoint                 string
-	nodeID                   string
-	cloudConfig              []string
-	cloudNames               []string
-	additionalTopologies     map[string]string
-	cluster                  string
-	httpEndpoint             string
+	endpoint             string
+	nodeID               string
+	cloudConfig          []string
+	cloudNames           []string
+	additionalTopologies map[string]string
+	cluster              string
+	httpEndpoint         string
+
+	searchOrder           string
+	rescanOnResize        bool
+	nodeVolumeAttachLimit int64
+
 	provideControllerService bool
 	provideNodeService       bool
 )
@@ -49,6 +54,17 @@ func main() {
 		Run: func(cmd *cobra.Command, args []string) {
 			handle()
 		},
+		PersistentPreRunE: func(cmd *cobra.Command, _ []string) error {
+			f := cmd.Flags()
+
+			if provideControllerService {
+				if configs, err := f.GetStringSlice("cloud-config"); err != nil || len(configs) == 0 {
+					klog.Fatalf("Unable to mark flag cloud-config to be required")
+				}
+			}
+
+			return nil
+		},
 		Version: version.Version,
 	}
 
@@ -62,19 +78,22 @@ func main() {
 		klog.Fatalf("Unable to mark flag endpoint to be required: %v", err)
 	}
 
-	cmd.PersistentFlags().StringSliceVar(&cloudConfig, "cloud-config", nil, "CSI driver cloud config. This option can be given multiple times")
-	if err := cmd.MarkPersistentFlagRequired("cloud-config"); err != nil {
-		klog.Fatalf("Unable to mark flag cloud-config to be required: %v", err)
-	}
+	cmd.Flags().BoolVar(&provideControllerService, "provide-controller-service", true, "If set to true then the CSI driver does provide the controller service")
+	cmd.Flags().BoolVar(&provideNodeService, "provide-node-service", true, "If set to true then the CSI driver does provide the node service")
+
+	cmd.Flags().StringSliceVar(&cloudConfig, "cloud-config", nil, "CSI driver cloud config. This option can be given multiple times")
 
 	cmd.PersistentFlags().StringSliceVar(&cloudNames, "cloud-name", []string{""}, "Cloud name to instruct CSI driver to read additional OpenStack cloud credentials from the configuration subsections. This option can be specified multiple times to manage multiple OpenStack clouds.")
 	cmd.PersistentFlags().StringToStringVar(&additionalTopologies, "additional-topology", map[string]string{}, "Additional CSI driver topology keys, for example topology.kubernetes.io/region=REGION1. This option can be specified multiple times to add multiple additional topology keys.")
 
 	cmd.PersistentFlags().StringVar(&cluster, "cluster", "", "The identifier of the cluster that the plugin is running in.")
 	cmd.PersistentFlags().StringVar(&httpEndpoint, "http-endpoint", "", "The TCP network address where the HTTP server for providing metrics for diagnostics, will listen (example: `:8080`). The default is empty string, which means the server is disabled.")
 
-	cmd.PersistentFlags().BoolVar(&provideControllerService, "provide-controller-service", true, "If set to true then the CSI driver does provide the controller service (default: true)")
-	cmd.PersistentFlags().BoolVar(&provideNodeService, "provide-node-service", true, "If set to true then the CSI driver does provide the node service (default: true)")
+	cmd.PersistentFlags().StringVar(&searchOrder, "search-order", "configDrive,metadataService", "The search order for metadata service")
+
+	// Node specific flags
+	cmd.PersistentFlags().BoolVar(&rescanOnResize, "rescan-on-resize", false, "If set to true then the CSI driver will rescan the device on volume resize")
+	cmd.PersistentFlags().Int64Var(&nodeVolumeAttachLimit, "node-volume-attach-limit", 256, "The maximum number of volumes that can be attached to a node")
 
 	openstack.AddExtraFlags(pflag.CommandLine)
 
@@ -87,28 +106,46 @@ func handle() {
 	d := cinder.NewDriver(&cinder.DriverOpts{Endpoint: endpoint, ClusterID: cluster})
 
 	openstack.InitOpenStackProvider(cloudConfig, httpEndpoint)
-	var err error
-	clouds := make(map[string]openstack.IOpenStack)
-	for _, cloudName := range cloudNames {
-		clouds[cloudName], err = openstack.GetOpenStackProvider(cloudName)
-		if err != nil {
-			klog.Warningf("Failed to GetOpenStackProvider %s: %v", cloudName, err)
-			return
-		}
-	}
 
 	if provideControllerService {
+		clouds := make(map[string]openstack.IOpenStack, len(cloudNames))
+		for _, cloudName := range cloudNames {
+			var err error
+
+			clouds[cloudName], err = openstack.GetOpenStackProvider(cloudName)
+			if err != nil {
+				klog.Warningf("Failed to GetOpenStackProvider %s: %v", cloudName, err)
+				return
+			}
+		}
+
 		d.SetupControllerService(clouds)
 	}
 
 	if provideNodeService {
 		//Initialize mount
 		mount := mount.GetMountProvider()
 
+		//Backward compatibility, read [BlockStorage] parameters from cloud config
+		cfg, err := openstack.GetConfigFromFiles(cloudConfig)
+		if err == nil {
+			if cfg.Metadata.SearchOrder != "" {
+				searchOrder = cfg.Metadata.SearchOrder
+			}
+
+			if cfg.BlockStorage.RescanOnResize {
+				rescanOnResize = cfg.BlockStorage.RescanOnResize
+			}
+
+			if cfg.BlockStorage.NodeVolumeAttachLimit < nodeVolumeAttachLimit {
+				nodeVolumeAttachLimit = cfg.BlockStorage.NodeVolumeAttachLimit
+			}
+		}
+
 		//Initialize Metadata
-		metadata := metadata.GetMetadataProvider(clouds[cloudNames[0]].GetMetadataOpts().SearchOrder)
+		metadata := metadata.GetMetadataProvider(searchOrder)
 
-		d.SetupNodeService(clouds[cloudNames[0]], mount, metadata, additionalTopologies)
+		d.SetupNodeService(mount, metadata, rescanOnResize, nodeVolumeAttachLimit, additionalTopologies)
 	}
 
 	d.Run()

pkg/csi/cinder/controllerserver.go

@@ -1061,8 +1061,11 @@ func (cs *controllerServer) ControllerExpandVolume(ctx context.Context, req *csi
 func getCreateVolumeResponse(vol *volumes.Volume, ignoreVolumeAZ bool, accessibleTopologyReq *csi.TopologyRequirement) *csi.CreateVolumeResponse {
 
 	var volsrc *csi.VolumeContentSource
+	resizeRequired := false
 
 	if vol.SnapshotID != "" {
+		resizeRequired = true
+
 		volsrc = &csi.VolumeContentSource{
 			Type: &csi.VolumeContentSource_Snapshot{
 				Snapshot: &csi.VolumeContentSource_SnapshotSource{
@@ -1073,6 +1076,8 @@ func getCreateVolumeResponse(vol *volumes.Volume, ignoreVolumeAZ bool, accessibl
 	}
 
 	if vol.SourceVolID != "" {
+		resizeRequired = true
+
 		volsrc = &csi.VolumeContentSource{
 			Type: &csi.VolumeContentSource_Volume{
 				Volume: &csi.VolumeContentSource_VolumeSource{
@@ -1083,6 +1088,8 @@ func getCreateVolumeResponse(vol *volumes.Volume, ignoreVolumeAZ bool, accessibl
 	}
 
 	if vol.BackupID != nil && *vol.BackupID != "" {
+		resizeRequired = true
+
 		volsrc = &csi.VolumeContentSource{
 			Type: &csi.VolumeContentSource_Snapshot{
 				Snapshot: &csi.VolumeContentSource_SnapshotSource{
@@ -1107,12 +1114,18 @@ func getCreateVolumeResponse(vol *volumes.Volume, ignoreVolumeAZ bool, accessibl
 		}
 	}
 
+	volCnx := map[string]string{}
+	if resizeRequired {
+		volCnx[ResizeRequired] = "true"
+	}
+
 	resp := &csi.CreateVolumeResponse{
 		Volume: &csi.Volume{
 			VolumeId:           vol.ID,
 			CapacityBytes:      int64(vol.Size * 1024 * 1024 * 1024),
 			AccessibleTopology: accessibleTopology,
 			ContentSource:      volsrc,
+			VolumeContext:      volCnx,
 		},
 	}
 

pkg/csi/cinder/driver.go

@@ -32,6 +32,12 @@ import (
 const (
 	driverName  = "cinder.csi.openstack.org"
 	topologyKey = "topology." + driverName + "/zone"
+
+	// MaxVolumesPerNode is the maximum number of volumes that can be attached to a node
+	MaxVolumesPerNode = 256
+
+	// ResizeRequired parameter, if set to true, will trigger a resize on mount operation
+	ResizeRequired = "resizeRequired"
 )
 
 var (
@@ -177,9 +183,14 @@ func (d *Driver) SetupControllerService(clouds map[string]openstack.IOpenStack)
 	d.cs = NewControllerServer(d, clouds)
 }
 
-func (d *Driver) SetupNodeService(cloud openstack.IOpenStack, mount mount.IMount, metadata metadata.IMetadata, topologies map[string]string) {
+func (d *Driver) SetupNodeService(mount mount.IMount, metadata metadata.IMetadata, rescanOnResize bool, nodeVolumeAttachLimit int64, topologies map[string]string) {
 	klog.Info("Providing node service")
-	d.ns = NewNodeServer(d, mount, metadata, cloud, topologies)
+
+	if nodeVolumeAttachLimit < 0 || nodeVolumeAttachLimit > MaxVolumesPerNode {
+		nodeVolumeAttachLimit = MaxVolumesPerNode
+	}
+
+	d.ns = NewNodeServer(d, mount, metadata, nodeVolumeAttachLimit, rescanOnResize, topologies)
 }
 
 func (d *Driver) Run() {