Retain existing aws-auth mapRoles (#28)

* retain existing aws-auth mapRoles

* fix delete

* upsert/delete

* discard invalid auth maps

* fix discovery mechanism

* unit test fixes

* remove redundant code

* remove unused method

* improve logging

* fix tests

controllers/provisioners/ekscloudformation/bootstrap.go

@@ -16,55 +16,98 @@ limitations under the License.
 package ekscloudformation
 
 import (
-	"sort"
+	"reflect"
+
+	"github.com/orkaproj/instance-manager/api/v1alpha1"
 
-	"github.com/orkaproj/instance-manager/controllers/common"
 	yaml "gopkg.in/yaml.v2"
 	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
-func (ctx *EksCfInstanceGroupContext) getActiveNodeArns() []string {
-	var arnList []string
+func getNodeRole(arn string) AwsAuthConfig {
+	return AwsAuthConfig{
+		RoleARN:  arn,
+		Username: "system:node:{{EC2PrivateDNSName}}",
+		Groups: []string{
+			"system:bootstrappers",
+			"system:nodes",
+		},
+	}
+}
+
+func (ctx *EksCfInstanceGroupContext) getDiscoveryAuthMap() AwsAuthConfigMapRolesData {
+	var authMap AwsAuthConfigMapRolesData
 	discovery := ctx.GetDiscoveredState()
 	instanceGroups := discovery.GetInstanceGroups()
 	// Append ARNs from discovered state
 	for _, instanceGroup := range instanceGroups.Items {
-		if !common.ContainsString(arnList, instanceGroup.ARN) {
-			arnList = append(arnList, instanceGroup.ARN)
-		}
+		config := getNodeRole(instanceGroup.ARN)
+		authMap.AddUnique(config)
 	}
 	// Append ARNs from controller config
 	for _, arn := range ctx.DefaultARNList {
-		if !common.ContainsString(arnList, arn) {
-			arnList = append(arnList, arn)
+		config := getNodeRole(arn)
+		authMap.AddUnique(config)
+	}
+	return authMap
+}
+
+func (ctx *EksCfInstanceGroupContext) isRemovableConfiguration(config AwsAuthConfig) bool {
+	discovery := ctx.GetDiscoveredState()
+	selfGroup := discovery.GetSelfGroup()
+	selfARN := selfGroup.GetARN()
+	removableRole := getNodeRole(selfARN)
+	if ctx.GetState() == v1alpha1.ReconcileInitDelete {
+		if reflect.DeepEqual(removableRole, config) {
+			return true
 		}
 	}
-	sort.Strings(arnList)
-	return arnList
+	return false
 }
 
 func (ctx *EksCfInstanceGroupContext) updateAuthConfigMap() error {
-	arnList := ctx.getActiveNodeArns()
-	configList := []AwsAuthConfig{}
-	for _, arn := range arnList {
-		log.Infof("bootstrapping: %v\n", arn)
-		authConfig := AwsAuthConfig{
-			RoleARN:  arn,
-			Username: "system:node:{{EC2PrivateDNSName}}",
-			Groups: []string{
-				"system:bootstrappers",
-				"system:nodes",
-			},
+	var (
+		existingAuthMap          *corev1.ConfigMap
+		existingConfigurations   AwsAuthConfigMapRolesData
+		newConfigurations        AwsAuthConfigMapRolesData
+		discoveredConfigurations = ctx.getDiscoveryAuthMap()
+	)
+
+	existingAuthMap, err := getConfigMap(ctx.KubernetesClient.Kubernetes, "kube-system", "aws-auth", metav1.GetOptions{})
+	if err != nil {
+		if errors.IsNotFound(err) {
+			log.Infoln("auth configmap not found, creating it")
+			existingAuthMap, err = ctx.createEmptyNodesAuthConfigMap()
+			if err != nil {
+				return err
+			}
 		}
-		configList = append(configList, authConfig)
 	}
 
-	maproles := AwsAuthConfigMapRolesData{
-		MapRoles: configList,
+	err = yaml.Unmarshal([]byte(existingAuthMap.Data["mapRoles"]), &existingConfigurations.MapRoles)
+	if err != nil {
+		return err
+	}
+
+	// add existing roles
+	for _, configuration := range existingConfigurations.MapRoles {
+		if !ctx.isRemovableConfiguration(configuration) {
+			newConfigurations.AddUnique(configuration)
+		}
 	}
 
-	d, err := yaml.Marshal(&maproles.MapRoles)
+	// add discovered node roles
+	for _, configuration := range discoveredConfigurations.MapRoles {
+		if !ctx.isRemovableConfiguration(configuration) {
+			newConfigurations.AddUnique(configuration)
+		}
+	}
+
+	log.Infof("bootstrapping: %+v\n", newConfigurations)
+
+	d, err := yaml.Marshal(&newConfigurations.MapRoles)
 	if err != nil {
 		log.Errorf("error: %v", err)
 	}
@@ -88,14 +131,17 @@ func (ctx *EksCfInstanceGroupContext) updateAuthConfigMap() error {
 
 }
 
-func (ctx *EksCfInstanceGroupContext) createEmptyNodesAuthConfigMap() error {
+func (ctx *EksCfInstanceGroupContext) createEmptyNodesAuthConfigMap() (*corev1.ConfigMap, error) {
 	cm := &corev1.ConfigMap{
 		ObjectMeta: metav1.ObjectMeta{
 			Namespace: "kube-system",
 			Name:      "aws-auth",
 		},
 		Data: nil,
 	}
-	createConfigMap(ctx.KubernetesClient.Kubernetes, cm)
-	return nil
+	err := createConfigMap(ctx.KubernetesClient.Kubernetes, cm)
+	if err != nil {
+		return cm, err
+	}
+	return cm, nil
 }

controllers/provisioners/ekscloudformation/ekscloudformation.go

@@ -19,14 +19,15 @@ import (
 	"fmt"
 	"strings"
 
+	"k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/cloudformation"
 	"github.com/orkaproj/instance-manager/api/v1alpha1"
 	"github.com/orkaproj/instance-manager/controllers/common"
 	awsprovider "github.com/orkaproj/instance-manager/controllers/providers/aws"
 	"github.com/sirupsen/logrus"
-	"k8s.io/apimachinery/pkg/api/errors"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
 var (
@@ -37,6 +38,11 @@ var (
 	outputLaunchConfiguration = "LaunchConfigName"
 	outputScalingGroupName    = "AsgName"
 	outputGroupARN            = "NodeInstanceRole"
+	groupVersionResource      = schema.GroupVersionResource{
+		Group:    "instancemgr.orkaproj.io",
+		Version:  "v1alpha1",
+		Resource: "instancegroups",
+	}
 )
 
 const (
@@ -136,6 +142,12 @@ func (ctx *EksCfInstanceGroupContext) Delete() error {
 	var err error
 	instanceGroup := ctx.GetInstanceGroup()
 
+	err = ctx.updateAuthConfigMap()
+	if err != nil {
+		log.Errorf("failed to remove role from aws-auth configmap: %v", err)
+		return err
+	}
+
 	err = ctx.AwsWorker.DeleteCloudformationStack()
 	if err != nil {
 		log.Errorf("failed to submit DeleteStack: %v", err)
@@ -242,7 +254,6 @@ func (ctx *EksCfInstanceGroupContext) discoverInstanceGroups() {
 
 	for _, stack := range stacks {
 		var group DiscoveredInstanceGroup
-		group.ClusterName = aws.StringValue(stack.StackName)
 		group.StackName = aws.StringValue(stack.StackName)
 
 		for _, tag := range stack.Tags {
@@ -272,6 +283,23 @@ func (ctx *EksCfInstanceGroupContext) discoverInstanceGroups() {
 				group.ScalingGroupName = value
 			}
 		}
+
+		if group.Namespace != "" && group.Name != "" {
+			var groupDeleting bool
+			var groupDeleted bool
+			groupDeleting, err := ctx.isResourceDeleting(groupVersionResource, group.Namespace, group.Name)
+			if err != nil {
+				if errors.IsNotFound(err) {
+					groupDeleted = true
+				} else {
+					log.Errorf("failed to determine whether %v/%v is being deleted: %v", group.Namespace, group.Name, err)
+				}
+			}
+			if groupDeleting || groupDeleted {
+				group.IsClusterMember = false
+			}
+		}
+
 		if group.IsClusterMember {
 			groups.AddGroup(group)
 		}
@@ -339,14 +367,7 @@ func (ctx *EksCfInstanceGroupContext) CloudDiscovery() error {
 }
 
 func (ctx *EksCfInstanceGroupContext) BootstrapNodes() error {
-	_, err := getConfigMap(ctx.KubernetesClient.Kubernetes, "kube-system", "aws-auth", metav1.GetOptions{})
-	if err != nil {
-		if errors.IsNotFound(err) {
-			log.Infoln("auth configmap not found, creating it")
-			ctx.createEmptyNodesAuthConfigMap()
-		}
-	}
-	err = ctx.updateAuthConfigMap()
+	err := ctx.updateAuthConfigMap()
 	if err != nil {
 		log.Errorln("failed to update bootstrap config map")
 		return err