From f8628a717f61955302a741b189c7dcf1f2d07f96 Mon Sep 17 00:00:00 2001 From: Sai Saran Vaidyanathan Date: Wed, 17 Apr 2024 08:34:20 -0700 Subject: [PATCH 1/8] fix: extract and merge overrides --- internal/client/integrations/overrides.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/internal/client/integrations/overrides.go b/internal/client/integrations/overrides.go index 9efd3fdc..c02d90f2 100644 --- a/internal/client/integrations/overrides.go +++ b/internal/client/integrations/overrides.go @@ -50,6 +50,7 @@ type triggeroverrides struct { ProjectId *string `json:"projectId,omitempty"` TopicName *string `json:"topicName,omitempty"` APIPath *string `json:"apiPath,omitempty"` + ServiceAccount *string `json:"serviceAccount,omitempty"` Properties map[string]string `json:"properties,omitempty"` CloudSchedulerServiceAccount *string `json:"cloudSchedulerServiceAccount,omitempty"` CloudSchedulerLocation *string `json:"cloudSchedulerLocation,omitempty"` @@ -101,6 +102,7 @@ func mergeOverrides(eversion integrationVersionExternal, o overrides) (integrati } trigger.TriggerId = pubsubTrigger + *triggerOverride.ProjectId + "_" + *triggerOverride.TopicName trigger.Properties["Subscription name"] = *triggerOverride.ProjectId + "_" + *triggerOverride.TopicName + trigger.Properties["Service account"] = *triggerOverride.ServiceAccount case "API": if triggerOverride.APIPath == nil { return eversion, fmt.Errorf("the field apiPath is missing from the API Trigger in overrides") @@ -297,9 +299,11 @@ func extractOverrides(iversion integrationVersion) (overrides, error) { triggerOverride := triggeroverrides{} triggerOverride.ProjectId = new(string) triggerOverride.TopicName = new(string) + triggerOverride.ServiceAccount = new(string) *triggerOverride.ProjectId = strings.Split(subscription, "_")[0] *triggerOverride.TopicName = strings.Split(subscription, "_")[1] triggerOverride.TriggerNumber = triggerConfig.TriggerNumber + *triggerOverride.ServiceAccount = triggerConfig.Properties["Service account"] taskOverrides.TriggerOverrides = append(taskOverrides.TriggerOverrides, triggerOverride) case "CLOUD_SCHEDULER": triggerOverride := triggeroverrides{} From 86d7a9a35f3b491d736ff090ee634c9903a8521c Mon Sep 17 00:00:00 2001 From: srinandan <13950006+srinandan@users.noreply.github.com> Date: Wed, 17 Apr 2024 11:44:02 -0700 Subject: [PATCH 2/8] feat: supports override of service account #234 --- internal/apiclient/iam.go | 6 ++++++ internal/client/integrations/integrations.go | 6 +++--- internal/client/integrations/overrides.go | 16 ++++++++++++++-- internal/cmd/integrations/apply.go | 12 +++++++----- internal/cmd/integrations/create.go | 5 ++++- 5 files changed, 34 insertions(+), 11 deletions(-) diff --git a/internal/apiclient/iam.go b/internal/apiclient/iam.go index 1689272a..efec68b7 100644 --- a/internal/apiclient/iam.go +++ b/internal/apiclient/iam.go @@ -390,6 +390,12 @@ func SetCloudSpannerIAMPermission(project string, memberName string) (err error) return setProjectIAMPermission(project, memberName, role) } +// SetIntegrationInvokerPermission +func SetIntegrationInvokerPermission(project string, memberName string) (err error) { + const role = "roles/integrations.integrationInvoker" + return setProjectIAMPermission(project, memberName, role) +} + func getNameAndProject(iamFullName string) (projectid string, name string, err error) { riam := regexp.MustCompile(`^[a-zA-Z0-9-]{6,30}$`) diff --git a/internal/client/integrations/integrations.go b/internal/client/integrations/integrations.go index a8798550..14b65d38 100644 --- a/internal/client/integrations/integrations.go +++ b/internal/client/integrations/integrations.go @@ -262,7 +262,7 @@ type integrationConnection struct { // CreateVersion func CreateVersion(name string, content []byte, overridesContent []byte, snapshot string, - userlabel string, + userlabel string, grantPermission bool, ) (respBody []byte, err error) { iversion := integrationVersion{} if err = json.Unmarshal(content, &iversion); err != nil { @@ -289,7 +289,7 @@ func CreateVersion(name string, content []byte, overridesContent []byte, snapsho if err = json.Unmarshal(overridesContent, &o); err != nil { return nil, err } - if eversion, err = mergeOverrides(eversion, o); err != nil { + if eversion, err = mergeOverrides(eversion, o, grantPermission); err != nil { return nil, err } } @@ -1213,7 +1213,7 @@ func uploadAsync(name string, filePath string) error { return err } - if _, err := CreateVersion(name, content, nil, "", ""); err != nil { + if _, err := CreateVersion(name, content, nil, "", "", false); err != nil { return err } diff --git a/internal/client/integrations/overrides.go b/internal/client/integrations/overrides.go index c02d90f2..ac0589df 100644 --- a/internal/client/integrations/overrides.go +++ b/internal/client/integrations/overrides.go @@ -89,7 +89,7 @@ const ( const authConfigValue = "{ \"@type\": \"type.googleapis.com/enterprise.crm.eventbus.authconfig.AuthConfigTaskParam\",\"authConfigId\": \"" // mergeOverrides -func mergeOverrides(eversion integrationVersionExternal, o overrides) (integrationVersionExternal, error) { +func mergeOverrides(eversion integrationVersionExternal, o overrides, grantPermission bool) (integrationVersionExternal, error) { // apply trigger overrides for _, triggerOverride := range o.TriggerOverrides { foundOverride := false @@ -102,7 +102,19 @@ func mergeOverrides(eversion integrationVersionExternal, o overrides) (integrati } trigger.TriggerId = pubsubTrigger + *triggerOverride.ProjectId + "_" + *triggerOverride.TopicName trigger.Properties["Subscription name"] = *triggerOverride.ProjectId + "_" + *triggerOverride.TopicName - trigger.Properties["Service account"] = *triggerOverride.ServiceAccount + if triggerOverride.ServiceAccount != nil { + serviceAccountName := fmt.Sprintf("%s@%s.iam.gserviceaccount.com", *triggerOverride.ServiceAccount, *triggerOverride.ProjectId) + trigger.Properties["Service account"] = serviceAccountName + if grantPermission { + // create the SA if it doesn't exist + if err := apiclient.CreateServiceAccount(serviceAccountName); err != nil { + return eversion, err + } + if err := apiclient.SetIntegrationInvokerPermission(*triggerOverride.ProjectId, serviceAccountName); err != nil { + clilog.Warning.Printf("Unable to update permissions for the service account: %v\n", err) + } + } + } case "API": if triggerOverride.APIPath == nil { return eversion, fmt.Errorf("the field apiPath is missing from the API Trigger in overrides") diff --git a/internal/cmd/integrations/apply.go b/internal/cmd/integrations/apply.go index 8fc49823..415f8abe 100644 --- a/internal/cmd/integrations/apply.go +++ b/internal/cmd/integrations/apply.go @@ -134,7 +134,8 @@ var ApplyCmd = &cobra.Command{ return err } - if err = processIntegration(overridesFile, integrationFolder, configVarsFolder, pipeline); err != nil { + if err = processIntegration(overridesFile, integrationFolder, + configVarsFolder, pipeline, grantPermission); err != nil { return err } @@ -160,9 +161,9 @@ func init() { ApplyCmd.Flags().StringVarP(&userLabel, "userlabel", "u", "", "Integration version userlabel") ApplyCmd.Flags().StringVarP(&serviceAccountName, "sa", "", - "", "Service Account name for the connection") + "", "Service Account name for the connection or integration trigger") ApplyCmd.Flags().StringVarP(&serviceAccountProject, "sp", "", - "", "Service Account Project for the connection. Default is the connection's project id") + "", "Service Account Project for the connection or integraton trigger.") ApplyCmd.Flags().StringVarP(&encryptionKey, "encryption-keyid", "k", "", "Cloud KMS key for decrypting Auth Config; Format = locations/*/keyRings/*/cryptoKeys/*") ApplyCmd.Flags().StringVarP(&env, "env", "e", @@ -523,7 +524,8 @@ func processSfdcChannels(sfdcchannelsFolder string) (err error) { return nil } -func processIntegration(overridesFile string, integrationFolder string, configVarsFolder string, pipeline string) (err error) { +func processIntegration(overridesFile string, integrationFolder string, + configVarsFolder string, pipeline string, grantPermission bool) (err error) { rJSONFiles := regexp.MustCompile(`(\S*)\.json`) var integrationNames []string @@ -563,7 +565,7 @@ func processIntegration(overridesFile string, integrationFolder string, configVa } clilog.Info.Printf("Create integration %s\n", getFilenameWithoutExtension(integrationNames[0])) respBody, err := integrations.CreateVersion(getFilenameWithoutExtension(integrationNames[0]), - integrationBytes, overridesBytes, "", userLabel) + integrationBytes, overridesBytes, "", userLabel, grantPermission) if err != nil { return err } diff --git a/internal/cmd/integrations/create.go b/internal/cmd/integrations/create.go index 05974853..6e19fead 100644 --- a/internal/cmd/integrations/create.go +++ b/internal/cmd/integrations/create.go @@ -62,12 +62,13 @@ var CreateCmd = &cobra.Command{ } } - _, err = integrations.CreateVersion(name, content, overridesContent, snapshot, userLabel) + _, err = integrations.CreateVersion(name, content, overridesContent, snapshot, userLabel, grantPermission) return err }, } var integrationFile, overridesFile string +var grantPermission bool func init() { var name string @@ -82,6 +83,8 @@ func init() { "", "Integration version snapshot number") CreateCmd.Flags().StringVarP(&userLabel, "userlabel", "u", "", "Integration version userlabel") + CreateCmd.Flags().BoolVarP(&grantPermission, "grant-permission", "g", + false, "Grant the service account permission the integration trigger; default is false") _ = CreateCmd.MarkFlagRequired("name") _ = CreateCmd.MarkFlagRequired("file") From ad3fdf26c79c6839efe0cd70ef6c51c4cff57fc2 Mon Sep 17 00:00:00 2001 From: srinandan <13950006+srinandan@users.noreply.github.com> Date: Wed, 17 Apr 2024 11:47:48 -0700 Subject: [PATCH 3/8] chore: update sample for overrides #234 --- test/pubsub_overrides.json | 37 +++++++++++++++++++++---------------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/test/pubsub_overrides.json b/test/pubsub_overrides.json index ab2d9ac3..83ddc5ff 100644 --- a/test/pubsub_overrides.json +++ b/test/pubsub_overrides.json @@ -1,20 +1,25 @@ { - "trigger_overrides": [{ - "triggerNumber": "1", - "triggerType": "CLOUD_PUBSUB_EXTERNAL", - "projectId": "my-project", - "topicName": "topic" - }], - "task_overrides": [{ - "taskId": "1", - "task": "GenericRestV2Task", - "parameters": { - "url": { - "key": "url", - "value": { - "stringValue": "https://httpbin.org/ip" + "trigger_overrides": [ + { + "triggerNumber": "1", + "triggerType": "CLOUD_PUBSUB_EXTERNAL", + "projectId": "my-project", + "topicName": "topic", + "serviceAccount": "my-sa" + } + ], + "task_overrides": [ + { + "taskId": "1", + "task": "GenericRestV2Task", + "parameters": { + "url": { + "key": "url", + "value": { + "stringValue": "https://httpbin.org/ip" + } } } } - }] -} \ No newline at end of file + ] +} From 32d573e68b2d958ee5a4b95f5e8e190b61d4ad31 Mon Sep 17 00:00:00 2001 From: srinandan <13950006+srinandan@users.noreply.github.com> Date: Wed, 17 Apr 2024 11:55:45 -0700 Subject: [PATCH 4/8] chore: update description #234 --- internal/cmd/integrations/create.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/cmd/integrations/create.go b/internal/cmd/integrations/create.go index 6e19fead..db23555b 100644 --- a/internal/cmd/integrations/create.go +++ b/internal/cmd/integrations/create.go @@ -84,7 +84,7 @@ func init() { CreateCmd.Flags().StringVarP(&userLabel, "userlabel", "u", "", "Integration version userlabel") CreateCmd.Flags().BoolVarP(&grantPermission, "grant-permission", "g", - false, "Grant the service account permission the integration trigger; default is false") + false, "Grant the service account permission for integration triggers; default is false") _ = CreateCmd.MarkFlagRequired("name") _ = CreateCmd.MarkFlagRequired("file") From 740822351968650c2dfdd50a3280e3d11b372ac8 Mon Sep 17 00:00:00 2001 From: srinandan <13950006+srinandan@users.noreply.github.com> Date: Wed, 17 Apr 2024 12:00:53 -0700 Subject: [PATCH 5/8] bug: extract service name only #234 --- internal/client/integrations/overrides.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/client/integrations/overrides.go b/internal/client/integrations/overrides.go index ac0589df..96d8d704 100644 --- a/internal/client/integrations/overrides.go +++ b/internal/client/integrations/overrides.go @@ -315,7 +315,7 @@ func extractOverrides(iversion integrationVersion) (overrides, error) { *triggerOverride.ProjectId = strings.Split(subscription, "_")[0] *triggerOverride.TopicName = strings.Split(subscription, "_")[1] triggerOverride.TriggerNumber = triggerConfig.TriggerNumber - *triggerOverride.ServiceAccount = triggerConfig.Properties["Service account"] + *triggerOverride.ServiceAccount = strings.Split(triggerConfig.Properties["Service account"], "@")[0] taskOverrides.TriggerOverrides = append(taskOverrides.TriggerOverrides, triggerOverride) case "CLOUD_SCHEDULER": triggerOverride := triggeroverrides{} From a8dadf611222137cb9df334081470abce1f6dadd Mon Sep 17 00:00:00 2001 From: srinandan <13950006+srinandan@users.noreply.github.com> Date: Wed, 17 Apr 2024 12:08:04 -0700 Subject: [PATCH 6/8] bug: exclude default sa #234 --- internal/client/integrations/overrides.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/internal/client/integrations/overrides.go b/internal/client/integrations/overrides.go index 96d8d704..4a47ab18 100644 --- a/internal/client/integrations/overrides.go +++ b/internal/client/integrations/overrides.go @@ -315,7 +315,11 @@ func extractOverrides(iversion integrationVersion) (overrides, error) { *triggerOverride.ProjectId = strings.Split(subscription, "_")[0] *triggerOverride.TopicName = strings.Split(subscription, "_")[1] triggerOverride.TriggerNumber = triggerConfig.TriggerNumber - *triggerOverride.ServiceAccount = strings.Split(triggerConfig.Properties["Service account"], "@")[0] + if sa, err := apiclient.GetComputeEngineDefaultServiceAccount(); err != nil { + if sa != triggerConfig.Properties["Service account"] { + *triggerOverride.ServiceAccount = strings.Split(triggerConfig.Properties["Service account"], "@")[0] + } + } taskOverrides.TriggerOverrides = append(taskOverrides.TriggerOverrides, triggerOverride) case "CLOUD_SCHEDULER": triggerOverride := triggeroverrides{} From 8195a9346e6b3d76723a88b2b9c8b699666fccc7 Mon Sep 17 00:00:00 2001 From: srinandan <13950006+srinandan@users.noreply.github.com> Date: Wed, 17 Apr 2024 12:14:18 -0700 Subject: [PATCH 7/8] bug: fix param to get compute sa #234 --- internal/client/integrations/overrides.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/client/integrations/overrides.go b/internal/client/integrations/overrides.go index 4a47ab18..177087b3 100644 --- a/internal/client/integrations/overrides.go +++ b/internal/client/integrations/overrides.go @@ -315,7 +315,7 @@ func extractOverrides(iversion integrationVersion) (overrides, error) { *triggerOverride.ProjectId = strings.Split(subscription, "_")[0] *triggerOverride.TopicName = strings.Split(subscription, "_")[1] triggerOverride.TriggerNumber = triggerConfig.TriggerNumber - if sa, err := apiclient.GetComputeEngineDefaultServiceAccount(); err != nil { + if sa, err := apiclient.GetComputeEngineDefaultServiceAccount(apiclient.GetProjectID()); err != nil { if sa != triggerConfig.Properties["Service account"] { *triggerOverride.ServiceAccount = strings.Split(triggerConfig.Properties["Service account"], "@")[0] } From ca2c066896f07faf1cd99a70aa51bb61d994479e Mon Sep 17 00:00:00 2001 From: Sai Saran Vaidyanathan Date: Wed, 17 Apr 2024 12:34:23 -0700 Subject: [PATCH 8/8] fix: update IP Project Name --- internal/client/integrations/overrides.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/client/integrations/overrides.go b/internal/client/integrations/overrides.go index 177087b3..6f5091bd 100644 --- a/internal/client/integrations/overrides.go +++ b/internal/client/integrations/overrides.go @@ -102,6 +102,7 @@ func mergeOverrides(eversion integrationVersionExternal, o overrides, grantPermi } trigger.TriggerId = pubsubTrigger + *triggerOverride.ProjectId + "_" + *triggerOverride.TopicName trigger.Properties["Subscription name"] = *triggerOverride.ProjectId + "_" + *triggerOverride.TopicName + trigger.Properties["IP Project name"] = *triggerOverride.ProjectId if triggerOverride.ServiceAccount != nil { serviceAccountName := fmt.Sprintf("%s@%s.iam.gserviceaccount.com", *triggerOverride.ServiceAccount, *triggerOverride.ProjectId) trigger.Properties["Service account"] = serviceAccountName