Parallelize. Use Vault API. Logging enhancements. (#21)

* use a waitgroup to parallelize. improve logging

* improve logging

* fix error output

* use vault api and waitgroups for sync

* tweak v10 logging for easier debug

* Add note about logging

Dockerfile

@@ -1,4 +1,4 @@
-FROM hashicorp/vault:1.7.1
+FROM alpine:3
 
 USER nobody
 COPY vault-token-injector /

README.md

@@ -33,6 +33,11 @@ token_ttl: 10m
 token_refresh_interval: 1m
 ```
 
+## Logging
+
+You can adjust the logging level with the `-vX` flag where X can be 1-10.
+*WARNING* - Log level 10 will output secrets into the logs for debugging scenarios. Please do not do this in a production environment.
+
 Note that the time intervals are golang time.Duration strings
 
 ## Future Planned Enhancements

go.mod

@@ -5,6 +5,7 @@ go 1.16
 require (
 	github.com/fsnotify/fsnotify v1.4.9 // indirect
 	github.com/hashicorp/go-tfe v0.17.1
+	github.com/hashicorp/vault/api v1.1.1 // indirect
 	github.com/magiconair/properties v1.8.5 // indirect
 	github.com/mitchellh/mapstructure v1.4.1 // indirect
 	github.com/pelletier/go-toml v1.9.1 // indirect
@@ -14,7 +15,7 @@ require (
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
 	github.com/spf13/pflag v1.0.5
 	github.com/spf13/viper v1.7.1
-	github.com/stretchr/testify v1.4.0
+	github.com/stretchr/testify v1.7.0
 	golang.org/x/sys v0.0.0-20210601080250-7ecdf8ef093b // indirect
 	golang.org/x/text v0.3.6 // indirect
 	gopkg.in/ini.v1 v1.62.0 // indirect

pkg/app/app.go

@@ -5,6 +5,7 @@ import (
 	"os"
 	"os/signal"
 	"strings"
+	"sync"
 	"syscall"
 	"time"
 
@@ -15,22 +16,29 @@ import (
 	"github.com/fairwindsops/vault-token-injector/pkg/vault"
 )
 
+// App is the main application struct
 type App struct {
 	Config         *Config
 	CircleToken    string
 	VaultTokenFile string
+	VaultClient    *vault.Client
 	TFCloudToken   string
 }
 
-// Config represents the top level our applications config yaml file
+// Config represents the configuration file
 type Config struct {
-	CircleCI             []CircleCIConfig `mapstructure:"circleci"`
-	TFCloud              []TFCloudConfig  `mapstructure:"tfcloud"`
-	VaultAddress         string           `mapstructure:"vault_address"`
-	TokenVariable        string           `mapstructure:"token_variable"`
-	OrphanTokens         bool             `mapstructure:"orphan_tokens"`
-	TokenTTL             time.Duration    `mapstructure:"token_ttl"`
-	TokenRefreshInterval time.Duration    `mapstructure:"token_refresh_interval"`
+	CircleCI []CircleCIConfig `mapstructure:"circleci"`
+	TFCloud  []TFCloudConfig  `mapstructure:"tfcloud"`
+	// The address of the vault server to use when creating tokens
+	VaultAddress string `mapstructure:"vault_address"`
+	// The variable name to use when setting a vault token. Defaults to VAULT_ADDR
+	TokenVariable string `mapstructure:"token_variable"`
+	// If true, all tokens will be created with the orphan flag set to true
+	OrphanTokens bool `mapstructure:"orphan_tokens"`
+	// The TTL of the tokens that will be created. Defaults to 30 minutes
+	TokenTTL time.Duration `mapstructure:"token_ttl"`
+	// The interval at which the token will be refreshed. Defaults to 1 hour
+	TokenRefreshInterval time.Duration `mapstructure:"token_refresh_interval"`
 }
 
 // CircleCIConfig represents a specific instance of a CircleCI project we want to
@@ -44,11 +52,17 @@ type CircleCIConfig struct {
 // TFCloudConfig represents a specific instance of a TFCloud workspace we want to
 // update an environment variable for
 type TFCloudConfig struct {
-	Workspace     string   `mapstructure:"workspace"`
-	VaultRole     *string  `mapstructure:"vault_role"`
+	// Workspace is the ID of the workspace in tfcloud. Should begin with ws- and is required
+	Workspace string `mapstructure:"workspace"`
+	// Name is an optional field that can be used to identify a workspace
+	Name string `mapstructure:"name"`
+	// VaultRole is the vault role to use for the token in this workspace
+	VaultRole *string `mapstructure:"vault_role"`
+	// VaultPolicies is a list of policies that will be given to the token in this workspace
 	VaultPolicies []string `mapstructure:"vault_policies"`
 }
 
+// NewApp creates a new App from the given configuration options
 func NewApp(circleToken, vaultTokenFile, tfCloudToken string, config *Config) *App {
 	app := &App{
 		Config:         config,
@@ -57,11 +71,11 @@ func NewApp(circleToken, vaultTokenFile, tfCloudToken string, config *Config) *A
 		VaultTokenFile: vaultTokenFile,
 	}
 	if len(app.Config.CircleCI) > 0 && circleToken == "" {
-		klog.Warning("CircleCI is configured but no token was provided.")
+		klog.Error("CircleCI is configured but no token was provided.")
 	}
 
 	if len(app.Config.TFCloud) > 0 && tfCloudToken == "" {
-		klog.Warning("TFCloud is configured but no token was provided.")
+		klog.Error("TFCloud is configured but no token was provided.")
 	}
 	if app.Config.TokenVariable == "" {
 		app.Config.TokenVariable = "VAULT_TOKEN"
@@ -89,6 +103,7 @@ func NewApp(circleToken, vaultTokenFile, tfCloudToken string, config *Config) *A
 	return app
 }
 
+// Run starts the application
 func (a *App) Run() error {
 	sigs := make(chan os.Signal, 1)
 	signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
@@ -101,65 +116,82 @@ func (a *App) Run() error {
 	klog.Info("starting main application loop")
 	for {
 		if err := a.refreshVaultTokenFromFile(); err != nil {
-			klog.Error(err)
+			return err
+		}
+		var wg sync.WaitGroup
+		for _, workspace := range a.Config.TFCloud {
+
+			wg.Add(1)
+			go a.updateTFCloudInstance(workspace, &wg)
+		}
+		for _, project := range a.Config.CircleCI {
+			wg.Add(1)
+			go a.updateCircleCIInstance(project, &wg)
 		}
-		a.updateCircleCI()
-		a.updateTFCloud()
+		wg.Wait()
+
 		time.Sleep(a.Config.TokenRefreshInterval)
 	}
 }
 
-func (a *App) updateCircleCI() {
-	for _, project := range a.Config.CircleCI {
-		projName := project.Name
-		projVariableName := a.Config.TokenVariable
-		token, err := vault.CreateToken(project.VaultRole, project.VaultPolicies, a.Config.TokenTTL, a.Config.OrphanTokens)
-		if err != nil {
-			klog.Error(err)
-			continue
-		}
-		klog.Infof("setting env var %s to vault token value", projVariableName)
-		if err := circleci.UpdateEnvVar(projName, projVariableName, token.Auth.ClientToken, a.CircleToken); err != nil {
-			klog.Error(err)
-			continue
-		}
-		if err := circleci.UpdateEnvVar(projName, "VAULT_ADDR", a.Config.VaultAddress, a.CircleToken); err != nil {
-			klog.Error(err)
-			continue
-		}
+func (a *App) updateCircleCIInstance(project CircleCIConfig, wg *sync.WaitGroup) {
+	defer wg.Done()
+	projName := project.Name
+	projVariableName := a.Config.TokenVariable
+	token, err := a.VaultClient.CreateToken(project.VaultRole, project.VaultPolicies, a.Config.TokenTTL, a.Config.OrphanTokens)
+	if err != nil {
+		klog.Errorf("error making token for CircleCI project %s: %s", projName, err.Error())
+		return
+	}
+	klog.V(10).Infof("got token %s for CircleCI project %s", token.Auth.ClientToken, projName)
+	klog.Infof("setting env var %s to vault token value in CircleCI project %s", projVariableName, projName)
+	if err := circleci.UpdateEnvVar(projName, projVariableName, token.Auth.ClientToken, a.CircleToken); err != nil {
+		klog.Errorf("error updating CircleCI project %s with token value: %s", projName, err.Error)
+		return
+	}
+	if err := circleci.UpdateEnvVar(projName, "VAULT_ADDR", a.Config.VaultAddress, a.CircleToken); err != nil {
+		klog.Errorf("error updating VAULT_ADDR in CircleCI project %s: %s", projName, err)
+		return
 	}
 }
 
-func (a *App) updateTFCloud() {
-	for _, instance := range a.Config.TFCloud {
-		token, err := vault.CreateToken(instance.VaultRole, instance.VaultPolicies, a.Config.TokenTTL, a.Config.OrphanTokens)
-		if err != nil {
-			klog.Error(err)
-			continue
-		}
-		klog.Infof("setting env var %s to vault token value", a.Config.TokenVariable)
-		tokenVar := tfcloud.Variable{
-			Key:       a.Config.TokenVariable,
-			Value:     token.Auth.ClientToken,
-			Token:     a.TFCloudToken,
-			Sensitive: true,
-			Workspace: instance.Workspace,
-		}
-		if err := tokenVar.Update(); err != nil {
-			klog.Error(err)
-			continue
-		}
-		addressVar := tfcloud.Variable{
-			Key:       "VAULT_ADDR",
-			Value:     a.Config.VaultAddress,
-			Sensitive: false,
-			Token:     a.TFCloudToken,
-			Workspace: instance.Workspace,
-		}
-		if err := addressVar.Update(); err != nil {
-			klog.Error(err)
-			continue
-		}
+func (a *App) updateTFCloudInstance(instance TFCloudConfig, wg *sync.WaitGroup) {
+	defer wg.Done()
+	var workspaceLogIdentifier string
+	if instance.Name != "" {
+		workspaceLogIdentifier = instance.Name
+	} else {
+		workspaceLogIdentifier = instance.Workspace
+	}
+	token, err := a.VaultClient.CreateToken(instance.VaultRole, instance.VaultPolicies, a.Config.TokenTTL, a.Config.OrphanTokens)
+	if err != nil {
+		klog.Errorf("error getting vault token for TFCloud workspace %s: %s", workspaceLogIdentifier, err.Error())
+		return
+	}
+	klog.V(10).Infof("got token %v for tfcloud workspace %s", token.Auth.ClientToken, workspaceLogIdentifier)
+	klog.Infof("setting env var %s to vault token value", a.Config.TokenVariable)
+	tokenVar := tfcloud.Variable{
+		Key:       a.Config.TokenVariable,
+		Value:     token.Auth.ClientToken,
+		Token:     a.TFCloudToken,
+		Sensitive: true,
+		Workspace: instance.Workspace,
+	}
+	if err := tokenVar.Update(); err != nil {
+		klog.Errorf("error updating token for TFCloud workspace %s: %s", workspaceLogIdentifier, err.Error())
+		return
+	}
+	addressVar := tfcloud.Variable{
+		Key:                 "VAULT_ADDR",
+		Value:               a.Config.VaultAddress,
+		Sensitive:           false,
+		Token:               a.TFCloudToken,
+		Workspace:           instance.Workspace,
+		WorkspaceIdentifier: workspaceLogIdentifier,
+	}
+	if err := addressVar.Update(); err != nil {
+		klog.Errorf("error updating VAULT_ADDR for ws %s: %s", workspaceLogIdentifier, err.Error())
+		return
 	}
 }
 
@@ -174,6 +206,17 @@ func (a *App) refreshVaultTokenFromFile() error {
 		if err := os.Setenv("VAULT_TOKEN", token); err != nil {
 			return fmt.Errorf("could not set VAULT_TOKEN from file: %s", err.Error())
 		}
+		client, err := vault.NewClient(a.Config.VaultAddress, token)
+		if err != nil {
+			return err
+		}
+		a.VaultClient = client
+	} else {
+		client, err := vault.NewClient(a.Config.VaultAddress, os.Getenv("VAULT_TOKEN"))
+		if err != nil {
+			return err
+		}
+		a.VaultClient = client
 	}
 	return nil
 }

pkg/tfcloud/tfcloud.go

@@ -8,16 +8,17 @@ import (
 )
 
 type Variable struct {
-	Workspace string
-	Key       string
-	Value     string
-	Token     string
-	Sensitive bool
+	Workspace           string
+	WorkspaceIdentifier string
+	Key                 string
+	Value               string
+	Token               string
+	Sensitive           bool
 }
 
 // Update will update a variable in TFCloud.
 func (v Variable) Update() error {
-	klog.Infof("setting env var %s in TFCloud workspace %s", v.Key, v.Workspace)
+	klog.Infof("setting env var %s in TFCloud workspace %s", v.Key, v.WorkspaceIdentifier)
 	config := &tfe.Config{
 		Token: v.Token,
 	}
@@ -42,7 +43,7 @@ func (v Variable) Update() error {
 	}
 	for _, tfvar := range tfvars.Items {
 		if tfvar.Key == v.Key {
-			klog.Infof("var %s already exists, updating instead", v.Key)
+			klog.Infof("var %s already exists in TFCloud workspace %s, updating instead", v.Key, v.WorkspaceIdentifier)
 
 			_, err = client.Variables.Update(ctx, v.Workspace, tfvar.ID, tfe.VariableUpdateOptions{
 				Description: &description,