feat: new plugin for copying static files (#132)

examples/inheritance-test/envs/dev/env-data.ytt.yaml → examples/integration-tests/envs/dev/env-data.ytt.yaml

@@ -9,3 +9,4 @@ environment:
       name: helm-installation
     - proto: ytt-render-test
       name: ytt-installation
+    - proto: static-test

examples/integration-tests/prototypes/static-test/static/literal.yaml

@@ -0,0 +1,9 @@
+---
+# https://kubernetes.io/docs/concepts/configuration/configmap/
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: static-test-cm
+  namespace: default
+data:
+  key: value

examples/integration-tests/readme.md

@@ -0,0 +1,19 @@
+# Integration tests
+
+This directory contains configuration for myks' integration tests.
+
+## Test areas
+
+### Inheritance
+
+This examples tests the full power of myks inheritance for helm and ytt rendering leveraging:
+
+- config from the base app
+- config from environment group level
+- config from application level
+
+Both environment group level and application level are rendering with prototype override config as well as application specific config.
+
+### Static files
+
+This example tests the ability to correctly render static files from the `static` directories.

examples/inheritance-test/rendered/argocd/mykso-dev/app-helm-installation.yaml → examples/integration-tests/rendered/argocd/mykso-dev/app-helm-installation.yaml

@@ -11,7 +11,7 @@ spec:
     name: mykso-dev
     namespace: helm-installation
   source:
-    path: examples/inheritance-test/rendered/envs/mykso-dev/helm-installation
+    path: examples/integration-tests/rendered/envs/mykso-dev/helm-installation
     plugin:
       name: argocd-vault-plugin-v1.0.0
     repoURL: [email protected]:mykso/myks.git

examples/integration-tests/rendered/argocd/mykso-dev/app-static-test.yaml

@@ -0,0 +1,25 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: app-mykso-dev-static-test
+  namespace: system-argocd
+  finalizers:
+  - resources-finalizer.argocd.argoproj.io
+spec:
+  project: env-mykso-dev
+  destination:
+    name: mykso-dev
+    namespace: static-test
+  source:
+    path: examples/integration-tests/rendered/envs/mykso-dev/static-test
+    plugin:
+      name: argocd-vault-plugin-v1.0.0
+    repoURL: [email protected]:mykso/myks.git
+    targetRevision: main
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+    syncOptions:
+    - CreateNamespace=true
+    - ServerSideApply=true

examples/inheritance-test/rendered/argocd/mykso-dev/app-ytt-installation.yaml → examples/integration-tests/rendered/argocd/mykso-dev/app-ytt-installation.yaml

@@ -11,7 +11,7 @@ spec:
     name: mykso-dev
     namespace: ytt-installation
   source:
-    path: examples/inheritance-test/rendered/envs/mykso-dev/ytt-installation
+    path: examples/integration-tests/rendered/envs/mykso-dev/ytt-installation
     plugin:
       name: argocd-vault-plugin-v1.0.0
     repoURL: [email protected]:mykso/myks.git

examples/integration-tests/rendered/envs/mykso-dev/static-test/static/literal.yaml

@@ -0,0 +1,9 @@
+---
+# https://kubernetes.io/docs/concepts/configuration/configmap/
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: static-test-cm
+  namespace: default
+data:
+  key: value

internal/myks/environment.go

@@ -96,6 +96,10 @@ func (e *Environment) Render(asyncLevel int) error {
 			return err
 		}
 
+		if err := app.copyStaticFiles(); err != nil {
+			return err
+		}
+
 		return app.renderArgoCD()
 	})
 	if err != nil {
@@ -127,6 +131,11 @@ func (e *Environment) SyncAndRender(asyncLevel int, vendirSecrets string) error
 		if err := app.RenderAndSlice(yamlTemplatingTools); err != nil {
 			return err
 		}
+
+		if err := app.copyStaticFiles(); err != nil {
+			return err
+		}
+
 		return app.renderArgoCD()
 	})
 	if err != nil {

internal/myks/globe.go

@@ -53,6 +53,8 @@ type Globe struct {
 	MyksDataFileName string `default:"myks-data.ytt.yaml"`
 	// Rendered environment data file name
 	RenderedEnvironmentDataFileName string `default:"env-data.yaml"`
+	// Static files directory name
+	StaticFilesDirName string `default:"static"`
 	// Service directory name
 	ServiceDirName string `default:".myks"`
 	// Temporary directory name

internal/myks/plugin_static_files.go

@@ -0,0 +1,46 @@
+package myks
+
+import (
+	"path/filepath"
+
+	"github.com/rs/zerolog/log"
+)
+
+const StaticFilesStepName = "static-files"
+
+func (a *Application) copyStaticFiles() (err error) {
+	logStaticFiles := func(files []string) {
+		log.Trace().Strs("staticFiles", files).Msg(a.Msg(StaticFilesStepName, "Static files"))
+	}
+	staticFilesDirs := []string{}
+
+	// 1. Static files from the prototype
+	prototypeStaticFilesDir := filepath.Join(a.Prototype, a.e.g.StaticFilesDirName)
+	if ok, err := isExist(prototypeStaticFilesDir); err != nil {
+		return err
+	} else if ok {
+		staticFilesDirs = append(staticFilesDirs, prototypeStaticFilesDir)
+	}
+	logStaticFiles(staticFilesDirs)
+	// 2. Static files from prototype overrides
+	staticFilesDirs = append(staticFilesDirs, a.e.collectBySubpath(filepath.Join("_proto", a.prototypeDirName(), a.e.g.StaticFilesDirName))...)
+	logStaticFiles(staticFilesDirs)
+
+	// 3. Static files from the environment
+	staticFilesDirs = append(staticFilesDirs, a.e.collectBySubpath(filepath.Join("_env", a.e.g.StaticFilesDirName))...)
+	logStaticFiles(staticFilesDirs)
+	// 4. Static files from the application
+	staticFilesDirs = append(staticFilesDirs, a.e.collectBySubpath(filepath.Join("_apps", a.Name, a.e.g.StaticFilesDirName))...)
+	logStaticFiles(staticFilesDirs)
+
+	staticFilesDestination := filepath.Join(a.getDestinationDir(), a.e.g.StaticFilesDirName)
+
+	for _, staticFilesDir := range staticFilesDirs {
+		if err = copyDir(staticFilesDir, staticFilesDestination, true); err != nil {
+			log.Error().Err(err).Msg(a.Msg(StaticFilesStepName, "Unable to copy static files"))
+			return err
+		}
+	}
+
+	return nil
+}

internal/myks/plugin_static_files_test.go

@@ -0,0 +1,95 @@
+package myks
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/creasty/defaults"
+	"github.com/rs/zerolog/log"
+)
+
+func TestApplication_copyStaticFiles(t *testing.T) {
+	defer chdir(t, "../../testData/static-test")()
+
+	envName := "static-test-env"
+	appName := "static-test-app"
+	protoName := "static-test-proto"
+
+	globe := &Globe{}
+	if err := defaults.Set(globe); err != nil {
+		log.Fatal().Err(err).Msg("Unable to set defaults")
+	}
+
+	env := &Environment{
+		Id:  envName,
+		g:   globe,
+		Dir: filepath.Join(globe.EnvironmentBaseDir, envName),
+	}
+
+	app := &Application{
+		Name:      appName,
+		Prototype: fmt.Sprintf("%s/%s", globe.PrototypesDir, protoName),
+		e:         env,
+	}
+
+	targetDir := app.getDestinationDir()
+
+	isIgnored := func(info os.DirEntry) bool {
+		return info.IsDir() || info.Name() == ".keep" || info.Name() == ".gitignore"
+	}
+
+	// Cleanup copied files
+	defer func() {
+		if err := filepath.WalkDir(targetDir, func(path string, info os.DirEntry, err error) error {
+			if err != nil {
+				return err
+			}
+			if isIgnored(info) {
+				return nil
+			}
+			return os.Remove(path)
+		}); err != nil {
+			t.Errorf("copyStaticFiles() error = %v", err)
+		}
+	}()
+
+	if err := app.copyStaticFiles(); err != nil {
+		t.Errorf("copyStaticFiles() error = %v", err)
+	}
+
+	expectedFiles := map[string]string{
+		"/static/app.txt":            "",
+		"/static/app_override.txt":   "",
+		"/static/conflict.txt":       "application",
+		"/static/env.txt":            "",
+		"/static/env_override.txt":   "",
+		"/static/proto.txt":          "",
+		"/static/proto_override.txt": "",
+	}
+
+	generatedFiles := map[string]string{}
+	if err := filepath.WalkDir(targetDir, func(path string, info os.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+		if isIgnored(info) {
+			return nil
+		}
+
+		content, err := os.ReadFile(path)
+		if err != nil {
+			return err
+		}
+		path = strings.TrimPrefix(path, targetDir)
+		generatedFiles[path] = strings.Trim(string(content), "\n")
+
+		return nil
+	}); err != nil {
+		t.Errorf("copyStaticFiles() error = %v", err)
+	}
+
+	assertEqual(t, generatedFiles, expectedFiles)
+}

internal/myks/util.go

@@ -335,3 +335,67 @@ func collectBySubpath(rootDir string, targetDir string, subpath string) []string
 	}
 	return items
 }
+
+// copyDir copies a directory recursively, overwriting existing files if overwrite is true.
+// If overwrite is false, existing files will not be overwritten, an error will be returned instead.
+// The destination directory will be created if it does not exist.
+func copyDir(src, dst string, overwrite bool) (err error) {
+	if err = os.MkdirAll(dst, os.ModePerm); err != nil {
+		return
+	}
+
+	err = filepath.WalkDir(src, func(path string, d os.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+
+		relPath, err := filepath.Rel(src, path)
+		if err != nil {
+			return err
+		}
+
+		dstPath := filepath.Join(dst, relPath)
+
+		if d.IsDir() {
+			if err = os.MkdirAll(dstPath, os.ModePerm); err != nil {
+				return err
+			}
+		} else {
+			if !overwrite {
+				if _, err = os.Stat(dstPath); err == nil {
+					return nil
+				}
+			}
+
+			if err = copyFile(path, dstPath); err != nil {
+				return err
+			}
+		}
+
+		return nil
+	})
+
+	return err
+}
+
+// copyFile copies a file from src to dst.
+func copyFile(src, dst string) (err error) {
+	dstFile, err := os.Create(dst)
+	if err != nil {
+		return
+	}
+	defer dstFile.Close()
+
+	srcFile, err := os.Open(src)
+	if err != nil {
+		return
+	}
+	defer srcFile.Close()
+
+	_, err = io.Copy(dstFile, srcFile)
+	if err != nil {
+		return
+	}
+
+	return nil
+}