feat: allow multiple files for app and env-data on each level (#366)

internal/myks/application.go

@@ -119,13 +119,14 @@ func (a *Application) collectDataFiles() {
 	a.yttDataFiles = append(a.yttDataFiles, environmentDataFiles...)
 
 	protoDataFile := filepath.Join(a.Prototype, a.e.g.ApplicationDataFileName)
-	if ok, err := isExist(protoDataFile); ok && err == nil {
-		a.yttDataFiles = append(a.yttDataFiles, protoDataFile)
+	if files, err := filepath.Glob(protoDataFile); err == nil {
+		a.yttDataFiles = append(a.yttDataFiles, files...)
 	}
 
 	protoOverrideDataFiles := a.e.collectBySubpath(filepath.Join(a.e.g.PrototypeOverrideDir, a.prototypeDirName(), a.e.g.ApplicationDataFileName))
 	a.yttDataFiles = append(a.yttDataFiles, protoOverrideDataFiles...)
-	overrideDataFiles := append(protoOverrideDataFiles, a.e.collectBySubpath(filepath.Join(a.e.g.AppsDir, a.Name, a.e.g.ApplicationDataFileName))...)
+
+	overrideDataFiles := a.e.collectBySubpath(filepath.Join(a.e.g.AppsDir, a.Name, a.e.g.ApplicationDataFileName))
 	a.yttDataFiles = append(a.yttDataFiles, overrideDataFiles...)
 }
 

internal/myks/environment.go

@@ -34,9 +34,7 @@ type Environment struct {
 	foundApplications map[string]string
 }
 
-func NewEnvironment(g *Globe, dir string) (*Environment, error) {
-	envDataFile := filepath.Join(dir, g.EnvironmentDataFileName)
-
+func NewEnvironment(g *Globe, dir string, envDataFile string) (*Environment, error) {
 	env := &Environment{
 		Dir:                     dir,
 		EnvironmentDataFile:     envDataFile,
@@ -359,8 +357,8 @@ func (e *Environment) collectBySubpath(subpath string) []string {
 		}
 		currentPath = filepath.Join(currentPath, level)
 		item := filepath.Join(currentPath, subpath)
-		if ok, err := isExist(item); ok && err == nil {
-			items = append(items, item)
+		if files, err := filepath.Glob(item); err == nil {
+			items = append(items, files...)
 		}
 	}
 	return items

internal/myks/globe.go

@@ -46,9 +46,9 @@ type Globe struct {
 	// Data values schema file name
 	DataSchemaFileName string `default:"data-schema.ytt.yaml"`
 	// Application data file name
-	ApplicationDataFileName string `default:"app-data.ytt.yaml"`
+	ApplicationDataFileName string `default:"app-data*.yaml"`
 	// Environment data file name
-	EnvironmentDataFileName string `default:"env-data.ytt.yaml"`
+	EnvironmentDataFileName string `default:"env-data*.yaml"`
 	// Rendered environment data file name
 	RenderedEnvironmentDataFileName string `default:"env-data.yaml"`
 	// Myks runtime data file name
@@ -406,20 +406,27 @@ func (g *Globe) collectEnvironmentsInPath(searchPath string) []string {
 			return err
 		}
 		if d != nil && d.IsDir() {
-			if ok, err := isExist(filepath.Join(path, g.EnvironmentDataFileName)); err != nil {
+			files, err := filepath.Glob(filepath.Join(path, g.EnvironmentDataFileName))
+			if err != nil {
 				return err
-			} else if ok {
-				env, err := NewEnvironment(g, path)
+			}
+			if len(files) == 0 {
+				return nil
+			}
+			// Try all files in the directory until a valid environment is found
+			for _, file := range files {
+				env, err := NewEnvironment(g, path, file)
 				if err == nil {
 					g.environments[path] = env
 					result = append(result, path)
-				} else {
-					log.Debug().
-						Err(err).
-						Str("path", path).
-						Msg("Unable to collect environment, might be base or parent environment. Skipping")
+					return nil
 				}
 			}
+			log.Debug().
+				Str("path", path).
+				Strs("files", files).
+				Msg("Unable to collect environment, might be base or parent environment. Skipping")
+
 		}
 		return nil
 	})

internal/myks/smart_mode.go

@@ -66,6 +66,13 @@ func (g *Globe) runSmartMode(changedFiles ChangedFiles) EnvAppMap {
 		return regexp.MustCompile("^" + g.GitPathPrefix + sample + "$")
 	}
 
+	globToRegexp := func(glob string) string {
+		r := glob
+		r = strings.ReplaceAll(r, ".", "\\.")
+		r = strings.ReplaceAll(r, "*", ".*")
+		return r
+	}
+
 	// Subdirectories of apps and prototypes are named after plugins
 	plugins := []string{
 		g.ArgoCDDataDirName,
@@ -86,17 +93,17 @@ func (g *Globe) runSmartMode(changedFiles ChangedFiles) EnvAppMap {
 		"env": {
 			e("(" + g.EnvironmentBaseDir + ".*)/" + g.EnvsDir + "/" + g.YttStepDirName + "/.*"),
 			e("(" + g.EnvironmentBaseDir + ".*)/" + g.EnvsDir + "/" + g.ArgoCDDataDirName + "/.*"),
-			e("(" + g.EnvironmentBaseDir + ".*)/" + g.EnvironmentDataFileName),
+			e("(" + g.EnvironmentBaseDir + ".*)/" + globToRegexp(g.EnvironmentDataFileName)),
 		},
 		// Prototype name is the only submatch
 		"prototype": {
 			e(g.PrototypesDir + "/(.+)/" + pluginsPattern + "/.*"),
-			e(g.PrototypesDir + "/(.+)/" + g.ApplicationDataFileName),
+			e(g.PrototypesDir + "/(.+)/" + globToRegexp(g.ApplicationDataFileName)),
 		},
 		// Env path and app name are the submatches
 		"app": {
 			e("(" + g.EnvironmentBaseDir + ".*)/" + g.AppsDir + "/([^/]+)/" + pluginsPattern + "/.*"),
-			e("(" + g.EnvironmentBaseDir + ".*)/" + g.AppsDir + "/([^/]+)/" + g.ApplicationDataFileName),
+			e("(" + g.EnvironmentBaseDir + ".*)/" + g.AppsDir + "/([^/]+)/" + globToRegexp(g.ApplicationDataFileName)),
 		},
 	}
 

internal/myks/smart_mode_test.go

@@ -238,7 +238,7 @@ func TestGlobe_runSmartMode(t *testing.T) {
 		},
 		{
 			"change to app",
-			ChangedFiles{"envs/env1/_apps/app1/app-data.ytt.yaml": "M"},
+			ChangedFiles{"envs/env1/_apps/app1/app-data.variable.yaml": "M"},
 			renderedEnvApps,
 			EnvAppMap{
 				"envs/env1": {"app1"},