From c3e550d038964d190b5640cd2ecf6bdaf3f9e3e8 Mon Sep 17 00:00:00 2001 From: fritzduchardt Date: Mon, 4 Dec 2023 08:52:46 +0100 Subject: [PATCH] feat: implement prototype overwrites in the envs tree (#110) Makes configuration at environment group level for multiple apps installed to the same environment possible. Does so, by using the directory structure in the "prototypes" folder to look for helm and ytt configuration at env group level. closes #109 --- examples/charts/render-test-chart/Chart.yaml | 5 +++ .../render-test-chart/templates/output.yaml | 5 +++ examples/charts/render-test-chart/values.yaml | 2 ++ examples/inheritance-test/.gitignore | 1 + examples/inheritance-test/.myks.yaml | 7 ++++ .../_apps/helm-installation/app-data.ytt.yaml | 5 +++ .../helm-installation/helm/render-test.yaml | 5 +++ .../_apps/ytt-installation/app-data.ytt.yaml | 5 +++ .../envs/_apps/ytt-installation/ytt/app.yaml | 8 +++++ .../_proto/helm-render-test/app-data.ytt.yaml | 5 +++ .../helm-render-test/helm/render-test.yaml | 5 +++ .../_proto/ytt-render-test/app-data.ytt.yaml | 5 +++ .../_proto/ytt-render-test/ytt/proto.yaml | 8 +++++ .../.myks/helm/render-test.yaml | 9 +++++ .../helm-installation/.myks/vendir.lock.yaml | 7 ++++ .../helm-installation/.myks/vendir.sync.yaml | 1 + .../_apps/helm-installation/.myks/vendir.yaml | 9 +++++ .../_apps/helm-installation/app-data.ytt.yaml | 5 +++ .../helm-installation/helm/render-test.yaml | 5 +++ .../vendor/charts/render-test/Chart.yaml | 5 +++ .../charts/render-test/templates/output.yaml | 5 +++ .../vendor/charts/render-test/values.yaml | 2 ++ .../_apps/ytt-installation/app-data.ytt.yaml | 5 +++ .../dev/_apps/ytt-installation/ytt/app.yaml | 8 +++++ .../_proto/helm-render-test/app-data.ytt.yaml | 5 +++ .../helm-render-test/helm/render-test.yaml | 5 +++ .../_proto/ytt-render-test/app-data.ytt.yaml | 5 +++ .../dev/_proto/ytt-render-test/ytt/proto.yaml | 8 +++++ .../envs/dev/env-data.ytt.yaml | 11 +++++++ .../inheritance-test/envs/env-data.ytt.yaml | 28 ++++++++++++++++ .../helm-render-test/app-data.ytt.yaml | 5 +++ .../helm-render-test/helm/render-test.yaml | 5 +++ .../helm-render-test/vendir/base.ytt.yaml | 13 ++++++++ .../vendir/vendir-data.ytt.yaml | 9 +++++ .../ytt-render-test/app-data.ytt.yaml | 5 +++ .../prototypes/ytt-render-test/ytt/base.yaml | 8 +++++ examples/inheritance-test/readme.md | 9 +++++ .../mykso-dev/app-helm-installation.yaml | 25 ++++++++++++++ .../mykso-dev/app-ytt-installation.yaml | 25 ++++++++++++++ .../argocd/mykso-dev/env-mykso-dev.yaml | 33 +++++++++++++++++++ .../rendering-render-test.yaml | 13 ++++++++ .../ytt-installation/rendering-app.yaml | 6 ++++ .../ytt-installation/rendering-base.yaml | 6 ++++ .../ytt-installation/rendering-env-group.yaml | 6 ++++ .../ytt-installation/rendering-proto.yaml | 6 ++++ internal/integration/render_test.go | 3 ++ internal/myks/application.go | 13 ++++++-- internal/myks/application_test.go | 18 ++++++++++ internal/myks/globe.go | 4 +++ internal/myks/plugin_argocd.go | 2 +- internal/myks/render.go | 7 +++- internal/myks/render_test.go | 16 ++++++--- internal/myks/render_ytt.go | 15 ++++++++- internal/myks/smart_mode.go | 4 +-- internal/myks/sync.go | 10 ++++-- internal/myks/util.go | 15 +++++++++ 56 files changed, 455 insertions(+), 15 deletions(-) create mode 100644 examples/charts/render-test-chart/Chart.yaml create mode 100644 examples/charts/render-test-chart/templates/output.yaml create mode 100644 examples/charts/render-test-chart/values.yaml create mode 100644 examples/inheritance-test/.gitignore create mode 100644 examples/inheritance-test/.myks.yaml create mode 100644 examples/inheritance-test/envs/_apps/helm-installation/app-data.ytt.yaml create mode 100644 examples/inheritance-test/envs/_apps/helm-installation/helm/render-test.yaml create mode 100644 examples/inheritance-test/envs/_apps/ytt-installation/app-data.ytt.yaml create mode 100644 examples/inheritance-test/envs/_apps/ytt-installation/ytt/app.yaml create mode 100644 examples/inheritance-test/envs/_proto/helm-render-test/app-data.ytt.yaml create mode 100644 examples/inheritance-test/envs/_proto/helm-render-test/helm/render-test.yaml create mode 100644 examples/inheritance-test/envs/_proto/ytt-render-test/app-data.ytt.yaml create mode 100644 examples/inheritance-test/envs/_proto/ytt-render-test/ytt/proto.yaml create mode 100644 examples/inheritance-test/envs/dev/_apps/helm-installation/.myks/helm/render-test.yaml create mode 100644 examples/inheritance-test/envs/dev/_apps/helm-installation/.myks/vendir.lock.yaml create mode 100644 examples/inheritance-test/envs/dev/_apps/helm-installation/.myks/vendir.sync.yaml create mode 100644 examples/inheritance-test/envs/dev/_apps/helm-installation/.myks/vendir.yaml create mode 100644 examples/inheritance-test/envs/dev/_apps/helm-installation/app-data.ytt.yaml create mode 100644 examples/inheritance-test/envs/dev/_apps/helm-installation/helm/render-test.yaml create mode 100644 examples/inheritance-test/envs/dev/_apps/helm-installation/vendor/charts/render-test/Chart.yaml create mode 100644 examples/inheritance-test/envs/dev/_apps/helm-installation/vendor/charts/render-test/templates/output.yaml create mode 100644 examples/inheritance-test/envs/dev/_apps/helm-installation/vendor/charts/render-test/values.yaml create mode 100644 examples/inheritance-test/envs/dev/_apps/ytt-installation/app-data.ytt.yaml create mode 100644 examples/inheritance-test/envs/dev/_apps/ytt-installation/ytt/app.yaml create mode 100644 examples/inheritance-test/envs/dev/_proto/helm-render-test/app-data.ytt.yaml create mode 100644 examples/inheritance-test/envs/dev/_proto/helm-render-test/helm/render-test.yaml create mode 100644 examples/inheritance-test/envs/dev/_proto/ytt-render-test/app-data.ytt.yaml create mode 100644 examples/inheritance-test/envs/dev/_proto/ytt-render-test/ytt/proto.yaml create mode 100644 examples/inheritance-test/envs/dev/env-data.ytt.yaml create mode 100644 examples/inheritance-test/envs/env-data.ytt.yaml create mode 100644 examples/inheritance-test/prototypes/helm-render-test/app-data.ytt.yaml create mode 100644 examples/inheritance-test/prototypes/helm-render-test/helm/render-test.yaml create mode 100644 examples/inheritance-test/prototypes/helm-render-test/vendir/base.ytt.yaml create mode 100644 examples/inheritance-test/prototypes/helm-render-test/vendir/vendir-data.ytt.yaml create mode 100644 examples/inheritance-test/prototypes/ytt-render-test/app-data.ytt.yaml create mode 100644 examples/inheritance-test/prototypes/ytt-render-test/ytt/base.yaml create mode 100644 examples/inheritance-test/readme.md create mode 100644 examples/inheritance-test/rendered/argocd/mykso-dev/app-helm-installation.yaml create mode 100644 examples/inheritance-test/rendered/argocd/mykso-dev/app-ytt-installation.yaml create mode 100644 examples/inheritance-test/rendered/argocd/mykso-dev/env-mykso-dev.yaml create mode 100644 examples/inheritance-test/rendered/envs/mykso-dev/helm-installation/rendering-render-test.yaml create mode 100644 examples/inheritance-test/rendered/envs/mykso-dev/ytt-installation/rendering-app.yaml create mode 100644 examples/inheritance-test/rendered/envs/mykso-dev/ytt-installation/rendering-base.yaml create mode 100644 examples/inheritance-test/rendered/envs/mykso-dev/ytt-installation/rendering-env-group.yaml create mode 100644 examples/inheritance-test/rendered/envs/mykso-dev/ytt-installation/rendering-proto.yaml diff --git a/examples/charts/render-test-chart/Chart.yaml b/examples/charts/render-test-chart/Chart.yaml new file mode 100644 index 00000000..f4ea219c --- /dev/null +++ b/examples/charts/render-test-chart/Chart.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +appVersion: v0.0.1 +description: Local Helm chart to test rendering. +name: local-renderer +version: 0.0.1 diff --git a/examples/charts/render-test-chart/templates/output.yaml b/examples/charts/render-test-chart/templates/output.yaml new file mode 100644 index 00000000..add0ffbf --- /dev/null +++ b/examples/charts/render-test-chart/templates/output.yaml @@ -0,0 +1,5 @@ +kind: rendering +metadata: + name: render-test +outputYaml: +{{ toYaml .Values.outputYaml | indent 2 }} diff --git a/examples/charts/render-test-chart/values.yaml b/examples/charts/render-test-chart/values.yaml new file mode 100644 index 00000000..8b85ef15 --- /dev/null +++ b/examples/charts/render-test-chart/values.yaml @@ -0,0 +1,2 @@ +outputYaml: + fromChartDefaultValues: true diff --git a/examples/inheritance-test/.gitignore b/examples/inheritance-test/.gitignore new file mode 100644 index 00000000..c0c2ea92 --- /dev/null +++ b/examples/inheritance-test/.gitignore @@ -0,0 +1 @@ +**/.myks/tmp/ diff --git a/examples/inheritance-test/.myks.yaml b/examples/inheritance-test/.myks.yaml new file mode 100644 index 00000000..78059104 --- /dev/null +++ b/examples/inheritance-test/.myks.yaml @@ -0,0 +1,7 @@ +--- +# Sets the number of applications to be processed in parallel. +# The default (0) is no limit. +async: 0 +# One of the zerolog log levels. +# See: https://github.com/rs/zerolog#leveled-logging +log-level: info diff --git a/examples/inheritance-test/envs/_apps/helm-installation/app-data.ytt.yaml b/examples/inheritance-test/envs/_apps/helm-installation/app-data.ytt.yaml new file mode 100644 index 00000000..9021ecaf --- /dev/null +++ b/examples/inheritance-test/envs/_apps/helm-installation/app-data.ytt.yaml @@ -0,0 +1,5 @@ +#@data/values-schema +#@overlay/match-child-defaults missing_ok=True +--- +application: + envGroupValue: true diff --git a/examples/inheritance-test/envs/_apps/helm-installation/helm/render-test.yaml b/examples/inheritance-test/envs/_apps/helm-installation/helm/render-test.yaml new file mode 100644 index 00000000..76101c0b --- /dev/null +++ b/examples/inheritance-test/envs/_apps/helm-installation/helm/render-test.yaml @@ -0,0 +1,5 @@ +#@ load("@ytt:data", "data") +--- +outputYaml: + fromEnvGroup: true + fromEnvGroupAppData: #@ data.values.application.envGroupValue diff --git a/examples/inheritance-test/envs/_apps/ytt-installation/app-data.ytt.yaml b/examples/inheritance-test/envs/_apps/ytt-installation/app-data.ytt.yaml new file mode 100644 index 00000000..9021ecaf --- /dev/null +++ b/examples/inheritance-test/envs/_apps/ytt-installation/app-data.ytt.yaml @@ -0,0 +1,5 @@ +#@data/values-schema +#@overlay/match-child-defaults missing_ok=True +--- +application: + envGroupValue: true diff --git a/examples/inheritance-test/envs/_apps/ytt-installation/ytt/app.yaml b/examples/inheritance-test/envs/_apps/ytt-installation/ytt/app.yaml new file mode 100644 index 00000000..3634e7a0 --- /dev/null +++ b/examples/inheritance-test/envs/_apps/ytt-installation/ytt/app.yaml @@ -0,0 +1,8 @@ +#@ load("@ytt:data", "data") +--- +kind: rendering +metadata: + name: env-group +outputYaml: + fromEnvGroup: true + fromEnvGroupAppData: #@ data.values.application.envGroupValue diff --git a/examples/inheritance-test/envs/_proto/helm-render-test/app-data.ytt.yaml b/examples/inheritance-test/envs/_proto/helm-render-test/app-data.ytt.yaml new file mode 100644 index 00000000..5ebe063e --- /dev/null +++ b/examples/inheritance-test/envs/_proto/helm-render-test/app-data.ytt.yaml @@ -0,0 +1,5 @@ +#@data/values-schema +#@overlay/match-child-defaults missing_ok=True +--- +application: + protoOverwriteValue: true diff --git a/examples/inheritance-test/envs/_proto/helm-render-test/helm/render-test.yaml b/examples/inheritance-test/envs/_proto/helm-render-test/helm/render-test.yaml new file mode 100644 index 00000000..ffbbbac9 --- /dev/null +++ b/examples/inheritance-test/envs/_proto/helm-render-test/helm/render-test.yaml @@ -0,0 +1,5 @@ +#@ load("@ytt:data", "data") +--- +outputYaml: + fromPrototypeOverride: true + fromPrototypeOverrideFromAppData: #@ data.values.application.protoOverwriteValue diff --git a/examples/inheritance-test/envs/_proto/ytt-render-test/app-data.ytt.yaml b/examples/inheritance-test/envs/_proto/ytt-render-test/app-data.ytt.yaml new file mode 100644 index 00000000..5ebe063e --- /dev/null +++ b/examples/inheritance-test/envs/_proto/ytt-render-test/app-data.ytt.yaml @@ -0,0 +1,5 @@ +#@data/values-schema +#@overlay/match-child-defaults missing_ok=True +--- +application: + protoOverwriteValue: true diff --git a/examples/inheritance-test/envs/_proto/ytt-render-test/ytt/proto.yaml b/examples/inheritance-test/envs/_proto/ytt-render-test/ytt/proto.yaml new file mode 100644 index 00000000..49c5358b --- /dev/null +++ b/examples/inheritance-test/envs/_proto/ytt-render-test/ytt/proto.yaml @@ -0,0 +1,8 @@ +#@ load("@ytt:data", "data") +--- +kind: rendering +metadata: + name: proto +outputYaml: + fromPrototypeOverride: true + fromPrototypeOverrideFromAppData: #@ data.values.application.protoOverwriteValue diff --git a/examples/inheritance-test/envs/dev/_apps/helm-installation/.myks/helm/render-test.yaml b/examples/inheritance-test/envs/dev/_apps/helm-installation/.myks/helm/render-test.yaml new file mode 100644 index 00000000..7f714736 --- /dev/null +++ b/examples/inheritance-test/envs/dev/_apps/helm-installation/.myks/helm/render-test.yaml @@ -0,0 +1,9 @@ +outputYaml: + fromPrototype: true + fromPrototypeAppData: true + fromPrototypeOverride: true + fromPrototypeOverrideFromAppData: true + fromEnvGroup: true + fromEnvGroupAppData: true + fromAppConfig: true + fromAppConfigAppData: true diff --git a/examples/inheritance-test/envs/dev/_apps/helm-installation/.myks/vendir.lock.yaml b/examples/inheritance-test/envs/dev/_apps/helm-installation/.myks/vendir.lock.yaml new file mode 100644 index 00000000..fbb7cb6c --- /dev/null +++ b/examples/inheritance-test/envs/dev/_apps/helm-installation/.myks/vendir.lock.yaml @@ -0,0 +1,7 @@ +apiVersion: vendir.k14s.io/v1alpha1 +directories: +- contents: + - directory: {} + path: . + path: charts/render-test +kind: LockConfig diff --git a/examples/inheritance-test/envs/dev/_apps/helm-installation/.myks/vendir.sync.yaml b/examples/inheritance-test/envs/dev/_apps/helm-installation/.myks/vendir.sync.yaml new file mode 100644 index 00000000..c00a1828 --- /dev/null +++ b/examples/inheritance-test/envs/dev/_apps/helm-installation/.myks/vendir.sync.yaml @@ -0,0 +1 @@ +charts/render-test: f6e95b613adfa1959faef2c0d661b198dcfa7b9058d89dd76418ed58d67b5f95 diff --git a/examples/inheritance-test/envs/dev/_apps/helm-installation/.myks/vendir.yaml b/examples/inheritance-test/envs/dev/_apps/helm-installation/.myks/vendir.yaml new file mode 100644 index 00000000..a266e602 --- /dev/null +++ b/examples/inheritance-test/envs/dev/_apps/helm-installation/.myks/vendir.yaml @@ -0,0 +1,9 @@ +apiVersion: vendir.k14s.io/v1alpha1 +kind: Config +directories: +- path: charts/render-test + contents: + - path: . + directory: + name: render-test + path: ../../../../../../charts/render-test-chart diff --git a/examples/inheritance-test/envs/dev/_apps/helm-installation/app-data.ytt.yaml b/examples/inheritance-test/envs/dev/_apps/helm-installation/app-data.ytt.yaml new file mode 100644 index 00000000..f7f4cd8a --- /dev/null +++ b/examples/inheritance-test/envs/dev/_apps/helm-installation/app-data.ytt.yaml @@ -0,0 +1,5 @@ +#@data/values-schema +#@overlay/match-child-defaults missing_ok=True +--- +application: + appValue: true diff --git a/examples/inheritance-test/envs/dev/_apps/helm-installation/helm/render-test.yaml b/examples/inheritance-test/envs/dev/_apps/helm-installation/helm/render-test.yaml new file mode 100644 index 00000000..df9c4b87 --- /dev/null +++ b/examples/inheritance-test/envs/dev/_apps/helm-installation/helm/render-test.yaml @@ -0,0 +1,5 @@ +#@ load("@ytt:data", "data") +--- +outputYaml: + fromAppConfig: true + fromAppConfigAppData: #@ data.values.application.appValue diff --git a/examples/inheritance-test/envs/dev/_apps/helm-installation/vendor/charts/render-test/Chart.yaml b/examples/inheritance-test/envs/dev/_apps/helm-installation/vendor/charts/render-test/Chart.yaml new file mode 100644 index 00000000..f4ea219c --- /dev/null +++ b/examples/inheritance-test/envs/dev/_apps/helm-installation/vendor/charts/render-test/Chart.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +appVersion: v0.0.1 +description: Local Helm chart to test rendering. +name: local-renderer +version: 0.0.1 diff --git a/examples/inheritance-test/envs/dev/_apps/helm-installation/vendor/charts/render-test/templates/output.yaml b/examples/inheritance-test/envs/dev/_apps/helm-installation/vendor/charts/render-test/templates/output.yaml new file mode 100644 index 00000000..add0ffbf --- /dev/null +++ b/examples/inheritance-test/envs/dev/_apps/helm-installation/vendor/charts/render-test/templates/output.yaml @@ -0,0 +1,5 @@ +kind: rendering +metadata: + name: render-test +outputYaml: +{{ toYaml .Values.outputYaml | indent 2 }} diff --git a/examples/inheritance-test/envs/dev/_apps/helm-installation/vendor/charts/render-test/values.yaml b/examples/inheritance-test/envs/dev/_apps/helm-installation/vendor/charts/render-test/values.yaml new file mode 100644 index 00000000..8b85ef15 --- /dev/null +++ b/examples/inheritance-test/envs/dev/_apps/helm-installation/vendor/charts/render-test/values.yaml @@ -0,0 +1,2 @@ +outputYaml: + fromChartDefaultValues: true diff --git a/examples/inheritance-test/envs/dev/_apps/ytt-installation/app-data.ytt.yaml b/examples/inheritance-test/envs/dev/_apps/ytt-installation/app-data.ytt.yaml new file mode 100644 index 00000000..f7f4cd8a --- /dev/null +++ b/examples/inheritance-test/envs/dev/_apps/ytt-installation/app-data.ytt.yaml @@ -0,0 +1,5 @@ +#@data/values-schema +#@overlay/match-child-defaults missing_ok=True +--- +application: + appValue: true diff --git a/examples/inheritance-test/envs/dev/_apps/ytt-installation/ytt/app.yaml b/examples/inheritance-test/envs/dev/_apps/ytt-installation/ytt/app.yaml new file mode 100644 index 00000000..c79e733f --- /dev/null +++ b/examples/inheritance-test/envs/dev/_apps/ytt-installation/ytt/app.yaml @@ -0,0 +1,8 @@ +#@ load("@ytt:data", "data") +--- +kind: rendering +metadata: + name: app +outputYaml: + fromAppConfig: true + fromAppConfigAppData: #@ data.values.application.appValue diff --git a/examples/inheritance-test/envs/dev/_proto/helm-render-test/app-data.ytt.yaml b/examples/inheritance-test/envs/dev/_proto/helm-render-test/app-data.ytt.yaml new file mode 100644 index 00000000..5ebe063e --- /dev/null +++ b/examples/inheritance-test/envs/dev/_proto/helm-render-test/app-data.ytt.yaml @@ -0,0 +1,5 @@ +#@data/values-schema +#@overlay/match-child-defaults missing_ok=True +--- +application: + protoOverwriteValue: true diff --git a/examples/inheritance-test/envs/dev/_proto/helm-render-test/helm/render-test.yaml b/examples/inheritance-test/envs/dev/_proto/helm-render-test/helm/render-test.yaml new file mode 100644 index 00000000..ffbbbac9 --- /dev/null +++ b/examples/inheritance-test/envs/dev/_proto/helm-render-test/helm/render-test.yaml @@ -0,0 +1,5 @@ +#@ load("@ytt:data", "data") +--- +outputYaml: + fromPrototypeOverride: true + fromPrototypeOverrideFromAppData: #@ data.values.application.protoOverwriteValue diff --git a/examples/inheritance-test/envs/dev/_proto/ytt-render-test/app-data.ytt.yaml b/examples/inheritance-test/envs/dev/_proto/ytt-render-test/app-data.ytt.yaml new file mode 100644 index 00000000..5ebe063e --- /dev/null +++ b/examples/inheritance-test/envs/dev/_proto/ytt-render-test/app-data.ytt.yaml @@ -0,0 +1,5 @@ +#@data/values-schema +#@overlay/match-child-defaults missing_ok=True +--- +application: + protoOverwriteValue: true diff --git a/examples/inheritance-test/envs/dev/_proto/ytt-render-test/ytt/proto.yaml b/examples/inheritance-test/envs/dev/_proto/ytt-render-test/ytt/proto.yaml new file mode 100644 index 00000000..49c5358b --- /dev/null +++ b/examples/inheritance-test/envs/dev/_proto/ytt-render-test/ytt/proto.yaml @@ -0,0 +1,8 @@ +#@ load("@ytt:data", "data") +--- +kind: rendering +metadata: + name: proto +outputYaml: + fromPrototypeOverride: true + fromPrototypeOverrideFromAppData: #@ data.values.application.protoOverwriteValue diff --git a/examples/inheritance-test/envs/dev/env-data.ytt.yaml b/examples/inheritance-test/envs/dev/env-data.ytt.yaml new file mode 100644 index 00000000..2d667290 --- /dev/null +++ b/examples/inheritance-test/envs/dev/env-data.ytt.yaml @@ -0,0 +1,11 @@ +#@data/values +--- +environment: + id: mykso-dev + #! applications: # already defined one level above + #! - proto: httpbingo # already defined one level above + applications: + - proto: helm-render-test + name: helm-installation + - proto: ytt-render-test + name: ytt-installation diff --git a/examples/inheritance-test/envs/env-data.ytt.yaml b/examples/inheritance-test/envs/env-data.ytt.yaml new file mode 100644 index 00000000..ddc95a8e --- /dev/null +++ b/examples/inheritance-test/envs/env-data.ytt.yaml @@ -0,0 +1,28 @@ +#! This is an example of a data values file for the root environment. +#! All nested environments inherit these values. The values can be overridden in nested environments. +#! This file is a good place to define default values for all environments. +#! +#! To change the schema of the data values, use `data/values-schema` annotation instead of `data/values`. +#! Refer to the documentation of ytt overlays and data values for more information. + +#@data/values +--- +argocd: + namespace: system-argocd + app: + prefix: app- + #! Disable finalizers to preserve resources after deleting the ArgoCD application. + finalizers: [] + source: + plugin: + name: argocd-vault-plugin-v1.0.0 + #! Fixed config to run tests successfull in pipeline + targetRevision: main + repoURL: git@github.com:mykso/myks.git + project: + prefix: env- + +#! Fixed git config to run tests successfull in pipeline. +myks: + gitRepoBranch: "main" + gitRepoUrl: "git@github.com:mykso/myks.git" \ No newline at end of file diff --git a/examples/inheritance-test/prototypes/helm-render-test/app-data.ytt.yaml b/examples/inheritance-test/prototypes/helm-render-test/app-data.ytt.yaml new file mode 100644 index 00000000..8e4d7fd0 --- /dev/null +++ b/examples/inheritance-test/prototypes/helm-render-test/app-data.ytt.yaml @@ -0,0 +1,5 @@ +#@data/values-schema +#@overlay/match-child-defaults missing_ok=True +--- +application: + baseValue: true diff --git a/examples/inheritance-test/prototypes/helm-render-test/helm/render-test.yaml b/examples/inheritance-test/prototypes/helm-render-test/helm/render-test.yaml new file mode 100644 index 00000000..1e7b7ee9 --- /dev/null +++ b/examples/inheritance-test/prototypes/helm-render-test/helm/render-test.yaml @@ -0,0 +1,5 @@ +#@ load("@ytt:data", "data") +--- +outputYaml: + fromPrototype: true + fromPrototypeAppData: #@ data.values.application.baseValue diff --git a/examples/inheritance-test/prototypes/helm-render-test/vendir/base.ytt.yaml b/examples/inheritance-test/prototypes/helm-render-test/vendir/base.ytt.yaml new file mode 100644 index 00000000..39cfb15c --- /dev/null +++ b/examples/inheritance-test/prototypes/helm-render-test/vendir/base.ytt.yaml @@ -0,0 +1,13 @@ +#@ load("@ytt:data", "data") + +#@ app = data.values.application +--- +apiVersion: vendir.k14s.io/v1alpha1 +kind: Config +directories: + - path: #@ "charts/" + app.name + contents: + - path: . + directory: + name: #@ app.name + path: ../../../../../../charts/render-test-chart diff --git a/examples/inheritance-test/prototypes/helm-render-test/vendir/vendir-data.ytt.yaml b/examples/inheritance-test/prototypes/helm-render-test/vendir/vendir-data.ytt.yaml new file mode 100644 index 00000000..afe13abb --- /dev/null +++ b/examples/inheritance-test/prototypes/helm-render-test/vendir/vendir-data.ytt.yaml @@ -0,0 +1,9 @@ +#@data/values-schema +--- +#@overlay/match-child-defaults missing_ok=True +application: + #! WARNING: The order of the keys (alphabetical) is important for renovate. + #! When changed, renovate won't be able to detect the new version. + #! See renovate.json for more details. + #! renovate: datasource=helm + name: render-test diff --git a/examples/inheritance-test/prototypes/ytt-render-test/app-data.ytt.yaml b/examples/inheritance-test/prototypes/ytt-render-test/app-data.ytt.yaml new file mode 100644 index 00000000..8e4d7fd0 --- /dev/null +++ b/examples/inheritance-test/prototypes/ytt-render-test/app-data.ytt.yaml @@ -0,0 +1,5 @@ +#@data/values-schema +#@overlay/match-child-defaults missing_ok=True +--- +application: + baseValue: true diff --git a/examples/inheritance-test/prototypes/ytt-render-test/ytt/base.yaml b/examples/inheritance-test/prototypes/ytt-render-test/ytt/base.yaml new file mode 100644 index 00000000..b725dc4b --- /dev/null +++ b/examples/inheritance-test/prototypes/ytt-render-test/ytt/base.yaml @@ -0,0 +1,8 @@ +#@ load("@ytt:data", "data") +--- +kind: rendering +metadata: + name: base +outputYaml: + fromPrototype: true + fromPrototypeAppData: #@ data.values.application.baseValue diff --git a/examples/inheritance-test/readme.md b/examples/inheritance-test/readme.md new file mode 100644 index 00000000..6821066c --- /dev/null +++ b/examples/inheritance-test/readme.md @@ -0,0 +1,9 @@ +# Inheritance test + +This examples tests the full power of myks inheritance for helm and ytt rendering leveraging: + +- config from the base app +- config from environment group level +- config from application level + +Both environment group level and application level are rendering with prototype override config as well as appliction specific config. diff --git a/examples/inheritance-test/rendered/argocd/mykso-dev/app-helm-installation.yaml b/examples/inheritance-test/rendered/argocd/mykso-dev/app-helm-installation.yaml new file mode 100644 index 00000000..408f0b7a --- /dev/null +++ b/examples/inheritance-test/rendered/argocd/mykso-dev/app-helm-installation.yaml @@ -0,0 +1,25 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: app-mykso-dev-helm-installation + namespace: system-argocd + finalizers: + - resources-finalizer.argocd.argoproj.io +spec: + project: env-mykso-dev + destination: + name: mykso-dev + namespace: helm-installation + source: + path: examples/inheritance-test/rendered/envs/mykso-dev/helm-installation + plugin: + name: argocd-vault-plugin-v1.0.0 + repoURL: git@github.com:mykso/myks.git + targetRevision: main + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - CreateNamespace=true + - ServerSideApply=true diff --git a/examples/inheritance-test/rendered/argocd/mykso-dev/app-ytt-installation.yaml b/examples/inheritance-test/rendered/argocd/mykso-dev/app-ytt-installation.yaml new file mode 100644 index 00000000..230f89a8 --- /dev/null +++ b/examples/inheritance-test/rendered/argocd/mykso-dev/app-ytt-installation.yaml @@ -0,0 +1,25 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: app-mykso-dev-ytt-installation + namespace: system-argocd + finalizers: + - resources-finalizer.argocd.argoproj.io +spec: + project: env-mykso-dev + destination: + name: mykso-dev + namespace: ytt-installation + source: + path: examples/inheritance-test/rendered/envs/mykso-dev/ytt-installation + plugin: + name: argocd-vault-plugin-v1.0.0 + repoURL: git@github.com:mykso/myks.git + targetRevision: main + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - CreateNamespace=true + - ServerSideApply=true diff --git a/examples/inheritance-test/rendered/argocd/mykso-dev/env-mykso-dev.yaml b/examples/inheritance-test/rendered/argocd/mykso-dev/env-mykso-dev.yaml new file mode 100644 index 00000000..a6c5f830 --- /dev/null +++ b/examples/inheritance-test/rendered/argocd/mykso-dev/env-mykso-dev.yaml @@ -0,0 +1,33 @@ +apiVersion: argoproj.io/v1alpha1 +kind: AppProject +metadata: + name: env-mykso-dev + namespace: system-argocd + finalizers: + - resources-finalizer.argocd.argoproj.io +spec: + description: Project for "mykso-dev" environment + clusterResourceWhitelist: + - group: '*' + kind: '*' + destinations: + - namespace: '*' + name: mykso-dev + namespaceResourceWhitelist: + - group: '*' + kind: '*' + sourceRepos: + - '*' +--- +apiVersion: v1 +kind: Secret +metadata: + labels: + argocd.argoproj.io/secret-type: cluster + name: mykso-dev + namespace: system-argocd +stringData: + config: ARGOCD_CLUSTER_CONNECT_CONFIG + name: mykso-dev + project: env-mykso-dev + server: ARGOCD_CLUSTER_SERVER_URL diff --git a/examples/inheritance-test/rendered/envs/mykso-dev/helm-installation/rendering-render-test.yaml b/examples/inheritance-test/rendered/envs/mykso-dev/helm-installation/rendering-render-test.yaml new file mode 100644 index 00000000..f968b56d --- /dev/null +++ b/examples/inheritance-test/rendered/envs/mykso-dev/helm-installation/rendering-render-test.yaml @@ -0,0 +1,13 @@ +kind: rendering +metadata: + name: render-test +outputYaml: + fromAppConfig: true + fromAppConfigAppData: true + fromChartDefaultValues: true + fromEnvGroup: true + fromEnvGroupAppData: true + fromPrototype: true + fromPrototypeAppData: true + fromPrototypeOverride: true + fromPrototypeOverrideFromAppData: true diff --git a/examples/inheritance-test/rendered/envs/mykso-dev/ytt-installation/rendering-app.yaml b/examples/inheritance-test/rendered/envs/mykso-dev/ytt-installation/rendering-app.yaml new file mode 100644 index 00000000..b1e16575 --- /dev/null +++ b/examples/inheritance-test/rendered/envs/mykso-dev/ytt-installation/rendering-app.yaml @@ -0,0 +1,6 @@ +kind: rendering +metadata: + name: app +outputYaml: + fromAppConfig: true + fromAppConfigAppData: true diff --git a/examples/inheritance-test/rendered/envs/mykso-dev/ytt-installation/rendering-base.yaml b/examples/inheritance-test/rendered/envs/mykso-dev/ytt-installation/rendering-base.yaml new file mode 100644 index 00000000..2be09b29 --- /dev/null +++ b/examples/inheritance-test/rendered/envs/mykso-dev/ytt-installation/rendering-base.yaml @@ -0,0 +1,6 @@ +kind: rendering +metadata: + name: base +outputYaml: + fromPrototype: true + fromPrototypeAppData: true diff --git a/examples/inheritance-test/rendered/envs/mykso-dev/ytt-installation/rendering-env-group.yaml b/examples/inheritance-test/rendered/envs/mykso-dev/ytt-installation/rendering-env-group.yaml new file mode 100644 index 00000000..a1c5863b --- /dev/null +++ b/examples/inheritance-test/rendered/envs/mykso-dev/ytt-installation/rendering-env-group.yaml @@ -0,0 +1,6 @@ +kind: rendering +metadata: + name: env-group +outputYaml: + fromEnvGroup: true + fromEnvGroupAppData: true diff --git a/examples/inheritance-test/rendered/envs/mykso-dev/ytt-installation/rendering-proto.yaml b/examples/inheritance-test/rendered/envs/mykso-dev/ytt-installation/rendering-proto.yaml new file mode 100644 index 00000000..2fac5357 --- /dev/null +++ b/examples/inheritance-test/rendered/envs/mykso-dev/ytt-installation/rendering-proto.yaml @@ -0,0 +1,6 @@ +kind: rendering +metadata: + name: proto +outputYaml: + fromPrototypeOverride: true + fromPrototypeOverrideFromAppData: true diff --git a/internal/integration/render_test.go b/internal/integration/render_test.go index 6850d0be..e7ae0cc3 100644 --- a/internal/integration/render_test.go +++ b/internal/integration/render_test.go @@ -32,6 +32,9 @@ func findRepos(t *testing.T, basefolder string) []testRepo { if !d.IsDir() { continue } + if d.Name() == "charts" { + continue + } repos = append(repos, testRepo{ name: d.Name(), dir: filepath.Join(basefolder, d.Name()), diff --git a/internal/myks/application.go b/internal/myks/application.go index 115ed1a7..b196cd3b 100644 --- a/internal/myks/application.go +++ b/internal/myks/application.go @@ -5,6 +5,7 @@ import ( "fmt" "io" "path/filepath" + "strings" "github.com/rs/zerolog/log" yaml "gopkg.in/yaml.v3" @@ -107,11 +108,11 @@ func (a *Application) Init() error { } func (a *Application) expandPath(path string) string { - return filepath.Join(a.e.Dir, "_apps", a.Name, path) + return filepath.Join(a.e.Dir, a.e.g.AppsDir, a.Name, path) } func (a *Application) expandServicePath(path string) string { - return filepath.Join(a.e.Dir, "_apps", a.Name, a.e.g.ServiceDirName, path) + return filepath.Join(a.e.Dir, a.e.g.AppsDir, a.Name, a.e.g.ServiceDirName, path) } func (a *Application) expandTempPath(path string) string { @@ -135,7 +136,9 @@ func (a *Application) collectDataFiles() { a.yttDataFiles = append(a.yttDataFiles, protoDataFile) } - overrideDataFiles := a.e.collectBySubpath(filepath.Join("_apps", a.Name, a.e.g.ApplicationDataFileName)) + protoOverrideDataFiles := a.e.collectBySubpath(filepath.Join(a.e.g.PrototypeOverrideDir, a.prototypeDirName(), a.e.g.ApplicationDataFileName)) + a.yttDataFiles = append(a.yttDataFiles, protoOverrideDataFiles...) + overrideDataFiles := append(protoOverrideDataFiles, a.e.collectBySubpath(filepath.Join(a.e.g.AppsDir, a.Name, a.e.g.ApplicationDataFileName))...) a.yttDataFiles = append(a.yttDataFiles, overrideDataFiles...) } @@ -195,3 +198,7 @@ func (a *Application) yttS(step string, purpose string, paths []string, stdin io log.Debug().Msg(a.Msg(step, msgRunCmd(purpose, name, args))) }, args...) } + +func (a *Application) prototypeDirName() string { + return strings.TrimPrefix(a.Prototype, a.e.g.PrototypesDir+string(filepath.Separator)) +} diff --git a/internal/myks/application_test.go b/internal/myks/application_test.go index ea5f861a..20f9fce6 100644 --- a/internal/myks/application_test.go +++ b/internal/myks/application_test.go @@ -1,6 +1,7 @@ package myks import ( + "reflect" "strings" "testing" ) @@ -30,3 +31,20 @@ func TestApplication_renderDataYaml(t *testing.T) { }) } } + +func TestApplication_prototypeDir(t *testing.T) { + tests := []struct { + name string + want string + }{ + {"happy path", "test-app"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := testApp.prototypeDirName() + if !reflect.DeepEqual(string(got), tt.want) { + t.Errorf("prototypeDir() got = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/internal/myks/globe.go b/internal/myks/globe.go index 68a1b71c..9ac84f3e 100644 --- a/internal/myks/globe.go +++ b/internal/myks/globe.go @@ -20,6 +20,10 @@ const GlobalLogFormat = "\033[1m[global]\033[0m %s" type Globe struct { /// Globe configuration + // Directory of application-specific configuration + AppsDir string `default:"_apps"` + // Directory of application-specific prototype overwrites + PrototypeOverrideDir string `default:"_proto"` // Base directory for environments EnvironmentBaseDir string `default:"envs"` // Main branch name diff --git a/internal/myks/plugin_argocd.go b/internal/myks/plugin_argocd.go index 4decf040..00b4478c 100644 --- a/internal/myks/plugin_argocd.go +++ b/internal/myks/plugin_argocd.go @@ -85,7 +85,7 @@ func (a *Application) renderArgoCD() (err error) { // 4. Collection of environment argocd-specific data values and schemas, and overlays yttFiles = append(yttFiles, a.e.collectBySubpath(filepath.Join("_env", a.e.g.ArgoCDDataDirName))...) // 5. Collection of application argocd-specific data values and schemas, and overlays - yttFiles = append(yttFiles, a.e.collectBySubpath(filepath.Join("_apps", a.Name, a.e.g.ArgoCDDataDirName))...) + yttFiles = append(yttFiles, a.e.collectBySubpath(filepath.Join(a.e.g.AppsDir, a.Name, a.e.g.ArgoCDDataDirName))...) res, err := a.yttS( "argocd", diff --git a/internal/myks/render.go b/internal/myks/render.go index 66b6399d..bda7c78b 100644 --- a/internal/myks/render.go +++ b/internal/myks/render.go @@ -170,6 +170,7 @@ func (a *Application) prepareValuesFile(dirName string, resourceName string) (st var valuesFiles []string + // add values file from base dir prototypeValuesFile := filepath.Join(a.Prototype, valuesFileName) if ok, err := isExist(prototypeValuesFile); err != nil { return "", err @@ -177,7 +178,11 @@ func (a *Application) prepareValuesFile(dirName string, resourceName string) (st valuesFiles = append(valuesFiles, prototypeValuesFile) } - valuesFiles = append(valuesFiles, a.e.collectBySubpath(filepath.Join("_apps", a.Name, valuesFileName))...) + // add prototype overwrites value file from env dir groups + valuesFiles = append(valuesFiles, a.e.collectBySubpath(filepath.Join(a.e.g.PrototypeOverrideDir, a.prototypeDirName(), valuesFileName))...) + + // add application values file from env dir and groups + valuesFiles = append(valuesFiles, a.e.collectBySubpath(filepath.Join(a.e.g.AppsDir, a.Name, valuesFileName))...) if len(valuesFiles) == 0 { log.Debug().Str("resource", resourceName).Msg(a.Msg(renderStepName, "No values files found")) diff --git a/internal/myks/render_test.go b/internal/myks/render_test.go index 71f4a135..1bada2fe 100644 --- a/internal/myks/render_test.go +++ b/internal/myks/render_test.go @@ -5,13 +5,19 @@ import ( "testing" ) +var prototypeDir = "prototypes" + +var appName = "test-app" + var testApp = &Application{ - Name: "", - Prototype: "", + Name: appName, + Prototype: prototypeDir + "/" + appName, e: &Environment{ Id: "test-env", g: &Globe{ - TempDirName: "/tmp", + TempDirName: "/tmp", + PrototypesDir: prototypeDir, + AppsDir: "_apps", }, Dir: "/tmp", }, @@ -69,7 +75,7 @@ func TestApplication_Render(t *testing.T) { renderedYaml: "step: Two", }, }, - want: "/tmp/_apps/tmp/steps/01-test-template-2.yaml", + want: "/tmp/_apps/" + appName + "/tmp/steps/01-test-template-2.yaml", wantYaml: "step: One\n---\nstep: Two", wantErr: false, }, @@ -89,7 +95,7 @@ func TestApplication_Render(t *testing.T) { renderedYaml: "step: Two", }, }, - want: "/tmp/_apps/tmp/steps/01-test-template-2.yaml", + want: "/tmp/_apps/" + appName + "/tmp/steps/01-test-template-2.yaml", wantYaml: "step: Two", wantErr: false, }, diff --git a/internal/myks/render_ytt.go b/internal/myks/render_ytt.go index 05b00a62..721489db 100644 --- a/internal/myks/render_ytt.go +++ b/internal/myks/render_ytt.go @@ -23,12 +23,18 @@ func (y *Ytt) Ident() string { func (y *Ytt) Render(previousStepFile string) (string, error) { var yttFiles []string + + // add environment, prototype, and application data files yttFiles = append(yttFiles, y.app.yttDataFiles...) + // if yamls were rendered during the last step, we might want to modify them during this step + // therefore, add them as well if previousStepFile != "" { yttFiles = append(yttFiles, previousStepFile) } + // we might have vendored some yamls or json files that we want to transform during this step + // therefore, add them as well vendorYttDir := y.app.expandPath(filepath.Join(y.app.e.g.VendorDirName, y.app.e.g.YttStepDirName)) if ok, err := isExist(vendorYttDir); err != nil { return "", err @@ -36,6 +42,7 @@ func (y *Ytt) Render(previousStepFile string) (string, error) { yttFiles = append(yttFiles, vendorYttDir) } + // we obviously want to add the ytt files from the prototype dir prototypeYttDir := filepath.Join(y.app.Prototype, y.app.e.g.YttStepDirName) if ok, err := isExist(prototypeYttDir); err != nil { return "", err @@ -43,7 +50,13 @@ func (y *Ytt) Render(previousStepFile string) (string, error) { yttFiles = append(yttFiles, prototypeYttDir) } - yttFiles = append(yttFiles, y.app.e.collectBySubpath(filepath.Join("_apps", y.app.Name, y.app.e.g.YttStepDirName))...) + // we might have some prototype overwrites in the environment group folders. + // let's iterate over the environment directory structure and add them + // these should follow the structure and naming using in the prototypes directory + yttFiles = append(yttFiles, collectBySubpath(y.app.e.g.RootDir, y.app.e.Dir, filepath.Join(y.app.e.g.PrototypeOverrideDir, y.app.prototypeDirName(), y.app.e.g.YttStepDirName))...) + + // finally, lets add the ytt directories from the application directory and the environment group folders + yttFiles = append(yttFiles, collectBySubpath(y.app.e.g.RootDir, y.app.e.Dir, filepath.Join(y.app.e.g.AppsDir, y.app.Name, y.app.e.g.YttStepDirName))...) if len(yttFiles) == 0 { log.Debug().Msg(y.app.Msg(yttStepName, "No local ytt directory found")) diff --git a/internal/myks/smart_mode.go b/internal/myks/smart_mode.go index 66c45163..3e20c63c 100644 --- a/internal/myks/smart_mode.go +++ b/internal/myks/smart_mode.go @@ -83,8 +83,8 @@ func (g *Globe) runSmartMode(changedFiles ChangedFiles) EnvAppMap { }, // Env path and app name are the submatches "app": { - e("(" + g.EnvironmentBaseDir + ".*)/_apps/([^/]+)/" + pluginsPattern + "/.*"), - e("(" + g.EnvironmentBaseDir + ".*)/_apps/([^/]+)/" + g.ApplicationDataFileName), + e("(" + g.EnvironmentBaseDir + ".*)/" + g.AppsDir + "/([^/]+)/" + pluginsPattern + "/.*"), + e("(" + g.EnvironmentBaseDir + ".*)/" + g.AppsDir + "/([^/]+)/" + g.ApplicationDataFileName), }, } diff --git a/internal/myks/sync.go b/internal/myks/sync.go index 0d9ae2ad..e1d4159e 100644 --- a/internal/myks/sync.go +++ b/internal/myks/sync.go @@ -44,7 +44,7 @@ func (a *Application) prepareSync() error { yttFiles = append(yttFiles, protoVendirDir) } - appVendirDirs := a.e.collectBySubpath(filepath.Join("_apps", a.Name, "vendir")) + appVendirDirs := a.e.collectBySubpath(filepath.Join(a.e.g.AppsDir, a.Name, "vendir")) yttFiles = append(yttFiles, appVendirDirs...) if len(yttFiles) == 0 { @@ -106,9 +106,15 @@ func (a *Application) doSync(vendirSecrets string) error { return err } + exist, err := isExist(vendorDir) + if err != nil { + log.Error().Err(err).Msg(a.Msg(syncStepName, "Unable to check if vendor dir exists")) + return err + } + // TODO sync retry // only sync vendir with directory flag, if the lock file matches the vendir config file and caching is enabled - if a.useCache && checkLockFileMatch(vendirDirHashes, lockFileDirHashes) { + if exist && a.useCache && checkLockFileMatch(vendirDirHashes, lockFileDirHashes) { for dir, hash := range vendirDirHashes { if checkVersionMatch(dir, hash, syncFileDirHashes) { log.Info().Str("vendir dir", dir).Msg(a.Msg(syncStepName, "Resource already synced")) diff --git a/internal/myks/util.go b/internal/myks/util.go index d6958d85..cc9ea528 100644 --- a/internal/myks/util.go +++ b/internal/myks/util.go @@ -320,3 +320,18 @@ func isExist(path string) (bool, error) { log.Error().Err(err).Msg("Unable to stat file") return false, err } + +func collectBySubpath(rootDir string, targetDir string, subpath string) []string { + items := []string{} + currentPath := rootDir + levels := []string{""} + levels = append(levels, strings.Split(targetDir, filepath.FromSlash("/"))...) + for _, level := range levels { + currentPath = filepath.Join(currentPath, level) + item := filepath.Join(currentPath, subpath) + if _, err := os.Stat(item); err == nil { + items = append(items, item) + } + } + return items +}