From b27ed3b12c2084bc1e8b085fc01e98a211260f40 Mon Sep 17 00:00:00 2001 From: razzle Date: Mon, 1 Jan 2024 21:54:40 -0500 Subject: [PATCH 001/113] required --> optional transition Signed-off-by: razzle --- src/internal/packager/validate/validate.go | 2 +- src/pkg/oci/pull.go | 2 +- src/pkg/packager/components.go | 4 ++-- src/pkg/packager/composer/override.go | 7 ++++++- src/types/component.go | 24 ++++++++++++++++++++-- 5 files changed, 32 insertions(+), 7 deletions(-) diff --git a/src/internal/packager/validate/validate.go b/src/internal/packager/validate/validate.go index dea997b635..bb69c34bb3 100644 --- a/src/internal/packager/validate/validate.go +++ b/src/internal/packager/validate/validate.go @@ -129,7 +129,7 @@ func validateComponent(pkg types.ZarfPackage, component types.ZarfComponent) err return fmt.Errorf(lang.PkgValidateErrComponentName, component.Name) } - if component.Required { + if component.IsRequired() { if component.Default { return fmt.Errorf(lang.PkgValidateErrComponentReqDefault, component.Name) } diff --git a/src/pkg/oci/pull.go b/src/pkg/oci/pull.go index 8bebcee533..e44db67a73 100644 --- a/src/pkg/oci/pull.go +++ b/src/pkg/oci/pull.go @@ -102,7 +102,7 @@ func (o *OrasRemote) LayersFromRequestedComponents(requestedComponents []string) } for _, component := range pkg.Components { // If we requested this component, or it is required, we need to pull its images and tarball - if slices.Contains(requestedComponents, component.Name) || component.Required { + if slices.Contains(requestedComponents, component.Name) || component.IsRequired() { for _, image := range component.Images { images[image] = true } diff --git a/src/pkg/packager/components.go b/src/pkg/packager/components.go index d393eb3e59..3d44a8291e 100644 --- a/src/pkg/packager/components.go +++ b/src/pkg/packager/components.go @@ -61,7 +61,7 @@ func (p *Packager) getSelectedComponents() []types.ZarfComponent { selectState, matchedRequest := includedOrExcluded(component, requestedComponents) - if !component.Required { + if !component.IsRequired() { if selectState == excluded { // If the component was explicitly excluded, record the match and continue matchedRequests[matchedRequest] = true @@ -117,7 +117,7 @@ func (p *Packager) getSelectedComponents() []types.ZarfComponent { } else { component := groupedComponents[groupKey][0] - if component.Required { + if component.IsRequired() { selectedComponents = append(selectedComponents, component) } else if selected := interactive.SelectOptionalComponent(component); selected { selectedComponents = append(selectedComponents, component) diff --git a/src/pkg/packager/composer/override.go b/src/pkg/packager/composer/override.go index 8d44dce935..faa156e0fe 100644 --- a/src/pkg/packager/composer/override.go +++ b/src/pkg/packager/composer/override.go @@ -13,7 +13,12 @@ import ( func overrideMetadata(c *types.ZarfComponent, override types.ZarfComponent) error { c.Name = override.Name c.Default = override.Default - c.Required = override.Required + if override.DeprecatedRequired != nil { + c.DeprecatedRequired = override.DeprecatedRequired + } + if override.Optional != nil { + c.Optional = override.Optional + } // Override description if it was provided. if override.Description != "" { diff --git a/src/types/component.go b/src/types/component.go index f63e6ab2a7..af981442ad 100644 --- a/src/types/component.go +++ b/src/types/component.go @@ -21,8 +21,10 @@ type ZarfComponent struct { // Default changes the default option when deploying this component Default bool `json:"default,omitempty" jsonschema:"description=Determines the default Y/N state for installing this component on package deploy"` - // Required makes this component mandatory for package deployment - Required bool `json:"required,omitempty" jsonschema:"description=Do not prompt user to install this component, always install on package deploy"` + // DeprecatedRequired makes this component mandatory for package deployment + DeprecatedRequired *bool `json:"required,omitempty" jsonschema:"description=[Deprecated] Do not prompt user to install this component, always install on package deploy. This will be removed in Zarf v1.0.0.,deprecated=true"` + + Optional *bool `json:"optional,omitempty" jsonschema:"description=Prompt user to install this component on package deploy, defaults to false"` // Only include compatible components during package deployment Only ZarfComponentOnlyTarget `json:"only,omitempty" jsonschema:"description=Filter when this component is included in package creation or deployment"` @@ -65,6 +67,24 @@ type ZarfComponent struct { Actions ZarfComponentActions `json:"actions,omitempty" jsonschema:"description=Custom commands to run at various stages of a package lifecycle"` } +func (c ZarfComponent) IsRequired() bool { + requiredExists := c.DeprecatedRequired != nil + optionalExists := c.Optional != nil + required := requiredExists && *c.DeprecatedRequired + optional := optionalExists && *c.Optional + + if requiredExists && optionalExists { + return required && !optional + } else if requiredExists { + return required + } else if optionalExists { + return !optional + } + + // If neither required nor optional are set, then the component is required + return true +} + // ZarfComponentOnlyTarget filters a component to only show it for a given local OS and cluster. type ZarfComponentOnlyTarget struct { LocalOS string `json:"localOS,omitempty" jsonschema:"description=Only deploy component to specified OS,enum=linux,enum=darwin,enum=windows"` From c93fabd7693cca3ef861c2d351d26b2fb9f03cd5 Mon Sep 17 00:00:00 2001 From: razzle Date: Mon, 1 Jan 2024 21:56:08 -0500 Subject: [PATCH 002/113] docs and scheme Signed-off-by: razzle --- docs/3-create-a-zarf-package/4-zarf-schema.md | 4 ++-- zarf.schema.json | 6 +++++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/docs/3-create-a-zarf-package/4-zarf-schema.md b/docs/3-create-a-zarf-package/4-zarf-schema.md index 8962ec9fc6..496a6ac74f 100644 --- a/docs/3-create-a-zarf-package/4-zarf-schema.md +++ b/docs/3-create-a-zarf-package/4-zarf-schema.md @@ -611,12 +611,12 @@ must respect the following conditions
- required + optional  
-**Description:** Do not prompt user to install this component +**Description:** Prompt user to install this component on package deploy | | | | -------- | --------- | diff --git a/zarf.schema.json b/zarf.schema.json index d849c1b39a..e91f979387 100644 --- a/zarf.schema.json +++ b/zarf.schema.json @@ -225,7 +225,11 @@ }, "required": { "type": "boolean", - "description": "Do not prompt user to install this component" + "description": "[Deprecated] Do not prompt user to install this component" + }, + "optional": { + "type": "boolean", + "description": "Prompt user to install this component on package deploy" }, "only": { "$schema": "http://json-schema.org/draft-04/schema#", From 7fe7919709e3ac1e7e259af72211681c1696f53b Mon Sep 17 00:00:00 2001 From: razzle Date: Mon, 1 Jan 2024 22:20:19 -0500 Subject: [PATCH 003/113] migrate Signed-off-by: razzle --- adr/0017-zarf-bundle.md | 1 - examples/argocd/zarf.yaml | 2 -- examples/big-bang/yolo/zarf.yaml | 2 -- examples/big-bang/zarf.yaml | 2 +- examples/component-webhooks/zarf.yaml | 1 - examples/composable-packages/zarf.yaml | 8 ++++---- examples/config-file/zarf.yaml | 1 - examples/dos-games/zarf.yaml | 1 - examples/helm-charts/zarf.yaml | 3 +-- examples/kiwix/zarf.yaml | 1 - examples/longhorn/zarf.yaml | 2 -- examples/manifests/zarf.yaml | 3 --- examples/package-flavors/zarf.yaml | 5 ----- examples/podinfo-flux/zarf.yaml | 2 -- examples/variables/README.md | 1 - examples/variables/zarf.yaml | 2 -- examples/wordpress/zarf.yaml | 1 - examples/yolo/zarf.yaml | 1 - packages/distros/eks/zarf.yaml | 3 ++- packages/zarf-agent/zarf.yaml | 1 - packages/zarf-registry/zarf.yaml | 3 +-- src/extensions/bigbang/test/package/zarf.yaml | 1 - src/test/e2e/09_component_compose_test.go | 1 - src/test/packages/00-helm-annotations/zarf.yaml | 1 - src/test/packages/00-kube-version-override/zarf.yaml | 1 - src/test/packages/00-remote-pull-fail/zarf.yaml | 1 - .../packages/04-file-folders-templating-sbom/zarf.yaml | 2 -- src/test/packages/05-multi-part/zarf.yaml | 1 - src/test/packages/08-differential-package/zarf.yaml | 2 -- .../09-composable-packages/bad-local-os/zarf.yaml | 1 - src/test/packages/09-composable-packages/zarf.yaml | 1 - src/test/packages/22-git-data/zarf.yaml | 4 ---- src/test/packages/24-evil-variables/zarf.yaml | 1 - src/test/packages/25-evil-chart-deps/zarf.yaml | 2 -- src/test/packages/25-evil-chart-lookup/zarf.yaml | 2 +- src/test/packages/25-evil-dos-games/zarf.yaml | 1 - src/test/packages/25-evil-templates/zarf.yaml | 1 - src/test/packages/25-local-tgz-chart/zarf.yaml | 1 - src/test/packages/26-image-dos-games/zarf.yaml | 1 - src/test/packages/28-helm-no-wait/zarf.yaml | 1 - .../packages/31-component-actions-edgecases/zarf.yaml | 1 - .../51-import-everything/big-bang-min/zarf.yaml | 1 - .../packages/51-import-everything/inception/zarf.yaml | 5 ----- .../packages/51-import-everything/oci-import/zarf.yaml | 2 +- src/test/packages/51-import-everything/zarf.yaml | 10 +++++----- src/test/upgrade/zarf.yaml | 1 - zarf.yaml | 7 +++---- 47 files changed, 19 insertions(+), 80 deletions(-) diff --git a/adr/0017-zarf-bundle.md b/adr/0017-zarf-bundle.md index 50c0e6a2b8..f51a114788 100644 --- a/adr/0017-zarf-bundle.md +++ b/adr/0017-zarf-bundle.md @@ -14,7 +14,6 @@ Currently there is no official way to enable the deployment, publishing, pulling ```yaml - name: init - required: true files: - source: zarf-init-amd64-v0.27.0.tar.zst target: zarf-init-amd64-v0.27.0.tar.zst diff --git a/examples/argocd/zarf.yaml b/examples/argocd/zarf.yaml index 853298970f..6ddd130e14 100644 --- a/examples/argocd/zarf.yaml +++ b/examples/argocd/zarf.yaml @@ -5,7 +5,6 @@ metadata: components: - name: argocd-helm-chart - required: true charts: - name: argo-cd version: 5.46.5 @@ -18,7 +17,6 @@ components: - public.ecr.aws/docker/library/redis:7.0.11-alpine - quay.io/argoproj/argocd:v2.8.4 - name: argocd-apps - required: true charts: - name: argocd-apps version: 1.4.1 diff --git a/examples/big-bang/yolo/zarf.yaml b/examples/big-bang/yolo/zarf.yaml index c5f59c301f..f3054a7f35 100644 --- a/examples/big-bang/yolo/zarf.yaml +++ b/examples/big-bang/yolo/zarf.yaml @@ -22,14 +22,12 @@ variables: components: - name: flux-private-registry - required: true manifests: - name: private-registry namespace: flux-system files: - private-registry.yaml - name: bigbang - required: true extensions: bigbang: # renovate: datasource=gitlab-releases depName=big-bang/bigbang versioning=semver registryUrl=https://repo1.dso.mil/ diff --git a/examples/big-bang/zarf.yaml b/examples/big-bang/zarf.yaml index cb2da3bb70..79e72ca598 100644 --- a/examples/big-bang/zarf.yaml +++ b/examples/big-bang/zarf.yaml @@ -15,7 +15,6 @@ variables: components: - name: bigbang - required: true actions: onRemove: before: @@ -57,6 +56,7 @@ components: # Values are merged in order, so this would override the above and disable everything if uncommented # - config/disable-all.yaml - name: gitea-virtual-service + optional: true description: > Expose the internal Zarf Gitea server through the Big Bang Istio deployment via a virtual service. (only applies if you are using the Zarf-provided Gitea deployment - not an externally configured git host) diff --git a/examples/component-webhooks/zarf.yaml b/examples/component-webhooks/zarf.yaml index 1baa3ab50d..5547a983d3 100644 --- a/examples/component-webhooks/zarf.yaml +++ b/examples/component-webhooks/zarf.yaml @@ -6,7 +6,6 @@ metadata: version: 0.0.1 components: - name: module - required: true manifests: - name: module namespace: pepr-system diff --git a/examples/composable-packages/zarf.yaml b/examples/composable-packages/zarf.yaml index 34f499d670..d83bfca4f0 100644 --- a/examples/composable-packages/zarf.yaml +++ b/examples/composable-packages/zarf.yaml @@ -5,8 +5,8 @@ metadata: components: - name: local-games-path - # The component logic keys ('required', 'group', and 'default') always override those of the imported package - required: true + # The component logic keys ('optional', 'group', and 'default') always override those of the imported package + optional: false # group: "" # the initial value overrides the child component # default: false # the initial value overrides the child component description: "Example of a local composed package with a unique description for this component" @@ -25,8 +25,8 @@ components: - quake-service.yaml - name: oci-games-url - # The component logic keys ('required', 'group', and 'default') always override those of the imported package - # required: false # the initial value overrides the child component + # The component logic keys ('optional', 'group', and 'default') always override those of the imported package + # optional: false # the initial value overrides the child component # group: "" # the initial value overrides the child component # default: false # the initial value overrides the child component import: diff --git a/examples/config-file/zarf.yaml b/examples/config-file/zarf.yaml index d18cf80d73..cee0eb1852 100644 --- a/examples/config-file/zarf.yaml +++ b/examples/config-file/zarf.yaml @@ -12,7 +12,6 @@ variables: components: - name: lion description: By default, only this component will be deployed when using the config file - required: true manifests: - name: config-example-configmap files: diff --git a/examples/dos-games/zarf.yaml b/examples/dos-games/zarf.yaml index f6dc923b75..f01f19ffae 100644 --- a/examples/dos-games/zarf.yaml +++ b/examples/dos-games/zarf.yaml @@ -6,7 +6,6 @@ metadata: components: - name: baseline - required: true manifests: - name: multi-games namespace: dos-games diff --git a/examples/helm-charts/zarf.yaml b/examples/helm-charts/zarf.yaml index 27a50c5595..fc4f13ff67 100644 --- a/examples/helm-charts/zarf.yaml +++ b/examples/helm-charts/zarf.yaml @@ -6,7 +6,6 @@ metadata: components: - name: demo-helm-charts - required: true charts: # Charts are organized in a list with unique chart names per component - note that a Zarf chart name does not need to match the chart name in a Chart.yaml - name: podinfo-local @@ -16,7 +15,7 @@ components: localPath: chart valuesFiles: - values.yaml - + - name: podinfo-oci version: 6.4.0 namespace: podinfo-from-oci diff --git a/examples/kiwix/zarf.yaml b/examples/kiwix/zarf.yaml index 8f3a300312..45f52ee902 100644 --- a/examples/kiwix/zarf.yaml +++ b/examples/kiwix/zarf.yaml @@ -8,7 +8,6 @@ metadata: components: - name: kiwix-serve - required: true manifests: - name: kiwix-serve namespace: kiwix diff --git a/examples/longhorn/zarf.yaml b/examples/longhorn/zarf.yaml index b447e8debe..f7146ba148 100644 --- a/examples/longhorn/zarf.yaml +++ b/examples/longhorn/zarf.yaml @@ -5,7 +5,6 @@ metadata: version: 1.4.0 components: - name: longhorn-environment-check - required: true files: - source: https://raw.githubusercontent.com/longhorn/longhorn/v1.4.0/scripts/environment_check.sh target: environment_check.sh @@ -26,7 +25,6 @@ components: chmod +x ./environment_check.sh ./environment_check.sh - name: longhorn - required: true description: Deploy Longhorn into a Kubernetes cluster. https://longhorn.io actions: # Set the delete confirmation flag for Longhorn diff --git a/examples/manifests/zarf.yaml b/examples/manifests/zarf.yaml index cb363312c0..116eb5c913 100644 --- a/examples/manifests/zarf.yaml +++ b/examples/manifests/zarf.yaml @@ -5,7 +5,6 @@ metadata: components: - name: httpd-local - required: true manifests: - name: simple-httpd-deployment namespace: httpd @@ -28,7 +27,6 @@ components: images: - httpd:alpine3.18 - name: nginx-remote - required: true manifests: - name: simple-nginx-deployment namespace: nginx @@ -54,7 +52,6 @@ components: images: - nginx:1.14.2 - name: podinfo-kustomize - required: true manifests: - name: simple-podinfo-deployment namespace: podinfo diff --git a/examples/package-flavors/zarf.yaml b/examples/package-flavors/zarf.yaml index 873b36f493..c92fe9cca3 100644 --- a/examples/package-flavors/zarf.yaml +++ b/examples/package-flavors/zarf.yaml @@ -6,7 +6,6 @@ metadata: components: - name: image - required: true description: "Sets the Enterprise Linux flavor to Rocky Linux" only: flavor: rocky-road @@ -20,7 +19,6 @@ components: - name: IMAGE - name: image - required: true description: "Sets the Enterprise Linux flavor to Oracle Linux" only: flavor: oracle-cookie-crunch @@ -34,7 +32,6 @@ components: - name: IMAGE - name: image - required: true description: "Sets the Enterprise Linux flavor to Alma Linux" only: flavor: vanilla-alma-nd @@ -48,7 +45,6 @@ components: - name: IMAGE - name: image - required: true description: "Sets the Enterprise Linux flavor to OpenSUSE" only: flavor: strawberry-suse @@ -63,7 +59,6 @@ components: - name: pod description: "The pod that runs the specified flavor of Enterprise Linux" - required: true manifests: - name: enterprise-linux namespace: enterprise-linux diff --git a/examples/podinfo-flux/zarf.yaml b/examples/podinfo-flux/zarf.yaml index 0f0e4b3336..428eec17c6 100644 --- a/examples/podinfo-flux/zarf.yaml +++ b/examples/podinfo-flux/zarf.yaml @@ -6,7 +6,6 @@ metadata: components: - name: flux description: Installs the flux CRDs / controllers to use flux-based deployments in the cluster - required: true manifests: - name: flux-crds namespace: flux @@ -18,7 +17,6 @@ components: - name: podinfo-via-flux description: Example deployment via flux using the famous podinfo example - required: true manifests: - name: podinfo-via-flux namespace: podinfo diff --git a/examples/variables/README.md b/examples/variables/README.md index fbb616aff6..f8e549cbf7 100644 --- a/examples/variables/README.md +++ b/examples/variables/README.md @@ -133,7 +133,6 @@ constants: components: - name: zarf-prompt-image - required: true images: - '###ZARF_PKG_TMPL_PROMPT_ON_CREATE###' ``` diff --git a/examples/variables/zarf.yaml b/examples/variables/zarf.yaml index f1a3564b5a..b709d05348 100644 --- a/examples/variables/zarf.yaml +++ b/examples/variables/zarf.yaml @@ -50,7 +50,6 @@ components: # NOTE: this component does not actually execute this file in this example - name: variables-with-terraform description: Change a value in a regular file with a Zarf variable. Set AWS_REGION variable to modify the file. - required: true files: - source: simple-terraform.tf target: modified-terraform.tf @@ -71,7 +70,6 @@ components: # The following component deploys nginx to the cluster using the defined variables - name: variables-with-nginx description: "This component deploys nginx version ###ZARF_PKG_TMPL_NGINX_VERSION### to the cluster" - required: true images: # This sets the nginx image tag to the same PKG_TMPL used for the constant above to keep the zarf.yaml and nginx-deployment.yaml in sync - "nginx:###ZARF_PKG_TMPL_NGINX_VERSION###" diff --git a/examples/wordpress/zarf.yaml b/examples/wordpress/zarf.yaml index 9900bbd0e0..96fa0d4700 100644 --- a/examples/wordpress/zarf.yaml +++ b/examples/wordpress/zarf.yaml @@ -40,7 +40,6 @@ components: - name: wordpress # specifies the name of our component and should be unique and unchanging through updates description: | # (optional) a human-readable description of the component you are defining Deploys the Bitnami-packaged WordPress chart into the cluster - required: true # (optional) sets the component as 'required' so that it is always deployed charts: - name: wordpress url: oci://registry-1.docker.io/bitnamicharts/wordpress diff --git a/examples/yolo/zarf.yaml b/examples/yolo/zarf.yaml index 2f7770c498..fa2e7c2a1f 100644 --- a/examples/yolo/zarf.yaml +++ b/examples/yolo/zarf.yaml @@ -6,7 +6,6 @@ metadata: components: - name: yolo-games - required: true manifests: - name: multi-games namespace: zarf-yolo-example diff --git a/packages/distros/eks/zarf.yaml b/packages/distros/eks/zarf.yaml index 11da41bba0..588d7e9866 100644 --- a/packages/distros/eks/zarf.yaml +++ b/packages/distros/eks/zarf.yaml @@ -28,7 +28,6 @@ variables: components: - name: load-eksctl - required: true files: - source: eks.yaml target: eks.yaml @@ -49,6 +48,7 @@ components: extractPath: eksctl - name: deploy-eks-cluster + optional: true description: Create an EKS cluster! actions: onDeploy: @@ -59,6 +59,7 @@ components: - cmd: ./binaries/eksctl_$(uname -s)_$(uname -m) utils write-kubeconfig -c ${ZARF_VAR_EKS_CLUSTER_NAME} - name: teardown-eks-cluster + optional: true description: Delete the EKS cluster that this package was used to create. actions: onDeploy: diff --git a/packages/zarf-agent/zarf.yaml b/packages/zarf-agent/zarf.yaml index 86799ff03f..4d2c3d2c5e 100644 --- a/packages/zarf-agent/zarf.yaml +++ b/packages/zarf-agent/zarf.yaml @@ -16,7 +16,6 @@ components: images and git repository references in Kubernetes manifests. This prevents the need to manually update URLs from their original sources to the Zarf-managed docker registry and git server. - required: true images: - "###ZARF_PKG_TMPL_AGENT_IMAGE_DOMAIN######ZARF_PKG_TMPL_AGENT_IMAGE###:###ZARF_PKG_TMPL_AGENT_IMAGE_TAG###" manifests: diff --git a/packages/zarf-registry/zarf.yaml b/packages/zarf-registry/zarf.yaml index 38a13006c1..da2b64e0fc 100644 --- a/packages/zarf-registry/zarf.yaml +++ b/packages/zarf-registry/zarf.yaml @@ -70,7 +70,6 @@ components: description: | Bootstraps a Kubernetes cluster by cloning a running pod in the cluster and hosting the registry image. Removed and destroyed after the Zarf Registry is self-hosting the registry image. - required: true only: cluster: architecture: amd64 @@ -85,7 +84,6 @@ components: description: | Bootstraps a Kubernetes cluster by cloning a running pod in the cluster and hosting the registry image. Removed and destroyed after the Zarf Registry is self-hosting the registry image. - required: true only: cluster: architecture: arm64 @@ -97,6 +95,7 @@ components: executable: true - name: zarf-seed-registry + optional: true description: | Deploys the Zarf Registry using the registry image provided by the Zarf Injector. charts: diff --git a/src/extensions/bigbang/test/package/zarf.yaml b/src/extensions/bigbang/test/package/zarf.yaml index 426fed8f54..478e3fc01c 100644 --- a/src/extensions/bigbang/test/package/zarf.yaml +++ b/src/extensions/bigbang/test/package/zarf.yaml @@ -14,7 +14,6 @@ variables: components: - name: bigbang - required: true extensions: bigbang: version: "###ZARF_PKG_TMPL_BB_VERSION###" diff --git a/src/test/e2e/09_component_compose_test.go b/src/test/e2e/09_component_compose_test.go index b996e6ea48..8698a6d36c 100644 --- a/src/test/e2e/09_component_compose_test.go +++ b/src/test/e2e/09_component_compose_test.go @@ -87,7 +87,6 @@ func (suite *CompositionSuite) Test_1_FullComposability() { suite.Contains(stdErr, ` - name: test-compose-package description: A contrived example for podinfo using many Zarf primitives for compose testing - required: true only: localOS: linux `) diff --git a/src/test/packages/00-helm-annotations/zarf.yaml b/src/test/packages/00-helm-annotations/zarf.yaml index c48384748a..b9742919a4 100644 --- a/src/test/packages/00-helm-annotations/zarf.yaml +++ b/src/test/packages/00-helm-annotations/zarf.yaml @@ -5,7 +5,6 @@ metadata: components: - name: istio-repo1-component description: Helm chart for the Istio Control Plane - required: true charts: - name: istio version: 1.17.2-bb.2 diff --git a/src/test/packages/00-kube-version-override/zarf.yaml b/src/test/packages/00-kube-version-override/zarf.yaml index 62521c1df8..c889bbb910 100644 --- a/src/test/packages/00-kube-version-override/zarf.yaml +++ b/src/test/packages/00-kube-version-override/zarf.yaml @@ -5,7 +5,6 @@ metadata: components: - name: cert-manager-component description: Cloud native certificate management - required: true charts: - name: cert-manager version: v1.11.1 diff --git a/src/test/packages/00-remote-pull-fail/zarf.yaml b/src/test/packages/00-remote-pull-fail/zarf.yaml index f44d99d4b1..4ef5d94112 100644 --- a/src/test/packages/00-remote-pull-fail/zarf.yaml +++ b/src/test/packages/00-remote-pull-fail/zarf.yaml @@ -3,6 +3,5 @@ metadata: name: doesnotexist components: - name: doesnotexist-docker - required: true images: - ghcr.io/defenseunicorns/doesnotexist:1.3.3.7 diff --git a/src/test/packages/04-file-folders-templating-sbom/zarf.yaml b/src/test/packages/04-file-folders-templating-sbom/zarf.yaml index 5aa4cb4ddf..b251d00300 100644 --- a/src/test/packages/04-file-folders-templating-sbom/zarf.yaml +++ b/src/test/packages/04-file-folders-templating-sbom/zarf.yaml @@ -9,7 +9,6 @@ variables: components: - name: folders - required: true files: - source: include-files target: temp/include-files @@ -20,7 +19,6 @@ components: - cmd: cat temp/include-files/something.yaml - cmd: rm -r temp - name: files - required: true files: - source: requirements.txt target: temp/requirements.txt diff --git a/src/test/packages/05-multi-part/zarf.yaml b/src/test/packages/05-multi-part/zarf.yaml index 78b47ba606..8017f355c2 100644 --- a/src/test/packages/05-multi-part/zarf.yaml +++ b/src/test/packages/05-multi-part/zarf.yaml @@ -5,7 +5,6 @@ metadata: components: - name: big-ol-file - required: true description: Single 5 MB file needed to demonstrate a multi-part package files: - source: https://zarf-public.s3-us-gov-west-1.amazonaws.com/examples/multi-part-demo.dat diff --git a/src/test/packages/08-differential-package/zarf.yaml b/src/test/packages/08-differential-package/zarf.yaml index 9d911a66d0..a2639b786e 100644 --- a/src/test/packages/08-differential-package/zarf.yaml +++ b/src/test/packages/08-differential-package/zarf.yaml @@ -6,7 +6,6 @@ metadata: components: - name: versioned-assets - required: true images: - ghcr.io/stefanprodan/podinfo:6.0.0 - ghcr.io/defenseunicorns/zarf/agent:###ZARF_PKG_TMPL_PACKAGE_VERSION### @@ -15,7 +14,6 @@ components: - https://github.com/defenseunicorns/zarf.git@refs/tags/###ZARF_PKG_TMPL_PACKAGE_VERSION### - name: generalized-assets - required: true images: - ghcr.io/stefanprodan/podinfo:latest repos: diff --git a/src/test/packages/09-composable-packages/bad-local-os/zarf.yaml b/src/test/packages/09-composable-packages/bad-local-os/zarf.yaml index d0772be32e..d2607a55d2 100644 --- a/src/test/packages/09-composable-packages/bad-local-os/zarf.yaml +++ b/src/test/packages/09-composable-packages/bad-local-os/zarf.yaml @@ -8,7 +8,6 @@ components: description: A contrived example for localOS compose testing only: localOS: windows - required: true import: path: ../sub-package name: test-compose-sub-package diff --git a/src/test/packages/09-composable-packages/zarf.yaml b/src/test/packages/09-composable-packages/zarf.yaml index 21e40e5df2..e502b8e69e 100644 --- a/src/test/packages/09-composable-packages/zarf.yaml +++ b/src/test/packages/09-composable-packages/zarf.yaml @@ -6,7 +6,6 @@ metadata: components: - name: test-compose-package description: A contrived example for podinfo using many Zarf primitives for compose testing - required: true import: path: sub-package name: test-compose-sub-package diff --git a/src/test/packages/22-git-data/zarf.yaml b/src/test/packages/22-git-data/zarf.yaml index 2e3b56d75c..c16bc55693 100644 --- a/src/test/packages/22-git-data/zarf.yaml +++ b/src/test/packages/22-git-data/zarf.yaml @@ -6,7 +6,6 @@ metadata: components: - name: full-repo - required: true repos: # Do a full Git Repo Mirror - https://github.com/defenseunicorns/zarf-public-test.git @@ -16,7 +15,6 @@ components: - https://github.com/defenseunicorns/golang-tekton-hello-world.git - name: specific-tag - required: true repos: # The following performs a tag Git Repo Mirror with `go-git` (internal to Zarf) - https://github.com/defenseunicorns/zarf-public-test.git@v0.0.1 @@ -33,7 +31,6 @@ components: maxRetries: 3 - name: specific-branch - required: true repos: # The following performs a branch Git Repo Mirror with `go-git` (internal to Zarf) - https://github.com/defenseunicorns/zarf-public-test.git@refs/heads/dragons @@ -48,7 +45,6 @@ components: maxRetries: 3 - name: specific-hash - required: true repos: # The following performs a SHA Git Repo Mirror with `go-git` (internal to Zarf) - https://github.com/defenseunicorns/zarf-public-test.git@01a23218923f24194133b5eb11268cf8d73ff1bb diff --git a/src/test/packages/24-evil-variables/zarf.yaml b/src/test/packages/24-evil-variables/zarf.yaml index 906bced7c5..6b4a15b505 100644 --- a/src/test/packages/24-evil-variables/zarf.yaml +++ b/src/test/packages/24-evil-variables/zarf.yaml @@ -6,7 +6,6 @@ metadata: components: - name: bad-set-variables-pattern description: "###ZARF_PKG_TMPL_NUMB3R5###" - required: true actions: onDeploy: before: diff --git a/src/test/packages/25-evil-chart-deps/zarf.yaml b/src/test/packages/25-evil-chart-deps/zarf.yaml index 1199c46b66..75afea1d1e 100644 --- a/src/test/packages/25-evil-chart-deps/zarf.yaml +++ b/src/test/packages/25-evil-chart-deps/zarf.yaml @@ -5,7 +5,6 @@ metadata: components: - name: good - required: true charts: - name: dogs localPath: good-chart @@ -13,7 +12,6 @@ components: namespace: good - name: bad - required: true charts: - name: cats localPath: bad-chart diff --git a/src/test/packages/25-evil-chart-lookup/zarf.yaml b/src/test/packages/25-evil-chart-lookup/zarf.yaml index b9f8a0441f..ee12ec8866 100644 --- a/src/test/packages/25-evil-chart-lookup/zarf.yaml +++ b/src/test/packages/25-evil-chart-lookup/zarf.yaml @@ -6,7 +6,7 @@ metadata: components: - name: bad-helm-repository-name - required: false + optional: true charts: - name: asdf version: 6.4.0 diff --git a/src/test/packages/25-evil-dos-games/zarf.yaml b/src/test/packages/25-evil-dos-games/zarf.yaml index 41ecd44716..f8188cf543 100644 --- a/src/test/packages/25-evil-dos-games/zarf.yaml +++ b/src/test/packages/25-evil-dos-games/zarf.yaml @@ -5,7 +5,6 @@ metadata: components: - name: baseline - required: true manifests: - name: multi-games namespace: dos-games diff --git a/src/test/packages/25-evil-templates/zarf.yaml b/src/test/packages/25-evil-templates/zarf.yaml index 66dbc71c30..b431ddc843 100644 --- a/src/test/packages/25-evil-templates/zarf.yaml +++ b/src/test/packages/25-evil-templates/zarf.yaml @@ -4,7 +4,6 @@ metadata: components: - name: test - required: true manifests: - name: configmap files: diff --git a/src/test/packages/25-local-tgz-chart/zarf.yaml b/src/test/packages/25-local-tgz-chart/zarf.yaml index 7609b45b45..a281174481 100644 --- a/src/test/packages/25-local-tgz-chart/zarf.yaml +++ b/src/test/packages/25-local-tgz-chart/zarf.yaml @@ -6,7 +6,6 @@ metadata: components: - name: demo-helm-local-chart - required: false charts: - name: podinfo version: 6.4.0 diff --git a/src/test/packages/26-image-dos-games/zarf.yaml b/src/test/packages/26-image-dos-games/zarf.yaml index 06131de09b..c64ec1ca78 100644 --- a/src/test/packages/26-image-dos-games/zarf.yaml +++ b/src/test/packages/26-image-dos-games/zarf.yaml @@ -5,7 +5,6 @@ metadata: components: - name: baseline - required: true images: - defenseunicorns/zarf-game:multi-tile-dark - defenseunicorns/zarf-game:multi-tile-dark@sha256:f78e442f0f3eb3e9459b5ae6b1a8fda62f8dfe818112e7d130a4e8ae72b3cbff diff --git a/src/test/packages/28-helm-no-wait/zarf.yaml b/src/test/packages/28-helm-no-wait/zarf.yaml index 177fe6792a..44b2157f13 100644 --- a/src/test/packages/28-helm-no-wait/zarf.yaml +++ b/src/test/packages/28-helm-no-wait/zarf.yaml @@ -5,7 +5,6 @@ metadata: components: - name: helm-no-wait - required: true manifests: - name: never-ready namespace: no-wait diff --git a/src/test/packages/31-component-actions-edgecases/zarf.yaml b/src/test/packages/31-component-actions-edgecases/zarf.yaml index 3f8e08264f..2d22ef34e9 100644 --- a/src/test/packages/31-component-actions-edgecases/zarf.yaml +++ b/src/test/packages/31-component-actions-edgecases/zarf.yaml @@ -5,7 +5,6 @@ metadata: components: - name: on-deploy - required: true actions: # runs during "zarf package remove" onDeploy: diff --git a/src/test/packages/51-import-everything/big-bang-min/zarf.yaml b/src/test/packages/51-import-everything/big-bang-min/zarf.yaml index 6fd483c848..e1540e9b57 100644 --- a/src/test/packages/51-import-everything/big-bang-min/zarf.yaml +++ b/src/test/packages/51-import-everything/big-bang-min/zarf.yaml @@ -14,7 +14,6 @@ variables: components: - name: bigbang - required: true extensions: bigbang: repo: https://github.com/DoD-Platform-One/big-bang.git diff --git a/src/test/packages/51-import-everything/inception/zarf.yaml b/src/test/packages/51-import-everything/inception/zarf.yaml index 7de7d43d81..78f5337ee2 100644 --- a/src/test/packages/51-import-everything/inception/zarf.yaml +++ b/src/test/packages/51-import-everything/inception/zarf.yaml @@ -6,26 +6,21 @@ metadata: components: - name: import-component-local - required: true import: url: oci://localhost:555/import-everything:0.0.1 - name: import-component-oci - required: true import: url: oci://localhost:555/import-everything:0.0.1 - name: import-big-bang - required: true import: url: oci://localhost:555/import-everything:0.0.1 - name: file-imports - required: true import: url: oci://localhost:555/import-everything:0.0.1 - name: local-chart-import - required: true import: url: oci://localhost:555/import-everything:0.0.1 diff --git a/src/test/packages/51-import-everything/oci-import/zarf.yaml b/src/test/packages/51-import-everything/oci-import/zarf.yaml index 42d4b79901..b544186e69 100644 --- a/src/test/packages/51-import-everything/oci-import/zarf.yaml +++ b/src/test/packages/51-import-everything/oci-import/zarf.yaml @@ -7,7 +7,7 @@ metadata: components: - name: import-component-oci description: "import-component-oci == ###ZARF_COMPONENT_NAME###" - required: false + optional: true import: name: demo-helm-charts url: oci://localhost:555/helm-charts:0.0.1 diff --git a/src/test/packages/51-import-everything/zarf.yaml b/src/test/packages/51-import-everything/zarf.yaml index b69ef73086..653d2ff1e6 100644 --- a/src/test/packages/51-import-everything/zarf.yaml +++ b/src/test/packages/51-import-everything/zarf.yaml @@ -8,7 +8,7 @@ components: # Test every simple primitive that Zarf has through a local import - name: import-component-local description: "import-component-local == ###ZARF_COMPONENT_NAME###" - required: false + optional: true import: path: ../09-composable-packages name: test-compose-package @@ -16,7 +16,7 @@ components: # Test nested local to oci imports - name: import-component-oci description: "import-component-oci == ###ZARF_COMPONENT_NAME###" - required: false + optional: true import: name: import-component-oci path: oci-import @@ -24,7 +24,7 @@ components: # Test big bang extension files - name: import-big-bang description: "import-big-bang == ###ZARF_COMPONENT_NAME###" - required: false + optional: true import: name: bigbang url: oci://localhost:555/big-bang-min:2.10.0 @@ -32,7 +32,7 @@ components: # Test file imports including cosignKeyPath - name: file-imports description: "file-imports == ###ZARF_COMPONENT_NAME###" - required: false + optional: true cosignKeyPath: ../../../../cosign.pub files: # Import of a local file @@ -57,7 +57,7 @@ components: # Test local charts (for skeletons) - name: local-chart-import description: "local-chart-import == ###ZARF_COMPONENT_NAME###" - required: false + optional: true charts: - name: podinfo localPath: ../../../../examples/helm-charts/chart diff --git a/src/test/upgrade/zarf.yaml b/src/test/upgrade/zarf.yaml index 5052a2bc55..3539fce943 100644 --- a/src/test/upgrade/zarf.yaml +++ b/src/test/upgrade/zarf.yaml @@ -7,7 +7,6 @@ metadata: components: - name: test-upgrade-package description: A semi-contrived example that deploys podinfo using many Zarf primitives and lets us upgrade it a few times. - required: true charts: - name: podinfo-upgrade version: "###ZARF_PKG_VAR_PODINFO_VERSION###" diff --git a/zarf.yaml b/zarf.yaml index 859813fde5..fc40c32283 100644 --- a/zarf.yaml +++ b/zarf.yaml @@ -5,39 +5,38 @@ metadata: components: - name: k3s + optional: true import: path: packages/distros/k3s # This package moves the injector & registries binaries - name: zarf-injector - required: true import: path: packages/zarf-registry # Creates the temporary seed-registry - name: zarf-seed-registry - required: true import: path: packages/zarf-registry # Creates the permanent registry - name: zarf-registry - required: true import: path: packages/zarf-registry # Creates the pod+git mutating webhook - name: zarf-agent - required: true import: path: packages/zarf-agent # (Optional) Adds logging to the cluster - name: logging + optional: true import: path: packages/logging-pgl # (Optional) Adds a git server to the cluster - name: git-server + optional: true import: path: packages/gitea From d3d2d9bfb35da7fe440039fcd99738442659fcba Mon Sep 17 00:00:00 2001 From: razzle Date: Mon, 1 Jan 2024 22:20:41 -0500 Subject: [PATCH 004/113] docs and scheme Signed-off-by: razzle --- .../static/docs/tutorials/differential_package_edit.html | 2 -- .../static/docs/tutorials/publish_and_deploy_manifest.html | 1 - .../tutorials/troubleshoot_version_required_no_version.html | 1 - docs/5-zarf-tutorials/0-creating-a-zarf-package.md | 2 +- docs/5-zarf-tutorials/6-big-bang.md | 4 +--- 5 files changed, 2 insertions(+), 8 deletions(-) diff --git a/docs-website/static/docs/tutorials/differential_package_edit.html b/docs-website/static/docs/tutorials/differential_package_edit.html index a6393041c1..cba5e373a0 100644 --- a/docs-website/static/docs/tutorials/differential_package_edit.html +++ b/docs-website/static/docs/tutorials/differential_package_edit.html @@ -59,7 +59,6 @@ version: "1.4.2" components: - name: longhorn-environment-check - required: true files: - source: https://raw.githubusercontent.com/longhorn/longhorn/v1.4.2/scripts/environment_check.sh target: environment_check.sh @@ -78,7 +77,6 @@ chmod +x ./environment_check.sh ./environment_check.sh - name: longhorn - required: true description: "Deploy Longhorn into a Kubernetes cluster. https://longhorn.io" actions: # Set the delete confirmation flag for Longhorn diff --git a/docs-website/static/docs/tutorials/publish_and_deploy_manifest.html b/docs-website/static/docs/tutorials/publish_and_deploy_manifest.html index 74e8928a4a..4d58b2ab98 100644 --- a/docs-website/static/docs/tutorials/publish_and_deploy_manifest.html +++ b/docs-website/static/docs/tutorials/publish_and_deploy_manifest.html @@ -62,7 +62,6 @@ components: - name: helm-oci-chart - required: true charts: - name: podinfo version: 6.3.3 diff --git a/docs-website/static/docs/tutorials/troubleshoot_version_required_no_version.html b/docs-website/static/docs/tutorials/troubleshoot_version_required_no_version.html index d2b396a540..e2b048154f 100644 --- a/docs-website/static/docs/tutorials/troubleshoot_version_required_no_version.html +++ b/docs-website/static/docs/tutorials/troubleshoot_version_required_no_version.html @@ -57,7 +57,6 @@ components: - name: helm-oci-chart - required: true charts: - name: podinfo version: 6.3.3 diff --git a/docs/5-zarf-tutorials/0-creating-a-zarf-package.md b/docs/5-zarf-tutorials/0-creating-a-zarf-package.md index e398df6933..ec7921d1a5 100644 --- a/docs/5-zarf-tutorials/0-creating-a-zarf-package.md +++ b/docs/5-zarf-tutorials/0-creating-a-zarf-package.md @@ -50,7 +50,7 @@ components: - name: wordpress # specifies the name of our component and should be unique and unchanging through updates description: | # (optional) a human-readable description of the component you are defining "Deploys the Bitnami-packaged WordPress chart into the cluster" - required: true # (optional) sets the component as 'required' so that it is always deployed + optional: true # (optional) sets the component as 'required' so that it is always deployed charts: - name: wordpress url: oci://registry-1.docker.io/bitnamicharts/wordpress diff --git a/docs/5-zarf-tutorials/6-big-bang.md b/docs/5-zarf-tutorials/6-big-bang.md index b8f8043230..1b3112336f 100644 --- a/docs/5-zarf-tutorials/6-big-bang.md +++ b/docs/5-zarf-tutorials/6-big-bang.md @@ -10,10 +10,9 @@ Zarf integrates with Big Bang through the use of an extension that simplifies th The current version of this extension requires Big Bang version `1.54.0` or later, and is not fully integrated into the `zarf package remove` lifecycle (see the [Big Bang example](../../examples/big-bang/README.md) for how to introduce those lifecycle hooks manually). Zarf also relies on [helm.sh/images annotations](https://github.com/helm/community/blob/main/hips/hip-0015.md) to discover images within charts (e.g. [GitLab](https://repo1.dso.mil/big-bang/product/packages/gitlab/-/blob/main/chart/Chart.yaml#L61)) - this is a relatively new feature for Big Bang so if you see an `ImagePullBackOff` error, check that these annotations are set correctly for the sub charts you are using. To work around this issue if you come across it, simply add the missing image to the images list on the same component that contains the Big Bang extension like the following: -``` +```yaml components: - name: bigbang - required: true images: - registry1.dso.mil/ironbank/gitlab/gitlab/gitlab-exporter:15.9.2 extensions: @@ -88,7 +87,6 @@ Within the `examples/big-bang` folder you will see a `zarf.yaml` that has the fo ```yaml components: - name: bigbang - required: true extensions: bigbang: version: 1.54.0 From 0e17d5268db6a56b7e2703e1c3e5402e942e1b28 Mon Sep 17 00:00:00 2001 From: razzle Date: Mon, 1 Jan 2024 22:42:43 -0500 Subject: [PATCH 005/113] unit tests Signed-off-by: razzle --- Makefile | 1 + src/types/component.go | 7 +-- src/types/component_test.go | 107 ++++++++++++++++++++++++++++++++++++ 3 files changed, 111 insertions(+), 4 deletions(-) create mode 100644 src/types/component_test.go diff --git a/Makefile b/Makefile index 5ab1738a69..2df92ab4b8 100644 --- a/Makefile +++ b/Makefile @@ -199,6 +199,7 @@ test-upgrade: ## Run the Zarf CLI E2E tests for an external registry and cluster .PHONY: test-unit test-unit: ## Run unit tests cd src/pkg && go test ./... -failfast -v -timeout 30m + cd src/types && go test ./... -failfast -v -timeout 30m cd src/internal && go test ./... -failfast -v timeout 30m cd src/extensions/bigbang && go test ./. -failfast -v timeout 30m diff --git a/src/types/component.go b/src/types/component.go index af981442ad..4b87c195cd 100644 --- a/src/types/component.go +++ b/src/types/component.go @@ -73,12 +73,11 @@ func (c ZarfComponent) IsRequired() bool { required := requiredExists && *c.DeprecatedRequired optional := optionalExists && *c.Optional - if requiredExists && optionalExists { - return required && !optional + // optional "wins" when defined + if optionalExists { + return !optional } else if requiredExists { return required - } else if optionalExists { - return !optional } // If neither required nor optional are set, then the component is required diff --git a/src/types/component_test.go b/src/types/component_test.go new file mode 100644 index 0000000000..d3e376e0d4 --- /dev/null +++ b/src/types/component_test.go @@ -0,0 +1,107 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package types contains all the types used by Zarf. +package types + +import "testing" + +func TestZarfComponent_IsRequired(t *testing.T) { + tests := []struct { + name string + component ZarfComponent + want bool + }{ + { + name: "Test when DeprecatedRequired is true and Optional is nil", + component: ZarfComponent{ + DeprecatedRequired: BoolPtr(true), + Optional: nil, + }, + want: true, + }, + { + name: "Test when DeprecatedRequired is true and Optional is false", + component: ZarfComponent{ + DeprecatedRequired: BoolPtr(true), + Optional: BoolPtr(false), + }, + want: true, + }, + { + name: "Test when DeprecatedRequired is true and Optional is true", + component: ZarfComponent{ + DeprecatedRequired: BoolPtr(true), + Optional: BoolPtr(true), + }, + // optional "wins" when defined + want: false, + }, + { + name: "Test when DeprecatedRequired is false and Optional is nil", + component: ZarfComponent{ + DeprecatedRequired: BoolPtr(false), + Optional: nil, + }, + want: false, + }, + { + name: "Test when DeprecatedRequired is false and Optional is false", + component: ZarfComponent{ + DeprecatedRequired: BoolPtr(false), + Optional: BoolPtr(false), + }, + // optional "wins" when defined + want: true, + }, + { + name: "Test when DeprecatedRequired is false and Optional is true", + component: ZarfComponent{ + DeprecatedRequired: BoolPtr(false), + Optional: BoolPtr(true), + }, + // optional "wins" when defined + want: false, + }, + { + name: "Test when DeprecatedRequired is nil and Optional is nil", + component: ZarfComponent{ + DeprecatedRequired: nil, + Optional: nil, + }, + // default is true (required: true || optional: false) + want: true, + }, + { + name: "Test when DeprecatedRequired is nil and Optional is false", + component: ZarfComponent{ + DeprecatedRequired: nil, + Optional: BoolPtr(false), + }, + // optional "wins" when defined + want: true, + }, + { + name: "Test when DeprecatedRequired is nil and Optional is true", + component: ZarfComponent{ + DeprecatedRequired: nil, + Optional: BoolPtr(true), + }, + // optional "wins" when defined + want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := tt.component.IsRequired(); got != tt.want { + t.Errorf("%q: ZarfComponent.IsRequired() = %v, want %v", tt.name, got, tt.want) + } + }) + } +} + +// BoolPtr is a helper function for creating a pointer to a bool. +func BoolPtr(b bool) *bool { + return &b +} From 3eb3a0ce1dad00a42da6c82553cca0a51a0922a8 Mon Sep 17 00:00:00 2001 From: razzle Date: Mon, 1 Jan 2024 23:11:26 -0500 Subject: [PATCH 006/113] cleaner override logic Signed-off-by: razzle --- src/pkg/packager/composer/override.go | 10 ++++++---- src/pkg/utils/helpers/misc.go | 5 +++++ src/types/component.go | 11 +++++++++++ 3 files changed, 22 insertions(+), 4 deletions(-) diff --git a/src/pkg/packager/composer/override.go b/src/pkg/packager/composer/override.go index faa156e0fe..bac1af94a5 100644 --- a/src/pkg/packager/composer/override.go +++ b/src/pkg/packager/composer/override.go @@ -7,16 +7,18 @@ package composer import ( "fmt" + "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "github.com/defenseunicorns/zarf/src/types" ) func overrideMetadata(c *types.ZarfComponent, override types.ZarfComponent) error { c.Name = override.Name c.Default = override.Default - if override.DeprecatedRequired != nil { - c.DeprecatedRequired = override.DeprecatedRequired - } - if override.Optional != nil { + // Only override "optional" with the value from the override + // if "required" is neither nil, nor false. + if override.DeprecatedRequired != nil && override.DeprecatedRequired == helpers.BoolPtr(true) { + c.Optional = helpers.BoolPtr(false) + } else if override.Optional != nil { c.Optional = override.Optional } diff --git a/src/pkg/utils/helpers/misc.go b/src/pkg/utils/helpers/misc.go index 9b6fefbbd8..24ab863dc7 100644 --- a/src/pkg/utils/helpers/misc.go +++ b/src/pkg/utils/helpers/misc.go @@ -11,6 +11,11 @@ import ( "time" ) +// BoolPtr returns a pointer to a bool. +func BoolPtr(b bool) *bool { + return &b +} + // Retry will retry a function until it succeeds or the timeout is reached, timeout == retries * delay. func Retry(fn func() error, retries int, delay time.Duration, logger func(format string, args ...any)) (err error) { for r := 0; r < retries; r++ { diff --git a/src/types/component.go b/src/types/component.go index 4b87c195cd..7938e63d48 100644 --- a/src/types/component.go +++ b/src/types/component.go @@ -67,6 +67,17 @@ type ZarfComponent struct { Actions ZarfComponentActions `json:"actions,omitempty" jsonschema:"description=Custom commands to run at various stages of a package lifecycle"` } +// IsRequired returns if the component is required or not. +// +// The logic is as follows: +// +// 1. If "optional" is set, then the component follows the inverse of that value +// +// 2. If "required" is set, then the component follows that value +// +// 3. If neither "optional" nor "required" are set, then the component is required +// +// Note: "required" is deprecated and will be removed in Zarf v1.0.0 func (c ZarfComponent) IsRequired() bool { requiredExists := c.DeprecatedRequired != nil optionalExists := c.Optional != nil From 5e6056f0720ab477da8dec143d697006b080b4bb Mon Sep 17 00:00:00 2001 From: razzle Date: Mon, 1 Jan 2024 23:17:03 -0500 Subject: [PATCH 007/113] fix zarf-seed-registry optional status Signed-off-by: razzle --- zarf.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/zarf.yaml b/zarf.yaml index fc40c32283..ee244d890f 100644 --- a/zarf.yaml +++ b/zarf.yaml @@ -16,6 +16,7 @@ components: # Creates the temporary seed-registry - name: zarf-seed-registry + optional: false import: path: packages/zarf-registry From 2a98254a476c8f09eeb92ec3e313671b1c269366 Mon Sep 17 00:00:00 2001 From: razzle Date: Mon, 1 Jan 2024 23:30:04 -0500 Subject: [PATCH 008/113] fix example Signed-off-by: razzle --- examples/component-choice/zarf.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/examples/component-choice/zarf.yaml b/examples/component-choice/zarf.yaml index 9c263ca7ba..fd422430b7 100644 --- a/examples/component-choice/zarf.yaml +++ b/examples/component-choice/zarf.yaml @@ -5,12 +5,16 @@ metadata: components: - name: first-choice + # if using a group, all members of the group must be optional + optional: true group: example-choice files: - source: blank-file.txt target: first-choice-file.txt - name: second-choice + # if using a group, all members of the group must be optional + optional: true group: example-choice default: true files: From 7b1379975f6f55ad06db00d635ed574b5c23d9ab Mon Sep 17 00:00:00 2001 From: razzle Date: Tue, 2 Jan 2024 12:39:52 -0500 Subject: [PATCH 009/113] fix tests + examples Signed-off-by: razzle --- examples/config-file/zarf.yaml | 2 ++ src/test/packages/00-no-components/zarf.yaml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/examples/config-file/zarf.yaml b/examples/config-file/zarf.yaml index cee0eb1852..979e8963a3 100644 --- a/examples/config-file/zarf.yaml +++ b/examples/config-file/zarf.yaml @@ -18,8 +18,10 @@ components: - simple-configmap.yaml - name: zebra + optional: true # Notice this string is wrapped in quotes, as Zarf variables use #### as a delimiter, which is also used in YAML for comments description: "This is a zebra and they have ###ZARF_PKG_TMPL_ZEBRA###" - name: leopard + optional: true description: "This is a leopard and they have ###ZARF_PKG_TMPL_LEOPARD###" diff --git a/src/test/packages/00-no-components/zarf.yaml b/src/test/packages/00-no-components/zarf.yaml index 78d89b650f..33fd9a27b8 100644 --- a/src/test/packages/00-no-components/zarf.yaml +++ b/src/test/packages/00-no-components/zarf.yaml @@ -4,6 +4,8 @@ metadata: components: - name: deselect-me + optional: true default: true - name: optional + optional: true From 2ffd606fbe2519eddf6c7822d42e7fd51d50c48d Mon Sep 17 00:00:00 2001 From: razzle Date: Tue, 2 Jan 2024 13:42:18 -0500 Subject: [PATCH 010/113] initial migration code Signed-off-by: razzle --- src/pkg/packager/composer/list.go | 10 +-- src/pkg/packager/composer/override.go | 9 +-- src/pkg/packager/deprecated/common.go | 45 +++++++----- .../deprecated/pluralize-set-variable.go | 73 +++++++++++-------- .../deprecated/required_to_optional.go | 42 +++++++++++ .../packager/deprecated/scripts-to-actions.go | 54 +++++++++----- src/pkg/packager/lint/lint.go | 30 ++++---- src/pkg/packager/yaml.go | 2 +- 8 files changed, 166 insertions(+), 99 deletions(-) create mode 100644 src/pkg/packager/deprecated/required_to_optional.go diff --git a/src/pkg/packager/composer/list.go b/src/pkg/packager/composer/list.go index f46235f212..ac4ad8a5f1 100644 --- a/src/pkg/packager/composer/list.go +++ b/src/pkg/packager/composer/list.go @@ -34,17 +34,17 @@ type Node struct { next *Node } -// GetIndex returns the .components index location for this node's source `zarf.yaml` -func (n *Node) GetIndex() int { +// Index returns the .components index location for this node's source `zarf.yaml` +func (n *Node) Index() int { return n.index } -// GetOriginalPackageName returns the .metadata.name of the zarf package the component originated from -func (n *Node) GetOriginalPackageName() string { +// OriginalPackageName returns the .metadata.name for this node's source `zarf.yaml` +func (n *Node) OriginalPackageName() string { return n.originalPackageName } -// ImportLocation gets the path from the base zarf file to the imported zarf file +// ImportLocation gets the path from the base `zarf.yaml` to the imported `zarf.yaml` func (n *Node) ImportLocation() string { if n.prev != nil { if n.prev.ZarfComponent.Import.URL != "" { diff --git a/src/pkg/packager/composer/override.go b/src/pkg/packager/composer/override.go index bac1af94a5..18eebf7677 100644 --- a/src/pkg/packager/composer/override.go +++ b/src/pkg/packager/composer/override.go @@ -7,20 +7,13 @@ package composer import ( "fmt" - "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "github.com/defenseunicorns/zarf/src/types" ) func overrideMetadata(c *types.ZarfComponent, override types.ZarfComponent) error { c.Name = override.Name c.Default = override.Default - // Only override "optional" with the value from the override - // if "required" is neither nil, nor false. - if override.DeprecatedRequired != nil && override.DeprecatedRequired == helpers.BoolPtr(true) { - c.Optional = helpers.BoolPtr(false) - } else if override.Optional != nil { - c.Optional = override.Optional - } + c.Optional = override.Optional // Override description if it was provided. if override.Description != "" { diff --git a/src/pkg/packager/deprecated/common.go b/src/pkg/packager/deprecated/common.go index 2fdbfdcaf3..2c3688923f 100644 --- a/src/pkg/packager/deprecated/common.go +++ b/src/pkg/packager/deprecated/common.go @@ -27,9 +27,10 @@ type BreakingChange struct { // List of migrations tracked in the zarf.yaml build data. const ( // This should be updated when a breaking change is introduced to the Zarf package structure. See: https://github.com/defenseunicorns/zarf/releases/tag/v0.27.0 - LastNonBreakingVersion = "v0.27.0" - ScriptsToActionsMigrated = "scripts-to-actions" - PluralizeSetVariable = "pluralize-set-variable" + LastNonBreakingVersion = "v0.27.0" + ScriptsToActions = "scripts-to-actions" + PluralizeSetVariable = "pluralize-set-variable" + RequiredToOptional = "required-to-optional" ) // List of breaking changes to warn the user of. @@ -41,30 +42,34 @@ var breakingChanges = []BreakingChange{ }, } +type migration interface { + name() string + postbuild() types.ZarfComponent + migrate() (types.ZarfComponent, string) +} + // MigrateComponent runs all migrations on a component. // Build should be empty on package create, but include just in case someone copied a zarf.yaml from a zarf package. func MigrateComponent(build types.ZarfBuildData, component types.ZarfComponent) (migratedComponent types.ZarfComponent, warnings []string) { migratedComponent = component - // If the component has already been migrated, clear the deprecated scripts. - if slices.Contains(build.Migrations, ScriptsToActionsMigrated) { - migratedComponent.DeprecatedScripts = types.DeprecatedZarfComponentScripts{} - } else { - // Otherwise, run the migration. - var warning string - if migratedComponent, warning = migrateScriptsToActions(migratedComponent); warning != "" { - warnings = append(warnings, warning) - } + migrations := []migration{ + migrateScriptsToActions{migratedComponent}, + migrateSetVariableToSetVariables{migratedComponent}, + migrateRequiredToOptional{migratedComponent}, } - // If the component has already been migrated, clear the setVariable definitions. - if slices.Contains(build.Migrations, PluralizeSetVariable) { - migratedComponent = clearSetVariables(migratedComponent) - } else { - // Otherwise, run the migration. - var warning string - if migratedComponent, warning = migrateSetVariableToSetVariables(migratedComponent); warning != "" { - warnings = append(warnings, warning) + // Run all migrations + for _, m := range migrations { + // If the component has already been migrated, run the postbuild function. + if slices.Contains(build.Migrations, m.name()) { + migratedComponent = m.postbuild() + } else { + // Otherwise, run the migration. + var warning string + if migratedComponent, warning = m.migrate(); warning != "" { + warnings = append(warnings, warning) + } } } diff --git a/src/pkg/packager/deprecated/pluralize-set-variable.go b/src/pkg/packager/deprecated/pluralize-set-variable.go index 9835c80a64..f0674a1464 100644 --- a/src/pkg/packager/deprecated/pluralize-set-variable.go +++ b/src/pkg/packager/deprecated/pluralize-set-variable.go @@ -10,7 +10,48 @@ import ( "github.com/defenseunicorns/zarf/src/types" ) -func migrateSetVariableToSetVariables(c types.ZarfComponent) (types.ZarfComponent, string) { +type migrateSetVariableToSetVariables struct { + component types.ZarfComponent +} + +func (m migrateSetVariableToSetVariables) name() string { + return PluralizeSetVariable +} + +// If the component has already been migrated, clear the deprecated setVariable. +func (m migrateSetVariableToSetVariables) postbuild() types.ZarfComponent { + c := m.component + clear := func(actions []types.ZarfComponentAction) []types.ZarfComponentAction { + for i := range actions { + actions[i].DeprecatedSetVariable = "" + } + + return actions + } + + // Clear OnCreate SetVariables + c.Actions.OnCreate.After = clear(c.Actions.OnCreate.After) + c.Actions.OnCreate.Before = clear(c.Actions.OnCreate.Before) + c.Actions.OnCreate.OnSuccess = clear(c.Actions.OnCreate.OnSuccess) + c.Actions.OnCreate.OnFailure = clear(c.Actions.OnCreate.OnFailure) + + // Clear OnDeploy SetVariables + c.Actions.OnDeploy.After = clear(c.Actions.OnDeploy.After) + c.Actions.OnDeploy.Before = clear(c.Actions.OnDeploy.Before) + c.Actions.OnDeploy.OnSuccess = clear(c.Actions.OnDeploy.OnSuccess) + c.Actions.OnDeploy.OnFailure = clear(c.Actions.OnDeploy.OnFailure) + + // Clear OnRemove SetVariables + c.Actions.OnRemove.After = clear(c.Actions.OnRemove.After) + c.Actions.OnRemove.Before = clear(c.Actions.OnRemove.Before) + c.Actions.OnRemove.OnSuccess = clear(c.Actions.OnRemove.OnSuccess) + c.Actions.OnRemove.OnFailure = clear(c.Actions.OnRemove.OnFailure) + + return c +} + +func (m migrateSetVariableToSetVariables) migrate() (types.ZarfComponent, string) { + c := m.component hasSetVariable := false migrate := func(actions []types.ZarfComponentAction) []types.ZarfComponentAction { @@ -54,33 +95,3 @@ func migrateSetVariableToSetVariables(c types.ZarfComponent) (types.ZarfComponen return c, "" } - -func clearSetVariables(c types.ZarfComponent) types.ZarfComponent { - clear := func(actions []types.ZarfComponentAction) []types.ZarfComponentAction { - for i := range actions { - actions[i].DeprecatedSetVariable = "" - } - - return actions - } - - // Clear OnCreate SetVariables - c.Actions.OnCreate.After = clear(c.Actions.OnCreate.After) - c.Actions.OnCreate.Before = clear(c.Actions.OnCreate.Before) - c.Actions.OnCreate.OnSuccess = clear(c.Actions.OnCreate.OnSuccess) - c.Actions.OnCreate.OnFailure = clear(c.Actions.OnCreate.OnFailure) - - // Clear OnDeploy SetVariables - c.Actions.OnDeploy.After = clear(c.Actions.OnDeploy.After) - c.Actions.OnDeploy.Before = clear(c.Actions.OnDeploy.Before) - c.Actions.OnDeploy.OnSuccess = clear(c.Actions.OnDeploy.OnSuccess) - c.Actions.OnDeploy.OnFailure = clear(c.Actions.OnDeploy.OnFailure) - - // Clear OnRemove SetVariables - c.Actions.OnRemove.After = clear(c.Actions.OnRemove.After) - c.Actions.OnRemove.Before = clear(c.Actions.OnRemove.Before) - c.Actions.OnRemove.OnSuccess = clear(c.Actions.OnRemove.OnSuccess) - c.Actions.OnRemove.OnFailure = clear(c.Actions.OnRemove.OnFailure) - - return c -} diff --git a/src/pkg/packager/deprecated/required_to_optional.go b/src/pkg/packager/deprecated/required_to_optional.go new file mode 100644 index 0000000000..c89efd1a81 --- /dev/null +++ b/src/pkg/packager/deprecated/required_to_optional.go @@ -0,0 +1,42 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package deprecated handles package deprecations and migrations +package deprecated + +import ( + "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" + "github.com/defenseunicorns/zarf/src/types" +) + +type migrateRequiredToOptional struct { + component types.ZarfComponent +} + +func (m migrateRequiredToOptional) name() string { + return RequiredToOptional +} + +// If the component has already been migrated, clear the deprecated required. +func (m migrateRequiredToOptional) postbuild() types.ZarfComponent { + c := m.component + c.DeprecatedRequired = nil + return c +} + +// migrate converts the deprecated required to the new optional +func (m migrateRequiredToOptional) migrate() (types.ZarfComponent, string) { + c := m.component + if c.DeprecatedRequired == nil { + return c, "" + } + + switch *c.DeprecatedRequired { + case true: + c.Optional = nil + case false: + c.Optional = helpers.BoolPtr(true) + } + + return c, "" +} diff --git a/src/pkg/packager/deprecated/scripts-to-actions.go b/src/pkg/packager/deprecated/scripts-to-actions.go index 2040e7eb90..5e241e2873 100644 --- a/src/pkg/packager/deprecated/scripts-to-actions.go +++ b/src/pkg/packager/deprecated/scripts-to-actions.go @@ -11,60 +11,76 @@ import ( "github.com/defenseunicorns/zarf/src/types" ) -// migrateScriptsToActions coverts the deprecated scripts to the new actions +type migrateScriptsToActions struct { + component types.ZarfComponent +} + +func (m migrateScriptsToActions) name() string { + return ScriptsToActions +} + +// If the component has already been migrated, clear the deprecated scripts. +func (m migrateScriptsToActions) postbuild() types.ZarfComponent { + mc := m.component + mc.DeprecatedScripts = types.DeprecatedZarfComponentScripts{} + return mc +} + +// migrate coverts the deprecated scripts to the new actions // The following have no migration: // - Actions.Create.After // - Actions.Remove.* // - Actions.*.OnSuccess // - Actions.*.OnFailure // - Actions.*.*.Env -func migrateScriptsToActions(c types.ZarfComponent) (types.ZarfComponent, string) { +func (m migrateScriptsToActions) migrate() (types.ZarfComponent, string) { + mc := m.component var hasScripts bool // Convert a script configs to action defaults. defaults := types.ZarfComponentActionDefaults{ // ShowOutput (default false) -> Mute (default false) - Mute: !c.DeprecatedScripts.ShowOutput, + Mute: !mc.DeprecatedScripts.ShowOutput, // TimeoutSeconds -> MaxSeconds - MaxTotalSeconds: c.DeprecatedScripts.TimeoutSeconds, + MaxTotalSeconds: mc.DeprecatedScripts.TimeoutSeconds, } // Retry is now an integer vs a boolean (implicit infinite retries), so set to an absurdly high number - if c.DeprecatedScripts.Retry { + if mc.DeprecatedScripts.Retry { defaults.MaxRetries = math.MaxInt } // Scripts.Prepare -> Actions.Create.Before - if len(c.DeprecatedScripts.Prepare) > 0 { + if len(mc.DeprecatedScripts.Prepare) > 0 { hasScripts = true - c.Actions.OnCreate.Defaults = defaults - for _, s := range c.DeprecatedScripts.Prepare { - c.Actions.OnCreate.Before = append(c.Actions.OnCreate.Before, types.ZarfComponentAction{Cmd: s}) + mc.Actions.OnCreate.Defaults = defaults + for _, s := range mc.DeprecatedScripts.Prepare { + mc.Actions.OnCreate.Before = append(mc.Actions.OnCreate.Before, types.ZarfComponentAction{Cmd: s}) } } // Scripts.Before -> Actions.Deploy.Before - if len(c.DeprecatedScripts.Before) > 0 { + if len(mc.DeprecatedScripts.Before) > 0 { hasScripts = true - c.Actions.OnDeploy.Defaults = defaults - for _, s := range c.DeprecatedScripts.Before { - c.Actions.OnDeploy.Before = append(c.Actions.OnDeploy.Before, types.ZarfComponentAction{Cmd: s}) + mc.Actions.OnDeploy.Defaults = defaults + for _, s := range mc.DeprecatedScripts.Before { + mc.Actions.OnDeploy.Before = append(mc.Actions.OnDeploy.Before, types.ZarfComponentAction{Cmd: s}) } } // Scripts.After -> Actions.Deploy.After - if len(c.DeprecatedScripts.After) > 0 { + if len(mc.DeprecatedScripts.After) > 0 { hasScripts = true - c.Actions.OnDeploy.Defaults = defaults - for _, s := range c.DeprecatedScripts.After { - c.Actions.OnDeploy.After = append(c.Actions.OnDeploy.After, types.ZarfComponentAction{Cmd: s}) + mc.Actions.OnDeploy.Defaults = defaults + for _, s := range mc.DeprecatedScripts.After { + mc.Actions.OnDeploy.After = append(mc.Actions.OnDeploy.After, types.ZarfComponentAction{Cmd: s}) } } // Leave deprecated scripts in place, but warn users if hasScripts { - return c, fmt.Sprintf("Component '%s' is using scripts which will be removed in Zarf v1.0.0. Please migrate to actions.", c.Name) + return mc, fmt.Sprintf("Component '%s' is using scripts which will be removed in Zarf v1.0.0. Please migrate to actions.", mc.Name) } - return c, "" + return mc, "" } diff --git a/src/pkg/packager/lint/lint.go b/src/pkg/packager/lint/lint.go index bc8b5df336..890df31f6d 100644 --- a/src/pkg/packager/lint/lint.go +++ b/src/pkg/packager/lint/lint.go @@ -110,7 +110,7 @@ func fillComponentTemplate(validator *Validator, node *composer.Node, createOpts validator.addWarning(validatorMessage{ description: err.Error(), packageRelPath: node.ImportLocation(), - packageName: node.GetOriginalPackageName(), + packageName: node.OriginalPackageName(), }) } templateMap := map[string]string{} @@ -121,7 +121,7 @@ func fillComponentTemplate(validator *Validator, node *composer.Node, createOpts validator.addWarning(validatorMessage{ description: err.Error(), packageRelPath: node.ImportLocation(), - packageName: node.GetOriginalPackageName(), + packageName: node.OriginalPackageName(), }) } @@ -130,7 +130,7 @@ func fillComponentTemplate(validator *Validator, node *composer.Node, createOpts validator.addWarning(validatorMessage{ description: fmt.Sprintf(lang.PkgValidateTemplateDeprecation, key, key, key), packageRelPath: node.ImportLocation(), - packageName: node.GetOriginalPackageName(), + packageName: node.OriginalPackageName(), }) } _, present := createOpts.SetVariables[key] @@ -138,7 +138,7 @@ func fillComponentTemplate(validator *Validator, node *composer.Node, createOpts validator.addWarning(validatorMessage{ description: lang.UnsetVarLintWarning, packageRelPath: node.ImportLocation(), - packageName: node.GetOriginalPackageName(), + packageName: node.OriginalPackageName(), }) } } @@ -179,12 +179,12 @@ func lintComponent(validator *Validator, node *composer.Node) { func checkForUnpinnedRepos(validator *Validator, node *composer.Node) { for j, repo := range node.Repos { - repoYqPath := fmt.Sprintf(".components.[%d].repos.[%d]", node.GetIndex(), j) + repoYqPath := fmt.Sprintf(".components.[%d].repos.[%d]", node.Index(), j) if !isPinnedRepo(repo) { validator.addWarning(validatorMessage{ yqPath: repoYqPath, packageRelPath: node.ImportLocation(), - packageName: node.GetOriginalPackageName(), + packageName: node.OriginalPackageName(), description: "Unpinned repository", item: repo, }) @@ -194,13 +194,13 @@ func checkForUnpinnedRepos(validator *Validator, node *composer.Node) { func checkForUnpinnedImages(validator *Validator, node *composer.Node) { for j, image := range node.Images { - imageYqPath := fmt.Sprintf(".components.[%d].images.[%d]", node.GetIndex(), j) + imageYqPath := fmt.Sprintf(".components.[%d].images.[%d]", node.Index(), j) pinnedImage, err := isPinnedImage(image) if err != nil { validator.addError(validatorMessage{ yqPath: imageYqPath, packageRelPath: node.ImportLocation(), - packageName: node.GetOriginalPackageName(), + packageName: node.OriginalPackageName(), description: "Invalid image reference", item: image, }) @@ -210,7 +210,7 @@ func checkForUnpinnedImages(validator *Validator, node *composer.Node) { validator.addWarning(validatorMessage{ yqPath: imageYqPath, packageRelPath: node.ImportLocation(), - packageName: node.GetOriginalPackageName(), + packageName: node.OriginalPackageName(), description: "Image not pinned with digest", item: image, }) @@ -220,12 +220,12 @@ func checkForUnpinnedImages(validator *Validator, node *composer.Node) { func checkForUnpinnedFiles(validator *Validator, node *composer.Node) { for j, file := range node.Files { - fileYqPath := fmt.Sprintf(".components.[%d].files.[%d]", node.GetIndex(), j) + fileYqPath := fmt.Sprintf(".components.[%d].files.[%d]", node.Index(), j) if file.Shasum == "" && helpers.IsURL(file.Source) { validator.addWarning(validatorMessage{ yqPath: fileYqPath, packageRelPath: node.ImportLocation(), - packageName: node.GetOriginalPackageName(), + packageName: node.OriginalPackageName(), description: "No shasum for remote file", item: file.Source, }) @@ -236,18 +236,18 @@ func checkForUnpinnedFiles(validator *Validator, node *composer.Node) { func checkForVarInComponentImport(validator *Validator, node *composer.Node) { if strings.Contains(node.Import.Path, types.ZarfPackageTemplatePrefix) { validator.addWarning(validatorMessage{ - yqPath: fmt.Sprintf(".components.[%d].import.path", node.GetIndex()), + yqPath: fmt.Sprintf(".components.[%d].import.path", node.Index()), packageRelPath: node.ImportLocation(), - packageName: node.GetOriginalPackageName(), + packageName: node.OriginalPackageName(), description: "Zarf does not evaluate variables at component.x.import.path", item: node.Import.Path, }) } if strings.Contains(node.Import.URL, types.ZarfPackageTemplatePrefix) { validator.addWarning(validatorMessage{ - yqPath: fmt.Sprintf(".components.[%d].import.url", node.GetIndex()), + yqPath: fmt.Sprintf(".components.[%d].import.url", node.Index()), packageRelPath: node.ImportLocation(), - packageName: node.GetOriginalPackageName(), + packageName: node.OriginalPackageName(), description: "Zarf does not evaluate variables at component.x.import.url", item: node.Import.URL, }) diff --git a/src/pkg/packager/yaml.go b/src/pkg/packager/yaml.go index ba38a31bf1..3f4c6f2a31 100644 --- a/src/pkg/packager/yaml.go +++ b/src/pkg/packager/yaml.go @@ -107,7 +107,7 @@ func (p *Packager) writeYaml() error { // Record the migrations that will be run on the package. p.cfg.Pkg.Build.Migrations = []string{ - deprecated.ScriptsToActionsMigrated, + deprecated.ScriptsToActions, deprecated.PluralizeSetVariable, } From d1d725877a37a9fb6f7e2e58d70e1670b46861ea Mon Sep 17 00:00:00 2001 From: razzle Date: Tue, 2 Jan 2024 14:00:49 -0500 Subject: [PATCH 011/113] add warning to migrate Signed-off-by: razzle --- .../0-creating-a-zarf-package.md | 2 +- src/pkg/packager/deprecated/common.go | 4 +- .../deprecated/pluralize-set-variable.go | 30 +++++++-------- .../deprecated/required_to_optional.go | 13 ++++--- .../packager/deprecated/scripts-to-actions.go | 38 +++++++++---------- 5 files changed, 45 insertions(+), 42 deletions(-) diff --git a/docs/5-zarf-tutorials/0-creating-a-zarf-package.md b/docs/5-zarf-tutorials/0-creating-a-zarf-package.md index ec7921d1a5..3d677d5408 100644 --- a/docs/5-zarf-tutorials/0-creating-a-zarf-package.md +++ b/docs/5-zarf-tutorials/0-creating-a-zarf-package.md @@ -50,7 +50,7 @@ components: - name: wordpress # specifies the name of our component and should be unique and unchanging through updates description: | # (optional) a human-readable description of the component you are defining "Deploys the Bitnami-packaged WordPress chart into the cluster" - optional: true # (optional) sets the component as 'required' so that it is always deployed + optional: false # (optional) sets the component as 'required' so that it is always deployed charts: - name: wordpress url: oci://registry-1.docker.io/bitnamicharts/wordpress diff --git a/src/pkg/packager/deprecated/common.go b/src/pkg/packager/deprecated/common.go index 2c3688923f..55afd1159a 100644 --- a/src/pkg/packager/deprecated/common.go +++ b/src/pkg/packager/deprecated/common.go @@ -44,7 +44,7 @@ var breakingChanges = []BreakingChange{ type migration interface { name() string - postbuild() types.ZarfComponent + clear() types.ZarfComponent migrate() (types.ZarfComponent, string) } @@ -63,7 +63,7 @@ func MigrateComponent(build types.ZarfBuildData, component types.ZarfComponent) for _, m := range migrations { // If the component has already been migrated, run the postbuild function. if slices.Contains(build.Migrations, m.name()) { - migratedComponent = m.postbuild() + migratedComponent = m.clear() } else { // Otherwise, run the migration. var warning string diff --git a/src/pkg/packager/deprecated/pluralize-set-variable.go b/src/pkg/packager/deprecated/pluralize-set-variable.go index f0674a1464..03a8a322a9 100644 --- a/src/pkg/packager/deprecated/pluralize-set-variable.go +++ b/src/pkg/packager/deprecated/pluralize-set-variable.go @@ -19,8 +19,8 @@ func (m migrateSetVariableToSetVariables) name() string { } // If the component has already been migrated, clear the deprecated setVariable. -func (m migrateSetVariableToSetVariables) postbuild() types.ZarfComponent { - c := m.component +func (m migrateSetVariableToSetVariables) clear() types.ZarfComponent { + mc := m.component clear := func(actions []types.ZarfComponentAction) []types.ZarfComponentAction { for i := range actions { actions[i].DeprecatedSetVariable = "" @@ -30,24 +30,24 @@ func (m migrateSetVariableToSetVariables) postbuild() types.ZarfComponent { } // Clear OnCreate SetVariables - c.Actions.OnCreate.After = clear(c.Actions.OnCreate.After) - c.Actions.OnCreate.Before = clear(c.Actions.OnCreate.Before) - c.Actions.OnCreate.OnSuccess = clear(c.Actions.OnCreate.OnSuccess) - c.Actions.OnCreate.OnFailure = clear(c.Actions.OnCreate.OnFailure) + mc.Actions.OnCreate.After = clear(mc.Actions.OnCreate.After) + mc.Actions.OnCreate.Before = clear(mc.Actions.OnCreate.Before) + mc.Actions.OnCreate.OnSuccess = clear(mc.Actions.OnCreate.OnSuccess) + mc.Actions.OnCreate.OnFailure = clear(mc.Actions.OnCreate.OnFailure) // Clear OnDeploy SetVariables - c.Actions.OnDeploy.After = clear(c.Actions.OnDeploy.After) - c.Actions.OnDeploy.Before = clear(c.Actions.OnDeploy.Before) - c.Actions.OnDeploy.OnSuccess = clear(c.Actions.OnDeploy.OnSuccess) - c.Actions.OnDeploy.OnFailure = clear(c.Actions.OnDeploy.OnFailure) + mc.Actions.OnDeploy.After = clear(mc.Actions.OnDeploy.After) + mc.Actions.OnDeploy.Before = clear(mc.Actions.OnDeploy.Before) + mc.Actions.OnDeploy.OnSuccess = clear(mc.Actions.OnDeploy.OnSuccess) + mc.Actions.OnDeploy.OnFailure = clear(mc.Actions.OnDeploy.OnFailure) // Clear OnRemove SetVariables - c.Actions.OnRemove.After = clear(c.Actions.OnRemove.After) - c.Actions.OnRemove.Before = clear(c.Actions.OnRemove.Before) - c.Actions.OnRemove.OnSuccess = clear(c.Actions.OnRemove.OnSuccess) - c.Actions.OnRemove.OnFailure = clear(c.Actions.OnRemove.OnFailure) + mc.Actions.OnRemove.After = clear(mc.Actions.OnRemove.After) + mc.Actions.OnRemove.Before = clear(mc.Actions.OnRemove.Before) + mc.Actions.OnRemove.OnSuccess = clear(mc.Actions.OnRemove.OnSuccess) + mc.Actions.OnRemove.OnFailure = clear(mc.Actions.OnRemove.OnFailure) - return c + return mc } func (m migrateSetVariableToSetVariables) migrate() (types.ZarfComponent, string) { diff --git a/src/pkg/packager/deprecated/required_to_optional.go b/src/pkg/packager/deprecated/required_to_optional.go index c89efd1a81..5e559d7f42 100644 --- a/src/pkg/packager/deprecated/required_to_optional.go +++ b/src/pkg/packager/deprecated/required_to_optional.go @@ -5,6 +5,8 @@ package deprecated import ( + "fmt" + "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "github.com/defenseunicorns/zarf/src/types" ) @@ -18,15 +20,16 @@ func (m migrateRequiredToOptional) name() string { } // If the component has already been migrated, clear the deprecated required. -func (m migrateRequiredToOptional) postbuild() types.ZarfComponent { - c := m.component - c.DeprecatedRequired = nil - return c +func (m migrateRequiredToOptional) clear() types.ZarfComponent { + mc := m.component + mc.DeprecatedRequired = nil + return mc } // migrate converts the deprecated required to the new optional func (m migrateRequiredToOptional) migrate() (types.ZarfComponent, string) { c := m.component + if c.DeprecatedRequired == nil { return c, "" } @@ -38,5 +41,5 @@ func (m migrateRequiredToOptional) migrate() (types.ZarfComponent, string) { c.Optional = helpers.BoolPtr(true) } - return c, "" + return c, fmt.Sprintf("Component %q is using \"required\" which will be removed in Zarf v1.0.0. Please migrate to \"optional\". Please migrate to \"optional\".", c.Name) } diff --git a/src/pkg/packager/deprecated/scripts-to-actions.go b/src/pkg/packager/deprecated/scripts-to-actions.go index 5e241e2873..632631c7a9 100644 --- a/src/pkg/packager/deprecated/scripts-to-actions.go +++ b/src/pkg/packager/deprecated/scripts-to-actions.go @@ -20,7 +20,7 @@ func (m migrateScriptsToActions) name() string { } // If the component has already been migrated, clear the deprecated scripts. -func (m migrateScriptsToActions) postbuild() types.ZarfComponent { +func (m migrateScriptsToActions) clear() types.ZarfComponent { mc := m.component mc.DeprecatedScripts = types.DeprecatedZarfComponentScripts{} return mc @@ -34,53 +34,53 @@ func (m migrateScriptsToActions) postbuild() types.ZarfComponent { // - Actions.*.OnFailure // - Actions.*.*.Env func (m migrateScriptsToActions) migrate() (types.ZarfComponent, string) { - mc := m.component + c := m.component var hasScripts bool // Convert a script configs to action defaults. defaults := types.ZarfComponentActionDefaults{ // ShowOutput (default false) -> Mute (default false) - Mute: !mc.DeprecatedScripts.ShowOutput, + Mute: !c.DeprecatedScripts.ShowOutput, // TimeoutSeconds -> MaxSeconds - MaxTotalSeconds: mc.DeprecatedScripts.TimeoutSeconds, + MaxTotalSeconds: c.DeprecatedScripts.TimeoutSeconds, } // Retry is now an integer vs a boolean (implicit infinite retries), so set to an absurdly high number - if mc.DeprecatedScripts.Retry { + if c.DeprecatedScripts.Retry { defaults.MaxRetries = math.MaxInt } // Scripts.Prepare -> Actions.Create.Before - if len(mc.DeprecatedScripts.Prepare) > 0 { + if len(c.DeprecatedScripts.Prepare) > 0 { hasScripts = true - mc.Actions.OnCreate.Defaults = defaults - for _, s := range mc.DeprecatedScripts.Prepare { - mc.Actions.OnCreate.Before = append(mc.Actions.OnCreate.Before, types.ZarfComponentAction{Cmd: s}) + c.Actions.OnCreate.Defaults = defaults + for _, s := range c.DeprecatedScripts.Prepare { + c.Actions.OnCreate.Before = append(c.Actions.OnCreate.Before, types.ZarfComponentAction{Cmd: s}) } } // Scripts.Before -> Actions.Deploy.Before - if len(mc.DeprecatedScripts.Before) > 0 { + if len(c.DeprecatedScripts.Before) > 0 { hasScripts = true - mc.Actions.OnDeploy.Defaults = defaults - for _, s := range mc.DeprecatedScripts.Before { - mc.Actions.OnDeploy.Before = append(mc.Actions.OnDeploy.Before, types.ZarfComponentAction{Cmd: s}) + c.Actions.OnDeploy.Defaults = defaults + for _, s := range c.DeprecatedScripts.Before { + c.Actions.OnDeploy.Before = append(c.Actions.OnDeploy.Before, types.ZarfComponentAction{Cmd: s}) } } // Scripts.After -> Actions.Deploy.After - if len(mc.DeprecatedScripts.After) > 0 { + if len(c.DeprecatedScripts.After) > 0 { hasScripts = true - mc.Actions.OnDeploy.Defaults = defaults - for _, s := range mc.DeprecatedScripts.After { - mc.Actions.OnDeploy.After = append(mc.Actions.OnDeploy.After, types.ZarfComponentAction{Cmd: s}) + c.Actions.OnDeploy.Defaults = defaults + for _, s := range c.DeprecatedScripts.After { + c.Actions.OnDeploy.After = append(c.Actions.OnDeploy.After, types.ZarfComponentAction{Cmd: s}) } } // Leave deprecated scripts in place, but warn users if hasScripts { - return mc, fmt.Sprintf("Component '%s' is using scripts which will be removed in Zarf v1.0.0. Please migrate to actions.", mc.Name) + return c, fmt.Sprintf("Component '%s' is using scripts which will be removed in Zarf v1.0.0. Please migrate to actions.", c.Name) } - return mc, "" + return c, "" } From d05c5154d29c13ee4180245ec00c0220b70c8343 Mon Sep 17 00:00:00 2001 From: razzle Date: Tue, 2 Jan 2024 14:15:40 -0500 Subject: [PATCH 012/113] cleanup Signed-off-by: razzle --- src/pkg/packager/yaml.go | 2 ++ src/pkg/utils/helpers/misc_test.go | 10 ++++++ .../packages/25-local-tgz-chart/zarf.yaml | 1 + src/types/component_test.go | 35 +++++++++---------- 4 files changed, 30 insertions(+), 18 deletions(-) diff --git a/src/pkg/packager/yaml.go b/src/pkg/packager/yaml.go index 3f4c6f2a31..aeadcd10e6 100644 --- a/src/pkg/packager/yaml.go +++ b/src/pkg/packager/yaml.go @@ -106,9 +106,11 @@ func (p *Packager) writeYaml() error { } // Record the migrations that will be run on the package. + // TODO: (@razzle) Think about a better way to keep track of migrations. p.cfg.Pkg.Build.Migrations = []string{ deprecated.ScriptsToActions, deprecated.PluralizeSetVariable, + deprecated.RequiredToOptional, } // Record the flavor of Zarf used to build this package (if any). diff --git a/src/pkg/utils/helpers/misc_test.go b/src/pkg/utils/helpers/misc_test.go index 74e42c40c5..80eaf1728c 100644 --- a/src/pkg/utils/helpers/misc_test.go +++ b/src/pkg/utils/helpers/misc_test.go @@ -187,6 +187,16 @@ func (suite *TestMiscSuite) Test_6_MergeNonZero() { suite.Equal("world", result.field3) } +func (suite *TestMiscSuite) TestBoolPtr() { + suite.Equal(true, *BoolPtr(true)) + suite.Equal(false, *BoolPtr(false)) + a := BoolPtr(true) + b := BoolPtr(true) + // This is a pointer comparison, not a value comparison + suite.False(a == b) + suite.True(*a == *b) +} + func TestMisc(t *testing.T) { suite.Run(t, new(TestMiscSuite)) } diff --git a/src/test/packages/25-local-tgz-chart/zarf.yaml b/src/test/packages/25-local-tgz-chart/zarf.yaml index a281174481..6dc3eabd62 100644 --- a/src/test/packages/25-local-tgz-chart/zarf.yaml +++ b/src/test/packages/25-local-tgz-chart/zarf.yaml @@ -6,6 +6,7 @@ metadata: components: - name: demo-helm-local-chart + optional: true charts: - name: podinfo version: 6.4.0 diff --git a/src/types/component_test.go b/src/types/component_test.go index d3e376e0d4..4961d18d4d 100644 --- a/src/types/component_test.go +++ b/src/types/component_test.go @@ -4,7 +4,11 @@ // Package types contains all the types used by Zarf. package types -import "testing" +import ( + "testing" + + "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" +) func TestZarfComponent_IsRequired(t *testing.T) { tests := []struct { @@ -15,7 +19,7 @@ func TestZarfComponent_IsRequired(t *testing.T) { { name: "Test when DeprecatedRequired is true and Optional is nil", component: ZarfComponent{ - DeprecatedRequired: BoolPtr(true), + DeprecatedRequired: helpers.BoolPtr(true), Optional: nil, }, want: true, @@ -23,16 +27,16 @@ func TestZarfComponent_IsRequired(t *testing.T) { { name: "Test when DeprecatedRequired is true and Optional is false", component: ZarfComponent{ - DeprecatedRequired: BoolPtr(true), - Optional: BoolPtr(false), + DeprecatedRequired: helpers.BoolPtr(true), + Optional: helpers.BoolPtr(false), }, want: true, }, { name: "Test when DeprecatedRequired is true and Optional is true", component: ZarfComponent{ - DeprecatedRequired: BoolPtr(true), - Optional: BoolPtr(true), + DeprecatedRequired: helpers.BoolPtr(true), + Optional: helpers.BoolPtr(true), }, // optional "wins" when defined want: false, @@ -40,7 +44,7 @@ func TestZarfComponent_IsRequired(t *testing.T) { { name: "Test when DeprecatedRequired is false and Optional is nil", component: ZarfComponent{ - DeprecatedRequired: BoolPtr(false), + DeprecatedRequired: helpers.BoolPtr(false), Optional: nil, }, want: false, @@ -48,8 +52,8 @@ func TestZarfComponent_IsRequired(t *testing.T) { { name: "Test when DeprecatedRequired is false and Optional is false", component: ZarfComponent{ - DeprecatedRequired: BoolPtr(false), - Optional: BoolPtr(false), + DeprecatedRequired: helpers.BoolPtr(false), + Optional: helpers.BoolPtr(false), }, // optional "wins" when defined want: true, @@ -57,8 +61,8 @@ func TestZarfComponent_IsRequired(t *testing.T) { { name: "Test when DeprecatedRequired is false and Optional is true", component: ZarfComponent{ - DeprecatedRequired: BoolPtr(false), - Optional: BoolPtr(true), + DeprecatedRequired: helpers.BoolPtr(false), + Optional: helpers.BoolPtr(true), }, // optional "wins" when defined want: false, @@ -76,7 +80,7 @@ func TestZarfComponent_IsRequired(t *testing.T) { name: "Test when DeprecatedRequired is nil and Optional is false", component: ZarfComponent{ DeprecatedRequired: nil, - Optional: BoolPtr(false), + Optional: helpers.BoolPtr(false), }, // optional "wins" when defined want: true, @@ -85,7 +89,7 @@ func TestZarfComponent_IsRequired(t *testing.T) { name: "Test when DeprecatedRequired is nil and Optional is true", component: ZarfComponent{ DeprecatedRequired: nil, - Optional: BoolPtr(true), + Optional: helpers.BoolPtr(true), }, // optional "wins" when defined want: false, @@ -100,8 +104,3 @@ func TestZarfComponent_IsRequired(t *testing.T) { }) } } - -// BoolPtr is a helper function for creating a pointer to a bool. -func BoolPtr(b bool) *bool { - return &b -} From 36bfdf4411663ad975c50631f8a53bbfed6985b9 Mon Sep 17 00:00:00 2001 From: razzle Date: Tue, 2 Jan 2024 14:17:38 -0500 Subject: [PATCH 013/113] cleanup Signed-off-by: razzle --- src/pkg/utils/helpers/misc_test.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/pkg/utils/helpers/misc_test.go b/src/pkg/utils/helpers/misc_test.go index 80eaf1728c..4942cdab64 100644 --- a/src/pkg/utils/helpers/misc_test.go +++ b/src/pkg/utils/helpers/misc_test.go @@ -46,7 +46,7 @@ func (suite *TestMiscSuite) SetupSuite() { } } -func (suite *TestMiscSuite) Test_0_Retry() { +func (suite *TestMiscSuite) TestRetry() { var count int countFn := func() error { count++ @@ -75,7 +75,7 @@ func (suite *TestMiscSuite) Test_0_Retry() { suite.Equal(3, logCount) } -func (suite *TestMiscSuite) Test_1_MergeMap() { +func (suite *TestMiscSuite) TestMergeMap() { expected := map[string]interface{}{ "different": "value", "hello": "it's me", @@ -90,7 +90,7 @@ func (suite *TestMiscSuite) Test_1_MergeMap() { suite.Equal(expected, result) } -func (suite *TestMiscSuite) Test_2_TransformMapKeys() { +func (suite *TestMiscSuite) TestTransformMapKeys() { expected := map[string]interface{}{ "HELLO": "world", "UNIQUE": "value", @@ -104,7 +104,7 @@ func (suite *TestMiscSuite) Test_2_TransformMapKeys() { suite.Equal(expected, result) } -func (suite *TestMiscSuite) Test_3_TransformAndMergeMap() { +func (suite *TestMiscSuite) TestTransformAndMergeMap() { expected := map[string]interface{}{ "DIFFERENT": "value", "HELLO": "it's me", @@ -119,7 +119,7 @@ func (suite *TestMiscSuite) Test_3_TransformAndMergeMap() { suite.Equal(expected, result) } -func (suite *TestMiscSuite) Test_4_MergeMapRecursive() { +func (suite *TestMiscSuite) TestMergeMapRecursive() { expected := map[string]interface{}{ "different": "value", "hello": "it's me", @@ -135,7 +135,7 @@ func (suite *TestMiscSuite) Test_4_MergeMapRecursive() { suite.Equal(expected, result) } -func (suite *TestMiscSuite) Test_5_IsNotZeroAndNotEqual() { +func (suite *TestMiscSuite) TestIsNotZeroAndNotEqual() { original := TestMiscStruct{ Field1: "hello", Field2: 100, @@ -159,7 +159,7 @@ func (suite *TestMiscSuite) Test_5_IsNotZeroAndNotEqual() { suite.Equal(true, result) } -func (suite *TestMiscSuite) Test_6_MergeNonZero() { +func (suite *TestMiscSuite) TestMergeNonZero() { original := TestMiscStruct{ Field1: "hello", Field2: 100, From 59e769f62576cc3fcf44d0f58e37891834fc914f Mon Sep 17 00:00:00 2001 From: razzle Date: Tue, 2 Jan 2024 16:57:13 -0500 Subject: [PATCH 014/113] fix example Signed-off-by: razzle --- examples/component-actions/zarf.yaml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/examples/component-actions/zarf.yaml b/examples/component-actions/zarf.yaml index b3a2d5a53f..92184ec5f6 100644 --- a/examples/component-actions/zarf.yaml +++ b/examples/component-actions/zarf.yaml @@ -10,6 +10,7 @@ variables: # See the example README.md in this folder or /adrs/0010-scripts-actions.md for more info. components: - name: on-create + optional: true actions: # runs during "zarf package create" onCreate: @@ -63,6 +64,7 @@ components: sleep 0.5 - name: on-deploy-and-remove + optional: true actions: # runs during "zarf package deploy" onDeploy: @@ -82,6 +84,7 @@ components: - cmd: rm test-deploy-after.txt - name: on-deploy-with-variable + optional: true actions: # runs during "zarf package deploy" onDeploy: @@ -90,6 +93,7 @@ components: - cmd: echo "the dog says ${ZARF_VAR_DOG_SOUND}" - name: on-deploy-with-dynamic-variable + optional: true actions: # runs during "zarf package deploy" onDeploy: @@ -104,6 +108,7 @@ components: - cmd: echo "the cat says ${ZARF_VAR_CAT_SOUND}" - name: on-deploy-with-multiple-variables + optional: true actions: # runs during "zarf package deploy" onDeploy: @@ -133,6 +138,7 @@ components: - cmd: echo "with a TF_VAR, the snake also says ${TF_VAR_snake_sound}" - name: on-deploy-with-template-use-of-variable + optional: true files: # this file will be copied to the target location and the cat, dog, and snake sounds will be replaced with their values # requires the on-deploy-with-dynamic-variable and on-deploy-with-multiple-variables components @@ -141,6 +147,7 @@ components: shasum: 3c0404e0c767ace905c361fadded6c4b91fdb88aa07d5c42d2a220a87564836d - name: on-deploy-with-timeout + optional: true description: This component will fail after 1 second actions: # runs during "zarf package deploy" @@ -156,6 +163,7 @@ components: - cmd: echo "😭😭😭 this action failed because it took too long to run 😭😭😭" - name: on-remove + optional: true # A manifest that we expect to be removed by Zarf manifests: - name: test-configmap @@ -172,6 +180,7 @@ components: - cmd: ./zarf tools kubectl get configmap -n zarf remove-test-configmap || echo "Not Found" - name: on-deploy-with-env-var + optional: true actions: onDeploy: before: @@ -181,6 +190,7 @@ components: - ZARF_VAR_TEST_FILENAME=test-filename-from-env.txt - name: on-create-with-network-wait-action + optional: true description: This component will wait for 15 seconds for a network resource to be available actions: onCreate: @@ -195,6 +205,7 @@ components: code: 200 - name: on-deploy-with-wait-action + optional: true description: This component will wait for 5 seconds for the test-configmap to be exist manifests: - name: test-configmap @@ -213,6 +224,7 @@ components: namespace: zarf - name: on-deploy-immediate-failure + optional: true description: This component will fail on the first error instead of continuing execution # the default for multi-line commands is set -e actions: From e88fa8277bf1ef802c7313e747ebd505c53529f2 Mon Sep 17 00:00:00 2001 From: razzle Date: Tue, 2 Jan 2024 17:47:49 -0500 Subject: [PATCH 015/113] fix migrations overwriting mutations Signed-off-by: razzle --- src/pkg/packager/composer/list.go | 2 +- src/pkg/packager/deprecated/common.go | 15 +++++++-------- .../packager/deprecated/pluralize-set-variable.go | 10 +++------- .../packager/deprecated/required_to_optional.go | 13 ++++--------- src/pkg/packager/deprecated/scripts-to-actions.go | 12 ++++-------- .../03-deprecated-component-scripts/zarf.yaml | 3 +++ 6 files changed, 22 insertions(+), 33 deletions(-) diff --git a/src/pkg/packager/composer/list.go b/src/pkg/packager/composer/list.go index ac4ad8a5f1..629712261a 100644 --- a/src/pkg/packager/composer/list.go +++ b/src/pkg/packager/composer/list.go @@ -264,7 +264,7 @@ func (ic *ImportChain) Migrate(build types.ZarfBuildData) (warnings []string) { node = node.next } if len(warnings) > 0 { - final := fmt.Sprintf("migrations were performed on the import chain of: %q", ic.head.Name) + final := fmt.Sprintf("Migrations were performed on the import chain of: %q", ic.head.Name) warnings = append(warnings, final) } return warnings diff --git a/src/pkg/packager/deprecated/common.go b/src/pkg/packager/deprecated/common.go index 55afd1159a..7f5d3da992 100644 --- a/src/pkg/packager/deprecated/common.go +++ b/src/pkg/packager/deprecated/common.go @@ -44,8 +44,8 @@ var breakingChanges = []BreakingChange{ type migration interface { name() string - clear() types.ZarfComponent - migrate() (types.ZarfComponent, string) + clear(mc types.ZarfComponent) types.ZarfComponent + run(c types.ZarfComponent) (types.ZarfComponent, string) } // MigrateComponent runs all migrations on a component. @@ -54,20 +54,19 @@ func MigrateComponent(build types.ZarfBuildData, component types.ZarfComponent) migratedComponent = component migrations := []migration{ - migrateScriptsToActions{migratedComponent}, - migrateSetVariableToSetVariables{migratedComponent}, - migrateRequiredToOptional{migratedComponent}, + migrateScriptsToActions{}, + migrateSetVariableToSetVariables{}, + migrateRequiredToOptional{}, } // Run all migrations for _, m := range migrations { - // If the component has already been migrated, run the postbuild function. if slices.Contains(build.Migrations, m.name()) { - migratedComponent = m.clear() + migratedComponent = m.clear(migratedComponent) } else { // Otherwise, run the migration. var warning string - if migratedComponent, warning = m.migrate(); warning != "" { + if migratedComponent, warning = m.run(migratedComponent); warning != "" { warnings = append(warnings, warning) } } diff --git a/src/pkg/packager/deprecated/pluralize-set-variable.go b/src/pkg/packager/deprecated/pluralize-set-variable.go index 03a8a322a9..98e4c61577 100644 --- a/src/pkg/packager/deprecated/pluralize-set-variable.go +++ b/src/pkg/packager/deprecated/pluralize-set-variable.go @@ -10,17 +10,14 @@ import ( "github.com/defenseunicorns/zarf/src/types" ) -type migrateSetVariableToSetVariables struct { - component types.ZarfComponent -} +type migrateSetVariableToSetVariables struct{} func (m migrateSetVariableToSetVariables) name() string { return PluralizeSetVariable } // If the component has already been migrated, clear the deprecated setVariable. -func (m migrateSetVariableToSetVariables) clear() types.ZarfComponent { - mc := m.component +func (m migrateSetVariableToSetVariables) clear(mc types.ZarfComponent) types.ZarfComponent { clear := func(actions []types.ZarfComponentAction) []types.ZarfComponentAction { for i := range actions { actions[i].DeprecatedSetVariable = "" @@ -50,8 +47,7 @@ func (m migrateSetVariableToSetVariables) clear() types.ZarfComponent { return mc } -func (m migrateSetVariableToSetVariables) migrate() (types.ZarfComponent, string) { - c := m.component +func (m migrateSetVariableToSetVariables) run(c types.ZarfComponent) (types.ZarfComponent, string) { hasSetVariable := false migrate := func(actions []types.ZarfComponentAction) []types.ZarfComponentAction { diff --git a/src/pkg/packager/deprecated/required_to_optional.go b/src/pkg/packager/deprecated/required_to_optional.go index 5e559d7f42..c408295832 100644 --- a/src/pkg/packager/deprecated/required_to_optional.go +++ b/src/pkg/packager/deprecated/required_to_optional.go @@ -11,25 +11,20 @@ import ( "github.com/defenseunicorns/zarf/src/types" ) -type migrateRequiredToOptional struct { - component types.ZarfComponent -} +type migrateRequiredToOptional struct{} func (m migrateRequiredToOptional) name() string { return RequiredToOptional } // If the component has already been migrated, clear the deprecated required. -func (m migrateRequiredToOptional) clear() types.ZarfComponent { - mc := m.component +func (m migrateRequiredToOptional) clear(mc types.ZarfComponent) types.ZarfComponent { mc.DeprecatedRequired = nil return mc } -// migrate converts the deprecated required to the new optional -func (m migrateRequiredToOptional) migrate() (types.ZarfComponent, string) { - c := m.component - +// run converts the deprecated required to the new optional +func (m migrateRequiredToOptional) run(c types.ZarfComponent) (types.ZarfComponent, string) { if c.DeprecatedRequired == nil { return c, "" } diff --git a/src/pkg/packager/deprecated/scripts-to-actions.go b/src/pkg/packager/deprecated/scripts-to-actions.go index 632631c7a9..903155ed80 100644 --- a/src/pkg/packager/deprecated/scripts-to-actions.go +++ b/src/pkg/packager/deprecated/scripts-to-actions.go @@ -11,30 +11,26 @@ import ( "github.com/defenseunicorns/zarf/src/types" ) -type migrateScriptsToActions struct { - component types.ZarfComponent -} +type migrateScriptsToActions struct{} func (m migrateScriptsToActions) name() string { return ScriptsToActions } // If the component has already been migrated, clear the deprecated scripts. -func (m migrateScriptsToActions) clear() types.ZarfComponent { - mc := m.component +func (m migrateScriptsToActions) clear(mc types.ZarfComponent) types.ZarfComponent { mc.DeprecatedScripts = types.DeprecatedZarfComponentScripts{} return mc } -// migrate coverts the deprecated scripts to the new actions +// run coverts the deprecated scripts to the new actions // The following have no migration: // - Actions.Create.After // - Actions.Remove.* // - Actions.*.OnSuccess // - Actions.*.OnFailure // - Actions.*.*.Env -func (m migrateScriptsToActions) migrate() (types.ZarfComponent, string) { - c := m.component +func (m migrateScriptsToActions) run(c types.ZarfComponent) (types.ZarfComponent, string) { var hasScripts bool // Convert a script configs to action defaults. diff --git a/src/test/packages/03-deprecated-component-scripts/zarf.yaml b/src/test/packages/03-deprecated-component-scripts/zarf.yaml index 57a7c8f838..e7845b5a3e 100644 --- a/src/test/packages/03-deprecated-component-scripts/zarf.yaml +++ b/src/test/packages/03-deprecated-component-scripts/zarf.yaml @@ -6,6 +6,7 @@ metadata: components: # Test that prepare scripts become onCreate actions - name: 1-test-deprecated-prepare-scripts + optional: true scripts: prepare: # on Windows, touch is replaced with New-Item @@ -13,6 +14,7 @@ components: # Test that deploy scripts become onCreate actions - name: 2-test-deprecated-deploy-scripts + optional: true scripts: before: - touch test-deprecated-deploy-before-hook.txt @@ -21,6 +23,7 @@ components: # Test that script timeouts still get set - name: 3-test-deprecated-timeout-scripts + optional: true scripts: timeoutSeconds: 1 before: From 0246af6144744c6ed3fb954001a2d915df22cdc5 Mon Sep 17 00:00:00 2001 From: razzle Date: Tue, 2 Jan 2024 19:30:16 -0500 Subject: [PATCH 016/113] upgrade flake Signed-off-by: razzle --- .github/workflows/test-upgrade.yml | 8 +++----- Makefile | 2 +- src/test/upgrade/previously_built_test.go | 6 ++++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/test-upgrade.yml b/.github/workflows/test-upgrade.yml index b5965f9bfc..ab08ddb10a 100644 --- a/.github/workflows/test-upgrade.yml +++ b/.github/workflows/test-upgrade.yml @@ -69,7 +69,7 @@ jobs: - name: Initialize the cluster with the release version # NOTE: "PATH=$PATH" preserves the default user $PATH. This is needed to maintain the version of zarf installed - # in a previous step. This test run will the current release to create a K3s cluster. + # in a previous step. This test run will use the current release to create a K3s cluster. run: | sudo env "PATH=$PATH" CI=true zarf init --components k3s,git-server,logging --nodeport 31337 --confirm @@ -79,7 +79,7 @@ jobs: - name: Create and deploy the upgrade test packages # NOTE: "PATH=$PATH" preserves the default user $PATH. This is needed to maintain the version of zarf installed - # in a previous step. This test run will the current release to create a K3s cluster. + # in a previous step. This test run will use the current release to create a K3s cluster. run: | zarf package create src/test/upgrade --set PODINFO_VERSION=6.3.3 --confirm sudo env "PATH=$PATH" CI=true zarf package deploy zarf-package-test-upgrade-package-amd64-6.3.3.tar.zst --confirm @@ -105,12 +105,10 @@ jobs: - name: Run the upgrade tests # NOTE: "PATH=$PATH" preserves the default user $PATH. This is needed to maintain the version of zarf installed - # in a previous step. This test run will the current release to create a K3s cluster. + # in a previous step. This test run will use the current release to create a K3s cluster. run: | sudo env "PATH=$PATH" CI=true zarf tools kubectl describe nodes - zarf package create src/test/upgrade --set PODINFO_VERSION=6.3.4 --confirm - sudo env "PATH=$PATH" CI=true make test-upgrade ARCH=amd64 - name: Save logs diff --git a/Makefile b/Makefile index 2df92ab4b8..821f459d61 100644 --- a/Makefile +++ b/Makefile @@ -193,7 +193,7 @@ test-upgrade: ## Run the Zarf CLI E2E tests for an external registry and cluster @test -s $(ZARF_BIN) || $(MAKE) build-cli [ -n "$(shell zarf version)" ] || (echo "Zarf must be installed prior to the upgrade test" && exit 1) [ -n "$(shell zarf package list 2>&1 | grep test-upgrade-package)" ] || (echo "Zarf must be initialized and have the 6.3.3 upgrade-test package installed prior to the upgrade test" && exit 1) - @test -s "zarf-package-test-upgrade-package-amd64-6.3.4.tar.zst" || zarf package create src/test/upgrade/ --set PODINFO_VERSION=6.3.4 --confirm + @test -s ./zarf-package-test-upgrade-package-$(ARCH)-6.3.4.tar.zst || zarf package create src/test/upgrade/ --set PODINFO_VERSION=6.3.4 --confirm cd src/test/upgrade && go test -failfast -v -timeout 30m .PHONY: test-unit diff --git a/src/test/upgrade/previously_built_test.go b/src/test/upgrade/previously_built_test.go index 179bad0432..7cd98c3644 100644 --- a/src/test/upgrade/previously_built_test.go +++ b/src/test/upgrade/previously_built_test.go @@ -6,9 +6,11 @@ package upgrade import ( "context" + "fmt" "path" "testing" + "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/pkg/utils/exec" test "github.com/defenseunicorns/zarf/src/test" "github.com/stretchr/testify/require" @@ -45,7 +47,7 @@ func TestPreviouslyBuiltZarfPackage(t *testing.T) { require.Equal(t, zarfGitServerSecret, podinfoGitServerSecret, "the zarf git server secret and podinfo-upgrade git server secret did not match") // We also expect a 6.3.4 package to have been previously built - previouslyBuiltPackage := "../../../zarf-package-test-upgrade-package-amd64-6.3.4.tar.zst" + previouslyBuiltPackage := fmt.Sprintf("../../../zarf-package-test-upgrade-package-%s-6.3.4.tar.zst", config.GetArch()) // Deploy the package. zarfDeployArgs := []string{"package", "deploy", previouslyBuiltPackage, "--confirm"} @@ -65,7 +67,7 @@ func TestPreviouslyBuiltZarfPackage(t *testing.T) { // We also want to build a new package. stdOut, stdErr, err = zarf("package", "create", "../../../src/test/upgrade", "--set", "PODINFO_VERSION=6.3.5", "--confirm") require.NoError(t, err, stdOut, stdErr) - newlyBuiltPackage := "zarf-package-test-upgrade-package-amd64-6.3.5.tar.zst" + newlyBuiltPackage := fmt.Sprintf("zarf-package-test-upgrade-package-%s-6.3.5.tar.zst", config.GetArch()) // Deploy the package. stdOut, stdErr, err = zarf("package", "deploy", newlyBuiltPackage, "--confirm") From acac49d36478c676cf06eac2e6ce518bab0f70cf Mon Sep 17 00:00:00 2001 From: razzle Date: Tue, 2 Jan 2024 20:56:15 -0600 Subject: [PATCH 017/113] revert upgrade package required Signed-off-by: razzle --- src/test/upgrade/zarf.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/src/test/upgrade/zarf.yaml b/src/test/upgrade/zarf.yaml index 3539fce943..34453ce6bc 100644 --- a/src/test/upgrade/zarf.yaml +++ b/src/test/upgrade/zarf.yaml @@ -6,6 +6,7 @@ metadata: components: - name: test-upgrade-package + required: true description: A semi-contrived example that deploys podinfo using many Zarf primitives and lets us upgrade it a few times. charts: - name: podinfo-upgrade From ac50291f5e9a512a7274c567c91c3171ac4ab134 Mon Sep 17 00:00:00 2001 From: razzle Date: Tue, 2 Jan 2024 21:26:05 -0600 Subject: [PATCH 018/113] init adr 23 Signed-off-by: razzle --- adr/0023-required-optional.md | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 adr/0023-required-optional.md diff --git a/adr/0023-required-optional.md b/adr/0023-required-optional.md new file mode 100644 index 0000000000..875206f658 --- /dev/null +++ b/adr/0023-required-optional.md @@ -0,0 +1,31 @@ +# 23. Required -> Optional + +Date: 2024-01-02 + +## Status + +Pending + +## Context + +> Feature request: + +Currently, all Zarf components default to being optional due to the `required` key being _optional_ in the yaml. This leads to behaviors where a package author must ensure they are careful to annotate this key for each component--though the validation doesn't require them to so assumes a sort of "all things are optional" default state. + +When Zarf was first created, we didn't really know how it would evolve and this key was introduced in those very early days. At this point it would be better to require all components by default--especially with the introduction of composability and the OCI skeleton work, there is plenty of flexibility in the API to compose bespoke packages assembled from other packages. + +A few ways to handle this: + +1. Simply force the `required` key to be a non-optional, so that package authors would be forced to specify it for each component, thereby removing any ambiguity--but also force one more key for every single component every created 🫠 + +2. Deprecate `required` and introduce an optional `optional` key, which would default to false. I do think this still feels strange if you did something like `optional: false`. + +3. Do something more significant like combine various condition-based things such as `only`, `optional` (instead of `required`), or `default`. + +## Decision + +> The change that we're proposing or have agreed to implement. + +## Consequences + +> What becomes easier or more difficult to do and any risks introduced by the change that will need to be mitigated. From adeebc347f58249f3f66266ef5640270f55d81b4 Mon Sep 17 00:00:00 2001 From: razzle Date: Wed, 3 Jan 2024 16:53:53 -0600 Subject: [PATCH 019/113] seasonal migrations Signed-off-by: razzle --- src/pkg/packager/deprecated/common.go | 49 ++++++++++--------- ...-variable.go => pluralize_set_variable.go} | 18 ++++--- .../deprecated/required_to_optional.go | 19 ++++--- .../packager/deprecated/scripts-to-actions.go | 19 ++++--- src/pkg/packager/yaml.go | 7 +-- 5 files changed, 65 insertions(+), 47 deletions(-) rename src/pkg/packager/deprecated/{pluralize-set-variable.go => pluralize_set_variable.go} (82%) diff --git a/src/pkg/packager/deprecated/common.go b/src/pkg/packager/deprecated/common.go index 7f5d3da992..87980b1400 100644 --- a/src/pkg/packager/deprecated/common.go +++ b/src/pkg/packager/deprecated/common.go @@ -24,14 +24,10 @@ type BreakingChange struct { mitigation string } -// List of migrations tracked in the zarf.yaml build data. -const ( - // This should be updated when a breaking change is introduced to the Zarf package structure. See: https://github.com/defenseunicorns/zarf/releases/tag/v0.27.0 - LastNonBreakingVersion = "v0.27.0" - ScriptsToActions = "scripts-to-actions" - PluralizeSetVariable = "pluralize-set-variable" - RequiredToOptional = "required-to-optional" -) +// LastNonBreakingVersion is the last version that did not have any breaking changes +// +// This should be updated when a breaking change is introduced to the Zarf package structure. See: https://github.com/defenseunicorns/zarf/releases/tag/v0.27.0 +const LastNonBreakingVersion = "v0.27.0" // List of breaking changes to warn the user of. var breakingChanges = []BreakingChange{ @@ -42,10 +38,25 @@ var breakingChanges = []BreakingChange{ }, } -type migration interface { - name() string - clear(mc types.ZarfComponent) types.ZarfComponent - run(c types.ZarfComponent) (types.ZarfComponent, string) +// Migration represents a migration that can be run on a component. +type Migration interface { + // ID returns the ID of the migration + ID() string + // Clear clears the deprecated configuration from the component + Clear(mc types.ZarfComponent) types.ZarfComponent + // Run runs the migration on the component + Run(c types.ZarfComponent) (types.ZarfComponent, string) +} + +// Migrations returns a list of all current migrations. +// +// This function operates as the source of truth for consuming migrations. +func Migrations() []Migration { + return []Migration{ + ScriptsToActions{}, + SetVariableToSetVariables{}, + RequiredToOptional{}, + } } // MigrateComponent runs all migrations on a component. @@ -53,20 +64,14 @@ type migration interface { func MigrateComponent(build types.ZarfBuildData, component types.ZarfComponent) (migratedComponent types.ZarfComponent, warnings []string) { migratedComponent = component - migrations := []migration{ - migrateScriptsToActions{}, - migrateSetVariableToSetVariables{}, - migrateRequiredToOptional{}, - } - // Run all migrations - for _, m := range migrations { - if slices.Contains(build.Migrations, m.name()) { - migratedComponent = m.clear(migratedComponent) + for _, m := range Migrations() { + if slices.Contains(build.Migrations, m.ID()) { + migratedComponent = m.Clear(migratedComponent) } else { // Otherwise, run the migration. var warning string - if migratedComponent, warning = m.run(migratedComponent); warning != "" { + if migratedComponent, warning = m.Run(migratedComponent); warning != "" { warnings = append(warnings, warning) } } diff --git a/src/pkg/packager/deprecated/pluralize-set-variable.go b/src/pkg/packager/deprecated/pluralize_set_variable.go similarity index 82% rename from src/pkg/packager/deprecated/pluralize-set-variable.go rename to src/pkg/packager/deprecated/pluralize_set_variable.go index 98e4c61577..3513422a4e 100644 --- a/src/pkg/packager/deprecated/pluralize-set-variable.go +++ b/src/pkg/packager/deprecated/pluralize_set_variable.go @@ -10,14 +10,19 @@ import ( "github.com/defenseunicorns/zarf/src/types" ) -type migrateSetVariableToSetVariables struct{} +// PluralizeSetVariableID is the ID of the SetVariableToSetVariables migration +const PluralizeSetVariableID = "pluralize-set-variable" -func (m migrateSetVariableToSetVariables) name() string { - return PluralizeSetVariable +// SetVariableToSetVariables migrates setVariable to setVariables +type SetVariableToSetVariables struct{} + +// ID returns the ID of the migration +func (m SetVariableToSetVariables) ID() string { + return PluralizeSetVariableID } -// If the component has already been migrated, clear the deprecated setVariable. -func (m migrateSetVariableToSetVariables) clear(mc types.ZarfComponent) types.ZarfComponent { +// Clear the deprecated setVariable. +func (m SetVariableToSetVariables) Clear(mc types.ZarfComponent) types.ZarfComponent { clear := func(actions []types.ZarfComponentAction) []types.ZarfComponentAction { for i := range actions { actions[i].DeprecatedSetVariable = "" @@ -47,7 +52,8 @@ func (m migrateSetVariableToSetVariables) clear(mc types.ZarfComponent) types.Za return mc } -func (m migrateSetVariableToSetVariables) run(c types.ZarfComponent) (types.ZarfComponent, string) { +// Run coverts the deprecated setVariable to the new setVariables +func (m SetVariableToSetVariables) Run(c types.ZarfComponent) (types.ZarfComponent, string) { hasSetVariable := false migrate := func(actions []types.ZarfComponentAction) []types.ZarfComponentAction { diff --git a/src/pkg/packager/deprecated/required_to_optional.go b/src/pkg/packager/deprecated/required_to_optional.go index c408295832..783f995d26 100644 --- a/src/pkg/packager/deprecated/required_to_optional.go +++ b/src/pkg/packager/deprecated/required_to_optional.go @@ -11,20 +11,25 @@ import ( "github.com/defenseunicorns/zarf/src/types" ) -type migrateRequiredToOptional struct{} +// RequiredToOptionalID is the ID of the RequiredToOptional migration +const RequiredToOptionalID = "required-to-optional" -func (m migrateRequiredToOptional) name() string { - return RequiredToOptional +// RequiredToOptional migrates required to optional +type RequiredToOptional struct{} + +// ID returns the ID of the migration +func (m RequiredToOptional) ID() string { + return RequiredToOptionalID } -// If the component has already been migrated, clear the deprecated required. -func (m migrateRequiredToOptional) clear(mc types.ZarfComponent) types.ZarfComponent { +// Clear the deprecated required. +func (m RequiredToOptional) Clear(mc types.ZarfComponent) types.ZarfComponent { mc.DeprecatedRequired = nil return mc } -// run converts the deprecated required to the new optional -func (m migrateRequiredToOptional) run(c types.ZarfComponent) (types.ZarfComponent, string) { +// Run converts the deprecated required to the new optional key +func (m RequiredToOptional) Run(c types.ZarfComponent) (types.ZarfComponent, string) { if c.DeprecatedRequired == nil { return c, "" } diff --git a/src/pkg/packager/deprecated/scripts-to-actions.go b/src/pkg/packager/deprecated/scripts-to-actions.go index 903155ed80..beba4e919b 100644 --- a/src/pkg/packager/deprecated/scripts-to-actions.go +++ b/src/pkg/packager/deprecated/scripts-to-actions.go @@ -11,26 +11,31 @@ import ( "github.com/defenseunicorns/zarf/src/types" ) -type migrateScriptsToActions struct{} +// ScriptsToActionsID is the ID of the ScriptsToActions migration +const ScriptsToActionsID = "scripts-to-actions" -func (m migrateScriptsToActions) name() string { - return ScriptsToActions +// ScriptsToActions migrates scripts to actions +type ScriptsToActions struct{} + +// ID returns the ID of the migration +func (m ScriptsToActions) ID() string { + return ScriptsToActionsID } -// If the component has already been migrated, clear the deprecated scripts. -func (m migrateScriptsToActions) clear(mc types.ZarfComponent) types.ZarfComponent { +// Clear the deprecated scripts. +func (m ScriptsToActions) Clear(mc types.ZarfComponent) types.ZarfComponent { mc.DeprecatedScripts = types.DeprecatedZarfComponentScripts{} return mc } -// run coverts the deprecated scripts to the new actions +// Run coverts the deprecated scripts to the new actions // The following have no migration: // - Actions.Create.After // - Actions.Remove.* // - Actions.*.OnSuccess // - Actions.*.OnFailure // - Actions.*.*.Env -func (m migrateScriptsToActions) run(c types.ZarfComponent) (types.ZarfComponent, string) { +func (m ScriptsToActions) Run(c types.ZarfComponent) (types.ZarfComponent, string) { var hasScripts bool // Convert a script configs to action defaults. diff --git a/src/pkg/packager/yaml.go b/src/pkg/packager/yaml.go index aeadcd10e6..38e893c7b5 100644 --- a/src/pkg/packager/yaml.go +++ b/src/pkg/packager/yaml.go @@ -106,11 +106,8 @@ func (p *Packager) writeYaml() error { } // Record the migrations that will be run on the package. - // TODO: (@razzle) Think about a better way to keep track of migrations. - p.cfg.Pkg.Build.Migrations = []string{ - deprecated.ScriptsToActions, - deprecated.PluralizeSetVariable, - deprecated.RequiredToOptional, + for _, m := range deprecated.Migrations() { + p.cfg.Pkg.Build.Migrations = append(p.cfg.Pkg.Build.Migrations, m.ID()) } // Record the flavor of Zarf used to build this package (if any). From 4d7de02ff9acb9117fb23803160d7bd9083b182a Mon Sep 17 00:00:00 2001 From: razzle Date: Wed, 3 Jan 2024 17:15:56 -0600 Subject: [PATCH 020/113] upgrade workflow and override and validate fixes Signed-off-by: razzle --- .github/workflows/test-upgrade.yml | 2 +- src/config/lang/english.go | 1 + src/internal/packager/validate/validate.go | 4 ++++ src/pkg/packager/composer/override.go | 3 +++ src/pkg/packager/deprecated/common.go | 8 ++++---- .../{scripts-to-actions.go => scripts_to_actions.go} | 0 src/test/upgrade/zarf.yaml | 1 - 7 files changed, 13 insertions(+), 6 deletions(-) rename src/pkg/packager/deprecated/{scripts-to-actions.go => scripts_to_actions.go} (100%) diff --git a/.github/workflows/test-upgrade.yml b/.github/workflows/test-upgrade.yml index ab08ddb10a..95c30b3ad7 100644 --- a/.github/workflows/test-upgrade.yml +++ b/.github/workflows/test-upgrade.yml @@ -82,7 +82,7 @@ jobs: # in a previous step. This test run will use the current release to create a K3s cluster. run: | zarf package create src/test/upgrade --set PODINFO_VERSION=6.3.3 --confirm - sudo env "PATH=$PATH" CI=true zarf package deploy zarf-package-test-upgrade-package-amd64-6.3.3.tar.zst --confirm + sudo env "PATH=$PATH" CI=true zarf package deploy zarf-package-test-upgrade-package-amd64-6.3.3.tar.zst --components="test-upgrade-package" --confirm sudo env "PATH=$PATH" CI=true zarf tools kubectl describe deployments -n=podinfo-upgrade sudo env "PATH=$PATH" CI=true zarf tools kubectl describe pods -n=podinfo-upgrade diff --git a/src/config/lang/english.go b/src/config/lang/english.go index 6a666474d7..a845347e63 100644 --- a/src/config/lang/english.go +++ b/src/config/lang/english.go @@ -649,6 +649,7 @@ const ( PkgValidateErrComponentName = "component name %q must be all lowercase and contain no special characters except '-' and cannot start with a '-'" PkgValidateErrComponentNameNotUnique = "component name %q is not unique" PkgValidateErrComponent = "invalid component %q: %w" + PkgValidateErrComponentRequired = "'required' key is deprecated on component %q, please use 'optional' instead" PkgValidateErrComponentReqDefault = "component %q cannot be both required and default" PkgValidateErrComponentReqGrouped = "component %q cannot be both required and grouped" PkgValidateErrComponentYOLO = "component %q incompatible with the online-only package flag (metadata.yolo): %w" diff --git a/src/internal/packager/validate/validate.go b/src/internal/packager/validate/validate.go index bb69c34bb3..9bd0b9d58a 100644 --- a/src/internal/packager/validate/validate.go +++ b/src/internal/packager/validate/validate.go @@ -129,6 +129,10 @@ func validateComponent(pkg types.ZarfPackage, component types.ZarfComponent) err return fmt.Errorf(lang.PkgValidateErrComponentName, component.Name) } + if component.DeprecatedRequired != nil { + return fmt.Errorf(lang.PkgValidateErrComponentRequired, component.Name) + } + if component.IsRequired() { if component.Default { return fmt.Errorf(lang.PkgValidateErrComponentReqDefault, component.Name) diff --git a/src/pkg/packager/composer/override.go b/src/pkg/packager/composer/override.go index 18eebf7677..c1a26b496e 100644 --- a/src/pkg/packager/composer/override.go +++ b/src/pkg/packager/composer/override.go @@ -13,6 +13,9 @@ import ( func overrideMetadata(c *types.ZarfComponent, override types.ZarfComponent) error { c.Name = override.Name c.Default = override.Default + if override.DeprecatedRequired != nil { + return fmt.Errorf("component %q \"required\" is deprecated, please migrate to \"optional\"", c.Name) + } c.Optional = override.Optional // Override description if it was provided. diff --git a/src/pkg/packager/deprecated/common.go b/src/pkg/packager/deprecated/common.go index 87980b1400..826bd0c153 100644 --- a/src/pkg/packager/deprecated/common.go +++ b/src/pkg/packager/deprecated/common.go @@ -17,8 +17,8 @@ import ( "github.com/pterm/pterm" ) -// BreakingChange represents a breaking change that happened on a specified Zarf version -type BreakingChange struct { +// breakingChange represents a breaking change that happened on a specified Zarf version +type breakingChange struct { version *semver.Version title string mitigation string @@ -30,7 +30,7 @@ type BreakingChange struct { const LastNonBreakingVersion = "v0.27.0" // List of breaking changes to warn the user of. -var breakingChanges = []BreakingChange{ +var breakingChanges = []breakingChange{ { version: semver.New(0, 26, 0, "", ""), title: "Zarf container images are now mutated based on tag instead of repository name.", @@ -94,7 +94,7 @@ func PrintBreakingChanges(deployedZarfVersion string) { return } - applicableBreakingChanges := []BreakingChange{} + applicableBreakingChanges := []breakingChange{} // Calculate the applicable breaking changes for _, breakingChange := range breakingChanges { diff --git a/src/pkg/packager/deprecated/scripts-to-actions.go b/src/pkg/packager/deprecated/scripts_to_actions.go similarity index 100% rename from src/pkg/packager/deprecated/scripts-to-actions.go rename to src/pkg/packager/deprecated/scripts_to_actions.go diff --git a/src/test/upgrade/zarf.yaml b/src/test/upgrade/zarf.yaml index 34453ce6bc..3539fce943 100644 --- a/src/test/upgrade/zarf.yaml +++ b/src/test/upgrade/zarf.yaml @@ -6,7 +6,6 @@ metadata: components: - name: test-upgrade-package - required: true description: A semi-contrived example that deploys podinfo using many Zarf primitives and lets us upgrade it a few times. charts: - name: podinfo-upgrade From 37c9f2873fc194c6857b9405ee5fdf8fd6585abc Mon Sep 17 00:00:00 2001 From: razzle Date: Wed, 3 Jan 2024 17:21:42 -0600 Subject: [PATCH 021/113] tweaks Signed-off-by: razzle --- src/config/lang/english.go | 2 +- src/pkg/packager/composer/override.go | 4 ++-- zarf.yaml | 1 - 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/config/lang/english.go b/src/config/lang/english.go index a845347e63..e834b222e6 100644 --- a/src/config/lang/english.go +++ b/src/config/lang/english.go @@ -649,7 +649,7 @@ const ( PkgValidateErrComponentName = "component name %q must be all lowercase and contain no special characters except '-' and cannot start with a '-'" PkgValidateErrComponentNameNotUnique = "component name %q is not unique" PkgValidateErrComponent = "invalid component %q: %w" - PkgValidateErrComponentRequired = "'required' key is deprecated on component %q, please use 'optional' instead" + PkgValidateErrComponentRequired = "component %q contains deprecated usage of required, please migrate to optional" PkgValidateErrComponentReqDefault = "component %q cannot be both required and default" PkgValidateErrComponentReqGrouped = "component %q cannot be both required and grouped" PkgValidateErrComponentYOLO = "component %q incompatible with the online-only package flag (metadata.yolo): %w" diff --git a/src/pkg/packager/composer/override.go b/src/pkg/packager/composer/override.go index c1a26b496e..b1a9fd7098 100644 --- a/src/pkg/packager/composer/override.go +++ b/src/pkg/packager/composer/override.go @@ -14,7 +14,7 @@ func overrideMetadata(c *types.ZarfComponent, override types.ZarfComponent) erro c.Name = override.Name c.Default = override.Default if override.DeprecatedRequired != nil { - return fmt.Errorf("component %q \"required\" is deprecated, please migrate to \"optional\"", c.Name) + return fmt.Errorf("component %q: \"required\" is deprecated, please migrate to \"optional\"", c.Name) } c.Optional = override.Optional @@ -25,7 +25,7 @@ func overrideMetadata(c *types.ZarfComponent, override types.ZarfComponent) erro if override.Only.LocalOS != "" { if c.Only.LocalOS != "" { - return fmt.Errorf("component %q \"only.localOS\" %q cannot be redefined as %q during compose", c.Name, c.Only.LocalOS, override.Only.LocalOS) + return fmt.Errorf("component %q: \"only.localOS\" %q cannot be redefined as %q during compose", c.Name, c.Only.LocalOS, override.Only.LocalOS) } c.Only.LocalOS = override.Only.LocalOS diff --git a/zarf.yaml b/zarf.yaml index ee244d890f..fc40c32283 100644 --- a/zarf.yaml +++ b/zarf.yaml @@ -16,7 +16,6 @@ components: # Creates the temporary seed-registry - name: zarf-seed-registry - optional: false import: path: packages/zarf-registry From 512182282a78e157fa489a046ead050ece8e9629 Mon Sep 17 00:00:00 2001 From: razzle Date: Wed, 3 Jan 2024 17:49:01 -0600 Subject: [PATCH 022/113] tweaks Signed-off-by: razzle --- src/pkg/packager/composer/override.go | 3 --- src/pkg/packager/deprecated/required_to_optional.go | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/src/pkg/packager/composer/override.go b/src/pkg/packager/composer/override.go index b1a9fd7098..ea25f92438 100644 --- a/src/pkg/packager/composer/override.go +++ b/src/pkg/packager/composer/override.go @@ -13,9 +13,6 @@ import ( func overrideMetadata(c *types.ZarfComponent, override types.ZarfComponent) error { c.Name = override.Name c.Default = override.Default - if override.DeprecatedRequired != nil { - return fmt.Errorf("component %q: \"required\" is deprecated, please migrate to \"optional\"", c.Name) - } c.Optional = override.Optional // Override description if it was provided. diff --git a/src/pkg/packager/deprecated/required_to_optional.go b/src/pkg/packager/deprecated/required_to_optional.go index 783f995d26..c0b4efe803 100644 --- a/src/pkg/packager/deprecated/required_to_optional.go +++ b/src/pkg/packager/deprecated/required_to_optional.go @@ -41,5 +41,5 @@ func (m RequiredToOptional) Run(c types.ZarfComponent) (types.ZarfComponent, str c.Optional = helpers.BoolPtr(true) } - return c, fmt.Sprintf("Component %q is using \"required\" which will be removed in Zarf v1.0.0. Please migrate to \"optional\". Please migrate to \"optional\".", c.Name) + return c, fmt.Sprintf("Component %q is using \"required\" which will be removed in Zarf v1.0.0. Please migrate to \"optional\".", c.Name) } From 829eecc3fe8d818893d2c64cdb4cdd84db1d966e Mon Sep 17 00:00:00 2001 From: razzle Date: Wed, 3 Jan 2024 19:26:38 -0600 Subject: [PATCH 023/113] dev migrate Signed-off-by: razzle --- src/cmd/dev.go | 49 +++++++++++++++++++ .../deprecated/required_to_optional.go | 25 +++++++--- 2 files changed, 67 insertions(+), 7 deletions(-) diff --git a/src/cmd/dev.go b/src/cmd/dev.go index 63426692ea..c5e5d0ebca 100644 --- a/src/cmd/dev.go +++ b/src/cmd/dev.go @@ -11,16 +11,21 @@ import ( "path/filepath" "strings" + goyaml "github.com/goccy/go-yaml" + "github.com/AlecAivazis/survey/v2" "github.com/defenseunicorns/zarf/src/cmd/common" "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/config/lang" + "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/packager" + "github.com/defenseunicorns/zarf/src/pkg/packager/deprecated" "github.com/defenseunicorns/zarf/src/pkg/packager/lint" "github.com/defenseunicorns/zarf/src/pkg/transform" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" + "github.com/defenseunicorns/zarf/src/types" "github.com/mholt/archiver/v3" "github.com/spf13/cobra" "github.com/spf13/viper" @@ -60,6 +65,49 @@ var devDeployCmd = &cobra.Command{ }, } +var devMigrateCmd = &cobra.Command{ + Use: "migrate", + // Short: lang.CmdDevMigrateShort, + // Long: lang.CmdDevMigrateLong, + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + dir := args[0] + var pkg types.ZarfPackage + cm := goyaml.CommentMap{} + + fi, err := os.Stat(filepath.Join(dir, layout.ZarfYAML)) + if err != nil { + return err + } + + b, err := os.ReadFile(filepath.Join(dir, layout.ZarfYAML)) + if err != nil { + return err + } + + if err := goyaml.UnmarshalWithOptions(b, &pkg, goyaml.CommentToMap(cm)); err != nil { + return err + } + + // Migrate the package definition + for idx, component := range pkg.Components { + for _, migration := range deprecated.Migrations() { + c, _ := migration.Run(component) + c = migration.Clear(c) + pkg.Components[idx] = c + } + } + + // Write the package definition back to disk + b, err = goyaml.MarshalWithOptions(pkg, goyaml.WithComment(cm), goyaml.IndentSequence(true), goyaml.UseSingleQuote(false)) + if err != nil { + return err + } + + return os.WriteFile(filepath.Join(dir, layout.ZarfYAML), b, fi.Mode()) + }, +} + var devTransformGitLinksCmd = &cobra.Command{ Use: "patch-git HOST FILE", Aliases: []string{"p"}, @@ -252,6 +300,7 @@ func init() { rootCmd.AddCommand(devCmd) devCmd.AddCommand(devDeployCmd) + devCmd.AddCommand(devMigrateCmd) devCmd.AddCommand(devTransformGitLinksCmd) devCmd.AddCommand(devSha256SumCmd) devCmd.AddCommand(devFindImagesCmd) diff --git a/src/pkg/packager/deprecated/required_to_optional.go b/src/pkg/packager/deprecated/required_to_optional.go index c0b4efe803..228cb3b8fa 100644 --- a/src/pkg/packager/deprecated/required_to_optional.go +++ b/src/pkg/packager/deprecated/required_to_optional.go @@ -34,12 +34,23 @@ func (m RequiredToOptional) Run(c types.ZarfComponent) (types.ZarfComponent, str return c, "" } - switch *c.DeprecatedRequired { - case true: - c.Optional = nil - case false: - c.Optional = helpers.BoolPtr(true) + warning := fmt.Sprintf("Component %q is using \"required\" which will be removed in Zarf v1.0.0. Please migrate to \"optional\".", c.Name) + + if *c.DeprecatedRequired { + if *c.Optional { + // ensure that optional wins over required + c.DeprecatedRequired = helpers.BoolPtr(false) + return c, warning + } else { + c.Optional = nil + return c, warning + } + } else { + if *c.Optional { + return c, warning + } else { + c.DeprecatedRequired = helpers.BoolPtr(true) + return c, warning + } } - - return c, fmt.Sprintf("Component %q is using \"required\" which will be removed in Zarf v1.0.0. Please migrate to \"optional\".", c.Name) } From 0179509cd686a551faf809d9579a12d379954e53 Mon Sep 17 00:00:00 2001 From: razzle Date: Wed, 3 Jan 2024 19:27:41 -0600 Subject: [PATCH 024/113] docs and schema Signed-off-by: razzle --- .../100-cli-commands/zarf_dev.md | 1 + .../100-cli-commands/zarf_dev_migrate.md | 31 +++++++++++++++++++ 2 files changed, 32 insertions(+) create mode 100644 docs/2-the-zarf-cli/100-cli-commands/zarf_dev_migrate.md diff --git a/docs/2-the-zarf-cli/100-cli-commands/zarf_dev.md b/docs/2-the-zarf-cli/100-cli-commands/zarf_dev.md index 0e44236b00..6f882cf7c5 100644 --- a/docs/2-the-zarf-cli/100-cli-commands/zarf_dev.md +++ b/docs/2-the-zarf-cli/100-cli-commands/zarf_dev.md @@ -29,6 +29,7 @@ Commands useful for developing packages * [zarf dev find-images](zarf_dev_find-images.md) - Evaluates components in a Zarf file to identify images specified in their helm charts and manifests * [zarf dev generate-config](zarf_dev_generate-config.md) - Generates a config file for Zarf * [zarf dev lint](zarf_dev_lint.md) - Lints the given package for valid schema and recommended practices +* [zarf dev migrate](zarf_dev_migrate.md) - * [zarf dev patch-git](zarf_dev_patch-git.md) - Converts all .git URLs to the specified Zarf HOST and with the Zarf URL pattern in a given FILE. NOTE: This should only be used for manifests that are not mutated by the Zarf Agent Mutating Webhook. * [zarf dev sha256sum](zarf_dev_sha256sum.md) - Generates a SHA256SUM for the given file diff --git a/docs/2-the-zarf-cli/100-cli-commands/zarf_dev_migrate.md b/docs/2-the-zarf-cli/100-cli-commands/zarf_dev_migrate.md new file mode 100644 index 0000000000..c54bd10ee9 --- /dev/null +++ b/docs/2-the-zarf-cli/100-cli-commands/zarf_dev_migrate.md @@ -0,0 +1,31 @@ +# zarf dev migrate + + + + +``` +zarf dev migrate [flags] +``` + +## Options + +``` + -h, --help help for migrate +``` + +## Options inherited from parent commands + +``` + -a, --architecture string Architecture for OCI images and Zarf packages + --insecure Allow access to insecure registries and disable other recommended security enforcements such as package checksum and signature validation. This flag should only be used if you have a specific reason and accept the reduced security posture. + -l, --log-level string Log level when running Zarf. Valid options are: warn, info, debug, trace (default "info") + --no-color Disable colors in output + --no-log-file Disable log file creation + --no-progress Disable fancy UI progress bars, spinners, logos, etc + --tmpdir string Specify the temporary directory to use for intermediate files + --zarf-cache string Specify the location of the Zarf cache directory (default "~/.zarf-cache") +``` + +## SEE ALSO + +* [zarf dev](zarf_dev.md) - Commands useful for developing packages From 7ab1b8b009a9045cee1667e9c479ec99e30d6487 Mon Sep 17 00:00:00 2001 From: razzle Date: Thu, 4 Jan 2024 12:16:54 -0600 Subject: [PATCH 025/113] lets get silly with it Signed-off-by: razzle --- src/cmd/dev.go | 41 ++++++++++++++++++++++-- src/types/component.go | 72 +++++++++++++++++++++--------------------- 2 files changed, 75 insertions(+), 38 deletions(-) diff --git a/src/cmd/dev.go b/src/cmd/dev.go index c5e5d0ebca..10ae96ae14 100644 --- a/src/cmd/dev.go +++ b/src/cmd/dev.go @@ -5,6 +5,8 @@ package cmd import ( + "bufio" + "bytes" "fmt" "io" "os" @@ -98,13 +100,48 @@ var devMigrateCmd = &cobra.Command{ } } - // Write the package definition back to disk b, err = goyaml.MarshalWithOptions(pkg, goyaml.WithComment(cm), goyaml.IndentSequence(true), goyaml.UseSingleQuote(false)) if err != nil { return err } - return os.WriteFile(filepath.Join(dir, layout.ZarfYAML), b, fi.Mode()) + scanner := bufio.NewScanner(bytes.NewReader(b)) + + var updated []byte + var commentLines []string + + // Some opinionated formatting for the zarf.yaml + for scanner.Scan() { + line := scanner.Text() + + if strings.HasPrefix(line, "components:") || strings.HasPrefix(line, " - name: ") || strings.HasPrefix(line, " name: ") { + if len(commentLines) > 0 { + commentText := strings.Join(commentLines, "\n") + updated = append(updated, []byte("\n"+commentText+"\n")...) + } else { + updated = append(updated, []byte("\n")...) + } + updated = append(updated, []byte(line+"\n")...) // Add "components:" line + commentLines = nil + } else { + if strings.HasPrefix(line, "#") || strings.HasPrefix(line, " - #") { + commentLines = append(commentLines, line) + } else { + if len(commentLines) > 0 { + commentText := strings.Join(commentLines, "\n") + updated = append(updated, []byte("\n"+commentText+"\n")...) + commentLines = nil + } + updated = append(updated, []byte(line+"\n")...) + } + } + } + + if err := scanner.Err(); err != nil { + return err + } + + return os.WriteFile(filepath.Join(dir, layout.ZarfYAML), updated, fi.Mode()) }, } diff --git a/src/types/component.go b/src/types/component.go index 7938e63d48..cb26b9ad23 100644 --- a/src/types/component.go +++ b/src/types/component.go @@ -39,18 +39,15 @@ type ZarfComponent struct { // Import refers to another zarf.yaml package component. Import ZarfComponentImport `json:"import,omitempty" jsonschema:"description=Import a component from another Zarf package"` - // (Deprecated) DeprecatedScripts are custom commands that run before or after package deployment - DeprecatedScripts DeprecatedZarfComponentScripts `json:"scripts,omitempty" jsonschema:"description=[Deprecated] (replaced by actions) Custom commands to run before or after package deployment. This will be removed in Zarf v1.0.0.,deprecated=true"` - // Files are files to place on disk during deploy Files []ZarfFile `json:"files,omitempty" jsonschema:"description=Files or folders to place on disk during package deployment"` - // Charts are helm charts to install during package deploy - Charts []ZarfChart `json:"charts,omitempty" jsonschema:"description=Helm charts to install during package deploy"` - // Manifests are raw manifests that get converted into zarf-generated helm charts during deploy Manifests []ZarfManifest `json:"manifests,omitempty" jsonschema:"description=Kubernetes manifests to be included in a generated Helm chart on package deploy"` + // Charts are helm charts to install during package deploy + Charts []ZarfChart `json:"charts,omitempty" jsonschema:"description=Helm charts to install during package deploy"` + // Images are the online images needed to be included in the zarf package Images []string `json:"images,omitempty" jsonschema:"description=List of OCI images to include in the package"` @@ -63,6 +60,9 @@ type ZarfComponent struct { // Extensions provide additional functionality to a component Extensions extensions.ZarfComponentExtensions `json:"extensions,omitempty" jsonschema:"description=Extend component functionality with additional features"` + // (Deprecated) DeprecatedScripts are custom commands that run before or after package deployment + DeprecatedScripts DeprecatedZarfComponentScripts `json:"scripts,omitempty" jsonschema:"description=[Deprecated] (replaced by actions) Custom commands to run before or after package deployment. This will be removed in Zarf v1.0.0.,deprecated=true"` + // Replaces scripts, fine-grained control over commands to run at various stages of a package lifecycle Actions ZarfComponentActions `json:"actions,omitempty" jsonschema:"description=Custom commands to run at various stages of a package lifecycle"` } @@ -95,6 +95,33 @@ func (c ZarfComponent) IsRequired() bool { return true } +// IsEmpty returns if the components fields (other than the fields we were told to ignore) are empty or set to the types zero-value +func (c ZarfComponent) IsEmpty(fieldsToIgnore []string) bool { + // Make a map for the fields we are going to ignore + ignoredFieldsMap := make(map[string]bool) + for _, field := range fieldsToIgnore { + ignoredFieldsMap[field] = true + } + + // Get a value representation of the component + componentReflectValue := reflect.Indirect(reflect.ValueOf(c)) + + // Loop through all of the Components struct fields + for i := 0; i < componentReflectValue.NumField(); i++ { + // If we were told to ignore this field, continue on.. + if ignoredFieldsMap[componentReflectValue.Type().Field(i).Name] { + continue + } + + // Check if this field is empty/zero + if !componentReflectValue.Field(i).IsZero() { + return false + } + } + + return true +} + // ZarfComponentOnlyTarget filters a component to only show it for a given local OS and cluster. type ZarfComponentOnlyTarget struct { LocalOS string `json:"localOS,omitempty" jsonschema:"description=Only deploy component to specified OS,enum=linux,enum=darwin,enum=windows"` @@ -203,10 +230,10 @@ type ZarfComponentAction struct { // ZarfComponentActionSetVariable represents a variable that is to be set via an action type ZarfComponentActionSetVariable struct { Name string `json:"name" jsonschema:"description=The name to be used for the variable,pattern=^[A-Z0-9_]+$"` + Type VariableType `json:"type,omitempty" jsonschema:"description=Changes the handling of a variable to load contents differently (i.e. from a file rather than as a raw variable - templated files should be kept below 1 MiB),enum=raw,enum=file"` + Pattern string `json:"pattern,omitempty" jsonschema:"description=An optional regex pattern that a variable value must match before a package deployment can continue."` Sensitive bool `json:"sensitive,omitempty" jsonschema:"description=Whether to mark this variable as sensitive to not print it in the Zarf log"` AutoIndent bool `json:"autoIndent,omitempty" jsonschema:"description=Whether to automatically indent the variable's value (if multiline) when templating. Based on the number of chars before the start of ###ZARF_VAR_."` - Pattern string `json:"pattern,omitempty" jsonschema:"description=An optional regex pattern that a variable value must match before a package deployment can continue."` - Type VariableType `json:"type,omitempty" jsonschema:"description=Changes the handling of a variable to load contents differently (i.e. from a file rather than as a raw variable - templated files should be kept below 1 MiB),enum=raw,enum=file"` } // ZarfComponentActionWait specifies a condition to wait for before continuing @@ -232,8 +259,8 @@ type ZarfComponentActionWaitNetwork struct { // ZarfContainerTarget defines the destination info for a ZarfData target type ZarfContainerTarget struct { - Namespace string `json:"namespace" jsonschema:"description=The namespace to target for data injection"` Selector string `json:"selector" jsonschema:"description=The K8s selector to target for data injection,example=app=data-injection"` + Namespace string `json:"namespace" jsonschema:"description=The namespace to target for data injection"` Container string `json:"container" jsonschema:"description=The container name to target for data injection"` Path string `json:"path" jsonschema:"description=The path within the container to copy the data into"` } @@ -253,30 +280,3 @@ type ZarfComponentImport struct { // For further explanation see https://regex101.com/r/nxX8vx/1 URL string `json:"url,omitempty" jsonschema:"description=[beta] The URL to a Zarf package to import via OCI,pattern=^oci://.*$"` } - -// IsEmpty returns if the components fields (other than the fields we were told to ignore) are empty or set to the types zero-value -func (c *ZarfComponent) IsEmpty(fieldsToIgnore []string) bool { - // Make a map for the fields we are going to ignore - ignoredFieldsMap := make(map[string]bool) - for _, field := range fieldsToIgnore { - ignoredFieldsMap[field] = true - } - - // Get a value representation of the component - componentReflectValue := reflect.Indirect(reflect.ValueOf(c)) - - // Loop through all of the Components struct fields - for i := 0; i < componentReflectValue.NumField(); i++ { - // If we were told to ignore this field, continue on.. - if ignoredFieldsMap[componentReflectValue.Type().Field(i).Name] { - continue - } - - // Check if this field is empty/zero - if !componentReflectValue.Field(i).IsZero() { - return false - } - } - - return true -} From f7b77b4d82a4dd0319f1fdf47bc970b9d161e2c5 Mon Sep 17 00:00:00 2001 From: razzle Date: Thu, 4 Jan 2024 13:50:28 -0600 Subject: [PATCH 026/113] docs and schema Signed-off-by: razzle --- docs/3-create-a-zarf-package/4-zarf-schema.md | 302 +++++++++--------- zarf.schema.json | 56 ++-- 2 files changed, 179 insertions(+), 179 deletions(-) diff --git a/docs/3-create-a-zarf-package/4-zarf-schema.md b/docs/3-create-a-zarf-package/4-zarf-schema.md index 496a6ac74f..23640b7f4e 100644 --- a/docs/3-create-a-zarf-package/4-zarf-schema.md +++ b/docs/3-create-a-zarf-package/4-zarf-schema.md @@ -969,14 +969,14 @@ Must be one of:
- charts + manifests  
- ## components > charts + ## components > manifests -**Description:** Helm charts to install during package deploy +**Description:** Kubernetes manifests to be included in a generated Helm chart on package deploy | | | | -------- | ------- | @@ -987,24 +987,24 @@ Must be one of: ![Item unicity: False](https://img.shields.io/badge/Item%20unicity%3A%20False-gold) ![Additional items: N/A](https://img.shields.io/badge/Additional%20items%3A%20N/A-gold) - ### ZarfChart + ### ZarfManifest | | | | ------------------------- | -------------------------------------------------------------------------------------------------------- | | **Type** | `object` | | **Additional properties** | [![Not allowed](https://img.shields.io/badge/Not%20allowed-red)](# "Additional Properties not allowed.") | -| **Defined in** | #/definitions/ZarfChart | +| **Defined in** | #/definitions/ZarfManifest |
- name * + name *  
![Required](https://img.shields.io/badge/Required-red) -**Description:** The name of the chart within Zarf; note that this must be unique and does not need to be the same as the name in the chart repo +**Description:** A name to give this collection of manifests; this will become the name of the dynamically-created helm chart | | | | -------- | -------- | @@ -1015,12 +1015,12 @@ Must be one of:
- version + namespace  
-**Description:** The version of the chart to deploy; for git-based charts this is also the tag of the git repo by default (when not using the '@' syntax for 'repos') +**Description:** The namespace to deploy the manifests to | | | | -------- | -------- | @@ -1031,88 +1031,131 @@ Must be one of:
- url + files  
-**Description:** The URL of the OCI registry, chart repository, or git repo where the helm chart is stored +**Description:** List of local K8s YAML files or remote URLs to deploy (in order) + +| | | +| -------- | ----------------- | +| **Type** | `array of string` | + +![Min Items: N/A](https://img.shields.io/badge/Min%20Items%3A%20N/A-gold) +![Max Items: N/A](https://img.shields.io/badge/Max%20Items%3A%20N/A-gold) +![Item unicity: False](https://img.shields.io/badge/Item%20unicity%3A%20False-gold) +![Additional items: N/A](https://img.shields.io/badge/Additional%20items%3A%20N/A-gold) + + ### files items | | | | -------- | -------- | | **Type** | `string` | -**Examples:** - - -"OCI registry: oci://ghcr.io/stefanprodan/charts/podinfo", "helm chart repo: https://stefanprodan.github.io/podinfo", "git repo: https://github.com/stefanprodan/podinfo (note the '@' syntax for 'repos' is supported here too)" -
- repoName + kustomizeAllowAnyDirectory  
-**Description:** The name of a chart within a Helm repository (defaults to the Zarf name of the chart) +**Description:** Allow traversing directory above the current directory if needed for kustomization -| | | -| -------- | -------- | -| **Type** | `string` | +| | | +| -------- | --------- | +| **Type** | `boolean` |
- gitPath + kustomizations  
-**Description:** (git repo only) The sub directory to the chart within a git repo +**Description:** List of local kustomization paths or remote URLs to include in the package + +| | | +| -------- | ----------------- | +| **Type** | `array of string` | + +![Min Items: N/A](https://img.shields.io/badge/Min%20Items%3A%20N/A-gold) +![Max Items: N/A](https://img.shields.io/badge/Max%20Items%3A%20N/A-gold) +![Item unicity: False](https://img.shields.io/badge/Item%20unicity%3A%20False-gold) +![Additional items: N/A](https://img.shields.io/badge/Additional%20items%3A%20N/A-gold) + + ### kustomizations items | | | | -------- | -------- | | **Type** | `string` | -**Example:** - - -"charts/your-chart" -
- localPath + noWait  
-**Description:** The path to a local chart's folder or .tgz archive +**Description:** Whether to not wait for manifest resources to be ready before continuing -| | | -| -------- | -------- | -| **Type** | `string` | +| | | +| -------- | --------- | +| **Type** | `boolean` |
+
+
+ +
+ + charts + +  +
+ + ## components > charts + +**Description:** Helm charts to install during package deploy + +| | | +| -------- | ------- | +| **Type** | `array` | + +![Min Items: N/A](https://img.shields.io/badge/Min%20Items%3A%20N/A-gold) +![Max Items: N/A](https://img.shields.io/badge/Max%20Items%3A%20N/A-gold) +![Item unicity: False](https://img.shields.io/badge/Item%20unicity%3A%20False-gold) +![Additional items: N/A](https://img.shields.io/badge/Additional%20items%3A%20N/A-gold) + + ### ZarfChart + +| | | +| ------------------------- | -------------------------------------------------------------------------------------------------------- | +| **Type** | `object` | +| **Additional properties** | [![Not allowed](https://img.shields.io/badge/Not%20allowed-red)](# "Additional Properties not allowed.") | +| **Defined in** | #/definitions/ZarfChart | +
- namespace * + name *  
![Required](https://img.shields.io/badge/Required-red) -**Description:** The namespace to deploy the chart to +**Description:** The name of the chart within Zarf; note that this must be unique and does not need to be the same as the name in the chart repo | | | | -------- | -------- | @@ -1123,12 +1166,12 @@ Must be one of:
- releaseName + version  
-**Description:** The name of the Helm release to create (defaults to the Zarf name of the chart) +**Description:** The version of the chart to deploy; for git-based charts this is also the tag of the git repo by default (when not using the '@' syntax for 'repos') | | | | -------- | -------- | @@ -1139,39 +1182,33 @@ Must be one of:
- noWait + url  
-**Description:** Whether to not wait for chart resources to be ready before continuing +**Description:** The URL of the OCI registry, chart repository, or git repo where the helm chart is stored -| | | -| -------- | --------- | -| **Type** | `boolean` | +| | | +| -------- | -------- | +| **Type** | `string` | + +**Examples:** + + +"OCI registry: oci://ghcr.io/stefanprodan/charts/podinfo", "helm chart repo: https://stefanprodan.github.io/podinfo", "git repo: https://github.com/stefanprodan/podinfo (note the '@' syntax for 'repos' is supported here too)"
- valuesFiles + repoName  
-**Description:** List of local values file paths or remote URLs to include in the package; these will be merged together when deployed - -| | | -| -------- | ----------------- | -| **Type** | `array of string` | - -![Min Items: N/A](https://img.shields.io/badge/Min%20Items%3A%20N/A-gold) -![Max Items: N/A](https://img.shields.io/badge/Max%20Items%3A%20N/A-gold) -![Item unicity: False](https://img.shields.io/badge/Item%20unicity%3A%20False-gold) -![Additional items: N/A](https://img.shields.io/badge/Additional%20items%3A%20N/A-gold) - - ### valuesFiles items +**Description:** The name of a chart within a Helm repository (defaults to the Zarf name of the chart) | | | | -------- | -------- | @@ -1180,47 +1217,35 @@ Must be one of:
-
-
- -
+
- manifests + gitPath  
- ## components > manifests - -**Description:** Kubernetes manifests to be included in a generated Helm chart on package deploy +**Description:** (git repo only) The sub directory to the chart within a git repo -| | | -| -------- | ------- | -| **Type** | `array` | +| | | +| -------- | -------- | +| **Type** | `string` | -![Min Items: N/A](https://img.shields.io/badge/Min%20Items%3A%20N/A-gold) -![Max Items: N/A](https://img.shields.io/badge/Max%20Items%3A%20N/A-gold) -![Item unicity: False](https://img.shields.io/badge/Item%20unicity%3A%20False-gold) -![Additional items: N/A](https://img.shields.io/badge/Additional%20items%3A%20N/A-gold) +**Example:** - ### ZarfManifest + +"charts/your-chart" -| | | -| ------------------------- | -------------------------------------------------------------------------------------------------------- | -| **Type** | `object` | -| **Additional properties** | [![Not allowed](https://img.shields.io/badge/Not%20allowed-red)](# "Additional Properties not allowed.") | -| **Defined in** | #/definitions/ZarfManifest | +
+
- name * + localPath  
-![Required](https://img.shields.io/badge/Required-red) - -**Description:** A name to give this collection of manifests; this will become the name of the dynamically-created helm chart +**Description:** The path to a local chart's folder or .tgz archive | | | | -------- | -------- | @@ -1231,12 +1256,14 @@ Must be one of:
- namespace + namespace *  
-**Description:** The namespace to deploy the manifests to +![Required](https://img.shields.io/badge/Required-red) + +**Description:** The namespace to deploy the chart to | | | | -------- | -------- | @@ -1247,23 +1274,12 @@ Must be one of:
- files + releaseName  
-**Description:** List of local K8s YAML files or remote URLs to deploy (in order) - -| | | -| -------- | ----------------- | -| **Type** | `array of string` | - -![Min Items: N/A](https://img.shields.io/badge/Min%20Items%3A%20N/A-gold) -![Max Items: N/A](https://img.shields.io/badge/Max%20Items%3A%20N/A-gold) -![Item unicity: False](https://img.shields.io/badge/Item%20unicity%3A%20False-gold) -![Additional items: N/A](https://img.shields.io/badge/Additional%20items%3A%20N/A-gold) - - ### files items +**Description:** The name of the Helm release to create (defaults to the Zarf name of the chart) | | | | -------- | -------- | @@ -1274,12 +1290,12 @@ Must be one of:
- kustomizeAllowAnyDirectory + noWait  
-**Description:** Allow traversing directory above the current directory if needed for kustomization +**Description:** Whether to not wait for chart resources to be ready before continuing | | | | -------- | --------- | @@ -1290,12 +1306,12 @@ Must be one of:
- kustomizations + valuesFiles  
-**Description:** List of local kustomization paths or remote URLs to include in the package +**Description:** List of local values file paths or remote URLs to include in the package; these will be merged together when deployed | | | | -------- | ----------------- | @@ -1306,7 +1322,7 @@ Must be one of: ![Item unicity: False](https://img.shields.io/badge/Item%20unicity%3A%20False-gold) ![Additional items: N/A](https://img.shields.io/badge/Additional%20items%3A%20N/A-gold) - ### kustomizations items + ### valuesFiles items | | | | -------- | -------- | @@ -1315,22 +1331,6 @@ Must be one of:
-
- - noWait - -  -
- -**Description:** Whether to not wait for manifest resources to be ready before continuing - -| | | -| -------- | --------- | -| **Type** | `boolean` | - -
-
-
@@ -1454,42 +1454,42 @@ Must be one of:
- namespace * + selector *  
![Required](https://img.shields.io/badge/Required-red) -**Description:** The namespace to target for data injection +**Description:** The K8s selector to target for data injection | | | | -------- | -------- | | **Type** | `string` | +**Example:** + + +"app=data-injection" +
- selector * + namespace *  
![Required](https://img.shields.io/badge/Required-red) -**Description:** The K8s selector to target for data injection +**Description:** The namespace to target for data injection | | | | -------- | -------- | | **Type** | `string` | -**Example:** - - -"app=data-injection" -
@@ -2130,70 +2130,70 @@ Must be one of:
- sensitive + type  
-**Description:** Whether to mark this variable as sensitive to not print it in the Zarf log +**Description:** Changes the handling of a variable to load contents differently (i.e. from a file rather than as a raw variable - templated files should be kept below 1 MiB) -| | | -| -------- | --------- | -| **Type** | `boolean` | +| | | +| -------- | ------------------ | +| **Type** | `enum (of string)` | + +:::note +Must be one of: +* "raw" +* "file" +:::
- autoIndent + pattern  
-**Description:** Whether to automatically indent the variable's value (if multiline) when templating. Based on the number of chars before the start of ###ZARF_VAR_. +**Description:** An optional regex pattern that a variable value must match before a package deployment can continue. -| | | -| -------- | --------- | -| **Type** | `boolean` | +| | | +| -------- | -------- | +| **Type** | `string` |
- pattern + sensitive  
-**Description:** An optional regex pattern that a variable value must match before a package deployment can continue. +**Description:** Whether to mark this variable as sensitive to not print it in the Zarf log -| | | -| -------- | -------- | -| **Type** | `string` | +| | | +| -------- | --------- | +| **Type** | `boolean` |
- type + autoIndent  
-**Description:** Changes the handling of a variable to load contents differently (i.e. from a file rather than as a raw variable - templated files should be kept below 1 MiB) - -| | | -| -------- | ------------------ | -| **Type** | `enum (of string)` | +**Description:** Whether to automatically indent the variable's value (if multiline) when templating. Based on the number of chars before the start of ###ZARF_VAR_. -:::note -Must be one of: -* "raw" -* "file" -::: +| | | +| -------- | --------- | +| **Type** | `boolean` |
diff --git a/zarf.schema.json b/zarf.schema.json index e91f979387..7ce8a33be6 100644 --- a/zarf.schema.json +++ b/zarf.schema.json @@ -249,11 +249,6 @@ "$ref": "#/definitions/ZarfComponentImport", "description": "Import a component from another Zarf package" }, - "scripts": { - "$schema": "http://json-schema.org/draft-04/schema#", - "$ref": "#/definitions/DeprecatedZarfComponentScripts", - "description": "[Deprecated] (replaced by actions) Custom commands to run before or after package deployment. This will be removed in Zarf v1.0.0." - }, "files": { "items": { "$schema": "http://json-schema.org/draft-04/schema#", @@ -262,21 +257,21 @@ "type": "array", "description": "Files or folders to place on disk during package deployment" }, - "charts": { + "manifests": { "items": { "$schema": "http://json-schema.org/draft-04/schema#", - "$ref": "#/definitions/ZarfChart" + "$ref": "#/definitions/ZarfManifest" }, "type": "array", - "description": "Helm charts to install during package deploy" + "description": "Kubernetes manifests to be included in a generated Helm chart on package deploy" }, - "manifests": { + "charts": { "items": { "$schema": "http://json-schema.org/draft-04/schema#", - "$ref": "#/definitions/ZarfManifest" + "$ref": "#/definitions/ZarfChart" }, "type": "array", - "description": "Kubernetes manifests to be included in a generated Helm chart on package deploy" + "description": "Helm charts to install during package deploy" }, "images": { "items": { @@ -305,6 +300,11 @@ "$ref": "#/definitions/ZarfComponentExtensions", "description": "Extend component functionality with additional features" }, + "scripts": { + "$schema": "http://json-schema.org/draft-04/schema#", + "$ref": "#/definitions/DeprecatedZarfComponentScripts", + "description": "[Deprecated] (replaced by actions) Custom commands to run before or after package deployment. This will be removed in Zarf v1.0.0." + }, "actions": { "$schema": "http://json-schema.org/draft-04/schema#", "$ref": "#/definitions/ZarfComponentActions", @@ -457,18 +457,6 @@ "type": "string", "description": "The name to be used for the variable" }, - "sensitive": { - "type": "boolean", - "description": "Whether to mark this variable as sensitive to not print it in the Zarf log" - }, - "autoIndent": { - "type": "boolean", - "description": "Whether to automatically indent the variable's value (if multiline) when templating. Based on the number of chars before the start of ###ZARF_VAR_." - }, - "pattern": { - "type": "string", - "description": "An optional regex pattern that a variable value must match before a package deployment can continue." - }, "type": { "enum": [ "raw", @@ -476,6 +464,18 @@ ], "type": "string", "description": "Changes the handling of a variable to load contents differently (i.e. from a file rather than as a raw variable - templated files should be kept below 1 MiB)" + }, + "pattern": { + "type": "string", + "description": "An optional regex pattern that a variable value must match before a package deployment can continue." + }, + "sensitive": { + "type": "boolean", + "description": "Whether to mark this variable as sensitive to not print it in the Zarf log" + }, + "autoIndent": { + "type": "boolean", + "description": "Whether to automatically indent the variable's value (if multiline) when templating. Based on the number of chars before the start of ###ZARF_VAR_." } }, "additionalProperties": false, @@ -706,16 +706,12 @@ }, "ZarfContainerTarget": { "required": [ - "namespace", "selector", + "namespace", "container", "path" ], "properties": { - "namespace": { - "type": "string", - "description": "The namespace to target for data injection" - }, "selector": { "type": "string", "description": "The K8s selector to target for data injection", @@ -723,6 +719,10 @@ "app\u0026#61;data-injection" ] }, + "namespace": { + "type": "string", + "description": "The namespace to target for data injection" + }, "container": { "type": "string", "description": "The container name to target for data injection" From 755f7ce9d427c7d0c2048e96c3277a60fba3b83e Mon Sep 17 00:00:00 2001 From: razzle Date: Thu, 4 Jan 2024 14:20:05 -0600 Subject: [PATCH 027/113] logic Signed-off-by: razzle --- src/internal/packager/validate/validate.go | 12 +++++----- src/pkg/packager/components.go | 6 ++--- src/pkg/packager/composer/override.go | 3 ++- src/pkg/packager/deprecated/common.go | 2 +- .../deprecated/required_to_optional.go | 24 ++++--------------- src/types/component.go | 9 +++---- 6 files changed, 22 insertions(+), 34 deletions(-) diff --git a/src/internal/packager/validate/validate.go b/src/internal/packager/validate/validate.go index 9bd0b9d58a..e2f1f28d07 100644 --- a/src/internal/packager/validate/validate.go +++ b/src/internal/packager/validate/validate.go @@ -62,14 +62,14 @@ func Run(pkg types.ZarfPackage) error { } // ensure groups don't have multiple defaults or only one component - if component.Group != "" { + if component.DeprecatedGroup != "" { if component.Default { - if _, ok := groupDefault[component.Group]; ok { - return fmt.Errorf(lang.PkgValidateErrGroupMultipleDefaults, component.Group, groupDefault[component.Group], component.Name) + if _, ok := groupDefault[component.DeprecatedGroup]; ok { + return fmt.Errorf(lang.PkgValidateErrGroupMultipleDefaults, component.DeprecatedGroup, groupDefault[component.DeprecatedGroup], component.Name) } - groupDefault[component.Group] = component.Name + groupDefault[component.DeprecatedGroup] = component.Name } - groupedComponents[component.Group] = append(groupedComponents[component.Group], component.Name) + groupedComponents[component.DeprecatedGroup] = append(groupedComponents[component.DeprecatedGroup], component.Name) } } @@ -137,7 +137,7 @@ func validateComponent(pkg types.ZarfPackage, component types.ZarfComponent) err if component.Default { return fmt.Errorf(lang.PkgValidateErrComponentReqDefault, component.Name) } - if component.Group != "" { + if component.DeprecatedGroup != "" { return fmt.Errorf(lang.PkgValidateErrComponentReqGrouped, component.Name) } } diff --git a/src/pkg/packager/components.go b/src/pkg/packager/components.go index 3d44a8291e..4e9bb7c0aa 100644 --- a/src/pkg/packager/components.go +++ b/src/pkg/packager/components.go @@ -32,8 +32,8 @@ func (p *Packager) getSelectedComponents() []types.ZarfComponent { // Group the components by Name and Group while maintaining order for _, component := range p.cfg.Pkg.Components { groupKey := component.Name - if component.Group != "" { - groupKey = component.Group + if component.DeprecatedGroup != "" { + groupKey = component.DeprecatedGroup } if !slices.Contains(orderedComponentGroups, groupKey) { @@ -81,7 +81,7 @@ func (p *Packager) getSelectedComponents() []types.ZarfComponent { // Then check for already selected groups if groupSelected != nil { - message.Fatalf(nil, lang.PkgDeployErrMultipleComponentsSameGroup, groupSelected.Name, component.Name, component.Group) + message.Fatalf(nil, lang.PkgDeployErrMultipleComponentsSameGroup, groupSelected.Name, component.Name, component.DeprecatedGroup) } // Then append to the final list diff --git a/src/pkg/packager/composer/override.go b/src/pkg/packager/composer/override.go index ea25f92438..3ae12c0068 100644 --- a/src/pkg/packager/composer/override.go +++ b/src/pkg/packager/composer/override.go @@ -14,6 +14,7 @@ func overrideMetadata(c *types.ZarfComponent, override types.ZarfComponent) erro c.Name = override.Name c.Default = override.Default c.Optional = override.Optional + c.DeprecatedRequired = override.DeprecatedRequired // Override description if it was provided. if override.Description != "" { @@ -37,7 +38,7 @@ func overrideDeprecated(c *types.ZarfComponent, override types.ZarfComponent) { c.DeprecatedCosignKeyPath = override.DeprecatedCosignKeyPath } - c.Group = override.Group + c.DeprecatedGroup = override.DeprecatedGroup // Merge deprecated scripts for backwards compatibility with older zarf binaries. c.DeprecatedScripts.Before = append(c.DeprecatedScripts.Before, override.DeprecatedScripts.Before...) diff --git a/src/pkg/packager/deprecated/common.go b/src/pkg/packager/deprecated/common.go index 826bd0c153..74d74da89a 100644 --- a/src/pkg/packager/deprecated/common.go +++ b/src/pkg/packager/deprecated/common.go @@ -78,7 +78,7 @@ func MigrateComponent(build types.ZarfBuildData, component types.ZarfComponent) } // Show a warning if the component contains a group as that has been deprecated and will be removed. - if component.Group != "" { + if component.DeprecatedGroup != "" { warnings = append(warnings, fmt.Sprintf("Component %s is using group which has been deprecated and will be removed in v1.0.0. Please migrate to another solution.", component.Name)) } diff --git a/src/pkg/packager/deprecated/required_to_optional.go b/src/pkg/packager/deprecated/required_to_optional.go index 228cb3b8fa..f579628180 100644 --- a/src/pkg/packager/deprecated/required_to_optional.go +++ b/src/pkg/packager/deprecated/required_to_optional.go @@ -7,7 +7,6 @@ package deprecated import ( "fmt" - "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "github.com/defenseunicorns/zarf/src/types" ) @@ -25,10 +24,13 @@ func (m RequiredToOptional) ID() string { // Clear the deprecated required. func (m RequiredToOptional) Clear(mc types.ZarfComponent) types.ZarfComponent { mc.DeprecatedRequired = nil + if mc.Optional != nil && *mc.Optional { + mc.Optional = nil + } return mc } -// Run converts the deprecated required to the new optional key +// Run is a no-op for this migration. func (m RequiredToOptional) Run(c types.ZarfComponent) (types.ZarfComponent, string) { if c.DeprecatedRequired == nil { return c, "" @@ -36,21 +38,5 @@ func (m RequiredToOptional) Run(c types.ZarfComponent) (types.ZarfComponent, str warning := fmt.Sprintf("Component %q is using \"required\" which will be removed in Zarf v1.0.0. Please migrate to \"optional\".", c.Name) - if *c.DeprecatedRequired { - if *c.Optional { - // ensure that optional wins over required - c.DeprecatedRequired = helpers.BoolPtr(false) - return c, warning - } else { - c.Optional = nil - return c, warning - } - } else { - if *c.Optional { - return c, warning - } else { - c.DeprecatedRequired = helpers.BoolPtr(true) - return c, warning - } - } + return c, warning } diff --git a/src/types/component.go b/src/types/component.go index cb26b9ad23..3f1b185a5d 100644 --- a/src/types/component.go +++ b/src/types/component.go @@ -29,11 +29,12 @@ type ZarfComponent struct { // Only include compatible components during package deployment Only ZarfComponentOnlyTarget `json:"only,omitempty" jsonschema:"description=Filter when this component is included in package creation or deployment"` - // Key to match other components to produce a user selector field, used to create a BOOLEAN XOR for a set of components + // DeprecatedGroup is a key to match other components to produce a user selector field, used to create a BOOLEAN XOR for a set of components + // // Note: ignores default and required flags - Group string `json:"group,omitempty" jsonschema:"description=[Deprecated] Create a user selector field based on all components in the same group. This will be removed in Zarf v1.0.0. Consider using 'only.flavor' instead.,deprecated=true"` + DeprecatedGroup string `json:"group,omitempty" jsonschema:"description=[Deprecated] Create a user selector field based on all components in the same group. This will be removed in Zarf v1.0.0. Consider using 'only.flavor' instead.,deprecated=true"` - // (Deprecated) Path to cosign public key for signed online resources + // DeprecatedCosignKeyPath to cosign public key for signed online resources DeprecatedCosignKeyPath string `json:"cosignKeyPath,omitempty" jsonschema:"description=[Deprecated] Specify a path to a public key to validate signed online resources. This will be removed in Zarf v1.0.0.,deprecated=true"` // Import refers to another zarf.yaml package component. @@ -60,7 +61,7 @@ type ZarfComponent struct { // Extensions provide additional functionality to a component Extensions extensions.ZarfComponentExtensions `json:"extensions,omitempty" jsonschema:"description=Extend component functionality with additional features"` - // (Deprecated) DeprecatedScripts are custom commands that run before or after package deployment + // DeprecatedScripts are custom commands that run before or after package deployment DeprecatedScripts DeprecatedZarfComponentScripts `json:"scripts,omitempty" jsonschema:"description=[Deprecated] (replaced by actions) Custom commands to run before or after package deployment. This will be removed in Zarf v1.0.0.,deprecated=true"` // Replaces scripts, fine-grained control over commands to run at various stages of a package lifecycle From a52c05646d5335367d3d730924f5364f463b1f4e Mon Sep 17 00:00:00 2001 From: razzle Date: Thu, 4 Jan 2024 15:18:39 -0600 Subject: [PATCH 028/113] move some stuff out of packger and into interactive Signed-off-by: razzle --- src/internal/packager/validate/validate.go | 2 +- src/pkg/interactive/components.go | 167 ++++++++++++++++++++ src/pkg/oci/pull.go | 21 +-- src/pkg/packager/components.go | 172 --------------------- src/pkg/packager/deploy.go | 3 +- src/pkg/packager/dev.go | 3 +- src/pkg/packager/mirror.go | 3 +- src/pkg/packager/remove.go | 5 +- src/pkg/packager/sources/oci.go | 11 +- 9 files changed, 191 insertions(+), 196 deletions(-) diff --git a/src/internal/packager/validate/validate.go b/src/internal/packager/validate/validate.go index e2f1f28d07..791b94ccb7 100644 --- a/src/internal/packager/validate/validate.go +++ b/src/internal/packager/validate/validate.go @@ -133,7 +133,7 @@ func validateComponent(pkg types.ZarfPackage, component types.ZarfComponent) err return fmt.Errorf(lang.PkgValidateErrComponentRequired, component.Name) } - if component.IsRequired() { + if component.Optional == nil || !*component.Optional { if component.Default { return fmt.Errorf(lang.PkgValidateErrComponentReqDefault, component.Name) } diff --git a/src/pkg/interactive/components.go b/src/pkg/interactive/components.go index bb8f244f75..0c76518747 100644 --- a/src/pkg/interactive/components.go +++ b/src/pkg/interactive/components.go @@ -6,6 +6,8 @@ package interactive import ( "fmt" + "path" + "slices" "strings" "github.com/AlecAivazis/survey/v2" @@ -13,10 +15,175 @@ import ( "github.com/defenseunicorns/zarf/src/config/lang" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/utils" + "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "github.com/defenseunicorns/zarf/src/types" "github.com/pterm/pterm" ) +type selectState int + +const ( + unknown selectState = iota + included + excluded +) + +func GetSelectedComponents(optionalComponents string, allComponents []types.ZarfComponent) []types.ZarfComponent { + var selectedComponents []types.ZarfComponent + groupedComponents := map[string][]types.ZarfComponent{} + orderedComponentGroups := []string{} + + // Group the components by Name and Group while maintaining order + for _, component := range allComponents { + groupKey := component.Name + if component.DeprecatedGroup != "" { + groupKey = component.DeprecatedGroup + } + + if !slices.Contains(orderedComponentGroups, groupKey) { + orderedComponentGroups = append(orderedComponentGroups, groupKey) + } + + groupedComponents[groupKey] = append(groupedComponents[groupKey], component) + } + + // Split the --components list as a comma-delimited list + requestedComponents := helpers.StringToSlice(optionalComponents) + isPartial := len(requestedComponents) > 0 && requestedComponents[0] != "" + + if isPartial { + matchedRequests := map[string]bool{} + + // NOTE: This does not use forIncludedComponents as it takes group, default and required status into account. + for _, groupKey := range orderedComponentGroups { + var groupDefault *types.ZarfComponent + var groupSelected *types.ZarfComponent + + for _, component := range groupedComponents[groupKey] { + // Ensure we have a local version of the component to point to (otherwise the pointer might change on us) + component := component + + selectState, matchedRequest := includedOrExcluded(component, requestedComponents) + + if !component.IsRequired() { + if selectState == excluded { + // If the component was explicitly excluded, record the match and continue + matchedRequests[matchedRequest] = true + continue + } else if selectState == unknown && component.Default && groupDefault == nil { + // If the component is default but not included or excluded, remember the default + groupDefault = &component + } + } else { + // Force the selectState to included for Required components + selectState = included + } + + if selectState == included { + // If the component was explicitly included, record the match + matchedRequests[matchedRequest] = true + + // Then check for already selected groups + if groupSelected != nil { + message.Fatalf(nil, lang.PkgDeployErrMultipleComponentsSameGroup, groupSelected.Name, component.Name, component.DeprecatedGroup) + } + + // Then append to the final list + selectedComponents = append(selectedComponents, component) + groupSelected = &component + } + } + + // If nothing was selected from a group, handle the default + if groupSelected == nil && groupDefault != nil { + selectedComponents = append(selectedComponents, *groupDefault) + } else if len(groupedComponents[groupKey]) > 1 && groupSelected == nil && groupDefault == nil { + // If no default component was found, give up + componentNames := []string{} + for _, component := range groupedComponents[groupKey] { + componentNames = append(componentNames, component.Name) + } + message.Fatalf(nil, lang.PkgDeployErrNoDefaultOrSelection, strings.Join(componentNames, ",")) + } + } + + // Check that we have matched against all requests + for _, requestedComponent := range requestedComponents { + if _, ok := matchedRequests[requestedComponent]; !ok { + message.Fatalf(nil, lang.PkgDeployErrNoCompatibleComponentsForSelection, requestedComponent) + } + } + } else { + for _, groupKey := range orderedComponentGroups { + if len(groupedComponents[groupKey]) > 1 { + component := SelectChoiceGroup(groupedComponents[groupKey]) + selectedComponents = append(selectedComponents, component) + } else { + component := groupedComponents[groupKey][0] + + if component.IsRequired() { + selectedComponents = append(selectedComponents, component) + } else if selected := SelectOptionalComponent(component); selected { + selectedComponents = append(selectedComponents, component) + } + } + } + } + + return selectedComponents +} + +func ForIncludedComponents(optionalComponents string, components []types.ZarfComponent, onIncluded func(types.ZarfComponent) error) error { + requestedComponents := helpers.StringToSlice(optionalComponents) + isPartial := len(requestedComponents) > 0 && requestedComponents[0] != "" + + for _, component := range components { + selectState := unknown + + if isPartial { + selectState, _ = includedOrExcluded(component, requestedComponents) + + if selectState == excluded { + continue + } + } else { + selectState = included + } + + if selectState == included { + if err := onIncluded(component); err != nil { + return err + } + } + } + + return nil +} + +func includedOrExcluded(component types.ZarfComponent, requestedComponentNames []string) (selectState, string) { + // Check if the component has a leading dash indicating it should be excluded - this is done first so that exclusions precede inclusions + for _, requestedComponent := range requestedComponentNames { + if strings.HasPrefix(requestedComponent, "-") { + // If the component glob matches one of the requested components, then return true + // This supports globbing with "path" in order to have the same behavior across OSes (if we ever allow namespaced components with /) + if matched, _ := path.Match(strings.TrimPrefix(requestedComponent, "-"), component.Name); matched { + return excluded, requestedComponent + } + } + } + // Check if the component matches a glob pattern and should be included + for _, requestedComponent := range requestedComponentNames { + // If the component glob matches one of the requested components, then return true + // This supports globbing with "path" in order to have the same behavior across OSes (if we ever allow namespaced components with /) + if matched, _ := path.Match(requestedComponent, component.Name); matched { + return included, requestedComponent + } + } + + // All other cases we don't know if we should include or exclude yet + return unknown, "" +} + // SelectOptionalComponent prompts to confirm optional components func SelectOptionalComponent(component types.ZarfComponent) (confirmComponent bool) { // Confirm flag passed, just use defaults diff --git a/src/pkg/oci/pull.go b/src/pkg/oci/pull.go index e44db67a73..dbc6b3f582 100644 --- a/src/pkg/oci/pull.go +++ b/src/pkg/oci/pull.go @@ -78,9 +78,7 @@ func (o *OrasRemote) LayersFromPaths(requestedPaths []string) (layers []ocispec. // LayersFromRequestedComponents returns the descriptors for the given components from the root manifest. // // It also retrieves the descriptors for all image layers that are required by the components. -// -// It also respects the `required` flag on components, and will retrieve all necessary layers for required components. -func (o *OrasRemote) LayersFromRequestedComponents(requestedComponents []string) (layers []ocispec.Descriptor, err error) { +func (o *OrasRemote) LayersFromRequestedComponents(requestedComponents []types.ZarfComponent) (layers []ocispec.Descriptor, err error) { root, err := o.FetchRoot() if err != nil { return nil, err @@ -92,22 +90,17 @@ func (o *OrasRemote) LayersFromRequestedComponents(requestedComponents []string) } images := map[string]bool{} tarballFormat := "%s.tar" - for _, name := range requestedComponents { + for _, rc := range requestedComponents { component := helpers.Find(pkg.Components, func(component types.ZarfComponent) bool { - return component.Name == name + return component.Name == rc.Name }) if component.Name == "" { - return nil, fmt.Errorf("component %s does not exist in this package", name) + return nil, fmt.Errorf("component %s does not exist in this package", rc.Name) } - } - for _, component := range pkg.Components { - // If we requested this component, or it is required, we need to pull its images and tarball - if slices.Contains(requestedComponents, component.Name) || component.IsRequired() { - for _, image := range component.Images { - images[image] = true - } - layers = append(layers, root.Locate(filepath.Join(layout.ComponentsDir, fmt.Sprintf(tarballFormat, component.Name)))) + for _, image := range component.Images { + images[image] = true } + layers = append(layers, root.Locate(filepath.Join(layout.ComponentsDir, fmt.Sprintf(tarballFormat, component.Name)))) } // Append the sboms.tar layer if it exists // diff --git a/src/pkg/packager/components.go b/src/pkg/packager/components.go index 4e9bb7c0aa..a23cd42f49 100644 --- a/src/pkg/packager/components.go +++ b/src/pkg/packager/components.go @@ -5,181 +5,9 @@ package packager import ( - "path" - "slices" - "strings" - - "github.com/defenseunicorns/zarf/src/config/lang" - "github.com/defenseunicorns/zarf/src/pkg/interactive" - "github.com/defenseunicorns/zarf/src/pkg/message" - "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "github.com/defenseunicorns/zarf/src/types" ) -type selectState int - -const ( - unknown selectState = iota - included - excluded -) - -func (p *Packager) getSelectedComponents() []types.ZarfComponent { - var selectedComponents []types.ZarfComponent - groupedComponents := map[string][]types.ZarfComponent{} - orderedComponentGroups := []string{} - - // Group the components by Name and Group while maintaining order - for _, component := range p.cfg.Pkg.Components { - groupKey := component.Name - if component.DeprecatedGroup != "" { - groupKey = component.DeprecatedGroup - } - - if !slices.Contains(orderedComponentGroups, groupKey) { - orderedComponentGroups = append(orderedComponentGroups, groupKey) - } - - groupedComponents[groupKey] = append(groupedComponents[groupKey], component) - } - - // Split the --components list as a comma-delimited list - requestedComponents := helpers.StringToSlice(p.cfg.PkgOpts.OptionalComponents) - isPartial := len(requestedComponents) > 0 && requestedComponents[0] != "" - - if isPartial { - matchedRequests := map[string]bool{} - - // NOTE: This does not use forIncludedComponents as it takes group, default and required status into account. - for _, groupKey := range orderedComponentGroups { - var groupDefault *types.ZarfComponent - var groupSelected *types.ZarfComponent - - for _, component := range groupedComponents[groupKey] { - // Ensure we have a local version of the component to point to (otherwise the pointer might change on us) - component := component - - selectState, matchedRequest := includedOrExcluded(component, requestedComponents) - - if !component.IsRequired() { - if selectState == excluded { - // If the component was explicitly excluded, record the match and continue - matchedRequests[matchedRequest] = true - continue - } else if selectState == unknown && component.Default && groupDefault == nil { - // If the component is default but not included or excluded, remember the default - groupDefault = &component - } - } else { - // Force the selectState to included for Required components - selectState = included - } - - if selectState == included { - // If the component was explicitly included, record the match - matchedRequests[matchedRequest] = true - - // Then check for already selected groups - if groupSelected != nil { - message.Fatalf(nil, lang.PkgDeployErrMultipleComponentsSameGroup, groupSelected.Name, component.Name, component.DeprecatedGroup) - } - - // Then append to the final list - selectedComponents = append(selectedComponents, component) - groupSelected = &component - } - } - - // If nothing was selected from a group, handle the default - if groupSelected == nil && groupDefault != nil { - selectedComponents = append(selectedComponents, *groupDefault) - } else if len(groupedComponents[groupKey]) > 1 && groupSelected == nil && groupDefault == nil { - // If no default component was found, give up - componentNames := []string{} - for _, component := range groupedComponents[groupKey] { - componentNames = append(componentNames, component.Name) - } - message.Fatalf(nil, lang.PkgDeployErrNoDefaultOrSelection, strings.Join(componentNames, ",")) - } - } - - // Check that we have matched against all requests - for _, requestedComponent := range requestedComponents { - if _, ok := matchedRequests[requestedComponent]; !ok { - message.Fatalf(nil, lang.PkgDeployErrNoCompatibleComponentsForSelection, requestedComponent) - } - } - } else { - for _, groupKey := range orderedComponentGroups { - if len(groupedComponents[groupKey]) > 1 { - component := interactive.SelectChoiceGroup(groupedComponents[groupKey]) - selectedComponents = append(selectedComponents, component) - } else { - component := groupedComponents[groupKey][0] - - if component.IsRequired() { - selectedComponents = append(selectedComponents, component) - } else if selected := interactive.SelectOptionalComponent(component); selected { - selectedComponents = append(selectedComponents, component) - } - } - } - } - - return selectedComponents -} - -func (p *Packager) forIncludedComponents(onIncluded func(types.ZarfComponent) error) error { - requestedComponents := helpers.StringToSlice(p.cfg.PkgOpts.OptionalComponents) - isPartial := len(requestedComponents) > 0 && requestedComponents[0] != "" - - for _, component := range p.cfg.Pkg.Components { - selectState := unknown - - if isPartial { - selectState, _ = includedOrExcluded(component, requestedComponents) - - if selectState == excluded { - continue - } - } else { - selectState = included - } - - if selectState == included { - if err := onIncluded(component); err != nil { - return err - } - } - } - - return nil -} - -func includedOrExcluded(component types.ZarfComponent, requestedComponentNames []string) (selectState, string) { - // Check if the component has a leading dash indicating it should be excluded - this is done first so that exclusions precede inclusions - for _, requestedComponent := range requestedComponentNames { - if strings.HasPrefix(requestedComponent, "-") { - // If the component glob matches one of the requested components, then return true - // This supports globbing with "path" in order to have the same behavior across OSes (if we ever allow namespaced components with /) - if matched, _ := path.Match(strings.TrimPrefix(requestedComponent, "-"), component.Name); matched { - return excluded, requestedComponent - } - } - } - // Check if the component matches a glob pattern and should be included - for _, requestedComponent := range requestedComponentNames { - // If the component glob matches one of the requested components, then return true - // This supports globbing with "path" in order to have the same behavior across OSes (if we ever allow namespaced components with /) - if matched, _ := path.Match(requestedComponent, component.Name); matched { - return included, requestedComponent - } - } - - // All other cases we don't know if we should include or exclude yet - return unknown, "" -} - func requiresCluster(component types.ZarfComponent) bool { hasImages := len(component.Images) > 0 hasCharts := len(component.Charts) > 0 diff --git a/src/pkg/packager/deploy.go b/src/pkg/packager/deploy.go index f76d20962a..43e1a85b99 100644 --- a/src/pkg/packager/deploy.go +++ b/src/pkg/packager/deploy.go @@ -20,6 +20,7 @@ import ( "github.com/defenseunicorns/zarf/src/internal/packager/images" "github.com/defenseunicorns/zarf/src/internal/packager/template" "github.com/defenseunicorns/zarf/src/pkg/cluster" + "github.com/defenseunicorns/zarf/src/pkg/interactive" "github.com/defenseunicorns/zarf/src/pkg/k8s" "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/message" @@ -93,7 +94,7 @@ func (p *Packager) Deploy() (err error) { // deployComponents loops through a list of ZarfComponents and deploys them. func (p *Packager) deployComponents() (deployedComponents []types.DeployedComponent, err error) { - componentsToDeploy := p.getSelectedComponents() + componentsToDeploy := interactive.GetSelectedComponents(p.cfg.PkgOpts.OptionalComponents, p.cfg.Pkg.Components) // Generate a value template if p.valueTemplate, err = template.Generate(p.cfg); err != nil { diff --git a/src/pkg/packager/dev.go b/src/pkg/packager/dev.go index b4e1744d46..b7d3892be2 100644 --- a/src/pkg/packager/dev.go +++ b/src/pkg/packager/dev.go @@ -10,6 +10,7 @@ import ( "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/internal/packager/validate" + "github.com/defenseunicorns/zarf/src/pkg/interactive" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/types" ) @@ -40,7 +41,7 @@ func (p *Packager) DevDeploy() error { // the user's selection and the component's `required` field // This is also different from regular package creation, where we still assemble and package up // all components and their dependencies, regardless of whether they are required or not - p.cfg.Pkg.Components = p.getSelectedComponents() + p.cfg.Pkg.Components = interactive.GetSelectedComponents(p.cfg.PkgOpts.OptionalComponents, p.cfg.Pkg.Components) if err := validate.Run(p.cfg.Pkg); err != nil { return fmt.Errorf("unable to validate package: %w", err) diff --git a/src/pkg/packager/mirror.go b/src/pkg/packager/mirror.go index 689ddcfbaa..6bc0ac64b0 100644 --- a/src/pkg/packager/mirror.go +++ b/src/pkg/packager/mirror.go @@ -9,6 +9,7 @@ import ( "strings" "github.com/defenseunicorns/zarf/src/config" + "github.com/defenseunicorns/zarf/src/pkg/interactive" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/types" ) @@ -44,7 +45,7 @@ func (p *Packager) Mirror() (err error) { p.filterComponents() // Run mirror for each requested component - return p.forIncludedComponents(p.mirrorComponent) + return interactive.ForIncludedComponents(p.cfg.PkgOpts.OptionalComponents, p.cfg.Pkg.Components, p.mirrorComponent) } // mirrorComponent mirrors a Zarf Component. diff --git a/src/pkg/packager/remove.go b/src/pkg/packager/remove.go index 2aaee0c3af..d12c3bc6ba 100644 --- a/src/pkg/packager/remove.go +++ b/src/pkg/packager/remove.go @@ -14,6 +14,7 @@ import ( "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/internal/packager/helm" "github.com/defenseunicorns/zarf/src/pkg/cluster" + "github.com/defenseunicorns/zarf/src/pkg/interactive" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/packager/sources" "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" @@ -49,7 +50,7 @@ func (p *Packager) Remove() (err error) { packageRequiresCluster := false // If components were provided; just remove the things we were asked to remove - p.forIncludedComponents(func(component types.ZarfComponent) error { + interactive.ForIncludedComponents(p.cfg.PkgOpts.OptionalComponents, p.cfg.Pkg.Components, (func(component types.ZarfComponent) error { componentsToRemove = append(componentsToRemove, component.Name) if requiresCluster(component) { @@ -57,7 +58,7 @@ func (p *Packager) Remove() (err error) { } return nil - }) + })) // Get or build the secret for the deployed package deployedPackage := &types.DeployedPackage{} diff --git a/src/pkg/packager/sources/oci.go b/src/pkg/packager/sources/oci.go index 371c53a7b5..f0bea6021f 100644 --- a/src/pkg/packager/sources/oci.go +++ b/src/pkg/packager/sources/oci.go @@ -12,11 +12,11 @@ import ( "strings" "github.com/defenseunicorns/zarf/src/config" + "github.com/defenseunicorns/zarf/src/pkg/interactive" "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/oci" "github.com/defenseunicorns/zarf/src/pkg/utils" - "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "github.com/defenseunicorns/zarf/src/types" "github.com/mholt/archiver/v3" ocispec "github.com/opencontainers/image-spec/specs-go/v1" @@ -40,11 +40,14 @@ func (s *OCISource) LoadPackage(dst *layout.PackagePaths, unarchiveAll bool) (er message.Debugf("Loading package from %q", s.PackageSource) - optionalComponents := helpers.StringToSlice(s.OptionalComponents) - // pull only needed layers if --confirm is set if config.CommonOptions.Confirm { - layersToPull, err = s.LayersFromRequestedComponents(optionalComponents) + pkg, err = s.FetchZarfYAML() + if err != nil { + return err + } + requested := interactive.GetSelectedComponents(s.OptionalComponents, pkg.Components) + layersToPull, err = s.LayersFromRequestedComponents(requested) if err != nil { return fmt.Errorf("unable to get published component image layers: %s", err.Error()) } From 2e4ac72c4f02c414c62740fd9aded68120a870d7 Mon Sep 17 00:00:00 2001 From: razzle Date: Thu, 4 Jan 2024 18:13:07 -0600 Subject: [PATCH 029/113] fix clear logic Signed-off-by: razzle --- src/pkg/packager/deprecated/required_to_optional.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pkg/packager/deprecated/required_to_optional.go b/src/pkg/packager/deprecated/required_to_optional.go index f579628180..384a5f9e7d 100644 --- a/src/pkg/packager/deprecated/required_to_optional.go +++ b/src/pkg/packager/deprecated/required_to_optional.go @@ -24,7 +24,7 @@ func (m RequiredToOptional) ID() string { // Clear the deprecated required. func (m RequiredToOptional) Clear(mc types.ZarfComponent) types.ZarfComponent { mc.DeprecatedRequired = nil - if mc.Optional != nil && *mc.Optional { + if mc.Optional != nil && !*mc.Optional { mc.Optional = nil } return mc From e3a74f2e7144c8e5c8e3dbb8ee674878b84b5286 Mon Sep 17 00:00:00 2001 From: razzle Date: Thu, 4 Jan 2024 18:16:41 -0600 Subject: [PATCH 030/113] re-org Signed-off-by: razzle --- src/pkg/interactive/components.go | 73 -------------------------- src/pkg/interactive/select.go | 86 +++++++++++++++++++++++++++++++ 2 files changed, 86 insertions(+), 73 deletions(-) create mode 100644 src/pkg/interactive/select.go diff --git a/src/pkg/interactive/components.go b/src/pkg/interactive/components.go index 0c76518747..be921448e8 100644 --- a/src/pkg/interactive/components.go +++ b/src/pkg/interactive/components.go @@ -5,19 +5,14 @@ package interactive import ( - "fmt" "path" "slices" "strings" - "github.com/AlecAivazis/survey/v2" - "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/config/lang" "github.com/defenseunicorns/zarf/src/pkg/message" - "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "github.com/defenseunicorns/zarf/src/types" - "github.com/pterm/pterm" ) type selectState int @@ -183,71 +178,3 @@ func includedOrExcluded(component types.ZarfComponent, requestedComponentNames [ // All other cases we don't know if we should include or exclude yet return unknown, "" } - -// SelectOptionalComponent prompts to confirm optional components -func SelectOptionalComponent(component types.ZarfComponent) (confirmComponent bool) { - // Confirm flag passed, just use defaults - if config.CommonOptions.Confirm { - return component.Default - } - - message.HorizontalRule() - - displayComponent := component - displayComponent.Description = "" - utils.ColorPrintYAML(displayComponent, nil, false) - if component.Description != "" { - message.Question(component.Description) - } - - prompt := &survey.Confirm{ - Message: fmt.Sprintf("Deploy the %s component?", component.Name), - Default: component.Default, - } - if err := survey.AskOne(prompt, &confirmComponent); err != nil { - message.Fatalf(nil, lang.PkgDeployErrComponentSelectionCanceled, err.Error()) - } - - return confirmComponent -} - -// SelectChoiceGroup prompts to select component groups -func SelectChoiceGroup(componentGroup []types.ZarfComponent) types.ZarfComponent { - // Confirm flag passed, just use defaults - if config.CommonOptions.Confirm { - var componentNames []string - for _, component := range componentGroup { - // If the component is default, then return it - if component.Default { - return component - } - // Add each component name to the list - componentNames = append(componentNames, component.Name) - } - // If no default component was found, give up - message.Fatalf(nil, lang.PkgDeployErrNoDefaultOrSelection, strings.Join(componentNames, ",")) - } - - message.HorizontalRule() - - var chosen int - var options []string - - for _, component := range componentGroup { - text := fmt.Sprintf("Name: %s\n Description: %s\n", component.Name, component.Description) - options = append(options, text) - } - - prompt := &survey.Select{ - Message: "Select a component to deploy:", - Options: options, - } - - pterm.Println() - - if err := survey.AskOne(prompt, &chosen); err != nil { - message.Fatalf(nil, lang.PkgDeployErrComponentSelectionCanceled, err.Error()) - } - - return componentGroup[chosen] -} diff --git a/src/pkg/interactive/select.go b/src/pkg/interactive/select.go new file mode 100644 index 0000000000..bb8f244f75 --- /dev/null +++ b/src/pkg/interactive/select.go @@ -0,0 +1,86 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package interactive contains functions for interacting with the user via STDIN. +package interactive + +import ( + "fmt" + "strings" + + "github.com/AlecAivazis/survey/v2" + "github.com/defenseunicorns/zarf/src/config" + "github.com/defenseunicorns/zarf/src/config/lang" + "github.com/defenseunicorns/zarf/src/pkg/message" + "github.com/defenseunicorns/zarf/src/pkg/utils" + "github.com/defenseunicorns/zarf/src/types" + "github.com/pterm/pterm" +) + +// SelectOptionalComponent prompts to confirm optional components +func SelectOptionalComponent(component types.ZarfComponent) (confirmComponent bool) { + // Confirm flag passed, just use defaults + if config.CommonOptions.Confirm { + return component.Default + } + + message.HorizontalRule() + + displayComponent := component + displayComponent.Description = "" + utils.ColorPrintYAML(displayComponent, nil, false) + if component.Description != "" { + message.Question(component.Description) + } + + prompt := &survey.Confirm{ + Message: fmt.Sprintf("Deploy the %s component?", component.Name), + Default: component.Default, + } + if err := survey.AskOne(prompt, &confirmComponent); err != nil { + message.Fatalf(nil, lang.PkgDeployErrComponentSelectionCanceled, err.Error()) + } + + return confirmComponent +} + +// SelectChoiceGroup prompts to select component groups +func SelectChoiceGroup(componentGroup []types.ZarfComponent) types.ZarfComponent { + // Confirm flag passed, just use defaults + if config.CommonOptions.Confirm { + var componentNames []string + for _, component := range componentGroup { + // If the component is default, then return it + if component.Default { + return component + } + // Add each component name to the list + componentNames = append(componentNames, component.Name) + } + // If no default component was found, give up + message.Fatalf(nil, lang.PkgDeployErrNoDefaultOrSelection, strings.Join(componentNames, ",")) + } + + message.HorizontalRule() + + var chosen int + var options []string + + for _, component := range componentGroup { + text := fmt.Sprintf("Name: %s\n Description: %s\n", component.Name, component.Description) + options = append(options, text) + } + + prompt := &survey.Select{ + Message: "Select a component to deploy:", + Options: options, + } + + pterm.Println() + + if err := survey.AskOne(prompt, &chosen); err != nil { + message.Fatalf(nil, lang.PkgDeployErrComponentSelectionCanceled, err.Error()) + } + + return componentGroup[chosen] +} From f05fd16d6922218e32952bb6a25666289eb2a8ff Mon Sep 17 00:00:00 2001 From: razzle Date: Thu, 4 Jan 2024 18:46:14 -0600 Subject: [PATCH 031/113] lets go the distance Signed-off-by: razzle --- go.mod | 2 +- src/config/lang/english.go | 6 +++--- src/pkg/interactive/components.go | 24 ++++++++++++++++++------ src/pkg/packager/deploy.go | 5 ++++- src/pkg/packager/dev.go | 7 +++++-- src/pkg/packager/sources/oci.go | 5 ++++- 6 files changed, 35 insertions(+), 14 deletions(-) diff --git a/go.mod b/go.mod index 36bc627772..231dd302f8 100644 --- a/go.mod +++ b/go.mod @@ -9,6 +9,7 @@ require ( cuelang.org/go v0.7.0 github.com/AlecAivazis/survey/v2 v2.3.7 github.com/Masterminds/semver/v3 v3.2.1 + github.com/agnivade/levenshtein v1.1.1 github.com/alecthomas/jsonschema v0.0.0-20220216202328-9eeeec9d044b github.com/anchore/clio v0.0.0-20231220164737-3e50431641a5 github.com/anchore/stereoscope v0.0.0-20231220161148-590920dabc54 @@ -108,7 +109,6 @@ require ( github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect github.com/acobaugh/osrelease v0.1.0 // indirect github.com/adrg/xdg v0.4.0 // indirect - github.com/agnivade/levenshtein v1.1.1 // indirect github.com/alibabacloud-go/alibabacloud-gateway-spi v0.0.4 // indirect github.com/alibabacloud-go/cr-20160607 v1.0.1 // indirect github.com/alibabacloud-go/cr-20181201 v1.0.10 // indirect diff --git a/src/config/lang/english.go b/src/config/lang/english.go index e834b222e6..4da00b8b6c 100644 --- a/src/config/lang/english.go +++ b/src/config/lang/english.go @@ -625,9 +625,9 @@ const ( // src/internal/packager/deploy. const ( - PkgDeployErrMultipleComponentsSameGroup = "You cannot specify multiple components (%q, %q) within the same group (%q) when using the --components flag." - PkgDeployErrNoDefaultOrSelection = "You must make a selection from %q with the --components flag as there is no default in their group." - PkgDeployErrNoCompatibleComponentsForSelection = "No compatible components found that matched %q. Please check spelling and try again." + PkgDeployErrMultipleComponentsSameGroup = "cannot specify multiple components (%q, %q) within the same group (%q) when using the --components flag" + PkgDeployErrNoDefaultOrSelection = "no selection made from %q with the --components flag and there is no default in the group" + PkgDeployErrNoCompatibleComponentsForSelection = "no compatible components found that matched %q, suggestion(s): %s" PkgDeployErrComponentSelectionCanceled = "Component selection canceled: %s" ) diff --git a/src/pkg/interactive/components.go b/src/pkg/interactive/components.go index be921448e8..314f1e95f4 100644 --- a/src/pkg/interactive/components.go +++ b/src/pkg/interactive/components.go @@ -5,12 +5,13 @@ package interactive import ( + "fmt" "path" "slices" "strings" + "github.com/agnivade/levenshtein" "github.com/defenseunicorns/zarf/src/config/lang" - "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "github.com/defenseunicorns/zarf/src/types" ) @@ -23,7 +24,8 @@ const ( excluded ) -func GetSelectedComponents(optionalComponents string, allComponents []types.ZarfComponent) []types.ZarfComponent { +// GetSelectedComponents prompts to select components based upon multiple conditions +func GetSelectedComponents(optionalComponents string, allComponents []types.ZarfComponent) ([]types.ZarfComponent, error) { var selectedComponents []types.ZarfComponent groupedComponents := map[string][]types.ZarfComponent{} orderedComponentGroups := []string{} @@ -80,7 +82,7 @@ func GetSelectedComponents(optionalComponents string, allComponents []types.Zarf // Then check for already selected groups if groupSelected != nil { - message.Fatalf(nil, lang.PkgDeployErrMultipleComponentsSameGroup, groupSelected.Name, component.Name, component.DeprecatedGroup) + return []types.ZarfComponent{}, fmt.Errorf(lang.PkgDeployErrMultipleComponentsSameGroup, groupSelected.Name, component.Name, component.DeprecatedGroup) } // Then append to the final list @@ -98,14 +100,23 @@ func GetSelectedComponents(optionalComponents string, allComponents []types.Zarf for _, component := range groupedComponents[groupKey] { componentNames = append(componentNames, component.Name) } - message.Fatalf(nil, lang.PkgDeployErrNoDefaultOrSelection, strings.Join(componentNames, ",")) + return []types.ZarfComponent{}, fmt.Errorf(lang.PkgDeployErrNoDefaultOrSelection, strings.Join(componentNames, ", ")) } } // Check that we have matched against all requests for _, requestedComponent := range requestedComponents { if _, ok := matchedRequests[requestedComponent]; !ok { - message.Fatalf(nil, lang.PkgDeployErrNoCompatibleComponentsForSelection, requestedComponent) + names := []string{} + for _, c := range allComponents { + d := levenshtein.ComputeDistance(c.Name, requestedComponent) + if d <= 5 { + return []types.ZarfComponent{}, fmt.Errorf(lang.PkgDeployErrNoCompatibleComponentsForSelection, requestedComponent, c.Name) + } + names = append(names, c.Name) + } + + return []types.ZarfComponent{}, fmt.Errorf(lang.PkgDeployErrNoCompatibleComponentsForSelection, requestedComponent, strings.Join(names, ", ")) } } } else { @@ -125,9 +136,10 @@ func GetSelectedComponents(optionalComponents string, allComponents []types.Zarf } } - return selectedComponents + return selectedComponents, nil } +// ForIncludedComponents iterates over components and calls onIncluded for each component that should be included func ForIncludedComponents(optionalComponents string, components []types.ZarfComponent, onIncluded func(types.ZarfComponent) error) error { requestedComponents := helpers.StringToSlice(optionalComponents) isPartial := len(requestedComponents) > 0 && requestedComponents[0] != "" diff --git a/src/pkg/packager/deploy.go b/src/pkg/packager/deploy.go index 43e1a85b99..4b91d03988 100644 --- a/src/pkg/packager/deploy.go +++ b/src/pkg/packager/deploy.go @@ -94,7 +94,10 @@ func (p *Packager) Deploy() (err error) { // deployComponents loops through a list of ZarfComponents and deploys them. func (p *Packager) deployComponents() (deployedComponents []types.DeployedComponent, err error) { - componentsToDeploy := interactive.GetSelectedComponents(p.cfg.PkgOpts.OptionalComponents, p.cfg.Pkg.Components) + componentsToDeploy, err := interactive.GetSelectedComponents(p.cfg.PkgOpts.OptionalComponents, p.cfg.Pkg.Components) + if err != nil { + return deployedComponents, fmt.Errorf("unable to get selected components: %w", err) + } // Generate a value template if p.valueTemplate, err = template.Generate(p.cfg); err != nil { diff --git a/src/pkg/packager/dev.go b/src/pkg/packager/dev.go index b7d3892be2..b93dfed8de 100644 --- a/src/pkg/packager/dev.go +++ b/src/pkg/packager/dev.go @@ -38,10 +38,13 @@ func (p *Packager) DevDeploy() error { // Also filter out components that are not required, nor requested via --components // This is different from the above filter, as it is not based on the system, but rather - // the user's selection and the component's `required` field + // the user's selection and the component's `required`/`optional` field // This is also different from regular package creation, where we still assemble and package up // all components and their dependencies, regardless of whether they are required or not - p.cfg.Pkg.Components = interactive.GetSelectedComponents(p.cfg.PkgOpts.OptionalComponents, p.cfg.Pkg.Components) + p.cfg.Pkg.Components, err = interactive.GetSelectedComponents(p.cfg.PkgOpts.OptionalComponents, p.cfg.Pkg.Components) + if err != nil { + return err + } if err := validate.Run(p.cfg.Pkg); err != nil { return fmt.Errorf("unable to validate package: %w", err) diff --git a/src/pkg/packager/sources/oci.go b/src/pkg/packager/sources/oci.go index f0bea6021f..78200fbc1b 100644 --- a/src/pkg/packager/sources/oci.go +++ b/src/pkg/packager/sources/oci.go @@ -46,7 +46,10 @@ func (s *OCISource) LoadPackage(dst *layout.PackagePaths, unarchiveAll bool) (er if err != nil { return err } - requested := interactive.GetSelectedComponents(s.OptionalComponents, pkg.Components) + requested, err := interactive.GetSelectedComponents(s.OptionalComponents, pkg.Components) + if err != nil { + return err + } layersToPull, err = s.LayersFromRequestedComponents(requested) if err != nil { return fmt.Errorf("unable to get published component image layers: %s", err.Error()) From 18f9a29e6c494354ee0fe36d50ba407ab3893bb1 Mon Sep 17 00:00:00 2001 From: razzle Date: Thu, 4 Jan 2024 19:38:23 -0600 Subject: [PATCH 032/113] docs and schema Signed-off-by: razzle --- docs/3-create-a-zarf-package/4-zarf-schema.md | 412 +++++++++--------- src/types/component.go | 14 +- zarf.schema.json | 42 +- 3 files changed, 234 insertions(+), 234 deletions(-) diff --git a/docs/3-create-a-zarf-package/4-zarf-schema.md b/docs/3-create-a-zarf-package/4-zarf-schema.md index 23640b7f4e..dc2807bdc5 100644 --- a/docs/3-create-a-zarf-package/4-zarf-schema.md +++ b/docs/3-create-a-zarf-package/4-zarf-schema.md @@ -825,148 +825,6 @@ Must be one of:
-
- - files - -  -
- - ## components > files - -**Description:** Files or folders to place on disk during package deployment - -| | | -| -------- | ------- | -| **Type** | `array` | - -![Min Items: N/A](https://img.shields.io/badge/Min%20Items%3A%20N/A-gold) -![Max Items: N/A](https://img.shields.io/badge/Max%20Items%3A%20N/A-gold) -![Item unicity: False](https://img.shields.io/badge/Item%20unicity%3A%20False-gold) -![Additional items: N/A](https://img.shields.io/badge/Additional%20items%3A%20N/A-gold) - - ### ZarfFile - -| | | -| ------------------------- | -------------------------------------------------------------------------------------------------------- | -| **Type** | `object` | -| **Additional properties** | [![Not allowed](https://img.shields.io/badge/Not%20allowed-red)](# "Additional Properties not allowed.") | -| **Defined in** | #/definitions/ZarfFile | - -
- - source * - -  -
- -![Required](https://img.shields.io/badge/Required-red) - -**Description:** Local folder or file path or remote URL to pull into the package - -| | | -| -------- | -------- | -| **Type** | `string` | - -
-
- -
- - shasum - -  -
- -**Description:** (files only) Optional SHA256 checksum of the file - -| | | -| -------- | -------- | -| **Type** | `string` | - -
-
- -
- - target * - -  -
- -![Required](https://img.shields.io/badge/Required-red) - -**Description:** The absolute or relative path where the file or folder should be copied to during package deploy - -| | | -| -------- | -------- | -| **Type** | `string` | - -
-
- -
- - executable - -  -
- -**Description:** (files only) Determines if the file should be made executable during package deploy - -| | | -| -------- | --------- | -| **Type** | `boolean` | - -
-
- -
- - symlinks - -  -
- -**Description:** List of symlinks to create during package deploy - -| | | -| -------- | ----------------- | -| **Type** | `array of string` | - -![Min Items: N/A](https://img.shields.io/badge/Min%20Items%3A%20N/A-gold) -![Max Items: N/A](https://img.shields.io/badge/Max%20Items%3A%20N/A-gold) -![Item unicity: False](https://img.shields.io/badge/Item%20unicity%3A%20False-gold) -![Additional items: N/A](https://img.shields.io/badge/Additional%20items%3A%20N/A-gold) - - ### symlinks items - -| | | -| -------- | -------- | -| **Type** | `string` | - -
-
- -
- - extractPath - -  -
- -**Description:** Local folder or file to be extracted from a 'source' archive - -| | | -| -------- | -------- | -| **Type** | `string` | - -
-
- -
-
-
manifests @@ -987,7 +845,7 @@ Must be one of: ![Item unicity: False](https://img.shields.io/badge/Item%20unicity%3A%20False-gold) ![Additional items: N/A](https://img.shields.io/badge/Additional%20items%3A%20N/A-gold) - ### ZarfManifest + ### ZarfManifest | | | | ------------------------- | -------------------------------------------------------------------------------------------------------- | @@ -1047,7 +905,7 @@ Must be one of: ![Item unicity: False](https://img.shields.io/badge/Item%20unicity%3A%20False-gold) ![Additional items: N/A](https://img.shields.io/badge/Additional%20items%3A%20N/A-gold) - ### files items + ### files items | | | | -------- | -------- | @@ -1090,7 +948,7 @@ Must be one of: ![Item unicity: False](https://img.shields.io/badge/Item%20unicity%3A%20False-gold) ![Additional items: N/A](https://img.shields.io/badge/Additional%20items%3A%20N/A-gold) - ### kustomizations items + ### kustomizations items | | | | -------- | -------- | @@ -1138,7 +996,7 @@ Must be one of: ![Item unicity: False](https://img.shields.io/badge/Item%20unicity%3A%20False-gold) ![Additional items: N/A](https://img.shields.io/badge/Additional%20items%3A%20N/A-gold) - ### ZarfChart + ### ZarfChart | | | | ------------------------- | -------------------------------------------------------------------------------------------------------- | @@ -1322,7 +1180,7 @@ Must be one of: ![Item unicity: False](https://img.shields.io/badge/Item%20unicity%3A%20False-gold) ![Additional items: N/A](https://img.shields.io/badge/Additional%20items%3A%20N/A-gold) - ### valuesFiles items + ### valuesFiles items | | | | -------- | -------- | @@ -1334,25 +1192,44 @@ Must be one of:
-
+
- images + dataInjections  
-**Description:** List of OCI images to include in the package + ## components > dataInjections -| | | -| -------- | ----------------- | -| **Type** | `array of string` | +**Description:** Datasets to inject into a container in the target cluster + +| | | +| -------- | ------- | +| **Type** | `array` | ![Min Items: N/A](https://img.shields.io/badge/Min%20Items%3A%20N/A-gold) ![Max Items: N/A](https://img.shields.io/badge/Max%20Items%3A%20N/A-gold) ![Item unicity: False](https://img.shields.io/badge/Item%20unicity%3A%20False-gold) ![Additional items: N/A](https://img.shields.io/badge/Additional%20items%3A%20N/A-gold) - ### images items + ### ZarfDataInjection + +| | | +| ------------------------- | -------------------------------------------------------------------------------------------------------- | +| **Type** | `object` | +| **Additional properties** | [![Not allowed](https://img.shields.io/badge/Not%20allowed-red)](# "Additional Properties not allowed.") | +| **Defined in** | #/definitions/ZarfDataInjection | + +
+ + source * + +  +
+ +![Required](https://img.shields.io/badge/Required-red) + +**Description:** Either a path to a local folder/file or a remote URL of a file to inject into the given target pod + container | | | | -------- | -------- | @@ -1361,25 +1238,93 @@ Must be one of:
+
+ + target * + +  +
+ + ## components > dataInjections > target +![Required](https://img.shields.io/badge/Required-red) + +**Description:** The target pod + container to inject the data into + +| | | +| ------------------------- | -------------------------------------------------------------------------------------------------------- | +| **Type** | `object` | +| **Additional properties** | [![Not allowed](https://img.shields.io/badge/Not%20allowed-red)](# "Additional Properties not allowed.") | +| **Defined in** | #/definitions/ZarfContainerTarget | +
- repos + namespace *  
-**Description:** List of git repos to include in the package +![Required](https://img.shields.io/badge/Required-red) -| | | -| -------- | ----------------- | -| **Type** | `array of string` | +**Description:** The namespace to target for data injection -![Min Items: N/A](https://img.shields.io/badge/Min%20Items%3A%20N/A-gold) -![Max Items: N/A](https://img.shields.io/badge/Max%20Items%3A%20N/A-gold) -![Item unicity: False](https://img.shields.io/badge/Item%20unicity%3A%20False-gold) -![Additional items: N/A](https://img.shields.io/badge/Additional%20items%3A%20N/A-gold) +| | | +| -------- | -------- | +| **Type** | `string` | + +
+
+ +
+ + selector * + +  +
+ +![Required](https://img.shields.io/badge/Required-red) + +**Description:** The K8s selector to target for data injection + +| | | +| -------- | -------- | +| **Type** | `string` | + +**Example:** + + +"app=data-injection" + +
+
+ +
+ + container * + +  +
+ +![Required](https://img.shields.io/badge/Required-red) + +**Description:** The container name to target for data injection + +| | | +| -------- | -------- | +| **Type** | `string` | - ### repos items +
+
+ +
+ + path * + +  +
+ +![Required](https://img.shields.io/badge/Required-red) + +**Description:** The path within the container to copy the data into | | | | -------- | -------- | @@ -1388,16 +1333,38 @@ Must be one of:
+
+
+ +
+ + compress + +  +
+ +**Description:** Compress the data before transmitting using gzip. Note: this requires support for tar/gzip locally and in the target image. + +| | | +| -------- | --------- | +| **Type** | `boolean` | + +
+
+ +
+
+
- dataInjections + files  
- ## components > dataInjections + ## components > files -**Description:** Datasets to inject into a container in the target cluster +**Description:** Files or folders to place on disk during package deployment | | | | -------- | ------- | @@ -1408,24 +1375,24 @@ Must be one of: ![Item unicity: False](https://img.shields.io/badge/Item%20unicity%3A%20False-gold) ![Additional items: N/A](https://img.shields.io/badge/Additional%20items%3A%20N/A-gold) - ### ZarfDataInjection + ### ZarfFile | | | | ------------------------- | -------------------------------------------------------------------------------------------------------- | | **Type** | `object` | | **Additional properties** | [![Not allowed](https://img.shields.io/badge/Not%20allowed-red)](# "Additional Properties not allowed.") | -| **Defined in** | #/definitions/ZarfDataInjection | +| **Defined in** | #/definitions/ZarfFile |
- source * + source *  
![Required](https://img.shields.io/badge/Required-red) -**Description:** Either a path to a local folder/file or a remote URL of a file to inject into the given target pod + container +**Description:** Local folder or file path or remote URL to pull into the package | | | | -------- | -------- | @@ -1434,57 +1401,75 @@ Must be one of:
-
+
- target * + shasum  
- ## components > dataInjections > target -![Required](https://img.shields.io/badge/Required-red) +**Description:** (files only) Optional SHA256 checksum of the file -**Description:** The target pod + container to inject the data into +| | | +| -------- | -------- | +| **Type** | `string` | -| | | -| ------------------------- | -------------------------------------------------------------------------------------------------------- | -| **Type** | `object` | -| **Additional properties** | [![Not allowed](https://img.shields.io/badge/Not%20allowed-red)](# "Additional Properties not allowed.") | -| **Defined in** | #/definitions/ZarfContainerTarget | +
+
- selector * + target *  
![Required](https://img.shields.io/badge/Required-red) -**Description:** The K8s selector to target for data injection +**Description:** The absolute or relative path where the file or folder should be copied to during package deploy | | | | -------- | -------- | | **Type** | `string` | -**Example:** +
+
- -"app=data-injection" +
+ + executable + +  +
+ +**Description:** (files only) Determines if the file should be made executable during package deploy + +| | | +| -------- | --------- | +| **Type** | `boolean` |
- namespace * + symlinks  
-![Required](https://img.shields.io/badge/Required-red) +**Description:** List of symlinks to create during package deploy -**Description:** The namespace to target for data injection +| | | +| -------- | ----------------- | +| **Type** | `array of string` | + +![Min Items: N/A](https://img.shields.io/badge/Min%20Items%3A%20N/A-gold) +![Max Items: N/A](https://img.shields.io/badge/Max%20Items%3A%20N/A-gold) +![Item unicity: False](https://img.shields.io/badge/Item%20unicity%3A%20False-gold) +![Additional items: N/A](https://img.shields.io/badge/Additional%20items%3A%20N/A-gold) + + ### symlinks items | | | | -------- | -------- | @@ -1495,14 +1480,12 @@ Must be one of:
- container * + extractPath  
-![Required](https://img.shields.io/badge/Required-red) - -**Description:** The container name to target for data injection +**Description:** Local folder or file to be extracted from a 'source' archive | | | | -------- | -------- | @@ -1511,16 +1494,28 @@ Must be one of:
+
+
+
- path * + images  
-![Required](https://img.shields.io/badge/Required-red) +**Description:** List of OCI images to include in the package -**Description:** The path within the container to copy the data into +| | | +| -------- | ----------------- | +| **Type** | `array of string` | + +![Min Items: N/A](https://img.shields.io/badge/Min%20Items%3A%20N/A-gold) +![Max Items: N/A](https://img.shields.io/badge/Max%20Items%3A%20N/A-gold) +![Item unicity: False](https://img.shields.io/badge/Item%20unicity%3A%20False-gold) +![Additional items: N/A](https://img.shields.io/badge/Additional%20items%3A%20N/A-gold) + + ### images items | | | | -------- | -------- | @@ -1529,24 +1524,29 @@ Must be one of:
-
-
-
- compress + repos  
-**Description:** Compress the data before transmitting using gzip. Note: this requires support for tar/gzip locally and in the target image. +**Description:** List of git repos to include in the package -| | | -| -------- | --------- | -| **Type** | `boolean` | +| | | +| -------- | ----------------- | +| **Type** | `array of string` | -
-
+![Min Items: N/A](https://img.shields.io/badge/Min%20Items%3A%20N/A-gold) +![Max Items: N/A](https://img.shields.io/badge/Max%20Items%3A%20N/A-gold) +![Item unicity: False](https://img.shields.io/badge/Item%20unicity%3A%20False-gold) +![Additional items: N/A](https://img.shields.io/badge/Additional%20items%3A%20N/A-gold) + + ### repos items + +| | | +| -------- | -------- | +| **Type** | `string` |
diff --git a/src/types/component.go b/src/types/component.go index 3f1b185a5d..9d77c549de 100644 --- a/src/types/component.go +++ b/src/types/component.go @@ -40,24 +40,24 @@ type ZarfComponent struct { // Import refers to another zarf.yaml package component. Import ZarfComponentImport `json:"import,omitempty" jsonschema:"description=Import a component from another Zarf package"` - // Files are files to place on disk during deploy - Files []ZarfFile `json:"files,omitempty" jsonschema:"description=Files or folders to place on disk during package deployment"` - // Manifests are raw manifests that get converted into zarf-generated helm charts during deploy Manifests []ZarfManifest `json:"manifests,omitempty" jsonschema:"description=Kubernetes manifests to be included in a generated Helm chart on package deploy"` // Charts are helm charts to install during package deploy Charts []ZarfChart `json:"charts,omitempty" jsonschema:"description=Helm charts to install during package deploy"` + // Data packages to push into a running cluster + DataInjections []ZarfDataInjection `json:"dataInjections,omitempty" jsonschema:"description=Datasets to inject into a container in the target cluster"` + + // Files are files to place on disk during deploy + Files []ZarfFile `json:"files,omitempty" jsonschema:"description=Files or folders to place on disk during package deployment"` + // Images are the online images needed to be included in the zarf package Images []string `json:"images,omitempty" jsonschema:"description=List of OCI images to include in the package"` // Repos are any git repos that need to be pushed into the git server Repos []string `json:"repos,omitempty" jsonschema:"description=List of git repos to include in the package"` - // Data packages to push into a running cluster - DataInjections []ZarfDataInjection `json:"dataInjections,omitempty" jsonschema:"description=Datasets to inject into a container in the target cluster"` - // Extensions provide additional functionality to a component Extensions extensions.ZarfComponentExtensions `json:"extensions,omitempty" jsonschema:"description=Extend component functionality with additional features"` @@ -260,8 +260,8 @@ type ZarfComponentActionWaitNetwork struct { // ZarfContainerTarget defines the destination info for a ZarfData target type ZarfContainerTarget struct { - Selector string `json:"selector" jsonschema:"description=The K8s selector to target for data injection,example=app=data-injection"` Namespace string `json:"namespace" jsonschema:"description=The namespace to target for data injection"` + Selector string `json:"selector" jsonschema:"description=The K8s selector to target for data injection,example=app=data-injection"` Container string `json:"container" jsonschema:"description=The container name to target for data injection"` Path string `json:"path" jsonschema:"description=The path within the container to copy the data into"` } diff --git a/zarf.schema.json b/zarf.schema.json index 7ce8a33be6..e26de4eb57 100644 --- a/zarf.schema.json +++ b/zarf.schema.json @@ -249,14 +249,6 @@ "$ref": "#/definitions/ZarfComponentImport", "description": "Import a component from another Zarf package" }, - "files": { - "items": { - "$schema": "http://json-schema.org/draft-04/schema#", - "$ref": "#/definitions/ZarfFile" - }, - "type": "array", - "description": "Files or folders to place on disk during package deployment" - }, "manifests": { "items": { "$schema": "http://json-schema.org/draft-04/schema#", @@ -273,6 +265,22 @@ "type": "array", "description": "Helm charts to install during package deploy" }, + "dataInjections": { + "items": { + "$schema": "http://json-schema.org/draft-04/schema#", + "$ref": "#/definitions/ZarfDataInjection" + }, + "type": "array", + "description": "Datasets to inject into a container in the target cluster" + }, + "files": { + "items": { + "$schema": "http://json-schema.org/draft-04/schema#", + "$ref": "#/definitions/ZarfFile" + }, + "type": "array", + "description": "Files or folders to place on disk during package deployment" + }, "images": { "items": { "type": "string" @@ -287,14 +295,6 @@ "type": "array", "description": "List of git repos to include in the package" }, - "dataInjections": { - "items": { - "$schema": "http://json-schema.org/draft-04/schema#", - "$ref": "#/definitions/ZarfDataInjection" - }, - "type": "array", - "description": "Datasets to inject into a container in the target cluster" - }, "extensions": { "$schema": "http://json-schema.org/draft-04/schema#", "$ref": "#/definitions/ZarfComponentExtensions", @@ -706,12 +706,16 @@ }, "ZarfContainerTarget": { "required": [ - "selector", "namespace", + "selector", "container", "path" ], "properties": { + "namespace": { + "type": "string", + "description": "The namespace to target for data injection" + }, "selector": { "type": "string", "description": "The K8s selector to target for data injection", @@ -719,10 +723,6 @@ "app\u0026#61;data-injection" ] }, - "namespace": { - "type": "string", - "description": "The namespace to target for data injection" - }, "container": { "type": "string", "description": "The container name to target for data injection" From c94845168510487475c665fa379430c1dd2ba7f5 Mon Sep 17 00:00:00 2001 From: razzle Date: Fri, 5 Jan 2024 16:19:47 -0600 Subject: [PATCH 033/113] a little cleaner error Signed-off-by: razzle --- src/config/lang/english.go | 4 ++-- src/pkg/interactive/components.go | 17 ++++++++++++----- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/src/config/lang/english.go b/src/config/lang/english.go index 4da00b8b6c..562271116c 100644 --- a/src/config/lang/english.go +++ b/src/config/lang/english.go @@ -623,7 +623,7 @@ const ( PkgCreateErrDifferentialSameVersion = "unable to create a differential package with the same version as the package you are using as a reference; the package version must be incremented" ) -// src/internal/packager/deploy. +// src/internal/packager/deploy const ( PkgDeployErrMultipleComponentsSameGroup = "cannot specify multiple components (%q, %q) within the same group (%q) when using the --components flag" PkgDeployErrNoDefaultOrSelection = "no selection made from %q with the --components flag and there is no default in the group" @@ -631,7 +631,7 @@ const ( PkgDeployErrComponentSelectionCanceled = "Component selection canceled: %s" ) -// src/internal/packager/validate. +// src/internal/packager/validate const ( PkgValidateTemplateDeprecation = "Package template %q is using the deprecated syntax ###ZARF_PKG_VAR_%s###. This will be removed in Zarf v1.0.0. Please update to ###ZARF_PKG_TMPL_%s###." PkgValidateMustBeUppercase = "variable name %q must be all uppercase and contain no special characters except _" diff --git a/src/pkg/interactive/components.go b/src/pkg/interactive/components.go index 314f1e95f4..32cd151fc3 100644 --- a/src/pkg/interactive/components.go +++ b/src/pkg/interactive/components.go @@ -105,20 +105,27 @@ func GetSelectedComponents(optionalComponents string, allComponents []types.Zarf } // Check that we have matched against all requests + var err error for _, requestedComponent := range requestedComponents { if _, ok := matchedRequests[requestedComponent]; !ok { - names := []string{} + closeEnough := []string{} for _, c := range allComponents { d := levenshtein.ComputeDistance(c.Name, requestedComponent) if d <= 5 { - return []types.ZarfComponent{}, fmt.Errorf(lang.PkgDeployErrNoCompatibleComponentsForSelection, requestedComponent, c.Name) + closeEnough = append(closeEnough, c.Name) } - names = append(names, c.Name) } - - return []types.ZarfComponent{}, fmt.Errorf(lang.PkgDeployErrNoCompatibleComponentsForSelection, requestedComponent, strings.Join(names, ", ")) + failure := fmt.Errorf(lang.PkgDeployErrNoCompatibleComponentsForSelection, requestedComponent, strings.Join(closeEnough, ", ")) + if err != nil { + err = fmt.Errorf("%w, %w", err, failure) + } else { + err = failure + } } } + if err != nil { + return []types.ZarfComponent{}, err + } } else { for _, groupKey := range orderedComponentGroups { if len(groupedComponents[groupKey]) > 1 { From 9f787c7303c889c2b8861998aa910a984e0ff1d9 Mon Sep 17 00:00:00 2001 From: razzle Date: Sat, 6 Jan 2024 10:58:11 -0600 Subject: [PATCH 034/113] resolve convos Signed-off-by: razzle --- adr/0023-required-optional.md | 6 +- .../0-creating-a-zarf-package.md | 55 +++++++++---------- 2 files changed, 30 insertions(+), 31 deletions(-) diff --git a/adr/0023-required-optional.md b/adr/0023-required-optional.md index 875206f658..a28a1043c6 100644 --- a/adr/0023-required-optional.md +++ b/adr/0023-required-optional.md @@ -10,15 +10,15 @@ Pending > Feature request: -Currently, all Zarf components default to being optional due to the `required` key being _optional_ in the yaml. This leads to behaviors where a package author must ensure they are careful to annotate this key for each component--though the validation doesn't require them to so assumes a sort of "all things are optional" default state. +Currently, all Zarf components default to being optional due to the `required` key being _optional_ in the yaml. This leads to package authors needing to ensure that they annotate this key for each component, and since nothing in the current validations prompts them about this they may be confused about the "all things are optional" default state. When Zarf was first created, we didn't really know how it would evolve and this key was introduced in those very early days. At this point it would be better to require all components by default--especially with the introduction of composability and the OCI skeleton work, there is plenty of flexibility in the API to compose bespoke packages assembled from other packages. A few ways to handle this: -1. Simply force the `required` key to be a non-optional, so that package authors would be forced to specify it for each component, thereby removing any ambiguity--but also force one more key for every single component every created 🫠 +1. Simply force the `required` key to be a non-optional, so that package authors would be forced to specify it for each component, thereby removing any ambiguity--but also force one more key for every single component ever created 🫠 -2. Deprecate `required` and introduce an optional `optional` key, which would default to false. I do think this still feels strange if you did something like `optional: false`. +2. Deprecate `required` and introduce an optional `optional` key, which would default to _false_. I do think this still feels strange if you did something like `optional: false`, (to be fair `required: false` has the same awkwardness). 3. Do something more significant like combine various condition-based things such as `only`, `optional` (instead of `required`), or `default`. diff --git a/docs/5-zarf-tutorials/0-creating-a-zarf-package.md b/docs/5-zarf-tutorials/0-creating-a-zarf-package.md index 3d677d5408..8eb6577aee 100644 --- a/docs/5-zarf-tutorials/0-creating-a-zarf-package.md +++ b/docs/5-zarf-tutorials/0-creating-a-zarf-package.md @@ -14,23 +14,23 @@ When creating a Zarf package, you must have a network connection so that Zarf ca Before beginning this tutorial you will need the following: -- Zarf binary installed on your $PATH: ([Installing Zarf](../1-getting-started/index.md#installing-zarf)) -- A text editor or development environment such as [VS Code](../3-create-a-zarf-package/8-vscode.md) +- Zarf binary installed on your $PATH: ([Installing Zarf](../1-getting-started/index.md#installing-zarf)) +- A text editor or development environment such as [VS Code](../3-create-a-zarf-package/8-vscode.md) ## Putting Together a Zarf Package -In order to create a Zarf package you first need to have an idea of what application(s) you want to package. In this example we will be using the [WordPress chart from Bitnami](https://artifacthub.io/packages/helm/bitnami/wordpress) but the steps and tools used below are very similar for other applications. +In order to create a Zarf package you first need to have an idea of what application(s) you want to package. In this example we will be using the [WordPress chart from Bitnami](https://artifacthub.io/packages/helm/bitnami/wordpress) but the steps and tools used below are very similar for other applications. ### Creating the Package Definition -A `zarf.yaml` file follows the [Zarf Package Schema](../3-create-a-zarf-package/4-zarf-schema.md) and allows us to specify package metadata and a set of components for us to deploy. We start a package definition with the `kind` of package we are making and `metadata` that describes the package. You can start our WordPress package by creating a new `zarf.yaml` with the following content: +A `zarf.yaml` file follows the [Zarf Package Schema](../3-create-a-zarf-package/4-zarf-schema.md) and allows us to specify package metadata and a set of components for us to deploy. We start a package definition with the `kind` of package we are making and `metadata` that describes the package. You can start our WordPress package by creating a new `zarf.yaml` with the following content: ```yaml kind: ZarfPackageConfig # ZarfPackageConfig is the package kind for most normal zarf packages metadata: - name: wordpress # specifies the name of our package and should be unique and unchanging through updates - version: 16.0.4 # (optional) a version we can track as we release updates or publish to a registry - description: | # (optional) a human-readable description of the package that you are creating + name: wordpress # specifies the name of our package and should be unique and unchanging through updates + version: 16.0.4 # (optional) a version we can track as we release updates or publish to a registry + description: | # (optional) a human-readable description of the package that you are creating "A Zarf Package that deploys the WordPress blogging and content management platform" ``` @@ -43,14 +43,13 @@ Additionally, you can run `zarf dev lint ` to validate aginst the [`z ### Adding the WordPress Component -Components are the unit of Zarf Packages that define an application stack. These are defined under the `components` key and allow many different resource types to be brought into a package. You can learn more about components on the [Understanding Zarf Components](../3-create-a-zarf-package/2-zarf-components.md) page. To add our WordPress component, add the following to the bottom of our `zarf.yaml`: +Components are the unit of Zarf Packages that define an application stack. These are defined under the `components` key and allow many different resource types to be brought into a package. You can learn more about components on the [Understanding Zarf Components](../3-create-a-zarf-package/2-zarf-components.md) page. To add our WordPress component, add the following to the bottom of our `zarf.yaml`: ```yaml components: - - name: wordpress # specifies the name of our component and should be unique and unchanging through updates - description: | # (optional) a human-readable description of the component you are defining + - name: wordpress # specifies the name of our component and should be unique and unchanging through updates + description: | # (optional) a human-readable description of the component you are defining "Deploys the Bitnami-packaged WordPress chart into the cluster" - optional: false # (optional) sets the component as 'required' so that it is always deployed charts: - name: wordpress url: oci://registry-1.docker.io/bitnamicharts/wordpress @@ -60,7 +59,7 @@ components: - wordpress-values.yaml ``` -In addition to this component definition, we also need to create the `valuesFiles` we have specified. In this case we need to create a file named `wordpress-values.yaml` in the same directory as our `zarf.yaml` with the following contents: +In addition to this component definition, we also need to create the `valuesFiles` we have specified. In this case we need to create a file named `wordpress-values.yaml` in the same directory as our `zarf.yaml` with the following contents: ```yaml # We are hard-coding these for now but will make them dynamic in Setting up Variables. @@ -89,13 +88,13 @@ We create any `values.yaml` file(s) at this stage because the `zarf dev find-ima :::caution -Note that we are explicitly defining the `wordpress` namespace for this deployment, this is strongly recommended to separate out the applications you deploy and to avoid issues with the Zarf Agent not being able to mutate your resources as it intentionally ignores resources in the `default` or `kube-system` namespaces. See [what happens to resources that exist before Zarf init](../8-faq.md#what-happens-to-resources-that-exist-in-the-cluster-before-zarf-init) for more information. +Note that we are explicitly defining the `wordpress` namespace for this deployment, this is strongly recommended to separate out the applications you deploy and to avoid issues with the Zarf Agent not being able to mutate your resources as it intentionally ignores resources in the `default` or `kube-system` namespaces. See [what happens to resources that exist before Zarf init](../8-faq.md#what-happens-to-resources-that-exist-in-the-cluster-before-zarf-init) for more information. ::: ### Finding the Images -Once you have the above defined we can now work on setting the images that we will need to bring with us into the air gap. For this, Zarf has a helper command you can run with `zarf dev find-images`. Running this command in the directory of your zarf.yaml will result in the following output: +Once you have the above defined we can now work on setting the images that we will need to bring with us into the air gap. For this, Zarf has a helper command you can run with `zarf dev find-images`. Running this command in the directory of your zarf.yaml will result in the following output: @@ -103,7 +102,7 @@ From here you can copy the `images` key and array of images into the `wordpress` :::note -Due to the way some applications are deployed, Zarf might not be able to find all of the images in this way (particularly with operators). For this you can look at the upstream charts or manifests and find them manually. +Due to the way some applications are deployed, Zarf might not be able to find all of the images in this way (particularly with operators). For this you can look at the upstream charts or manifests and find them manually. ::: @@ -115,13 +114,13 @@ Zarf has more `dev` commands you can learn about on the [dev CLI docs page](../3 ### Setting up Variables -We now have a deployable package definition, but it is currently not very configurable and might not fit every environment we want to deploy it to. If we deployed it as-is we would always have a Zarf Blog and a `zarf` user with an autogenerated password. +We now have a deployable package definition, but it is currently not very configurable and might not fit every environment we want to deploy it to. If we deployed it as-is we would always have a Zarf Blog and a `zarf` user with an autogenerated password. -To resolve this, we can add configuration options with [Zarf Deploy-Time Variables](../../examples/variables/README.md#deploy-time-variables-and-constants). For this package we will add a `variables` section to our `zarf.yaml` above `components` that will allow us to setup the user and the blog. +To resolve this, we can add configuration options with [Zarf Deploy-Time Variables](../../examples/variables/README.md#deploy-time-variables-and-constants). For this package we will add a `variables` section to our `zarf.yaml` above `components` that will allow us to setup the user and the blog. ```yaml variables: - # The unique name of the variable corresponding to the ###ZARF_VAR_### template + # The unique name of the variable corresponding to the ###ZARF_VAR_### template - name: WORDPRESS_USERNAME # A human-readable description of the variable shown during prompting description: The username that is used to login to the WordPress admin account @@ -152,7 +151,7 @@ variables: prompt: true ``` -To use these variables in our chart we must add their corresponding templates to our `wordpress-values.yaml` file. Zarf can template chart values, manifests, included text files and more. +To use these variables in our chart we must add their corresponding templates to our `wordpress-values.yaml` file. Zarf can template chart values, manifests, included text files and more. ```yaml wordpressUsername: ###ZARF_VAR_WORDPRESS_USERNAME### @@ -165,15 +164,15 @@ wordpressBlogName: ###ZARF_VAR_WORDPRESS_BLOG_NAME### :::caution -When dealing with `sensitive` values in Zarf it is strongly recommended to not include them directly inside of a Zarf Package and to only define them at deploy-time. You should also be aware of where you are using these values as they may be printed in `actions` you create or `files` that you place on disk. +When dealing with `sensitive` values in Zarf it is strongly recommended to not include them directly inside of a Zarf Package and to only define them at deploy-time. You should also be aware of where you are using these values as they may be printed in `actions` you create or `files` that you place on disk. ::: ### Setting up a Zarf Connect Service -As-is, our package could be configured to interface with an ingress provider to provide access to our blog, but this may not be desired for every service, particularly those that provide a backend for other frontend services. To help with debugging, Zarf allows you to specify Zarf Connect Services that will be displayed after package deployment to quickly connect into our deployed application. +As-is, our package could be configured to interface with an ingress provider to provide access to our blog, but this may not be desired for every service, particularly those that provide a backend for other frontend services. To help with debugging, Zarf allows you to specify Zarf Connect Services that will be displayed after package deployment to quickly connect into our deployed application. -For this package we will define two services, one for the blog and the other for the admin panel. These are normal Kubernetes services with special labels and annotations that Zarf watches out for, and to defined them create a `connect-services.yaml` with the following contents: +For this package we will define two services, one for the blog and the other for the admin panel. These are normal Kubernetes services with special labels and annotations that Zarf watches out for, and to defined them create a `connect-services.yaml` with the following contents: ```yaml apiVersion: v1 @@ -220,18 +219,18 @@ spec: To add this to our `zarf.yaml` we can simply specify it under our `wordpress` component using the `manifests` key: ```yaml - manifests: - - name: connect-services - namespace: wordpress - files: - - connect-services.yaml +manifests: + - name: connect-services + namespace: wordpress + files: + - connect-services.yaml ``` ### Creating the Package Once you have followed the above you should now have a `zarf.yaml` file that matches the one found on the [WordPress example page](../../examples/wordpress/README.md). -Creating this package is as simple as running the `zarf package create` command with the directory containing our `zarf.yaml`. Zarf will show us the `zarf.yaml` one last time asking if we would like to build the package, and upon confirmation Zarf will pull down all of the resources and bundle them into a package tarball. +Creating this package is as simple as running the `zarf package create` command with the directory containing our `zarf.yaml`. Zarf will show us the `zarf.yaml` one last time asking if we would like to build the package, and upon confirmation Zarf will pull down all of the resources and bundle them into a package tarball. ```bash zarf package create . From b45eaed60dc33ed33715a47748fea201c8c4eef5 Mon Sep 17 00:00:00 2001 From: razzle Date: Tue, 9 Jan 2024 10:54:28 -0600 Subject: [PATCH 035/113] cleanup Signed-off-by: razzle --- src/pkg/packager/components.go | 23 ----------------------- src/pkg/packager/deploy.go | 6 +++--- src/pkg/packager/remove.go | 2 +- src/pkg/utils/io.go | 4 +++- src/types/component.go | 15 +++++++++++++++ 5 files changed, 22 insertions(+), 28 deletions(-) delete mode 100644 src/pkg/packager/components.go diff --git a/src/pkg/packager/components.go b/src/pkg/packager/components.go deleted file mode 100644 index a23cd42f49..0000000000 --- a/src/pkg/packager/components.go +++ /dev/null @@ -1,23 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: 2021-Present The Zarf Authors - -// Package packager contains functions for interacting with, managing and deploying Zarf packages. -package packager - -import ( - "github.com/defenseunicorns/zarf/src/types" -) - -func requiresCluster(component types.ZarfComponent) bool { - hasImages := len(component.Images) > 0 - hasCharts := len(component.Charts) > 0 - hasManifests := len(component.Manifests) > 0 - hasRepos := len(component.Repos) > 0 - hasDataInjections := len(component.DataInjections) > 0 - - if hasImages || hasCharts || hasManifests || hasRepos || hasDataInjections { - return true - } - - return false -} diff --git a/src/pkg/packager/deploy.go b/src/pkg/packager/deploy.go index 4b91d03988..5383728b00 100644 --- a/src/pkg/packager/deploy.go +++ b/src/pkg/packager/deploy.go @@ -119,7 +119,7 @@ func (p *Packager) deployComponents() (deployedComponents []types.DeployedCompon } // If this component requires a cluster, connect to one - if requiresCluster(component) { + if component.RequiresCluster() { timeout := cluster.DefaultTimeout if p.isInitConfig() { timeout = 5 * time.Minute @@ -205,7 +205,7 @@ func (p *Packager) deployInitComponent(component types.ZarfComponent) (charts [] isAgent := component.Name == "zarf-agent" // Always init the state before the first component that requires the cluster (on most deployments, the zarf-seed-registry) - if requiresCluster(component) && p.cfg.State == nil { + if component.RequiresCluster() && p.cfg.State == nil { err = p.cluster.InitZarfState(p.cfg.InitOpts) if err != nil { return charts, fmt.Errorf("unable to initialize Zarf state: %w", err) @@ -269,7 +269,7 @@ func (p *Packager) deployComponent(component types.ZarfComponent, noImgChecksum } } - if !p.valueTemplate.Ready() && requiresCluster(component) { + if !p.valueTemplate.Ready() && component.RequiresCluster() { // Setup the state in the config and get the valuesTemplate p.valueTemplate, err = p.setupStateValuesTemplate() if err != nil { diff --git a/src/pkg/packager/remove.go b/src/pkg/packager/remove.go index d12c3bc6ba..e03db05b41 100644 --- a/src/pkg/packager/remove.go +++ b/src/pkg/packager/remove.go @@ -53,7 +53,7 @@ func (p *Packager) Remove() (err error) { interactive.ForIncludedComponents(p.cfg.PkgOpts.OptionalComponents, p.cfg.Pkg.Components, (func(component types.ZarfComponent) error { componentsToRemove = append(componentsToRemove, component.Name) - if requiresCluster(component) { + if component.RequiresCluster() { packageRequiresCluster = true } diff --git a/src/pkg/utils/io.go b/src/pkg/utils/io.go index c02ce6b55e..ed9a39780c 100755 --- a/src/pkg/utils/io.go +++ b/src/pkg/utils/io.go @@ -462,7 +462,9 @@ func CreateReproducibleTarballFromDir(dirPath, dirPrefix, tarballPath string) er if err != nil { return fmt.Errorf("error getting relative path: %w", err) } - header.Name = filepath.Join(dirPrefix, name) + name = filepath.Join(dirPrefix, name) + name = filepath.ToSlash(name) + header.Name = name // Write the header to the tarball if err := tw.WriteHeader(header); err != nil { diff --git a/src/types/component.go b/src/types/component.go index 9d77c549de..a888f92ea9 100644 --- a/src/types/component.go +++ b/src/types/component.go @@ -68,6 +68,21 @@ type ZarfComponent struct { Actions ZarfComponentActions `json:"actions,omitempty" jsonschema:"description=Custom commands to run at various stages of a package lifecycle"` } +// RequiresCluster returns if the component requires a cluster connection to deploy +func (c ZarfComponent) RequiresCluster() bool { + hasImages := len(c.Images) > 0 + hasCharts := len(c.Charts) > 0 + hasManifests := len(c.Manifests) > 0 + hasRepos := len(c.Repos) > 0 + hasDataInjections := len(c.DataInjections) > 0 + + if hasImages || hasCharts || hasManifests || hasRepos || hasDataInjections { + return true + } + + return false +} + // IsRequired returns if the component is required or not. // // The logic is as follows: From ab03c581317c10a5db4f56fac144bf4ab7c0a191 Mon Sep 17 00:00:00 2001 From: razzle Date: Tue, 9 Jan 2024 11:00:05 -0600 Subject: [PATCH 036/113] cli stuffs Signed-off-by: razzle --- src/cmd/dev.go | 8 ++++---- src/config/lang/english.go | 4 ++++ 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/cmd/dev.go b/src/cmd/dev.go index 10ae96ae14..3e68296197 100644 --- a/src/cmd/dev.go +++ b/src/cmd/dev.go @@ -68,10 +68,10 @@ var devDeployCmd = &cobra.Command{ } var devMigrateCmd = &cobra.Command{ - Use: "migrate", - // Short: lang.CmdDevMigrateShort, - // Long: lang.CmdDevMigrateLong, - Args: cobra.ExactArgs(1), + Use: "migrate", + Short: lang.CmdDevMigrateShort, + Long: lang.CmdDevMigrateLong, + Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { dir := args[0] var pkg types.ZarfPackage diff --git a/src/config/lang/english.go b/src/config/lang/english.go index 562271116c..919a280e46 100644 --- a/src/config/lang/english.go +++ b/src/config/lang/english.go @@ -356,6 +356,10 @@ $ zarf package pull oci://ghcr.io/defenseunicorns/packages/dos-games:1.0.0 -a sk CmdDevDeployFlagNoYolo = "Disable the YOLO mode default override and create / deploy the package as-defined" CmdDevDeployErr = "Failed to dev deploy: %s" + CmdDevMigrateShort = "[beta] Migrates the zarf.yaml in a given directory to the latest version of the zarf.yaml format" + CmdDevMigrateLong = "[beta] Migrates the zarf.yaml in a given directory to the latest version of the zarf.yaml format.\n\n" + + "This command modifies the original zarf.yaml file, performs a best effort attempt to preserve comments, and will format the contents in an opinionated manner." + CmdDevPatchGitShort = "Converts all .git URLs to the specified Zarf HOST and with the Zarf URL pattern in a given FILE. NOTE:\n" + "This should only be used for manifests that are not mutated by the Zarf Agent Mutating Webhook." CmdDevPatchGitOverwritePrompt = "Overwrite the file %s with these changes?" From c4ffd818579d5360f8fad5809f87ef85bc52e368 Mon Sep 17 00:00:00 2001 From: razzle Date: Tue, 9 Jan 2024 11:00:21 -0600 Subject: [PATCH 037/113] docs and schema Signed-off-by: razzle --- docs/2-the-zarf-cli/100-cli-commands/zarf_dev.md | 2 +- docs/2-the-zarf-cli/100-cli-commands/zarf_dev_migrate.md | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/docs/2-the-zarf-cli/100-cli-commands/zarf_dev.md b/docs/2-the-zarf-cli/100-cli-commands/zarf_dev.md index 6f882cf7c5..ef26a00800 100644 --- a/docs/2-the-zarf-cli/100-cli-commands/zarf_dev.md +++ b/docs/2-the-zarf-cli/100-cli-commands/zarf_dev.md @@ -29,7 +29,7 @@ Commands useful for developing packages * [zarf dev find-images](zarf_dev_find-images.md) - Evaluates components in a Zarf file to identify images specified in their helm charts and manifests * [zarf dev generate-config](zarf_dev_generate-config.md) - Generates a config file for Zarf * [zarf dev lint](zarf_dev_lint.md) - Lints the given package for valid schema and recommended practices -* [zarf dev migrate](zarf_dev_migrate.md) - +* [zarf dev migrate](zarf_dev_migrate.md) - [beta] Migrates the zarf.yaml in a given directory to the latest version of the zarf.yaml format * [zarf dev patch-git](zarf_dev_patch-git.md) - Converts all .git URLs to the specified Zarf HOST and with the Zarf URL pattern in a given FILE. NOTE: This should only be used for manifests that are not mutated by the Zarf Agent Mutating Webhook. * [zarf dev sha256sum](zarf_dev_sha256sum.md) - Generates a SHA256SUM for the given file diff --git a/docs/2-the-zarf-cli/100-cli-commands/zarf_dev_migrate.md b/docs/2-the-zarf-cli/100-cli-commands/zarf_dev_migrate.md index c54bd10ee9..979b78e3f7 100644 --- a/docs/2-the-zarf-cli/100-cli-commands/zarf_dev_migrate.md +++ b/docs/2-the-zarf-cli/100-cli-commands/zarf_dev_migrate.md @@ -1,7 +1,13 @@ # zarf dev migrate +[beta] Migrates the zarf.yaml in a given directory to the latest version of the zarf.yaml format +## Synopsis + +[beta] Migrates the zarf.yaml in a given directory to the latest version of the zarf.yaml format. + +This command modifies the original zarf.yaml file, performs a best effort attempt to preserve comments, and will format the contents in an opinionated manner. ``` zarf dev migrate [flags] From b91fcf49c76bfacc3c3a1c9e9ba511cf294e16d6 Mon Sep 17 00:00:00 2001 From: razzle Date: Tue, 9 Jan 2024 12:07:33 -0600 Subject: [PATCH 038/113] handle differences in filtering OCI based upon operation Signed-off-by: razzle --- src/pkg/interactive/components.go | 26 +++++++++++++++++++++----- src/pkg/packager/deploy.go | 6 ++++++ src/pkg/packager/mirror.go | 6 ++++++ src/pkg/packager/sources/new.go | 2 +- src/pkg/packager/sources/oci.go | 13 +++++++++---- 5 files changed, 43 insertions(+), 10 deletions(-) diff --git a/src/pkg/interactive/components.go b/src/pkg/interactive/components.go index 32cd151fc3..fb54c4c520 100644 --- a/src/pkg/interactive/components.go +++ b/src/pkg/interactive/components.go @@ -146,11 +146,13 @@ func GetSelectedComponents(optionalComponents string, allComponents []types.Zarf return selectedComponents, nil } -// ForIncludedComponents iterates over components and calls onIncluded for each component that should be included -func ForIncludedComponents(optionalComponents string, components []types.ZarfComponent, onIncluded func(types.ZarfComponent) error) error { +// GetOnlyIncludedComponents returns only the components that are included +func GetOnlyIncludedComponents(optionalComponents string, components []types.ZarfComponent) ([]types.ZarfComponent, error) { requestedComponents := helpers.StringToSlice(optionalComponents) isPartial := len(requestedComponents) > 0 && requestedComponents[0] != "" + result := []types.ZarfComponent{} + for _, component := range components { selectState := unknown @@ -165,9 +167,23 @@ func ForIncludedComponents(optionalComponents string, components []types.ZarfCom } if selectState == included { - if err := onIncluded(component); err != nil { - return err - } + result = append(result, component) + } + } + + return result, nil +} + +// ForIncludedComponents runs a function for each included component +func ForIncludedComponents(optionalComponents string, components []types.ZarfComponent, fn func(types.ZarfComponent) error) error { + included, err := GetOnlyIncludedComponents(optionalComponents, components) + if err != nil { + return err + } + + for _, component := range included { + if err := fn(component); err != nil { + return err } } diff --git a/src/pkg/packager/deploy.go b/src/pkg/packager/deploy.go index 5383728b00..e4356d74e0 100644 --- a/src/pkg/packager/deploy.go +++ b/src/pkg/packager/deploy.go @@ -24,6 +24,7 @@ import ( "github.com/defenseunicorns/zarf/src/pkg/k8s" "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/message" + "github.com/defenseunicorns/zarf/src/pkg/packager/sources" "github.com/defenseunicorns/zarf/src/pkg/transform" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" @@ -41,6 +42,11 @@ func (p *Packager) resetRegistryHPA() { // Deploy attempts to deploy the given PackageConfig. func (p *Packager) Deploy() (err error) { + if ociSource, ok := p.source.(*sources.OCISource); ok { + ociSource.ComponentSelectionFilter = interactive.GetSelectedComponents + p.source = ociSource + } + if err = p.source.LoadPackage(p.layout, true); err != nil { return fmt.Errorf("unable to load the package: %w", err) } diff --git a/src/pkg/packager/mirror.go b/src/pkg/packager/mirror.go index 6bc0ac64b0..9d8913eb07 100644 --- a/src/pkg/packager/mirror.go +++ b/src/pkg/packager/mirror.go @@ -11,6 +11,7 @@ import ( "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/pkg/interactive" "github.com/defenseunicorns/zarf/src/pkg/message" + "github.com/defenseunicorns/zarf/src/pkg/packager/sources" "github.com/defenseunicorns/zarf/src/types" ) @@ -19,6 +20,11 @@ func (p *Packager) Mirror() (err error) { spinner := message.NewProgressSpinner("Mirroring Zarf package %s", p.cfg.PkgOpts.PackageSource) defer spinner.Stop() + if ociSource, ok := p.source.(*sources.OCISource); ok { + ociSource.ComponentSelectionFilter = interactive.GetOnlyIncludedComponents + p.source = ociSource + } + if err = p.source.LoadPackage(p.layout, true); err != nil { return fmt.Errorf("unable to load the package: %w", err) } diff --git a/src/pkg/packager/sources/new.go b/src/pkg/packager/sources/new.go index 9cd8c07c1a..4ec16c65ed 100644 --- a/src/pkg/packager/sources/new.go +++ b/src/pkg/packager/sources/new.go @@ -71,7 +71,7 @@ func New(pkgOpts *types.ZarfPackageOptions) (PackageSource, error) { if err != nil { return nil, err } - source = &OCISource{pkgOpts, remote} + source = &OCISource{ZarfPackageOptions: pkgOpts, OrasRemote: remote} case "tarball": source = &TarballSource{pkgOpts} case "http", "https", "sget": diff --git a/src/pkg/packager/sources/oci.go b/src/pkg/packager/sources/oci.go index 78200fbc1b..b25e776b42 100644 --- a/src/pkg/packager/sources/oci.go +++ b/src/pkg/packager/sources/oci.go @@ -12,7 +12,6 @@ import ( "strings" "github.com/defenseunicorns/zarf/src/config" - "github.com/defenseunicorns/zarf/src/pkg/interactive" "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/oci" @@ -31,6 +30,7 @@ var ( type OCISource struct { *types.ZarfPackageOptions *oci.OrasRemote + ComponentSelectionFilter func(optionalComponents string, allComponents []types.ZarfComponent) ([]types.ZarfComponent, error) } // LoadPackage loads a package from an OCI registry. @@ -46,9 +46,14 @@ func (s *OCISource) LoadPackage(dst *layout.PackagePaths, unarchiveAll bool) (er if err != nil { return err } - requested, err := interactive.GetSelectedComponents(s.OptionalComponents, pkg.Components) - if err != nil { - return err + var requested []types.ZarfComponent + if s.ComponentSelectionFilter != nil { + requested, err = s.ComponentSelectionFilter(s.OptionalComponents, pkg.Components) + if err != nil { + return err + } + } else { + requested = pkg.Components } layersToPull, err = s.LayersFromRequestedComponents(requested) if err != nil { From 2c70de717a0367cd7b0fd8596021eec1c1b31845 Mon Sep 17 00:00:00 2001 From: razzle Date: Tue, 9 Jan 2024 12:37:29 -0600 Subject: [PATCH 039/113] rename symbol Signed-off-by: razzle --- src/pkg/interactive/components.go | 4 ++-- src/pkg/packager/deploy.go | 4 ++-- src/pkg/packager/dev.go | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/pkg/interactive/components.go b/src/pkg/interactive/components.go index fb54c4c520..dea4737a86 100644 --- a/src/pkg/interactive/components.go +++ b/src/pkg/interactive/components.go @@ -24,8 +24,8 @@ const ( excluded ) -// GetSelectedComponents prompts to select components based upon multiple conditions -func GetSelectedComponents(optionalComponents string, allComponents []types.ZarfComponent) ([]types.ZarfComponent, error) { +// GetComponentsForDeployment prompts to select components based upon multiple conditions +func GetComponentsForDeployment(optionalComponents string, allComponents []types.ZarfComponent) ([]types.ZarfComponent, error) { var selectedComponents []types.ZarfComponent groupedComponents := map[string][]types.ZarfComponent{} orderedComponentGroups := []string{} diff --git a/src/pkg/packager/deploy.go b/src/pkg/packager/deploy.go index e4356d74e0..035d91421c 100644 --- a/src/pkg/packager/deploy.go +++ b/src/pkg/packager/deploy.go @@ -43,7 +43,7 @@ func (p *Packager) resetRegistryHPA() { // Deploy attempts to deploy the given PackageConfig. func (p *Packager) Deploy() (err error) { if ociSource, ok := p.source.(*sources.OCISource); ok { - ociSource.ComponentSelectionFilter = interactive.GetSelectedComponents + ociSource.ComponentSelectionFilter = interactive.GetComponentsForDeployment p.source = ociSource } @@ -100,7 +100,7 @@ func (p *Packager) Deploy() (err error) { // deployComponents loops through a list of ZarfComponents and deploys them. func (p *Packager) deployComponents() (deployedComponents []types.DeployedComponent, err error) { - componentsToDeploy, err := interactive.GetSelectedComponents(p.cfg.PkgOpts.OptionalComponents, p.cfg.Pkg.Components) + componentsToDeploy, err := interactive.GetComponentsForDeployment(p.cfg.PkgOpts.OptionalComponents, p.cfg.Pkg.Components) if err != nil { return deployedComponents, fmt.Errorf("unable to get selected components: %w", err) } diff --git a/src/pkg/packager/dev.go b/src/pkg/packager/dev.go index b93dfed8de..3778740fee 100644 --- a/src/pkg/packager/dev.go +++ b/src/pkg/packager/dev.go @@ -41,7 +41,7 @@ func (p *Packager) DevDeploy() error { // the user's selection and the component's `required`/`optional` field // This is also different from regular package creation, where we still assemble and package up // all components and their dependencies, regardless of whether they are required or not - p.cfg.Pkg.Components, err = interactive.GetSelectedComponents(p.cfg.PkgOpts.OptionalComponents, p.cfg.Pkg.Components) + p.cfg.Pkg.Components, err = interactive.GetComponentsForDeployment(p.cfg.PkgOpts.OptionalComponents, p.cfg.Pkg.Components) if err != nil { return err } From 0e698fcf6c39a64ce98017c2eac48183d0305e9a Mon Sep 17 00:00:00 2001 From: razzle Date: Thu, 11 Jan 2024 13:00:21 -0600 Subject: [PATCH 040/113] my new insanity Signed-off-by: razzle --- src/pkg/packager/deploy.go | 13 +- src/pkg/packager/dev.go | 5 +- src/pkg/packager/filters/common.go | 7 ++ .../filters/deploy.go} | 111 ++++-------------- src/pkg/packager/filters/empty.go | 13 ++ src/pkg/packager/filters/included.go | 48 ++++++++ src/pkg/packager/filters/platform.go | 50 ++++++++ src/pkg/packager/filters/utils.go | 38 ++++++ src/pkg/packager/mirror.go | 22 ++-- src/pkg/packager/publish.go | 4 +- src/pkg/packager/remove.go | 14 ++- src/pkg/packager/sources/cluster.go | 3 +- src/pkg/packager/sources/new.go | 3 +- src/pkg/packager/sources/oci.go | 8 +- src/pkg/packager/sources/split.go | 5 +- src/pkg/packager/sources/tarball.go | 3 +- src/pkg/packager/sources/url.go | 5 +- src/pkg/packager/yaml.go | 35 +----- 18 files changed, 233 insertions(+), 154 deletions(-) create mode 100644 src/pkg/packager/filters/common.go rename src/pkg/{interactive/components.go => packager/filters/deploy.go} (54%) create mode 100644 src/pkg/packager/filters/empty.go create mode 100644 src/pkg/packager/filters/included.go create mode 100644 src/pkg/packager/filters/platform.go create mode 100644 src/pkg/packager/filters/utils.go diff --git a/src/pkg/packager/deploy.go b/src/pkg/packager/deploy.go index d10f819257..5c5b6ad65d 100644 --- a/src/pkg/packager/deploy.go +++ b/src/pkg/packager/deploy.go @@ -20,11 +20,10 @@ import ( "github.com/defenseunicorns/zarf/src/internal/packager/images" "github.com/defenseunicorns/zarf/src/internal/packager/template" "github.com/defenseunicorns/zarf/src/pkg/cluster" - "github.com/defenseunicorns/zarf/src/pkg/interactive" "github.com/defenseunicorns/zarf/src/pkg/k8s" "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/message" - "github.com/defenseunicorns/zarf/src/pkg/packager/sources" + "github.com/defenseunicorns/zarf/src/pkg/packager/filters" "github.com/defenseunicorns/zarf/src/pkg/transform" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" @@ -42,12 +41,9 @@ func (p *Packager) resetRegistryHPA() { // Deploy attempts to deploy the given PackageConfig. func (p *Packager) Deploy() (err error) { - if ociSource, ok := p.source.(*sources.OCISource); ok { - ociSource.ComponentSelectionFilter = interactive.GetComponentsForDeployment - p.source = ociSource - } + filter := filters.NewDeploymentFilter(p.cfg.PkgOpts.OptionalComponents) - if err = p.source.LoadPackage(p.layout, true); err != nil { + if err = p.source.LoadPackage(p.layout, filter, true); err != nil { return fmt.Errorf("unable to load the package: %w", err) } @@ -100,7 +96,8 @@ func (p *Packager) Deploy() (err error) { // deployComponents loops through a list of ZarfComponents and deploys them. func (p *Packager) deployComponents() (deployedComponents []types.DeployedComponent, err error) { - componentsToDeploy, err := interactive.GetComponentsForDeployment(p.cfg.PkgOpts.OptionalComponents, p.cfg.Pkg.Components) + filter := filters.NewDeploymentFilter(p.cfg.PkgOpts.OptionalComponents) + componentsToDeploy, err := filter.Apply(p.cfg.Pkg.Components) if err != nil { return deployedComponents, fmt.Errorf("unable to get selected components: %w", err) } diff --git a/src/pkg/packager/dev.go b/src/pkg/packager/dev.go index 3778740fee..ce1597ee82 100644 --- a/src/pkg/packager/dev.go +++ b/src/pkg/packager/dev.go @@ -10,8 +10,8 @@ import ( "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/internal/packager/validate" - "github.com/defenseunicorns/zarf/src/pkg/interactive" "github.com/defenseunicorns/zarf/src/pkg/message" + "github.com/defenseunicorns/zarf/src/pkg/packager/filters" "github.com/defenseunicorns/zarf/src/types" ) @@ -41,7 +41,8 @@ func (p *Packager) DevDeploy() error { // the user's selection and the component's `required`/`optional` field // This is also different from regular package creation, where we still assemble and package up // all components and their dependencies, regardless of whether they are required or not - p.cfg.Pkg.Components, err = interactive.GetComponentsForDeployment(p.cfg.PkgOpts.OptionalComponents, p.cfg.Pkg.Components) + filter := filters.NewDeploymentFilter(p.cfg.PkgOpts.OptionalComponents) + p.cfg.Pkg.Components, err = filter.Apply(p.cfg.Pkg.Components) if err != nil { return err } diff --git a/src/pkg/packager/filters/common.go b/src/pkg/packager/filters/common.go new file mode 100644 index 0000000000..21686a015b --- /dev/null +++ b/src/pkg/packager/filters/common.go @@ -0,0 +1,7 @@ +package filters + +import "github.com/defenseunicorns/zarf/src/types" + +type ComponentFilterStrategy interface { + Apply([]types.ZarfComponent) ([]types.ZarfComponent, error) +} diff --git a/src/pkg/interactive/components.go b/src/pkg/packager/filters/deploy.go similarity index 54% rename from src/pkg/interactive/components.go rename to src/pkg/packager/filters/deploy.go index dea4737a86..03cd5ddc7c 100644 --- a/src/pkg/interactive/components.go +++ b/src/pkg/packager/filters/deploy.go @@ -1,31 +1,34 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: 2021-Present The Zarf Authors - -// Package interactive contains functions for interacting with the user via STDIN. -package interactive +package filters import ( "fmt" - "path" "slices" "strings" "github.com/agnivade/levenshtein" "github.com/defenseunicorns/zarf/src/config/lang" + "github.com/defenseunicorns/zarf/src/pkg/interactive" "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "github.com/defenseunicorns/zarf/src/types" ) -type selectState int - -const ( - unknown selectState = iota - included - excluded +var ( + _ ComponentFilterStrategy = &DeploymentFilter{} ) -// GetComponentsForDeployment prompts to select components based upon multiple conditions -func GetComponentsForDeployment(optionalComponents string, allComponents []types.ZarfComponent) ([]types.ZarfComponent, error) { +func NewDeploymentFilter(optionalComponents string) *DeploymentFilter { + requested := helpers.StringToSlice(optionalComponents) + + return &DeploymentFilter{ + requested, + } +} + +type DeploymentFilter struct { + requestedComponents []string +} + +func (f *DeploymentFilter) Apply(allComponents []types.ZarfComponent) ([]types.ZarfComponent, error) { var selectedComponents []types.ZarfComponent groupedComponents := map[string][]types.ZarfComponent{} orderedComponentGroups := []string{} @@ -44,9 +47,7 @@ func GetComponentsForDeployment(optionalComponents string, allComponents []types groupedComponents[groupKey] = append(groupedComponents[groupKey], component) } - // Split the --components list as a comma-delimited list - requestedComponents := helpers.StringToSlice(optionalComponents) - isPartial := len(requestedComponents) > 0 && requestedComponents[0] != "" + isPartial := len(f.requestedComponents) > 0 && f.requestedComponents[0] != "" if isPartial { matchedRequests := map[string]bool{} @@ -60,7 +61,7 @@ func GetComponentsForDeployment(optionalComponents string, allComponents []types // Ensure we have a local version of the component to point to (otherwise the pointer might change on us) component := component - selectState, matchedRequest := includedOrExcluded(component, requestedComponents) + selectState, matchedRequest := includedOrExcluded(component.Name, f.requestedComponents) if !component.IsRequired() { if selectState == excluded { @@ -106,7 +107,7 @@ func GetComponentsForDeployment(optionalComponents string, allComponents []types // Check that we have matched against all requests var err error - for _, requestedComponent := range requestedComponents { + for _, requestedComponent := range f.requestedComponents { if _, ok := matchedRequests[requestedComponent]; !ok { closeEnough := []string{} for _, c := range allComponents { @@ -129,14 +130,14 @@ func GetComponentsForDeployment(optionalComponents string, allComponents []types } else { for _, groupKey := range orderedComponentGroups { if len(groupedComponents[groupKey]) > 1 { - component := SelectChoiceGroup(groupedComponents[groupKey]) + component := interactive.SelectChoiceGroup(groupedComponents[groupKey]) selectedComponents = append(selectedComponents, component) } else { component := groupedComponents[groupKey][0] if component.IsRequired() { selectedComponents = append(selectedComponents, component) - } else if selected := SelectOptionalComponent(component); selected { + } else if selected := interactive.SelectOptionalComponent(component); selected { selectedComponents = append(selectedComponents, component) } } @@ -145,71 +146,3 @@ func GetComponentsForDeployment(optionalComponents string, allComponents []types return selectedComponents, nil } - -// GetOnlyIncludedComponents returns only the components that are included -func GetOnlyIncludedComponents(optionalComponents string, components []types.ZarfComponent) ([]types.ZarfComponent, error) { - requestedComponents := helpers.StringToSlice(optionalComponents) - isPartial := len(requestedComponents) > 0 && requestedComponents[0] != "" - - result := []types.ZarfComponent{} - - for _, component := range components { - selectState := unknown - - if isPartial { - selectState, _ = includedOrExcluded(component, requestedComponents) - - if selectState == excluded { - continue - } - } else { - selectState = included - } - - if selectState == included { - result = append(result, component) - } - } - - return result, nil -} - -// ForIncludedComponents runs a function for each included component -func ForIncludedComponents(optionalComponents string, components []types.ZarfComponent, fn func(types.ZarfComponent) error) error { - included, err := GetOnlyIncludedComponents(optionalComponents, components) - if err != nil { - return err - } - - for _, component := range included { - if err := fn(component); err != nil { - return err - } - } - - return nil -} - -func includedOrExcluded(component types.ZarfComponent, requestedComponentNames []string) (selectState, string) { - // Check if the component has a leading dash indicating it should be excluded - this is done first so that exclusions precede inclusions - for _, requestedComponent := range requestedComponentNames { - if strings.HasPrefix(requestedComponent, "-") { - // If the component glob matches one of the requested components, then return true - // This supports globbing with "path" in order to have the same behavior across OSes (if we ever allow namespaced components with /) - if matched, _ := path.Match(strings.TrimPrefix(requestedComponent, "-"), component.Name); matched { - return excluded, requestedComponent - } - } - } - // Check if the component matches a glob pattern and should be included - for _, requestedComponent := range requestedComponentNames { - // If the component glob matches one of the requested components, then return true - // This supports globbing with "path" in order to have the same behavior across OSes (if we ever allow namespaced components with /) - if matched, _ := path.Match(requestedComponent, component.Name); matched { - return included, requestedComponent - } - } - - // All other cases we don't know if we should include or exclude yet - return unknown, "" -} diff --git a/src/pkg/packager/filters/empty.go b/src/pkg/packager/filters/empty.go new file mode 100644 index 0000000000..42cb0aedd2 --- /dev/null +++ b/src/pkg/packager/filters/empty.go @@ -0,0 +1,13 @@ +package filters + +import "github.com/defenseunicorns/zarf/src/types" + +var ( + _ ComponentFilterStrategy = &EmptyFilter{} +) + +type EmptyFilter struct{} + +func (f *EmptyFilter) Apply(components []types.ZarfComponent) ([]types.ZarfComponent, error) { + return components, nil +} diff --git a/src/pkg/packager/filters/included.go b/src/pkg/packager/filters/included.go new file mode 100644 index 0000000000..31a6570a22 --- /dev/null +++ b/src/pkg/packager/filters/included.go @@ -0,0 +1,48 @@ +package filters + +import ( + "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" + "github.com/defenseunicorns/zarf/src/types" +) + +var ( + _ ComponentFilterStrategy = &IncludedFilter{} +) + +func NewIncludedFilter(optionalComponents string) *IncludedFilter { + requested := helpers.StringToSlice(optionalComponents) + + return &IncludedFilter{ + requested, + } +} + +type IncludedFilter struct { + requestedComponents []string +} + +func (f *IncludedFilter) Apply(allComponents []types.ZarfComponent) ([]types.ZarfComponent, error) { + isPartial := len(f.requestedComponents) > 0 && f.requestedComponents[0] != "" + + result := []types.ZarfComponent{} + + for _, component := range allComponents { + selectState := unknown + + if isPartial { + selectState, _ = includedOrExcluded(component.Name, f.requestedComponents) + + if selectState == excluded { + continue + } + } else { + selectState = included + } + + if selectState == included { + result = append(result, component) + } + } + + return result, nil +} diff --git a/src/pkg/packager/filters/platform.go b/src/pkg/packager/filters/platform.go new file mode 100644 index 0000000000..d088adceef --- /dev/null +++ b/src/pkg/packager/filters/platform.go @@ -0,0 +1,50 @@ +package filters + +import ( + "github.com/defenseunicorns/zarf/src/pkg/message" + "github.com/defenseunicorns/zarf/src/types" +) + +var ( + _ ComponentFilterStrategy = &ArchAndOSFilter{} +) + +func NewArchAndOSFilter(arch string, os string) *ArchAndOSFilter { + return &ArchAndOSFilter{ + arch: arch, + os: os, + } +} + +type ArchAndOSFilter struct { + arch string + os string +} + +func (f *ArchAndOSFilter) Apply(components []types.ZarfComponent) (filtered []types.ZarfComponent, _ error) { + // Filter each component to only compatible platforms. + for _, component := range components { + // Ignore only filters that are empty + var validArch, validOS bool + + // Test for valid architecture + if component.Only.Cluster.Architecture == "" || component.Only.Cluster.Architecture == f.arch { + validArch = true + } else { + message.Debugf("Skipping component %s, %s is not compatible with %s", component.Name, component.Only.Cluster.Architecture, f.arch) + } + + // Test for a valid OS + if component.Only.LocalOS == "" || component.Only.LocalOS == f.os { + validOS = true + } else { + message.Debugf("Skipping component %s, %s is not compatible with %s", component.Name, component.Only.LocalOS, f.os) + } + + // If both the OS and architecture are valid, add the component to the filtered list + if validArch && validOS { + filtered = append(filtered, component) + } + } + return filtered, nil +} diff --git a/src/pkg/packager/filters/utils.go b/src/pkg/packager/filters/utils.go new file mode 100644 index 0000000000..31fa6370a0 --- /dev/null +++ b/src/pkg/packager/filters/utils.go @@ -0,0 +1,38 @@ +package filters + +import ( + "path" + "strings" +) + +type selectState int + +const ( + unknown selectState = iota + included + excluded +) + +func includedOrExcluded(componentName string, requestedComponentNames []string) (selectState, string) { + // Check if the component has a leading dash indicating it should be excluded - this is done first so that exclusions precede inclusions + for _, requestedComponent := range requestedComponentNames { + if strings.HasPrefix(requestedComponent, "-") { + // If the component glob matches one of the requested components, then return true + // This supports globbing with "path" in order to have the same behavior across OSes (if we ever allow namespaced components with /) + if matched, _ := path.Match(strings.TrimPrefix(requestedComponent, "-"), componentName); matched { + return excluded, requestedComponent + } + } + } + // Check if the component matches a glob pattern and should be included + for _, requestedComponent := range requestedComponentNames { + // If the component glob matches one of the requested components, then return true + // This supports globbing with "path" in order to have the same behavior across OSes (if we ever allow namespaced components with /) + if matched, _ := path.Match(requestedComponent, componentName); matched { + return included, requestedComponent + } + } + + // All other cases we don't know if we should include or exclude yet + return unknown, "" +} diff --git a/src/pkg/packager/mirror.go b/src/pkg/packager/mirror.go index 9d8913eb07..58432cd0fc 100644 --- a/src/pkg/packager/mirror.go +++ b/src/pkg/packager/mirror.go @@ -9,9 +9,8 @@ import ( "strings" "github.com/defenseunicorns/zarf/src/config" - "github.com/defenseunicorns/zarf/src/pkg/interactive" "github.com/defenseunicorns/zarf/src/pkg/message" - "github.com/defenseunicorns/zarf/src/pkg/packager/sources" + "github.com/defenseunicorns/zarf/src/pkg/packager/filters" "github.com/defenseunicorns/zarf/src/types" ) @@ -20,12 +19,9 @@ func (p *Packager) Mirror() (err error) { spinner := message.NewProgressSpinner("Mirroring Zarf package %s", p.cfg.PkgOpts.PackageSource) defer spinner.Stop() - if ociSource, ok := p.source.(*sources.OCISource); ok { - ociSource.ComponentSelectionFilter = interactive.GetOnlyIncludedComponents - p.source = ociSource - } + filter := filters.NewIncludedFilter(p.cfg.PkgOpts.OptionalComponents) - if err = p.source.LoadPackage(p.layout, true); err != nil { + if err = p.source.LoadPackage(p.layout, filter, true); err != nil { return fmt.Errorf("unable to load the package: %w", err) } if err = p.readZarfYAML(p.layout.ZarfYAML); err != nil { @@ -51,7 +47,17 @@ func (p *Packager) Mirror() (err error) { p.filterComponents() // Run mirror for each requested component - return interactive.ForIncludedComponents(p.cfg.PkgOpts.OptionalComponents, p.cfg.Pkg.Components, p.mirrorComponent) + included, err := filter.Apply(p.cfg.Pkg.Components) + if err != nil { + return err + } + + for _, component := range included { + if err := p.mirrorComponent(component); err != nil { + return err + } + } + return nil } // mirrorComponent mirrors a Zarf Component. diff --git a/src/pkg/packager/publish.go b/src/pkg/packager/publish.go index 6022eeada2..09a875a66c 100644 --- a/src/pkg/packager/publish.go +++ b/src/pkg/packager/publish.go @@ -14,6 +14,7 @@ import ( "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/oci" + "github.com/defenseunicorns/zarf/src/pkg/packager/filters" "github.com/defenseunicorns/zarf/src/pkg/packager/sources" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" @@ -98,7 +99,8 @@ func (p *Packager) Publish() (err error) { return err } } else { - if err = p.source.LoadPackage(p.layout, false); err != nil { + filter := &filters.EmptyFilter{} + if err = p.source.LoadPackage(p.layout, filter, false); err != nil { return fmt.Errorf("unable to load the package: %w", err) } if err = p.readZarfYAML(p.layout.ZarfYAML); err != nil { diff --git a/src/pkg/packager/remove.go b/src/pkg/packager/remove.go index e03db05b41..7e2151814d 100644 --- a/src/pkg/packager/remove.go +++ b/src/pkg/packager/remove.go @@ -14,8 +14,8 @@ import ( "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/internal/packager/helm" "github.com/defenseunicorns/zarf/src/pkg/cluster" - "github.com/defenseunicorns/zarf/src/pkg/interactive" "github.com/defenseunicorns/zarf/src/pkg/message" + "github.com/defenseunicorns/zarf/src/pkg/packager/filters" "github.com/defenseunicorns/zarf/src/pkg/packager/sources" "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "github.com/defenseunicorns/zarf/src/types" @@ -50,15 +50,19 @@ func (p *Packager) Remove() (err error) { packageRequiresCluster := false // If components were provided; just remove the things we were asked to remove - interactive.ForIncludedComponents(p.cfg.PkgOpts.OptionalComponents, p.cfg.Pkg.Components, (func(component types.ZarfComponent) error { + filter := filters.NewIncludedFilter(p.cfg.PkgOpts.OptionalComponents) + included, err := filter.Apply(p.cfg.Pkg.Components) + if err != nil { + return err + } + + for _, component := range included { componentsToRemove = append(componentsToRemove, component.Name) if component.RequiresCluster() { packageRequiresCluster = true } - - return nil - })) + } // Get or build the secret for the deployed package deployedPackage := &types.DeployedPackage{} diff --git a/src/pkg/packager/sources/cluster.go b/src/pkg/packager/sources/cluster.go index 4718092474..d7f54bf17c 100644 --- a/src/pkg/packager/sources/cluster.go +++ b/src/pkg/packager/sources/cluster.go @@ -11,6 +11,7 @@ import ( "github.com/defenseunicorns/zarf/src/internal/packager/validate" "github.com/defenseunicorns/zarf/src/pkg/cluster" "github.com/defenseunicorns/zarf/src/pkg/layout" + "github.com/defenseunicorns/zarf/src/pkg/packager/filters" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/types" ) @@ -41,7 +42,7 @@ type ClusterSource struct { // LoadPackage loads a package from a cluster. // // This is not implemented. -func (s *ClusterSource) LoadPackage(_ *layout.PackagePaths, _ bool) error { +func (s *ClusterSource) LoadPackage(_ *layout.PackagePaths, _ filters.ComponentFilterStrategy, _ bool) error { return fmt.Errorf("not implemented") } diff --git a/src/pkg/packager/sources/new.go b/src/pkg/packager/sources/new.go index 4ec16c65ed..5f8e634640 100644 --- a/src/pkg/packager/sources/new.go +++ b/src/pkg/packager/sources/new.go @@ -13,6 +13,7 @@ import ( "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/oci" + "github.com/defenseunicorns/zarf/src/pkg/packager/filters" "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "github.com/defenseunicorns/zarf/src/types" ) @@ -28,7 +29,7 @@ import ( // `sources.ValidatePackageSignature` and `sources.ValidatePackageIntegrity` can be leveraged for this purpose. type PackageSource interface { // LoadPackage loads a package from a source. - LoadPackage(dst *layout.PackagePaths, unarchiveAll bool) error + LoadPackage(dst *layout.PackagePaths, filter filters.ComponentFilterStrategy, unarchiveAll bool) error // LoadPackageMetadata loads a package's metadata from a source. LoadPackageMetadata(dst *layout.PackagePaths, wantSBOM bool, skipValidation bool) error diff --git a/src/pkg/packager/sources/oci.go b/src/pkg/packager/sources/oci.go index b25e776b42..cd792a877f 100644 --- a/src/pkg/packager/sources/oci.go +++ b/src/pkg/packager/sources/oci.go @@ -15,6 +15,7 @@ import ( "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/oci" + "github.com/defenseunicorns/zarf/src/pkg/packager/filters" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/types" "github.com/mholt/archiver/v3" @@ -30,11 +31,10 @@ var ( type OCISource struct { *types.ZarfPackageOptions *oci.OrasRemote - ComponentSelectionFilter func(optionalComponents string, allComponents []types.ZarfComponent) ([]types.ZarfComponent, error) } // LoadPackage loads a package from an OCI registry. -func (s *OCISource) LoadPackage(dst *layout.PackagePaths, unarchiveAll bool) (err error) { +func (s *OCISource) LoadPackage(dst *layout.PackagePaths, filter filters.ComponentFilterStrategy, unarchiveAll bool) (err error) { var pkg types.ZarfPackage layersToPull := []ocispec.Descriptor{} @@ -47,8 +47,8 @@ func (s *OCISource) LoadPackage(dst *layout.PackagePaths, unarchiveAll bool) (er return err } var requested []types.ZarfComponent - if s.ComponentSelectionFilter != nil { - requested, err = s.ComponentSelectionFilter(s.OptionalComponents, pkg.Components) + if filter != nil { + requested, err = filter.Apply(pkg.Components) if err != nil { return err } diff --git a/src/pkg/packager/sources/split.go b/src/pkg/packager/sources/split.go index 1aade2eee5..9897e02a40 100644 --- a/src/pkg/packager/sources/split.go +++ b/src/pkg/packager/sources/split.go @@ -15,6 +15,7 @@ import ( "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/message" + "github.com/defenseunicorns/zarf/src/pkg/packager/filters" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/types" ) @@ -103,7 +104,7 @@ func (s *SplitTarballSource) Collect(dir string) (string, error) { } // LoadPackage loads a package from a split tarball. -func (s *SplitTarballSource) LoadPackage(dst *layout.PackagePaths, unarchiveAll bool) (err error) { +func (s *SplitTarballSource) LoadPackage(dst *layout.PackagePaths, _ filters.ComponentFilterStrategy, unarchiveAll bool) (err error) { tb, err := s.Collect(filepath.Dir(s.PackageSource)) if err != nil { return err @@ -117,7 +118,7 @@ func (s *SplitTarballSource) LoadPackage(dst *layout.PackagePaths, unarchiveAll ts := &TarballSource{ s.ZarfPackageOptions, } - return ts.LoadPackage(dst, unarchiveAll) + return ts.LoadPackage(dst, &filters.EmptyFilter{}, unarchiveAll) } // LoadPackageMetadata loads a package's metadata from a split tarball. diff --git a/src/pkg/packager/sources/tarball.go b/src/pkg/packager/sources/tarball.go index e65d0b962a..b57f99d4f2 100644 --- a/src/pkg/packager/sources/tarball.go +++ b/src/pkg/packager/sources/tarball.go @@ -15,6 +15,7 @@ import ( "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/oci" + "github.com/defenseunicorns/zarf/src/pkg/packager/filters" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/types" "github.com/mholt/archiver/v3" @@ -31,7 +32,7 @@ type TarballSource struct { } // LoadPackage loads a package from a tarball. -func (s *TarballSource) LoadPackage(dst *layout.PackagePaths, unarchiveAll bool) (err error) { +func (s *TarballSource) LoadPackage(dst *layout.PackagePaths, _ filters.ComponentFilterStrategy, unarchiveAll bool) (err error) { var pkg types.ZarfPackage spinner := message.NewProgressSpinner("Loading package from %q", s.PackageSource) diff --git a/src/pkg/packager/sources/url.go b/src/pkg/packager/sources/url.go index 8d65cf6a9c..5a32a011e6 100644 --- a/src/pkg/packager/sources/url.go +++ b/src/pkg/packager/sources/url.go @@ -12,6 +12,7 @@ import ( "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/pkg/layout" + "github.com/defenseunicorns/zarf/src/pkg/packager/filters" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "github.com/defenseunicorns/zarf/src/types" @@ -49,7 +50,7 @@ func (s *URLSource) Collect(dir string) (string, error) { } // LoadPackage loads a package from an http, https or sget URL. -func (s *URLSource) LoadPackage(dst *layout.PackagePaths, unarchiveAll bool) (err error) { +func (s *URLSource) LoadPackage(dst *layout.PackagePaths, _ filters.ComponentFilterStrategy, unarchiveAll bool) (err error) { tmp, err := utils.MakeTempDir(config.CommonOptions.TempDirectory) if err != nil { return err @@ -69,7 +70,7 @@ func (s *URLSource) LoadPackage(dst *layout.PackagePaths, unarchiveAll bool) (er s.ZarfPackageOptions, } - return ts.LoadPackage(dst, unarchiveAll) + return ts.LoadPackage(dst, &filters.EmptyFilter{}, unarchiveAll) } // LoadPackageMetadata loads a package's metadata from an http, https or sget URL. diff --git a/src/pkg/packager/yaml.go b/src/pkg/packager/yaml.go index 38e893c7b5..ec73e94618 100644 --- a/src/pkg/packager/yaml.go +++ b/src/pkg/packager/yaml.go @@ -10,10 +10,9 @@ import ( "time" "github.com/defenseunicorns/zarf/src/config" - "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/packager/deprecated" + "github.com/defenseunicorns/zarf/src/pkg/packager/filters" "github.com/defenseunicorns/zarf/src/pkg/utils" - "github.com/defenseunicorns/zarf/src/types" ) // readZarfYAML reads a Zarf YAML file. @@ -42,35 +41,11 @@ func (p *Packager) readZarfYAML(path string) error { return nil } -// filterComponents removes components not matching the current OS if filterByOS is set. +// filterComponents removes components not matching the current OS and architecture. func (p *Packager) filterComponents() { - // Filter each component to only compatible platforms. - filteredComponents := []types.ZarfComponent{} - for _, component := range p.cfg.Pkg.Components { - // Ignore only filters that are empty - var validArch, validOS bool - - // Test for valid architecture - if component.Only.Cluster.Architecture == "" || component.Only.Cluster.Architecture == p.arch { - validArch = true - } else { - message.Debugf("Skipping component %s, %s is not compatible with %s", component.Name, component.Only.Cluster.Architecture, p.arch) - } - - // Test for a valid OS - if component.Only.LocalOS == "" || component.Only.LocalOS == runtime.GOOS { - validOS = true - } else { - message.Debugf("Skipping component %s, %s is not compatible with %s", component.Name, component.Only.LocalOS, runtime.GOOS) - } - - // If both the OS and architecture are valid, add the component to the filtered list - if validArch && validOS { - filteredComponents = append(filteredComponents, component) - } - } - // Update the active package with the filtered components. - p.cfg.Pkg.Components = filteredComponents + filter := filters.NewArchAndOSFilter(p.arch, runtime.GOOS) + // Currently the arch and os filter will never return an error. + p.cfg.Pkg.Components, _ = filter.Apply(p.cfg.Pkg.Components) } // writeYaml adds build information and writes the config to the temp directory. From a758a27010de999db235b76661541570452db720 Mon Sep 17 00:00:00 2001 From: razzle Date: Thu, 11 Jan 2024 13:01:44 -0600 Subject: [PATCH 041/113] spdx Signed-off-by: razzle --- src/pkg/packager/filters/common.go | 4 ++++ src/pkg/packager/filters/deploy.go | 4 ++++ src/pkg/packager/filters/empty.go | 4 ++++ src/pkg/packager/filters/included.go | 4 ++++ src/pkg/packager/filters/platform.go | 4 ++++ src/pkg/packager/filters/utils.go | 4 ++++ 6 files changed, 24 insertions(+) diff --git a/src/pkg/packager/filters/common.go b/src/pkg/packager/filters/common.go index 21686a015b..8a7861ef11 100644 --- a/src/pkg/packager/filters/common.go +++ b/src/pkg/packager/filters/common.go @@ -1,3 +1,7 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package filters contains core implementations of the ComponentFilterStrategy interface. package filters import "github.com/defenseunicorns/zarf/src/types" diff --git a/src/pkg/packager/filters/deploy.go b/src/pkg/packager/filters/deploy.go index 03cd5ddc7c..d017e017b4 100644 --- a/src/pkg/packager/filters/deploy.go +++ b/src/pkg/packager/filters/deploy.go @@ -1,3 +1,7 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package filters contains core implementations of the ComponentFilterStrategy interface. package filters import ( diff --git a/src/pkg/packager/filters/empty.go b/src/pkg/packager/filters/empty.go index 42cb0aedd2..af5e856467 100644 --- a/src/pkg/packager/filters/empty.go +++ b/src/pkg/packager/filters/empty.go @@ -1,3 +1,7 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package filters contains core implementations of the ComponentFilterStrategy interface. package filters import "github.com/defenseunicorns/zarf/src/types" diff --git a/src/pkg/packager/filters/included.go b/src/pkg/packager/filters/included.go index 31a6570a22..112f916010 100644 --- a/src/pkg/packager/filters/included.go +++ b/src/pkg/packager/filters/included.go @@ -1,3 +1,7 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package filters contains core implementations of the ComponentFilterStrategy interface. package filters import ( diff --git a/src/pkg/packager/filters/platform.go b/src/pkg/packager/filters/platform.go index d088adceef..ff9badc589 100644 --- a/src/pkg/packager/filters/platform.go +++ b/src/pkg/packager/filters/platform.go @@ -1,3 +1,7 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package filters contains core implementations of the ComponentFilterStrategy interface. package filters import ( diff --git a/src/pkg/packager/filters/utils.go b/src/pkg/packager/filters/utils.go index 31fa6370a0..3b33f9b594 100644 --- a/src/pkg/packager/filters/utils.go +++ b/src/pkg/packager/filters/utils.go @@ -1,3 +1,7 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package filters contains core implementations of the ComponentFilterStrategy interface. package filters import ( From c254463a743a855d943879896480a9d017ef810f Mon Sep 17 00:00:00 2001 From: razzle Date: Thu, 11 Jan 2024 13:06:05 -0600 Subject: [PATCH 042/113] lint Signed-off-by: razzle --- src/pkg/packager/filters/common.go | 1 + src/pkg/packager/filters/deploy.go | 3 +++ src/pkg/packager/filters/empty.go | 2 ++ src/pkg/packager/filters/included.go | 3 +++ src/pkg/packager/filters/platform.go | 3 +++ 5 files changed, 12 insertions(+) diff --git a/src/pkg/packager/filters/common.go b/src/pkg/packager/filters/common.go index 8a7861ef11..dd2b369c76 100644 --- a/src/pkg/packager/filters/common.go +++ b/src/pkg/packager/filters/common.go @@ -6,6 +6,7 @@ package filters import "github.com/defenseunicorns/zarf/src/types" +// ComponentFilterStrategy is a strategy interface for filtering components. type ComponentFilterStrategy interface { Apply([]types.ZarfComponent) ([]types.ZarfComponent, error) } diff --git a/src/pkg/packager/filters/deploy.go b/src/pkg/packager/filters/deploy.go index d017e017b4..abda6d859c 100644 --- a/src/pkg/packager/filters/deploy.go +++ b/src/pkg/packager/filters/deploy.go @@ -20,6 +20,7 @@ var ( _ ComponentFilterStrategy = &DeploymentFilter{} ) +// NewDeploymentFilter creates a new deployment filter. func NewDeploymentFilter(optionalComponents string) *DeploymentFilter { requested := helpers.StringToSlice(optionalComponents) @@ -28,10 +29,12 @@ func NewDeploymentFilter(optionalComponents string) *DeploymentFilter { } } +// DeploymentFilter is the default filter for deployments. type DeploymentFilter struct { requestedComponents []string } +// Apply applies the filter. func (f *DeploymentFilter) Apply(allComponents []types.ZarfComponent) ([]types.ZarfComponent, error) { var selectedComponents []types.ZarfComponent groupedComponents := map[string][]types.ZarfComponent{} diff --git a/src/pkg/packager/filters/empty.go b/src/pkg/packager/filters/empty.go index af5e856467..95499ea3a4 100644 --- a/src/pkg/packager/filters/empty.go +++ b/src/pkg/packager/filters/empty.go @@ -10,8 +10,10 @@ var ( _ ComponentFilterStrategy = &EmptyFilter{} ) +// EmptyFilter is a filter that does nothing. type EmptyFilter struct{} +// Apply returns the components unchanged. func (f *EmptyFilter) Apply(components []types.ZarfComponent) ([]types.ZarfComponent, error) { return components, nil } diff --git a/src/pkg/packager/filters/included.go b/src/pkg/packager/filters/included.go index 112f916010..838b9b54cd 100644 --- a/src/pkg/packager/filters/included.go +++ b/src/pkg/packager/filters/included.go @@ -13,6 +13,7 @@ var ( _ ComponentFilterStrategy = &IncludedFilter{} ) +// NewIncludedFilter creates a new simple included filter. func NewIncludedFilter(optionalComponents string) *IncludedFilter { requested := helpers.StringToSlice(optionalComponents) @@ -21,10 +22,12 @@ func NewIncludedFilter(optionalComponents string) *IncludedFilter { } } +// IncludedFilter sorts based purely on the internal included state of the component. type IncludedFilter struct { requestedComponents []string } +// Apply applies the filter. func (f *IncludedFilter) Apply(allComponents []types.ZarfComponent) ([]types.ZarfComponent, error) { isPartial := len(f.requestedComponents) > 0 && f.requestedComponents[0] != "" diff --git a/src/pkg/packager/filters/platform.go b/src/pkg/packager/filters/platform.go index ff9badc589..1e3cbf6033 100644 --- a/src/pkg/packager/filters/platform.go +++ b/src/pkg/packager/filters/platform.go @@ -13,6 +13,7 @@ var ( _ ComponentFilterStrategy = &ArchAndOSFilter{} ) +// NewArchAndOSFilter creates an architecture and OS filter. func NewArchAndOSFilter(arch string, os string) *ArchAndOSFilter { return &ArchAndOSFilter{ arch: arch, @@ -20,11 +21,13 @@ func NewArchAndOSFilter(arch string, os string) *ArchAndOSFilter { } } +// ArchAndOSFilter filters components based on OS/architecture. type ArchAndOSFilter struct { arch string os string } +// Apply applies the filter. func (f *ArchAndOSFilter) Apply(components []types.ZarfComponent) (filtered []types.ZarfComponent, _ error) { // Filter each component to only compatible platforms. for _, component := range components { From 5bcf902c8f8a2c510d69af424d067d5cf987c363 Mon Sep 17 00:00:00 2001 From: razzle Date: Fri, 12 Jan 2024 13:35:25 -0600 Subject: [PATCH 043/113] filter manager Signed-off-by: razzle --- src/pkg/packager/deploy.go | 3 ++- src/pkg/packager/filters/common.go | 34 ++++++++++++++++++++++++++++- src/pkg/packager/filters/deploy.go | 11 ++++++++++ src/pkg/packager/mirror.go | 3 ++- src/pkg/packager/publish.go | 3 ++- src/pkg/packager/sources/cluster.go | 2 +- src/pkg/packager/sources/new.go | 2 +- src/pkg/packager/sources/oci.go | 8 ++++--- src/pkg/packager/sources/split.go | 4 ++-- src/pkg/packager/sources/tarball.go | 2 +- src/pkg/packager/sources/url.go | 4 ++-- 11 files changed, 62 insertions(+), 14 deletions(-) diff --git a/src/pkg/packager/deploy.go b/src/pkg/packager/deploy.go index 5c5b6ad65d..6b7e42b7e4 100644 --- a/src/pkg/packager/deploy.go +++ b/src/pkg/packager/deploy.go @@ -42,8 +42,9 @@ func (p *Packager) resetRegistryHPA() { // Deploy attempts to deploy the given PackageConfig. func (p *Packager) Deploy() (err error) { filter := filters.NewDeploymentFilter(p.cfg.PkgOpts.OptionalComponents) + fm := filters.NewFilterManager(filter) - if err = p.source.LoadPackage(p.layout, filter, true); err != nil { + if err = p.source.LoadPackage(p.layout, fm, true); err != nil { return fmt.Errorf("unable to load the package: %w", err) } diff --git a/src/pkg/packager/filters/common.go b/src/pkg/packager/filters/common.go index dd2b369c76..c5d09aa546 100644 --- a/src/pkg/packager/filters/common.go +++ b/src/pkg/packager/filters/common.go @@ -4,9 +4,41 @@ // Package filters contains core implementations of the ComponentFilterStrategy interface. package filters -import "github.com/defenseunicorns/zarf/src/types" +import ( + "github.com/Masterminds/semver/v3" + "github.com/defenseunicorns/zarf/src/types" +) // ComponentFilterStrategy is a strategy interface for filtering components. type ComponentFilterStrategy interface { Apply([]types.ZarfComponent) ([]types.ZarfComponent, error) } + +// VersionBehavior is an interface for setting version behavior on filters +type VersionBehavior interface { + UseVersionBehavior(*semver.Version) +} + +func NewFilterManager(strategy ComponentFilterStrategy) *FilterManager { + m := &FilterManager{} + m.SetStrategy(strategy) + return m +} + +type FilterManager struct { + strategy ComponentFilterStrategy +} + +func (m *FilterManager) SetStrategy(strategy ComponentFilterStrategy) { + m.strategy = strategy +} + +func (m *FilterManager) SetVersionBehavior(buildVersion *semver.Version) { + if v, ok := m.strategy.(VersionBehavior); ok { + v.UseVersionBehavior(buildVersion) + } +} + +func (m *FilterManager) Execute(components []types.ZarfComponent) ([]types.ZarfComponent, error) { + return m.strategy.Apply(components) +} diff --git a/src/pkg/packager/filters/deploy.go b/src/pkg/packager/filters/deploy.go index abda6d859c..9a5f180c08 100644 --- a/src/pkg/packager/filters/deploy.go +++ b/src/pkg/packager/filters/deploy.go @@ -9,6 +9,7 @@ import ( "slices" "strings" + "github.com/Masterminds/semver/v3" "github.com/agnivade/levenshtein" "github.com/defenseunicorns/zarf/src/config/lang" "github.com/defenseunicorns/zarf/src/pkg/interactive" @@ -18,6 +19,7 @@ import ( var ( _ ComponentFilterStrategy = &DeploymentFilter{} + _ VersionBehavior = &DeploymentFilter{} ) // NewDeploymentFilter creates a new deployment filter. @@ -25,15 +27,24 @@ func NewDeploymentFilter(optionalComponents string) *DeploymentFilter { requested := helpers.StringToSlice(optionalComponents) return &DeploymentFilter{ + false, requested, } } // DeploymentFilter is the default filter for deployments. type DeploymentFilter struct { + useRequiredLogic bool requestedComponents []string } +// UseVersionBehavior sets the version behavior for the filter. +func (f *DeploymentFilter) UseVersionBehavior(buildVersion *semver.Version) { + if buildVersion.LessThan(semver.MustParse("v0.32.1")) { + f.useRequiredLogic = true + } +} + // Apply applies the filter. func (f *DeploymentFilter) Apply(allComponents []types.ZarfComponent) ([]types.ZarfComponent, error) { var selectedComponents []types.ZarfComponent diff --git a/src/pkg/packager/mirror.go b/src/pkg/packager/mirror.go index 58432cd0fc..807a647ff0 100644 --- a/src/pkg/packager/mirror.go +++ b/src/pkg/packager/mirror.go @@ -20,8 +20,9 @@ func (p *Packager) Mirror() (err error) { defer spinner.Stop() filter := filters.NewIncludedFilter(p.cfg.PkgOpts.OptionalComponents) + fm := filters.NewFilterManager(filter) - if err = p.source.LoadPackage(p.layout, filter, true); err != nil { + if err = p.source.LoadPackage(p.layout, fm, true); err != nil { return fmt.Errorf("unable to load the package: %w", err) } if err = p.readZarfYAML(p.layout.ZarfYAML); err != nil { diff --git a/src/pkg/packager/publish.go b/src/pkg/packager/publish.go index 09a875a66c..7aeb2812e0 100644 --- a/src/pkg/packager/publish.go +++ b/src/pkg/packager/publish.go @@ -100,7 +100,8 @@ func (p *Packager) Publish() (err error) { } } else { filter := &filters.EmptyFilter{} - if err = p.source.LoadPackage(p.layout, filter, false); err != nil { + fm := filters.NewFilterManager(filter) + if err = p.source.LoadPackage(p.layout, fm, false); err != nil { return fmt.Errorf("unable to load the package: %w", err) } if err = p.readZarfYAML(p.layout.ZarfYAML); err != nil { diff --git a/src/pkg/packager/sources/cluster.go b/src/pkg/packager/sources/cluster.go index d7f54bf17c..76682227d9 100644 --- a/src/pkg/packager/sources/cluster.go +++ b/src/pkg/packager/sources/cluster.go @@ -42,7 +42,7 @@ type ClusterSource struct { // LoadPackage loads a package from a cluster. // // This is not implemented. -func (s *ClusterSource) LoadPackage(_ *layout.PackagePaths, _ filters.ComponentFilterStrategy, _ bool) error { +func (s *ClusterSource) LoadPackage(_ *layout.PackagePaths, _ *filters.FilterManager, _ bool) error { return fmt.Errorf("not implemented") } diff --git a/src/pkg/packager/sources/new.go b/src/pkg/packager/sources/new.go index 5f8e634640..f5fd4e76f9 100644 --- a/src/pkg/packager/sources/new.go +++ b/src/pkg/packager/sources/new.go @@ -29,7 +29,7 @@ import ( // `sources.ValidatePackageSignature` and `sources.ValidatePackageIntegrity` can be leveraged for this purpose. type PackageSource interface { // LoadPackage loads a package from a source. - LoadPackage(dst *layout.PackagePaths, filter filters.ComponentFilterStrategy, unarchiveAll bool) error + LoadPackage(dst *layout.PackagePaths, fm *filters.FilterManager, unarchiveAll bool) error // LoadPackageMetadata loads a package's metadata from a source. LoadPackageMetadata(dst *layout.PackagePaths, wantSBOM bool, skipValidation bool) error diff --git a/src/pkg/packager/sources/oci.go b/src/pkg/packager/sources/oci.go index cd792a877f..cfb991e1bd 100644 --- a/src/pkg/packager/sources/oci.go +++ b/src/pkg/packager/sources/oci.go @@ -11,6 +11,7 @@ import ( "path/filepath" "strings" + "github.com/Masterminds/semver/v3" "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/message" @@ -34,7 +35,7 @@ type OCISource struct { } // LoadPackage loads a package from an OCI registry. -func (s *OCISource) LoadPackage(dst *layout.PackagePaths, filter filters.ComponentFilterStrategy, unarchiveAll bool) (err error) { +func (s *OCISource) LoadPackage(dst *layout.PackagePaths, fm *filters.FilterManager, unarchiveAll bool) (err error) { var pkg types.ZarfPackage layersToPull := []ocispec.Descriptor{} @@ -47,8 +48,9 @@ func (s *OCISource) LoadPackage(dst *layout.PackagePaths, filter filters.Compone return err } var requested []types.ZarfComponent - if filter != nil { - requested, err = filter.Apply(pkg.Components) + if fm != nil { + fm.SetVersionBehavior(semver.MustParse(pkg.Build.Version)) + requested, err = fm.Execute(pkg.Components) if err != nil { return err } diff --git a/src/pkg/packager/sources/split.go b/src/pkg/packager/sources/split.go index 9897e02a40..6219ee1b88 100644 --- a/src/pkg/packager/sources/split.go +++ b/src/pkg/packager/sources/split.go @@ -104,7 +104,7 @@ func (s *SplitTarballSource) Collect(dir string) (string, error) { } // LoadPackage loads a package from a split tarball. -func (s *SplitTarballSource) LoadPackage(dst *layout.PackagePaths, _ filters.ComponentFilterStrategy, unarchiveAll bool) (err error) { +func (s *SplitTarballSource) LoadPackage(dst *layout.PackagePaths, fm *filters.FilterManager, unarchiveAll bool) (err error) { tb, err := s.Collect(filepath.Dir(s.PackageSource)) if err != nil { return err @@ -118,7 +118,7 @@ func (s *SplitTarballSource) LoadPackage(dst *layout.PackagePaths, _ filters.Com ts := &TarballSource{ s.ZarfPackageOptions, } - return ts.LoadPackage(dst, &filters.EmptyFilter{}, unarchiveAll) + return ts.LoadPackage(dst, fm, unarchiveAll) } // LoadPackageMetadata loads a package's metadata from a split tarball. diff --git a/src/pkg/packager/sources/tarball.go b/src/pkg/packager/sources/tarball.go index b57f99d4f2..70bc03fd1c 100644 --- a/src/pkg/packager/sources/tarball.go +++ b/src/pkg/packager/sources/tarball.go @@ -32,7 +32,7 @@ type TarballSource struct { } // LoadPackage loads a package from a tarball. -func (s *TarballSource) LoadPackage(dst *layout.PackagePaths, _ filters.ComponentFilterStrategy, unarchiveAll bool) (err error) { +func (s *TarballSource) LoadPackage(dst *layout.PackagePaths, _ *filters.FilterManager, unarchiveAll bool) (err error) { var pkg types.ZarfPackage spinner := message.NewProgressSpinner("Loading package from %q", s.PackageSource) diff --git a/src/pkg/packager/sources/url.go b/src/pkg/packager/sources/url.go index 5a32a011e6..c9fc6dd6e6 100644 --- a/src/pkg/packager/sources/url.go +++ b/src/pkg/packager/sources/url.go @@ -50,7 +50,7 @@ func (s *URLSource) Collect(dir string) (string, error) { } // LoadPackage loads a package from an http, https or sget URL. -func (s *URLSource) LoadPackage(dst *layout.PackagePaths, _ filters.ComponentFilterStrategy, unarchiveAll bool) (err error) { +func (s *URLSource) LoadPackage(dst *layout.PackagePaths, fm *filters.FilterManager, unarchiveAll bool) (err error) { tmp, err := utils.MakeTempDir(config.CommonOptions.TempDirectory) if err != nil { return err @@ -70,7 +70,7 @@ func (s *URLSource) LoadPackage(dst *layout.PackagePaths, _ filters.ComponentFil s.ZarfPackageOptions, } - return ts.LoadPackage(dst, &filters.EmptyFilter{}, unarchiveAll) + return ts.LoadPackage(dst, fm, unarchiveAll) } // LoadPackageMetadata loads a package's metadata from an http, https or sget URL. From 1dd33c8cc3c57bb54aff069eb5d0a14c1fa1bf91 Mon Sep 17 00:00:00 2001 From: razzle Date: Fri, 12 Jan 2024 14:08:21 -0600 Subject: [PATCH 044/113] move unit tests Signed-off-by: razzle --- src/pkg/packager/filters/common.go | 5 ++ src/pkg/packager/filters/deploy.go | 4 +- src/pkg/packager/filters/utils.go | 37 ++++++++++ .../packager/filters/utils_test.go} | 74 ++++++++++++++----- src/types/component.go | 28 ------- 5 files changed, 100 insertions(+), 48 deletions(-) rename src/{types/component_test.go => pkg/packager/filters/utils_test.go} (52%) diff --git a/src/pkg/packager/filters/common.go b/src/pkg/packager/filters/common.go index c5d09aa546..13db4a1505 100644 --- a/src/pkg/packager/filters/common.go +++ b/src/pkg/packager/filters/common.go @@ -19,26 +19,31 @@ type VersionBehavior interface { UseVersionBehavior(*semver.Version) } +// NewFilterManager creates a new filter manager for the given strategy. func NewFilterManager(strategy ComponentFilterStrategy) *FilterManager { m := &FilterManager{} m.SetStrategy(strategy) return m } +// FilterManager manages a filter strategy. type FilterManager struct { strategy ComponentFilterStrategy } +// SetStrategy sets the strategy for the filter manager. func (m *FilterManager) SetStrategy(strategy ComponentFilterStrategy) { m.strategy = strategy } +// SetVersionBehavior sets the version behavior for the filter strategy. func (m *FilterManager) SetVersionBehavior(buildVersion *semver.Version) { if v, ok := m.strategy.(VersionBehavior); ok { v.UseVersionBehavior(buildVersion) } } +// Execute executes the filter strategy. func (m *FilterManager) Execute(components []types.ZarfComponent) ([]types.ZarfComponent, error) { return m.strategy.Apply(components) } diff --git a/src/pkg/packager/filters/deploy.go b/src/pkg/packager/filters/deploy.go index 9a5f180c08..caf040a124 100644 --- a/src/pkg/packager/filters/deploy.go +++ b/src/pkg/packager/filters/deploy.go @@ -81,7 +81,7 @@ func (f *DeploymentFilter) Apply(allComponents []types.ZarfComponent) ([]types.Z selectState, matchedRequest := includedOrExcluded(component.Name, f.requestedComponents) - if !component.IsRequired() { + if !isRequired(component, f.useRequiredLogic) { if selectState == excluded { // If the component was explicitly excluded, record the match and continue matchedRequests[matchedRequest] = true @@ -153,7 +153,7 @@ func (f *DeploymentFilter) Apply(allComponents []types.ZarfComponent) ([]types.Z } else { component := groupedComponents[groupKey][0] - if component.IsRequired() { + if isRequired(component, f.useRequiredLogic) { selectedComponents = append(selectedComponents, component) } else if selected := interactive.SelectOptionalComponent(component); selected { selectedComponents = append(selectedComponents, component) diff --git a/src/pkg/packager/filters/utils.go b/src/pkg/packager/filters/utils.go index 3b33f9b594..39dbcc644a 100644 --- a/src/pkg/packager/filters/utils.go +++ b/src/pkg/packager/filters/utils.go @@ -7,6 +7,8 @@ package filters import ( "path" "strings" + + "github.com/defenseunicorns/zarf/src/types" ) type selectState int @@ -40,3 +42,38 @@ func includedOrExcluded(componentName string, requestedComponentNames []string) // All other cases we don't know if we should include or exclude yet return unknown, "" } + +// isRequired returns if the component is required or not. +// +// The logic is as follows: +// +// 1. If "optional" is set, then the component follows the inverse of that value +// +// 2. If "required" is set, then the component follows that value +// +// 3. If neither "optional" nor "required" are set, then the component is required +// +// Note: "required" is deprecated and will be removed in Zarf v1.0.0 +func isRequired(c types.ZarfComponent, useRequiredLogic bool) bool { + requiredExists := c.DeprecatedRequired != nil + optionalExists := c.Optional != nil + required := requiredExists && *c.DeprecatedRequired + optional := optionalExists && *c.Optional + + if useRequiredLogic { + // old required logic does not know about optional + if requiredExists { + return required + } + return false + } else { + // optional "wins" when defined + if optionalExists { + return !optional + } else if requiredExists { + return required + } + // If neither required nor optional are set, then the component is required + return true + } +} diff --git a/src/types/component_test.go b/src/pkg/packager/filters/utils_test.go similarity index 52% rename from src/types/component_test.go rename to src/pkg/packager/filters/utils_test.go index 4961d18d4d..321aba8967 100644 --- a/src/types/component_test.go +++ b/src/pkg/packager/filters/utils_test.go @@ -1,24 +1,27 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: 2021-Present The Zarf Authors -// Package types contains all the types used by Zarf. -package types +// Package filters contains core implementations of the ComponentFilterStrategy interface. +package filters import ( "testing" "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" + "github.com/defenseunicorns/zarf/src/types" + "github.com/stretchr/testify/require" ) -func TestZarfComponent_IsRequired(t *testing.T) { +func Test_isRequired(t *testing.T) { tests := []struct { - name string - component ZarfComponent - want bool + name string + component types.ZarfComponent + useRequiredLogic bool + want bool }{ { name: "Test when DeprecatedRequired is true and Optional is nil", - component: ZarfComponent{ + component: types.ZarfComponent{ DeprecatedRequired: helpers.BoolPtr(true), Optional: nil, }, @@ -26,7 +29,7 @@ func TestZarfComponent_IsRequired(t *testing.T) { }, { name: "Test when DeprecatedRequired is true and Optional is false", - component: ZarfComponent{ + component: types.ZarfComponent{ DeprecatedRequired: helpers.BoolPtr(true), Optional: helpers.BoolPtr(false), }, @@ -34,7 +37,7 @@ func TestZarfComponent_IsRequired(t *testing.T) { }, { name: "Test when DeprecatedRequired is true and Optional is true", - component: ZarfComponent{ + component: types.ZarfComponent{ DeprecatedRequired: helpers.BoolPtr(true), Optional: helpers.BoolPtr(true), }, @@ -43,7 +46,7 @@ func TestZarfComponent_IsRequired(t *testing.T) { }, { name: "Test when DeprecatedRequired is false and Optional is nil", - component: ZarfComponent{ + component: types.ZarfComponent{ DeprecatedRequired: helpers.BoolPtr(false), Optional: nil, }, @@ -51,7 +54,7 @@ func TestZarfComponent_IsRequired(t *testing.T) { }, { name: "Test when DeprecatedRequired is false and Optional is false", - component: ZarfComponent{ + component: types.ZarfComponent{ DeprecatedRequired: helpers.BoolPtr(false), Optional: helpers.BoolPtr(false), }, @@ -60,7 +63,7 @@ func TestZarfComponent_IsRequired(t *testing.T) { }, { name: "Test when DeprecatedRequired is false and Optional is true", - component: ZarfComponent{ + component: types.ZarfComponent{ DeprecatedRequired: helpers.BoolPtr(false), Optional: helpers.BoolPtr(true), }, @@ -69,7 +72,7 @@ func TestZarfComponent_IsRequired(t *testing.T) { }, { name: "Test when DeprecatedRequired is nil and Optional is nil", - component: ZarfComponent{ + component: types.ZarfComponent{ DeprecatedRequired: nil, Optional: nil, }, @@ -78,7 +81,7 @@ func TestZarfComponent_IsRequired(t *testing.T) { }, { name: "Test when DeprecatedRequired is nil and Optional is false", - component: ZarfComponent{ + component: types.ZarfComponent{ DeprecatedRequired: nil, Optional: helpers.BoolPtr(false), }, @@ -87,20 +90,55 @@ func TestZarfComponent_IsRequired(t *testing.T) { }, { name: "Test when DeprecatedRequired is nil and Optional is true", - component: ZarfComponent{ + component: types.ZarfComponent{ DeprecatedRequired: nil, Optional: helpers.BoolPtr(true), }, // optional "wins" when defined want: false, }, + { + name: "Test when DeprecatedRequired is true, Optional is true and useRequiredLogic is true", + component: types.ZarfComponent{ + DeprecatedRequired: helpers.BoolPtr(true), + Optional: helpers.BoolPtr(true), + }, + useRequiredLogic: true, + want: true, + }, + { + name: "Test when DeprecatedRequired is true, Optional is false and useRequiredLogic is false", + component: types.ZarfComponent{ + DeprecatedRequired: helpers.BoolPtr(true), + Optional: helpers.BoolPtr(false), + }, + useRequiredLogic: false, + want: true, + }, + { + name: "Test when DeprecatedRequired is false, Optional is true and useRequiredLogic is true", + component: types.ZarfComponent{ + DeprecatedRequired: helpers.BoolPtr(false), + Optional: helpers.BoolPtr(true), + }, + useRequiredLogic: true, + want: false, + }, + { + name: "Test when DeprecatedRequired is false, Optional is false and useRequiredLogic is false", + component: types.ZarfComponent{ + DeprecatedRequired: helpers.BoolPtr(false), + Optional: helpers.BoolPtr(false), + }, + useRequiredLogic: false, + want: true, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if got := tt.component.IsRequired(); got != tt.want { - t.Errorf("%q: ZarfComponent.IsRequired() = %v, want %v", tt.name, got, tt.want) - } + got := isRequired(tt.component, tt.useRequiredLogic) + require.Equal(t, tt.want, got) }) } } diff --git a/src/types/component.go b/src/types/component.go index a888f92ea9..181735b46e 100644 --- a/src/types/component.go +++ b/src/types/component.go @@ -83,34 +83,6 @@ func (c ZarfComponent) RequiresCluster() bool { return false } -// IsRequired returns if the component is required or not. -// -// The logic is as follows: -// -// 1. If "optional" is set, then the component follows the inverse of that value -// -// 2. If "required" is set, then the component follows that value -// -// 3. If neither "optional" nor "required" are set, then the component is required -// -// Note: "required" is deprecated and will be removed in Zarf v1.0.0 -func (c ZarfComponent) IsRequired() bool { - requiredExists := c.DeprecatedRequired != nil - optionalExists := c.Optional != nil - required := requiredExists && *c.DeprecatedRequired - optional := optionalExists && *c.Optional - - // optional "wins" when defined - if optionalExists { - return !optional - } else if requiredExists { - return required - } - - // If neither required nor optional are set, then the component is required - return true -} - // IsEmpty returns if the components fields (other than the fields we were told to ignore) are empty or set to the types zero-value func (c ZarfComponent) IsEmpty(fieldsToIgnore []string) bool { // Make a map for the fields we are going to ignore From e6127653c3784e818a4554712112ddad3427c1fc Mon Sep 17 00:00:00 2001 From: razzle Date: Sat, 13 Jan 2024 23:11:20 -0600 Subject: [PATCH 045/113] fix deploy Signed-off-by: razzle --- src/pkg/packager/deploy.go | 2 ++ src/pkg/packager/filters/deploy.go | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/pkg/packager/deploy.go b/src/pkg/packager/deploy.go index 6b7e42b7e4..2657193041 100644 --- a/src/pkg/packager/deploy.go +++ b/src/pkg/packager/deploy.go @@ -13,6 +13,7 @@ import ( "sync" "time" + "github.com/Masterminds/semver/v3" "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/config/lang" "github.com/defenseunicorns/zarf/src/internal/packager/git" @@ -98,6 +99,7 @@ func (p *Packager) Deploy() (err error) { // deployComponents loops through a list of ZarfComponents and deploys them. func (p *Packager) deployComponents() (deployedComponents []types.DeployedComponent, err error) { filter := filters.NewDeploymentFilter(p.cfg.PkgOpts.OptionalComponents) + filter.UseVersionBehavior(semver.MustParse(p.cfg.Pkg.Build.Version)) componentsToDeploy, err := filter.Apply(p.cfg.Pkg.Components) if err != nil { return deployedComponents, fmt.Errorf("unable to get selected components: %w", err) diff --git a/src/pkg/packager/filters/deploy.go b/src/pkg/packager/filters/deploy.go index caf040a124..b061778646 100644 --- a/src/pkg/packager/filters/deploy.go +++ b/src/pkg/packager/filters/deploy.go @@ -40,7 +40,7 @@ type DeploymentFilter struct { // UseVersionBehavior sets the version behavior for the filter. func (f *DeploymentFilter) UseVersionBehavior(buildVersion *semver.Version) { - if buildVersion.LessThan(semver.MustParse("v0.32.1")) { + if buildVersion.LessThan(semver.MustParse("v0.33.0")) { f.useRequiredLogic = true } } From 4037da6d7563cda8c4cfca408fbb3b62b00251ab Mon Sep 17 00:00:00 2001 From: razzle Date: Sun, 14 Jan 2024 14:42:57 -0500 Subject: [PATCH 046/113] smarter Signed-off-by: razzle --- src/pkg/packager/deploy.go | 7 ++---- src/pkg/packager/dev.go | 2 +- src/pkg/packager/filters/common.go | 37 +--------------------------- src/pkg/packager/filters/deploy.go | 22 +++++++---------- src/pkg/packager/filters/empty.go | 4 +-- src/pkg/packager/filters/included.go | 4 +-- src/pkg/packager/filters/platform.go | 5 ++-- src/pkg/packager/mirror.go | 5 ++-- src/pkg/packager/publish.go | 3 +-- src/pkg/packager/remove.go | 2 +- src/pkg/packager/sources/cluster.go | 2 +- src/pkg/packager/sources/new.go | 2 +- src/pkg/packager/sources/oci.go | 8 +++--- src/pkg/packager/sources/split.go | 4 +-- src/pkg/packager/sources/tarball.go | 2 +- src/pkg/packager/sources/url.go | 4 +-- src/pkg/packager/yaml.go | 3 +-- 17 files changed, 35 insertions(+), 81 deletions(-) diff --git a/src/pkg/packager/deploy.go b/src/pkg/packager/deploy.go index 2657193041..83fba1a9fd 100644 --- a/src/pkg/packager/deploy.go +++ b/src/pkg/packager/deploy.go @@ -13,7 +13,6 @@ import ( "sync" "time" - "github.com/Masterminds/semver/v3" "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/config/lang" "github.com/defenseunicorns/zarf/src/internal/packager/git" @@ -43,9 +42,8 @@ func (p *Packager) resetRegistryHPA() { // Deploy attempts to deploy the given PackageConfig. func (p *Packager) Deploy() (err error) { filter := filters.NewDeploymentFilter(p.cfg.PkgOpts.OptionalComponents) - fm := filters.NewFilterManager(filter) - if err = p.source.LoadPackage(p.layout, fm, true); err != nil { + if err = p.source.LoadPackage(p.layout, filter, true); err != nil { return fmt.Errorf("unable to load the package: %w", err) } @@ -99,8 +97,7 @@ func (p *Packager) Deploy() (err error) { // deployComponents loops through a list of ZarfComponents and deploys them. func (p *Packager) deployComponents() (deployedComponents []types.DeployedComponent, err error) { filter := filters.NewDeploymentFilter(p.cfg.PkgOpts.OptionalComponents) - filter.UseVersionBehavior(semver.MustParse(p.cfg.Pkg.Build.Version)) - componentsToDeploy, err := filter.Apply(p.cfg.Pkg.Components) + componentsToDeploy, err := filter.Apply(p.cfg.Pkg) if err != nil { return deployedComponents, fmt.Errorf("unable to get selected components: %w", err) } diff --git a/src/pkg/packager/dev.go b/src/pkg/packager/dev.go index ce1597ee82..0d5fc1d636 100644 --- a/src/pkg/packager/dev.go +++ b/src/pkg/packager/dev.go @@ -42,7 +42,7 @@ func (p *Packager) DevDeploy() error { // This is also different from regular package creation, where we still assemble and package up // all components and their dependencies, regardless of whether they are required or not filter := filters.NewDeploymentFilter(p.cfg.PkgOpts.OptionalComponents) - p.cfg.Pkg.Components, err = filter.Apply(p.cfg.Pkg.Components) + p.cfg.Pkg.Components, err = filter.Apply(p.cfg.Pkg) if err != nil { return err } diff --git a/src/pkg/packager/filters/common.go b/src/pkg/packager/filters/common.go index 13db4a1505..d8d01e4262 100644 --- a/src/pkg/packager/filters/common.go +++ b/src/pkg/packager/filters/common.go @@ -5,45 +5,10 @@ package filters import ( - "github.com/Masterminds/semver/v3" "github.com/defenseunicorns/zarf/src/types" ) // ComponentFilterStrategy is a strategy interface for filtering components. type ComponentFilterStrategy interface { - Apply([]types.ZarfComponent) ([]types.ZarfComponent, error) -} - -// VersionBehavior is an interface for setting version behavior on filters -type VersionBehavior interface { - UseVersionBehavior(*semver.Version) -} - -// NewFilterManager creates a new filter manager for the given strategy. -func NewFilterManager(strategy ComponentFilterStrategy) *FilterManager { - m := &FilterManager{} - m.SetStrategy(strategy) - return m -} - -// FilterManager manages a filter strategy. -type FilterManager struct { - strategy ComponentFilterStrategy -} - -// SetStrategy sets the strategy for the filter manager. -func (m *FilterManager) SetStrategy(strategy ComponentFilterStrategy) { - m.strategy = strategy -} - -// SetVersionBehavior sets the version behavior for the filter strategy. -func (m *FilterManager) SetVersionBehavior(buildVersion *semver.Version) { - if v, ok := m.strategy.(VersionBehavior); ok { - v.UseVersionBehavior(buildVersion) - } -} - -// Execute executes the filter strategy. -func (m *FilterManager) Execute(components []types.ZarfComponent) ([]types.ZarfComponent, error) { - return m.strategy.Apply(components) + Apply(types.ZarfPackage) ([]types.ZarfComponent, error) } diff --git a/src/pkg/packager/filters/deploy.go b/src/pkg/packager/filters/deploy.go index b061778646..cd29114a22 100644 --- a/src/pkg/packager/filters/deploy.go +++ b/src/pkg/packager/filters/deploy.go @@ -19,7 +19,6 @@ import ( var ( _ ComponentFilterStrategy = &DeploymentFilter{} - _ VersionBehavior = &DeploymentFilter{} ) // NewDeploymentFilter creates a new deployment filter. @@ -27,32 +26,29 @@ func NewDeploymentFilter(optionalComponents string) *DeploymentFilter { requested := helpers.StringToSlice(optionalComponents) return &DeploymentFilter{ - false, requested, } } // DeploymentFilter is the default filter for deployments. type DeploymentFilter struct { - useRequiredLogic bool requestedComponents []string } -// UseVersionBehavior sets the version behavior for the filter. -func (f *DeploymentFilter) UseVersionBehavior(buildVersion *semver.Version) { +// Apply applies the filter. +func (f *DeploymentFilter) Apply(pkg types.ZarfPackage) ([]types.ZarfComponent, error) { + useRequiredLogic := false + buildVersion := semver.MustParse(pkg.Build.Version) if buildVersion.LessThan(semver.MustParse("v0.33.0")) { - f.useRequiredLogic = true + useRequiredLogic = true } -} -// Apply applies the filter. -func (f *DeploymentFilter) Apply(allComponents []types.ZarfComponent) ([]types.ZarfComponent, error) { var selectedComponents []types.ZarfComponent groupedComponents := map[string][]types.ZarfComponent{} orderedComponentGroups := []string{} // Group the components by Name and Group while maintaining order - for _, component := range allComponents { + for _, component := range pkg.Components { groupKey := component.Name if component.DeprecatedGroup != "" { groupKey = component.DeprecatedGroup @@ -81,7 +77,7 @@ func (f *DeploymentFilter) Apply(allComponents []types.ZarfComponent) ([]types.Z selectState, matchedRequest := includedOrExcluded(component.Name, f.requestedComponents) - if !isRequired(component, f.useRequiredLogic) { + if !isRequired(component, useRequiredLogic) { if selectState == excluded { // If the component was explicitly excluded, record the match and continue matchedRequests[matchedRequest] = true @@ -128,7 +124,7 @@ func (f *DeploymentFilter) Apply(allComponents []types.ZarfComponent) ([]types.Z for _, requestedComponent := range f.requestedComponents { if _, ok := matchedRequests[requestedComponent]; !ok { closeEnough := []string{} - for _, c := range allComponents { + for _, c := range pkg.Components { d := levenshtein.ComputeDistance(c.Name, requestedComponent) if d <= 5 { closeEnough = append(closeEnough, c.Name) @@ -153,7 +149,7 @@ func (f *DeploymentFilter) Apply(allComponents []types.ZarfComponent) ([]types.Z } else { component := groupedComponents[groupKey][0] - if isRequired(component, f.useRequiredLogic) { + if isRequired(component, useRequiredLogic) { selectedComponents = append(selectedComponents, component) } else if selected := interactive.SelectOptionalComponent(component); selected { selectedComponents = append(selectedComponents, component) diff --git a/src/pkg/packager/filters/empty.go b/src/pkg/packager/filters/empty.go index 95499ea3a4..6ecdb3f6be 100644 --- a/src/pkg/packager/filters/empty.go +++ b/src/pkg/packager/filters/empty.go @@ -14,6 +14,6 @@ var ( type EmptyFilter struct{} // Apply returns the components unchanged. -func (f *EmptyFilter) Apply(components []types.ZarfComponent) ([]types.ZarfComponent, error) { - return components, nil +func (f *EmptyFilter) Apply(pkg types.ZarfPackage) ([]types.ZarfComponent, error) { + return pkg.Components, nil } diff --git a/src/pkg/packager/filters/included.go b/src/pkg/packager/filters/included.go index 838b9b54cd..92776886ca 100644 --- a/src/pkg/packager/filters/included.go +++ b/src/pkg/packager/filters/included.go @@ -28,12 +28,12 @@ type IncludedFilter struct { } // Apply applies the filter. -func (f *IncludedFilter) Apply(allComponents []types.ZarfComponent) ([]types.ZarfComponent, error) { +func (f *IncludedFilter) Apply(pkg types.ZarfPackage) ([]types.ZarfComponent, error) { isPartial := len(f.requestedComponents) > 0 && f.requestedComponents[0] != "" result := []types.ZarfComponent{} - for _, component := range allComponents { + for _, component := range pkg.Components { selectState := unknown if isPartial { diff --git a/src/pkg/packager/filters/platform.go b/src/pkg/packager/filters/platform.go index 1e3cbf6033..49ad6a4d6d 100644 --- a/src/pkg/packager/filters/platform.go +++ b/src/pkg/packager/filters/platform.go @@ -28,9 +28,10 @@ type ArchAndOSFilter struct { } // Apply applies the filter. -func (f *ArchAndOSFilter) Apply(components []types.ZarfComponent) (filtered []types.ZarfComponent, _ error) { +func (f *ArchAndOSFilter) Apply(pkg types.ZarfPackage) ([]types.ZarfComponent, error) { + filtered := []types.ZarfComponent{} // Filter each component to only compatible platforms. - for _, component := range components { + for _, component := range pkg.Components { // Ignore only filters that are empty var validArch, validOS bool diff --git a/src/pkg/packager/mirror.go b/src/pkg/packager/mirror.go index 807a647ff0..e1711c9cc5 100644 --- a/src/pkg/packager/mirror.go +++ b/src/pkg/packager/mirror.go @@ -20,9 +20,8 @@ func (p *Packager) Mirror() (err error) { defer spinner.Stop() filter := filters.NewIncludedFilter(p.cfg.PkgOpts.OptionalComponents) - fm := filters.NewFilterManager(filter) - if err = p.source.LoadPackage(p.layout, fm, true); err != nil { + if err = p.source.LoadPackage(p.layout, filter, true); err != nil { return fmt.Errorf("unable to load the package: %w", err) } if err = p.readZarfYAML(p.layout.ZarfYAML); err != nil { @@ -48,7 +47,7 @@ func (p *Packager) Mirror() (err error) { p.filterComponents() // Run mirror for each requested component - included, err := filter.Apply(p.cfg.Pkg.Components) + included, err := filter.Apply(p.cfg.Pkg) if err != nil { return err } diff --git a/src/pkg/packager/publish.go b/src/pkg/packager/publish.go index 7aeb2812e0..09a875a66c 100644 --- a/src/pkg/packager/publish.go +++ b/src/pkg/packager/publish.go @@ -100,8 +100,7 @@ func (p *Packager) Publish() (err error) { } } else { filter := &filters.EmptyFilter{} - fm := filters.NewFilterManager(filter) - if err = p.source.LoadPackage(p.layout, fm, false); err != nil { + if err = p.source.LoadPackage(p.layout, filter, false); err != nil { return fmt.Errorf("unable to load the package: %w", err) } if err = p.readZarfYAML(p.layout.ZarfYAML); err != nil { diff --git a/src/pkg/packager/remove.go b/src/pkg/packager/remove.go index 7e2151814d..1b394bad52 100644 --- a/src/pkg/packager/remove.go +++ b/src/pkg/packager/remove.go @@ -51,7 +51,7 @@ func (p *Packager) Remove() (err error) { // If components were provided; just remove the things we were asked to remove filter := filters.NewIncludedFilter(p.cfg.PkgOpts.OptionalComponents) - included, err := filter.Apply(p.cfg.Pkg.Components) + included, err := filter.Apply(p.cfg.Pkg) if err != nil { return err } diff --git a/src/pkg/packager/sources/cluster.go b/src/pkg/packager/sources/cluster.go index 76682227d9..d7f54bf17c 100644 --- a/src/pkg/packager/sources/cluster.go +++ b/src/pkg/packager/sources/cluster.go @@ -42,7 +42,7 @@ type ClusterSource struct { // LoadPackage loads a package from a cluster. // // This is not implemented. -func (s *ClusterSource) LoadPackage(_ *layout.PackagePaths, _ *filters.FilterManager, _ bool) error { +func (s *ClusterSource) LoadPackage(_ *layout.PackagePaths, _ filters.ComponentFilterStrategy, _ bool) error { return fmt.Errorf("not implemented") } diff --git a/src/pkg/packager/sources/new.go b/src/pkg/packager/sources/new.go index f5fd4e76f9..6604c5c581 100644 --- a/src/pkg/packager/sources/new.go +++ b/src/pkg/packager/sources/new.go @@ -29,7 +29,7 @@ import ( // `sources.ValidatePackageSignature` and `sources.ValidatePackageIntegrity` can be leveraged for this purpose. type PackageSource interface { // LoadPackage loads a package from a source. - LoadPackage(dst *layout.PackagePaths, fm *filters.FilterManager, unarchiveAll bool) error + LoadPackage(dst *layout.PackagePaths, fm filters.ComponentFilterStrategy, unarchiveAll bool) error // LoadPackageMetadata loads a package's metadata from a source. LoadPackageMetadata(dst *layout.PackagePaths, wantSBOM bool, skipValidation bool) error diff --git a/src/pkg/packager/sources/oci.go b/src/pkg/packager/sources/oci.go index cfb991e1bd..7d564a9930 100644 --- a/src/pkg/packager/sources/oci.go +++ b/src/pkg/packager/sources/oci.go @@ -11,7 +11,6 @@ import ( "path/filepath" "strings" - "github.com/Masterminds/semver/v3" "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/message" @@ -35,7 +34,7 @@ type OCISource struct { } // LoadPackage loads a package from an OCI registry. -func (s *OCISource) LoadPackage(dst *layout.PackagePaths, fm *filters.FilterManager, unarchiveAll bool) (err error) { +func (s *OCISource) LoadPackage(dst *layout.PackagePaths, filter filters.ComponentFilterStrategy, unarchiveAll bool) (err error) { var pkg types.ZarfPackage layersToPull := []ocispec.Descriptor{} @@ -48,9 +47,8 @@ func (s *OCISource) LoadPackage(dst *layout.PackagePaths, fm *filters.FilterMana return err } var requested []types.ZarfComponent - if fm != nil { - fm.SetVersionBehavior(semver.MustParse(pkg.Build.Version)) - requested, err = fm.Execute(pkg.Components) + if filter != nil { + requested, err = filter.Apply(pkg) if err != nil { return err } diff --git a/src/pkg/packager/sources/split.go b/src/pkg/packager/sources/split.go index 6219ee1b88..53b1def772 100644 --- a/src/pkg/packager/sources/split.go +++ b/src/pkg/packager/sources/split.go @@ -104,7 +104,7 @@ func (s *SplitTarballSource) Collect(dir string) (string, error) { } // LoadPackage loads a package from a split tarball. -func (s *SplitTarballSource) LoadPackage(dst *layout.PackagePaths, fm *filters.FilterManager, unarchiveAll bool) (err error) { +func (s *SplitTarballSource) LoadPackage(dst *layout.PackagePaths, filter filters.ComponentFilterStrategy, unarchiveAll bool) (err error) { tb, err := s.Collect(filepath.Dir(s.PackageSource)) if err != nil { return err @@ -118,7 +118,7 @@ func (s *SplitTarballSource) LoadPackage(dst *layout.PackagePaths, fm *filters.F ts := &TarballSource{ s.ZarfPackageOptions, } - return ts.LoadPackage(dst, fm, unarchiveAll) + return ts.LoadPackage(dst, filter, unarchiveAll) } // LoadPackageMetadata loads a package's metadata from a split tarball. diff --git a/src/pkg/packager/sources/tarball.go b/src/pkg/packager/sources/tarball.go index 70bc03fd1c..b57f99d4f2 100644 --- a/src/pkg/packager/sources/tarball.go +++ b/src/pkg/packager/sources/tarball.go @@ -32,7 +32,7 @@ type TarballSource struct { } // LoadPackage loads a package from a tarball. -func (s *TarballSource) LoadPackage(dst *layout.PackagePaths, _ *filters.FilterManager, unarchiveAll bool) (err error) { +func (s *TarballSource) LoadPackage(dst *layout.PackagePaths, _ filters.ComponentFilterStrategy, unarchiveAll bool) (err error) { var pkg types.ZarfPackage spinner := message.NewProgressSpinner("Loading package from %q", s.PackageSource) diff --git a/src/pkg/packager/sources/url.go b/src/pkg/packager/sources/url.go index c9fc6dd6e6..8e82650424 100644 --- a/src/pkg/packager/sources/url.go +++ b/src/pkg/packager/sources/url.go @@ -50,7 +50,7 @@ func (s *URLSource) Collect(dir string) (string, error) { } // LoadPackage loads a package from an http, https or sget URL. -func (s *URLSource) LoadPackage(dst *layout.PackagePaths, fm *filters.FilterManager, unarchiveAll bool) (err error) { +func (s *URLSource) LoadPackage(dst *layout.PackagePaths, filter filters.ComponentFilterStrategy, unarchiveAll bool) (err error) { tmp, err := utils.MakeTempDir(config.CommonOptions.TempDirectory) if err != nil { return err @@ -70,7 +70,7 @@ func (s *URLSource) LoadPackage(dst *layout.PackagePaths, fm *filters.FilterMana s.ZarfPackageOptions, } - return ts.LoadPackage(dst, fm, unarchiveAll) + return ts.LoadPackage(dst, filter, unarchiveAll) } // LoadPackageMetadata loads a package's metadata from an http, https or sget URL. diff --git a/src/pkg/packager/yaml.go b/src/pkg/packager/yaml.go index ec73e94618..bf912d103d 100644 --- a/src/pkg/packager/yaml.go +++ b/src/pkg/packager/yaml.go @@ -43,9 +43,8 @@ func (p *Packager) readZarfYAML(path string) error { // filterComponents removes components not matching the current OS and architecture. func (p *Packager) filterComponents() { - filter := filters.NewArchAndOSFilter(p.arch, runtime.GOOS) // Currently the arch and os filter will never return an error. - p.cfg.Pkg.Components, _ = filter.Apply(p.cfg.Pkg.Components) + p.cfg.Pkg.Components, _ = filters.NewArchAndOSFilter(p.arch, runtime.GOOS).Apply(p.cfg.Pkg) } // writeYaml adds build information and writes the config to the temp directory. From f82b0f3ba78897f5b24b8a3e1c7b2d7a35a2ac45 Mon Sep 17 00:00:00 2001 From: razzle Date: Tue, 16 Jan 2024 12:35:20 -0600 Subject: [PATCH 047/113] tweaks Signed-off-by: razzle --- adr/0023-required-optional.md | 12 ++++++++++-- src/cmd/dev.go | 2 +- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/adr/0023-required-optional.md b/adr/0023-required-optional.md index a28a1043c6..2d22645f07 100644 --- a/adr/0023-required-optional.md +++ b/adr/0023-required-optional.md @@ -18,7 +18,7 @@ A few ways to handle this: 1. Simply force the `required` key to be a non-optional, so that package authors would be forced to specify it for each component, thereby removing any ambiguity--but also force one more key for every single component ever created 🫠 -2. Deprecate `required` and introduce an optional `optional` key, which would default to _false_. I do think this still feels strange if you did something like `optional: false`, (to be fair `required: false` has the same awkwardness). +2. Deprecate `required` and introduce an optional `optional` key, which would default to _false_. 3. Do something more significant like combine various condition-based things such as `only`, `optional` (instead of `required`), or `default`. @@ -26,6 +26,14 @@ A few ways to handle this: > The change that we're proposing or have agreed to implement. +Option 2: deprecate `required` and introduce an optional `optional` key, which defaults to _false_. + +Components are now **required** by default, instead of **optional**. + ## Consequences -> What becomes easier or more difficult to do and any risks introduced by the change that will need to be mitigated. +`zarf package create` will fail if any usage of `required` is detected in the `zarf.yaml`, resulting in some thrash for package creators. + +Packages created w/ Zarf v0.33.0+ will have their implicit _required_ logic flipped from previous versions (implicit `required: false` --> implicit `optional: false`). + +A `required-to-optional` migration. diff --git a/src/cmd/dev.go b/src/cmd/dev.go index 3e68296197..49370d8122 100644 --- a/src/cmd/dev.go +++ b/src/cmd/dev.go @@ -71,7 +71,7 @@ var devMigrateCmd = &cobra.Command{ Use: "migrate", Short: lang.CmdDevMigrateShort, Long: lang.CmdDevMigrateLong, - Args: cobra.ExactArgs(1), + Args: cobra.RangeArgs(1, 2), RunE: func(cmd *cobra.Command, args []string) error { dir := args[0] var pkg types.ZarfPackage From 28dbfc996909b53fdd017363b309cc41c165fa4f Mon Sep 17 00:00:00 2001 From: razzle Date: Tue, 16 Jan 2024 14:19:08 -0600 Subject: [PATCH 048/113] dev migrate cli autocomplete Signed-off-by: razzle --- src/cmd/dev.go | 44 ++++++++++++++++++++++++++++++--- src/pkg/packager/sources/new.go | 2 +- 2 files changed, 42 insertions(+), 4 deletions(-) diff --git a/src/cmd/dev.go b/src/cmd/dev.go index 49370d8122..d334c554ed 100644 --- a/src/cmd/dev.go +++ b/src/cmd/dev.go @@ -34,6 +34,7 @@ import ( ) var extractPath string +var migrationsToRun []string var devCmd = &cobra.Command{ Use: "dev", @@ -71,9 +72,14 @@ var devMigrateCmd = &cobra.Command{ Use: "migrate", Short: lang.CmdDevMigrateShort, Long: lang.CmdDevMigrateLong, - Args: cobra.RangeArgs(1, 2), + Args: cobra.MaximumNArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - dir := args[0] + var dir string + if len(args) == 0 { + dir = "." + } else { + dir = args[0] + } var pkg types.ZarfPackage cm := goyaml.CommentMap{} @@ -91,13 +97,36 @@ var devMigrateCmd = &cobra.Command{ return err } + all := deprecated.Migrations() + + migrations := []deprecated.Migration{} + + // Only run the specified migrations + for _, migrationToRun := range migrationsToRun { + for _, migration := range all { + if migration.ID() == migrationToRun { + migrations = []deprecated.Migration{migration} + } + } + } + + if len(migrations) == 0 { + // Run all migrations + migrations = all + } + // Migrate the package definition for idx, component := range pkg.Components { - for _, migration := range deprecated.Migrations() { + ran := []string{} + for _, migration := range migrations { + ran = append(ran, migration.ID()) c, _ := migration.Run(component) c = migration.Clear(c) pkg.Components[idx] = c } + if len(ran) > 0 { + message.Successf("Ran %s on %q", strings.Join(ran, ", "), component.Name) + } } b, err = goyaml.MarshalWithOptions(pkg, goyaml.WithComment(cm), goyaml.IndentSequence(true), goyaml.UseSingleQuote(false)) @@ -346,6 +375,15 @@ func init() { bindDevDeployFlags(v) + allMigrations := []string{} + for _, migration := range deprecated.Migrations() { + allMigrations = append(allMigrations, migration.ID()) + } + devMigrateCmd.Flags().StringArrayVar(&migrationsToRun, "run", []string{}, fmt.Sprintf("migrations to run (default: all, available: %s)", strings.Join(allMigrations, ", "))) + devMigrateCmd.RegisterFlagCompletionFunc("run", func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + return allMigrations, cobra.ShellCompDirectiveNoFileComp + }) + devSha256SumCmd.Flags().StringVarP(&extractPath, "extract-path", "e", "", lang.CmdDevFlagExtractPath) devFindImagesCmd.Flags().StringVarP(&pkgConfig.FindImagesOpts.RepoHelmChartPath, "repo-chart-path", "p", "", lang.CmdDevFlagRepoChartPath) diff --git a/src/pkg/packager/sources/new.go b/src/pkg/packager/sources/new.go index 6604c5c581..5f8e634640 100644 --- a/src/pkg/packager/sources/new.go +++ b/src/pkg/packager/sources/new.go @@ -29,7 +29,7 @@ import ( // `sources.ValidatePackageSignature` and `sources.ValidatePackageIntegrity` can be leveraged for this purpose. type PackageSource interface { // LoadPackage loads a package from a source. - LoadPackage(dst *layout.PackagePaths, fm filters.ComponentFilterStrategy, unarchiveAll bool) error + LoadPackage(dst *layout.PackagePaths, filter filters.ComponentFilterStrategy, unarchiveAll bool) error // LoadPackageMetadata loads a package's metadata from a source. LoadPackageMetadata(dst *layout.PackagePaths, wantSBOM bool, skipValidation bool) error From 0db59040eb8c91722f006187e41ab65f89587751 Mon Sep 17 00:00:00 2001 From: razzle Date: Tue, 16 Jan 2024 14:34:05 -0600 Subject: [PATCH 049/113] handle unset cli version Signed-off-by: razzle --- src/config/config.go | 4 +++- src/pkg/packager/filters/deploy.go | 12 +++++++++--- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/src/config/config.go b/src/config/config.go index df4deb9030..eec9773b92 100644 --- a/src/config/config.go +++ b/src/config/config.go @@ -64,12 +64,14 @@ const ( ZarfInClusterArtifactServiceURL = ZarfInClusterGitServiceURL + "/api/packages/" + ZarfGitPushUser ZarfLoggingUser = "zarf-admin" + + UnsetCLIVersion = "unset" ) // Zarf Global Configuration Variables. var ( // CLIVersion track the version of the CLI - CLIVersion = "unset" + CLIVersion = UnsetCLIVersion // ActionsUseSystemZarf sets whether to use Zarf from the system path if Zarf is being used as a library ActionsUseSystemZarf = false diff --git a/src/pkg/packager/filters/deploy.go b/src/pkg/packager/filters/deploy.go index cd29114a22..b2024b0ab2 100644 --- a/src/pkg/packager/filters/deploy.go +++ b/src/pkg/packager/filters/deploy.go @@ -11,6 +11,7 @@ import ( "github.com/Masterminds/semver/v3" "github.com/agnivade/levenshtein" + "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/config/lang" "github.com/defenseunicorns/zarf/src/pkg/interactive" "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" @@ -38,9 +39,14 @@ type DeploymentFilter struct { // Apply applies the filter. func (f *DeploymentFilter) Apply(pkg types.ZarfPackage) ([]types.ZarfComponent, error) { useRequiredLogic := false - buildVersion := semver.MustParse(pkg.Build.Version) - if buildVersion.LessThan(semver.MustParse("v0.33.0")) { - useRequiredLogic = true + if pkg.Build.Version != config.UnsetCLIVersion { + buildVersion, err := semver.NewVersion(pkg.Build.Version) + if err != nil { + return []types.ZarfComponent{}, fmt.Errorf("unable to parse package version: %w", err) + } + if buildVersion.LessThan(semver.MustParse("v0.33.0")) { + useRequiredLogic = true + } } var selectedComponents []types.ZarfComponent From 826bb09cebbf051c0f64a55dcdb5cb151a216a7e Mon Sep 17 00:00:00 2001 From: razzle Date: Tue, 16 Jan 2024 16:39:28 -0600 Subject: [PATCH 050/113] docs and schema Signed-off-by: razzle --- Makefile | 2 +- docs/2-the-zarf-cli/100-cli-commands/zarf_dev_migrate.md | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 2bdbd3cf07..e52c658a0f 100644 --- a/Makefile +++ b/Makefile @@ -30,7 +30,7 @@ else endif endif -CLI_VERSION ?= $(if $(shell git describe --tags),$(shell git describe --tags),"UnknownVersion") +CLI_VERSION ?= $(if $(shell git describe --tags),$(shell git describe --tags),"unset") BUILD_ARGS := -s -w -X github.com/defenseunicorns/zarf/src/config.CLIVersion=$(CLI_VERSION) K8S_MODULES_VER=$(subst ., ,$(subst v,,$(shell go list -f '{{.Version}}' -m k8s.io/client-go))) K8S_MODULES_MAJOR_VER=$(shell echo $$(($(firstword $(K8S_MODULES_VER)) + 1))) diff --git a/docs/2-the-zarf-cli/100-cli-commands/zarf_dev_migrate.md b/docs/2-the-zarf-cli/100-cli-commands/zarf_dev_migrate.md index 979b78e3f7..92d451e079 100644 --- a/docs/2-the-zarf-cli/100-cli-commands/zarf_dev_migrate.md +++ b/docs/2-the-zarf-cli/100-cli-commands/zarf_dev_migrate.md @@ -16,7 +16,8 @@ zarf dev migrate [flags] ## Options ``` - -h, --help help for migrate + -h, --help help for migrate + --run stringArray migrations to run (default: all, available: scripts-to-actions, pluralize-set-variable, required-to-optional) ``` ## Options inherited from parent commands From 3a2dc368255bf1fc863a2ce02aba3f66f52f7f34 Mon Sep 17 00:00:00 2001 From: razzle Date: Tue, 16 Jan 2024 17:18:55 -0600 Subject: [PATCH 051/113] fix dev migrate --run --run Signed-off-by: razzle --- src/cmd/dev.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cmd/dev.go b/src/cmd/dev.go index d334c554ed..ab3c320b1d 100644 --- a/src/cmd/dev.go +++ b/src/cmd/dev.go @@ -105,7 +105,7 @@ var devMigrateCmd = &cobra.Command{ for _, migrationToRun := range migrationsToRun { for _, migration := range all { if migration.ID() == migrationToRun { - migrations = []deprecated.Migration{migration} + migrations = append(migrations, migration) } } } From 44fe5ec9aedcb84cac8004e2c8cd158d82108bf1 Mon Sep 17 00:00:00 2001 From: razzle Date: Fri, 19 Jan 2024 21:31:18 -0600 Subject: [PATCH 052/113] fix dev deploy version Signed-off-by: razzle --- src/pkg/packager/dev.go | 3 +++ src/pkg/packager/filters/deploy.go | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/pkg/packager/dev.go b/src/pkg/packager/dev.go index 0d5fc1d636..d0963ac054 100644 --- a/src/pkg/packager/dev.go +++ b/src/pkg/packager/dev.go @@ -33,6 +33,9 @@ func (p *Packager) DevDeploy() error { return err } + // Set the package version to the CLI version + p.cfg.Pkg.Build.Version = config.CLIVersion + // Filter out components that are not compatible with this system p.filterComponents() diff --git a/src/pkg/packager/filters/deploy.go b/src/pkg/packager/filters/deploy.go index b2024b0ab2..c808cd0651 100644 --- a/src/pkg/packager/filters/deploy.go +++ b/src/pkg/packager/filters/deploy.go @@ -42,7 +42,7 @@ func (f *DeploymentFilter) Apply(pkg types.ZarfPackage) ([]types.ZarfComponent, if pkg.Build.Version != config.UnsetCLIVersion { buildVersion, err := semver.NewVersion(pkg.Build.Version) if err != nil { - return []types.ZarfComponent{}, fmt.Errorf("unable to parse package version: %w", err) + return []types.ZarfComponent{}, fmt.Errorf("unable to parse package version %q: %w", pkg.Build.Version, err) } if buildVersion.LessThan(semver.MustParse("v0.33.0")) { useRequiredLogic = true From 78bdb71da52f59a2321578bba1856539bfbf49c4 Mon Sep 17 00:00:00 2001 From: razzle Date: Sat, 20 Jan 2024 23:15:23 -0600 Subject: [PATCH 053/113] fix upgrade test, little ugly Signed-off-by: razzle --- .github/workflows/test-upgrade.yml | 2 +- Makefile | 2 +- src/test/upgrade/pre-0.33/zarf.yaml | 11 +++++++++++ 3 files changed, 13 insertions(+), 2 deletions(-) create mode 100644 src/test/upgrade/pre-0.33/zarf.yaml diff --git a/.github/workflows/test-upgrade.yml b/.github/workflows/test-upgrade.yml index 95c30b3ad7..01c5d569b8 100644 --- a/.github/workflows/test-upgrade.yml +++ b/.github/workflows/test-upgrade.yml @@ -81,7 +81,7 @@ jobs: # NOTE: "PATH=$PATH" preserves the default user $PATH. This is needed to maintain the version of zarf installed # in a previous step. This test run will use the current release to create a K3s cluster. run: | - zarf package create src/test/upgrade --set PODINFO_VERSION=6.3.3 --confirm + zarf package create src/test/upgrade/pre-0.33 --set PODINFO_VERSION=6.3.3 --confirm sudo env "PATH=$PATH" CI=true zarf package deploy zarf-package-test-upgrade-package-amd64-6.3.3.tar.zst --components="test-upgrade-package" --confirm sudo env "PATH=$PATH" CI=true zarf tools kubectl describe deployments -n=podinfo-upgrade sudo env "PATH=$PATH" CI=true zarf tools kubectl describe pods -n=podinfo-upgrade diff --git a/Makefile b/Makefile index e52c658a0f..45834d4475 100644 --- a/Makefile +++ b/Makefile @@ -193,7 +193,7 @@ test-upgrade: ## Run the Zarf CLI E2E tests for an external registry and cluster @test -s $(ZARF_BIN) || $(MAKE) build-cli [ -n "$(shell zarf version)" ] || (echo "Zarf must be installed prior to the upgrade test" && exit 1) [ -n "$(shell zarf package list 2>&1 | grep test-upgrade-package)" ] || (echo "Zarf must be initialized and have the 6.3.3 upgrade-test package installed prior to the upgrade test" && exit 1) - @test -s ./zarf-package-test-upgrade-package-$(ARCH)-6.3.4.tar.zst || zarf package create src/test/upgrade/ --set PODINFO_VERSION=6.3.4 --confirm + @test -s ./zarf-package-test-upgrade-package-$(ARCH)-6.3.4.tar.zst || zarf package create src/test/upgrade/pre-0.33 --set PODINFO_VERSION=6.3.4 --confirm cd src/test/upgrade && go test -failfast -v -timeout 30m .PHONY: test-unit diff --git a/src/test/upgrade/pre-0.33/zarf.yaml b/src/test/upgrade/pre-0.33/zarf.yaml new file mode 100644 index 0000000000..114e101ad3 --- /dev/null +++ b/src/test/upgrade/pre-0.33/zarf.yaml @@ -0,0 +1,11 @@ +kind: ZarfPackageConfig +metadata: + name: test-upgrade-package + description: Deploy podinfo using a Helm OCI chart + version: "###ZARF_PKG_VAR_PODINFO_VERSION###" + +components: + - name: test-upgrade-package + required: true + import: + path: .. From e5965c49801f56eee55968d32aad603bcc023bc6 Mon Sep 17 00:00:00 2001 From: razzle Date: Mon, 22 Jan 2024 11:56:35 -0600 Subject: [PATCH 054/113] rename string Signed-off-by: razzle --- Makefile | 2 +- src/config/config.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 45834d4475..43e2f062b8 100644 --- a/Makefile +++ b/Makefile @@ -30,7 +30,7 @@ else endif endif -CLI_VERSION ?= $(if $(shell git describe --tags),$(shell git describe --tags),"unset") +CLI_VERSION ?= $(if $(shell git describe --tags),$(shell git describe --tags),"unset-development-only") BUILD_ARGS := -s -w -X github.com/defenseunicorns/zarf/src/config.CLIVersion=$(CLI_VERSION) K8S_MODULES_VER=$(subst ., ,$(subst v,,$(shell go list -f '{{.Version}}' -m k8s.io/client-go))) K8S_MODULES_MAJOR_VER=$(shell echo $$(($(firstword $(K8S_MODULES_VER)) + 1))) diff --git a/src/config/config.go b/src/config/config.go index eec9773b92..fa4495bd17 100644 --- a/src/config/config.go +++ b/src/config/config.go @@ -65,7 +65,7 @@ const ( ZarfLoggingUser = "zarf-admin" - UnsetCLIVersion = "unset" + UnsetCLIVersion = "unset-development-only" ) // Zarf Global Configuration Variables. From 1ad2e58ebebde8e82b30d516d9445a2adcd24527 Mon Sep 17 00:00:00 2001 From: razzle Date: Sat, 27 Jan 2024 18:35:49 -0600 Subject: [PATCH 055/113] slighly different filter based on isInteractive Signed-off-by: razzle --- src/pkg/packager/deploy.go | 4 ++-- src/pkg/packager/deprecated/common.go | 4 ++-- src/pkg/packager/dev.go | 2 +- src/pkg/packager/filters/deploy.go | 18 ++++++++++++++---- 4 files changed, 19 insertions(+), 9 deletions(-) diff --git a/src/pkg/packager/deploy.go b/src/pkg/packager/deploy.go index 83fba1a9fd..018eba4195 100644 --- a/src/pkg/packager/deploy.go +++ b/src/pkg/packager/deploy.go @@ -41,7 +41,7 @@ func (p *Packager) resetRegistryHPA() { // Deploy attempts to deploy the given PackageConfig. func (p *Packager) Deploy() (err error) { - filter := filters.NewDeploymentFilter(p.cfg.PkgOpts.OptionalComponents) + filter := filters.NewDeploymentFilter(p.cfg.PkgOpts.OptionalComponents, config.CommonOptions.Confirm) if err = p.source.LoadPackage(p.layout, filter, true); err != nil { return fmt.Errorf("unable to load the package: %w", err) @@ -96,7 +96,7 @@ func (p *Packager) Deploy() (err error) { // deployComponents loops through a list of ZarfComponents and deploys them. func (p *Packager) deployComponents() (deployedComponents []types.DeployedComponent, err error) { - filter := filters.NewDeploymentFilter(p.cfg.PkgOpts.OptionalComponents) + filter := filters.NewDeploymentFilter(p.cfg.PkgOpts.OptionalComponents, false) componentsToDeploy, err := filter.Apply(p.cfg.Pkg) if err != nil { return deployedComponents, fmt.Errorf("unable to get selected components: %w", err) diff --git a/src/pkg/packager/deprecated/common.go b/src/pkg/packager/deprecated/common.go index 74d74da89a..c3f87d1bef 100644 --- a/src/pkg/packager/deprecated/common.go +++ b/src/pkg/packager/deprecated/common.go @@ -26,8 +26,8 @@ type breakingChange struct { // LastNonBreakingVersion is the last version that did not have any breaking changes // -// This should be updated when a breaking change is introduced to the Zarf package structure. See: https://github.com/defenseunicorns/zarf/releases/tag/v0.27.0 -const LastNonBreakingVersion = "v0.27.0" +// This should be updated when a breaking change is introduced to the Zarf package structure. See: https://github.com/defenseunicorns/zarf/releases/tag/v0.32.2 +const LastNonBreakingVersion = "v0.32.2" // List of breaking changes to warn the user of. var breakingChanges = []breakingChange{ diff --git a/src/pkg/packager/dev.go b/src/pkg/packager/dev.go index d0963ac054..26f88deb1c 100644 --- a/src/pkg/packager/dev.go +++ b/src/pkg/packager/dev.go @@ -44,7 +44,7 @@ func (p *Packager) DevDeploy() error { // the user's selection and the component's `required`/`optional` field // This is also different from regular package creation, where we still assemble and package up // all components and their dependencies, regardless of whether they are required or not - filter := filters.NewDeploymentFilter(p.cfg.PkgOpts.OptionalComponents) + filter := filters.NewDeploymentFilter(p.cfg.PkgOpts.OptionalComponents, false) p.cfg.Pkg.Components, err = filter.Apply(p.cfg.Pkg) if err != nil { return err diff --git a/src/pkg/packager/filters/deploy.go b/src/pkg/packager/filters/deploy.go index c808cd0651..1e76df06e7 100644 --- a/src/pkg/packager/filters/deploy.go +++ b/src/pkg/packager/filters/deploy.go @@ -23,17 +23,19 @@ var ( ) // NewDeploymentFilter creates a new deployment filter. -func NewDeploymentFilter(optionalComponents string) *DeploymentFilter { +func NewDeploymentFilter(optionalComponents string, isInteractive bool) *DeploymentFilter { requested := helpers.StringToSlice(optionalComponents) return &DeploymentFilter{ requested, + isInteractive, } } // DeploymentFilter is the default filter for deployments. type DeploymentFilter struct { requestedComponents []string + isInteractive bool } // Apply applies the filter. @@ -150,15 +152,23 @@ func (f *DeploymentFilter) Apply(pkg types.ZarfPackage) ([]types.ZarfComponent, } else { for _, groupKey := range orderedComponentGroups { if len(groupedComponents[groupKey]) > 1 { - component := interactive.SelectChoiceGroup(groupedComponents[groupKey]) - selectedComponents = append(selectedComponents, component) + if f.isInteractive { + component := interactive.SelectChoiceGroup(groupedComponents[groupKey]) + selectedComponents = append(selectedComponents, component) + } else { + return []types.ZarfComponent{}, fmt.Errorf(lang.PkgDeployErrMultipleComponentsSameGroup, groupedComponents[groupKey][0].Name, groupedComponents[groupKey][1].Name, groupedComponents[groupKey][0].DeprecatedGroup) + } } else { component := groupedComponents[groupKey][0] if isRequired(component, useRequiredLogic) { selectedComponents = append(selectedComponents, component) - } else if selected := interactive.SelectOptionalComponent(component); selected { + } else if component.Default && config.CommonOptions.Confirm { selectedComponents = append(selectedComponents, component) + } else if f.isInteractive { + if selected := interactive.SelectOptionalComponent(component); selected { + selectedComponents = append(selectedComponents, component) + } } } } From c9ba849a59833fd3d1c74ae44d13f1e0f2b5d489 Mon Sep 17 00:00:00 2001 From: razzle Date: Sat, 27 Jan 2024 19:47:19 -0600 Subject: [PATCH 056/113] fix component choice Signed-off-by: razzle --- src/pkg/packager/deploy.go | 4 ++-- src/pkg/packager/filters/deploy.go | 26 +++++++++++++++++++++----- 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/src/pkg/packager/deploy.go b/src/pkg/packager/deploy.go index 018eba4195..2b75b2ee29 100644 --- a/src/pkg/packager/deploy.go +++ b/src/pkg/packager/deploy.go @@ -41,7 +41,7 @@ func (p *Packager) resetRegistryHPA() { // Deploy attempts to deploy the given PackageConfig. func (p *Packager) Deploy() (err error) { - filter := filters.NewDeploymentFilter(p.cfg.PkgOpts.OptionalComponents, config.CommonOptions.Confirm) + filter := filters.NewDeploymentFilter(p.cfg.PkgOpts.OptionalComponents, !config.CommonOptions.Confirm) if err = p.source.LoadPackage(p.layout, filter, true); err != nil { return fmt.Errorf("unable to load the package: %w", err) @@ -96,7 +96,7 @@ func (p *Packager) Deploy() (err error) { // deployComponents loops through a list of ZarfComponents and deploys them. func (p *Packager) deployComponents() (deployedComponents []types.DeployedComponent, err error) { - filter := filters.NewDeploymentFilter(p.cfg.PkgOpts.OptionalComponents, false) + filter := filters.NewDeploymentFilter(p.cfg.PkgOpts.OptionalComponents, !config.CommonOptions.Confirm) componentsToDeploy, err := filter.Apply(p.cfg.Pkg) if err != nil { return deployedComponents, fmt.Errorf("unable to get selected components: %w", err) diff --git a/src/pkg/packager/filters/deploy.go b/src/pkg/packager/filters/deploy.go index 1e76df06e7..b28168155f 100644 --- a/src/pkg/packager/filters/deploy.go +++ b/src/pkg/packager/filters/deploy.go @@ -151,21 +151,37 @@ func (f *DeploymentFilter) Apply(pkg types.ZarfPackage) ([]types.ZarfComponent, } } else { for _, groupKey := range orderedComponentGroups { - if len(groupedComponents[groupKey]) > 1 { + group := groupedComponents[groupKey] + if len(group) > 1 { if f.isInteractive { - component := interactive.SelectChoiceGroup(groupedComponents[groupKey]) + component := interactive.SelectChoiceGroup(group) selectedComponents = append(selectedComponents, component) } else { - return []types.ZarfComponent{}, fmt.Errorf(lang.PkgDeployErrMultipleComponentsSameGroup, groupedComponents[groupKey][0].Name, groupedComponents[groupKey][1].Name, groupedComponents[groupKey][0].DeprecatedGroup) + foundDefault := false + componentNames := []string{} + for _, component := range group { + // If the component is default, then use it + if component.Default { + selectedComponents = append(selectedComponents, component) + foundDefault = true + break + } + // Add each component name to the list + componentNames = append(componentNames, component.Name) + } + if !foundDefault { + // If no default component was found, give up + return []types.ZarfComponent{}, fmt.Errorf(lang.PkgDeployErrNoDefaultOrSelection, strings.Join(componentNames, ", ")) + } } } else { component := groupedComponents[groupKey][0] if isRequired(component, useRequiredLogic) { selectedComponents = append(selectedComponents, component) - } else if component.Default && config.CommonOptions.Confirm { + } else if component.Default && !f.isInteractive { selectedComponents = append(selectedComponents, component) - } else if f.isInteractive { + } else { if selected := interactive.SelectOptionalComponent(component); selected { selectedComponents = append(selectedComponents, component) } From b4adcbe4539a33f46f7805b3e436f5e380b96e2c Mon Sep 17 00:00:00 2001 From: razzle Date: Sun, 28 Jan 2024 13:19:17 -0600 Subject: [PATCH 057/113] remove usage of required Signed-off-by: razzle --- src/test/packages/external-helm-auth/zarf.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/src/test/packages/external-helm-auth/zarf.yaml b/src/test/packages/external-helm-auth/zarf.yaml index d373670825..26e5ba2782 100644 --- a/src/test/packages/external-helm-auth/zarf.yaml +++ b/src/test/packages/external-helm-auth/zarf.yaml @@ -4,7 +4,6 @@ metadata: version: 0.0.1 components: - name: private-chart - required: true charts: - name: podinfo version: 6.4.0 From 44367a6faa649071106286a93ff6e60259cc1f8d Mon Sep 17 00:00:00 2001 From: razzle Date: Tue, 30 Jan 2024 11:33:09 -0600 Subject: [PATCH 058/113] optional agrees w/ required if required is set and optional dne Signed-off-by: razzle --- src/pkg/packager/deprecated/required_to_optional.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/pkg/packager/deprecated/required_to_optional.go b/src/pkg/packager/deprecated/required_to_optional.go index 384a5f9e7d..d3ff92435f 100644 --- a/src/pkg/packager/deprecated/required_to_optional.go +++ b/src/pkg/packager/deprecated/required_to_optional.go @@ -7,6 +7,7 @@ package deprecated import ( "fmt" + "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "github.com/defenseunicorns/zarf/src/types" ) @@ -38,5 +39,15 @@ func (m RequiredToOptional) Run(c types.ZarfComponent) (types.ZarfComponent, str warning := fmt.Sprintf("Component %q is using \"required\" which will be removed in Zarf v1.0.0. Please migrate to \"optional\".", c.Name) + // Make optional agree with required if not set + if c.Optional == nil && c.DeprecatedRequired != nil { + switch *c.DeprecatedRequired { + case true: + c.Optional = helpers.BoolPtr(false) + case false: + c.Optional = helpers.BoolPtr(true) + } + } + return c, warning } From a92602a3c34b2ac63013a7c8e2ccdc0b53cca6a3 Mon Sep 17 00:00:00 2001 From: razzle Date: Fri, 2 Feb 2024 13:04:40 -0600 Subject: [PATCH 059/113] changes from review Signed-off-by: razzle --- src/config/lang/english.go | 2 +- src/pkg/packager/filters/utils.go | 18 +++++++++--------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/config/lang/english.go b/src/config/lang/english.go index 919a280e46..e125516558 100644 --- a/src/config/lang/english.go +++ b/src/config/lang/english.go @@ -653,7 +653,7 @@ const ( PkgValidateErrComponentName = "component name %q must be all lowercase and contain no special characters except '-' and cannot start with a '-'" PkgValidateErrComponentNameNotUnique = "component name %q is not unique" PkgValidateErrComponent = "invalid component %q: %w" - PkgValidateErrComponentRequired = "component %q contains deprecated usage of required, please migrate to optional" + PkgValidateErrComponentRequired = "component %q contains deprecated usage of required. Please use `zarf dev migrate --run required-to-optional` to migrate your zarf.yaml" PkgValidateErrComponentReqDefault = "component %q cannot be both required and default" PkgValidateErrComponentReqGrouped = "component %q cannot be both required and grouped" PkgValidateErrComponentYOLO = "component %q incompatible with the online-only package flag (metadata.yolo): %w" diff --git a/src/pkg/packager/filters/utils.go b/src/pkg/packager/filters/utils.go index 39dbcc644a..cf166b27f4 100644 --- a/src/pkg/packager/filters/utils.go +++ b/src/pkg/packager/filters/utils.go @@ -66,14 +66,14 @@ func isRequired(c types.ZarfComponent, useRequiredLogic bool) bool { return required } return false - } else { - // optional "wins" when defined - if optionalExists { - return !optional - } else if requiredExists { - return required - } - // If neither required nor optional are set, then the component is required - return true } + + // optional "wins" when defined + if optionalExists { + return !optional + } else if requiredExists { + return required + } + // If neither required nor optional are set, then the component is required + return true } From db974ce60d75d251551e6ba8e928acd868024286 Mon Sep 17 00:00:00 2001 From: razzle Date: Mon, 5 Feb 2024 16:22:56 -0600 Subject: [PATCH 060/113] cleanup Signed-off-by: razzle --- src/pkg/packager/deploy.go | 4 +++- src/pkg/packager/dev.go | 4 +++- src/pkg/packager/mirror.go | 4 +++- src/pkg/packager/remove.go | 4 +++- src/pkg/packager/yaml.go | 8 ++++---- 5 files changed, 16 insertions(+), 8 deletions(-) diff --git a/src/pkg/packager/deploy.go b/src/pkg/packager/deploy.go index 2b75b2ee29..dad5fc3761 100644 --- a/src/pkg/packager/deploy.go +++ b/src/pkg/packager/deploy.go @@ -75,7 +75,9 @@ func (p *Packager) Deploy() (err error) { defer p.resetRegistryHPA() // Filter out components that are not compatible with this system - p.filterComponents() + if err := p.filterComponentsByOSAndArch(); err != nil { + return err + } // Get a list of all the components we are deploying and actually deploy them deployedComponents, err := p.deployComponents() diff --git a/src/pkg/packager/dev.go b/src/pkg/packager/dev.go index 26f88deb1c..aa8f99a188 100644 --- a/src/pkg/packager/dev.go +++ b/src/pkg/packager/dev.go @@ -37,7 +37,9 @@ func (p *Packager) DevDeploy() error { p.cfg.Pkg.Build.Version = config.CLIVersion // Filter out components that are not compatible with this system - p.filterComponents() + if err := p.filterComponentsByOSAndArch(); err != nil { + return err + } // Also filter out components that are not required, nor requested via --components // This is different from the above filter, as it is not based on the system, but rather diff --git a/src/pkg/packager/mirror.go b/src/pkg/packager/mirror.go index e1711c9cc5..330a91ecb8 100644 --- a/src/pkg/packager/mirror.go +++ b/src/pkg/packager/mirror.go @@ -44,7 +44,9 @@ func (p *Packager) Mirror() (err error) { p.cfg.State = state // Filter out components that are not compatible with this system if we have loaded from a tarball - p.filterComponents() + if err := p.filterComponentsByOSAndArch(); err != nil { + return err + } // Run mirror for each requested component included, err := filter.Apply(p.cfg.Pkg) diff --git a/src/pkg/packager/remove.go b/src/pkg/packager/remove.go index 1b394bad52..3f2e7063f3 100644 --- a/src/pkg/packager/remove.go +++ b/src/pkg/packager/remove.go @@ -42,7 +42,9 @@ func (p *Packager) Remove() (err error) { if err = p.readZarfYAML(p.layout.ZarfYAML); err != nil { return err } - p.filterComponents() + if err := p.filterComponentsByOSAndArch(); err != nil { + return err + } packageName = p.cfg.Pkg.Metadata.Name // Build a list of components to remove and determine if we need a cluster connection diff --git a/src/pkg/packager/yaml.go b/src/pkg/packager/yaml.go index bf912d103d..8c3fc35556 100644 --- a/src/pkg/packager/yaml.go +++ b/src/pkg/packager/yaml.go @@ -41,10 +41,10 @@ func (p *Packager) readZarfYAML(path string) error { return nil } -// filterComponents removes components not matching the current OS and architecture. -func (p *Packager) filterComponents() { - // Currently the arch and os filter will never return an error. - p.cfg.Pkg.Components, _ = filters.NewArchAndOSFilter(p.arch, runtime.GOOS).Apply(p.cfg.Pkg) +// filterComponentsByOSAndArch removes components not matching the current OS and architecture. +func (p *Packager) filterComponentsByOSAndArch() (err error) { + p.cfg.Pkg.Components, err = filters.NewArchAndOSFilter(p.arch, runtime.GOOS).Apply(p.cfg.Pkg) + return err } // writeYaml adds build information and writes the config to the temp directory. From e14b48e4d82d3236549c92fcb21447e76146fc3d Mon Sep 17 00:00:00 2001 From: razzle Date: Mon, 5 Feb 2024 16:33:23 -0600 Subject: [PATCH 061/113] utils fn Signed-off-by: razzle --- src/cmd/dev.go | 39 +++------------------------------------ src/pkg/utils/yaml.go | 37 +++++++++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 36 deletions(-) diff --git a/src/cmd/dev.go b/src/cmd/dev.go index ab3c320b1d..00d483cf5c 100644 --- a/src/cmd/dev.go +++ b/src/cmd/dev.go @@ -5,8 +5,6 @@ package cmd import ( - "bufio" - "bytes" "fmt" "io" "os" @@ -134,43 +132,12 @@ var devMigrateCmd = &cobra.Command{ return err } - scanner := bufio.NewScanner(bytes.NewReader(b)) - - var updated []byte - var commentLines []string - - // Some opinionated formatting for the zarf.yaml - for scanner.Scan() { - line := scanner.Text() - - if strings.HasPrefix(line, "components:") || strings.HasPrefix(line, " - name: ") || strings.HasPrefix(line, " name: ") { - if len(commentLines) > 0 { - commentText := strings.Join(commentLines, "\n") - updated = append(updated, []byte("\n"+commentText+"\n")...) - } else { - updated = append(updated, []byte("\n")...) - } - updated = append(updated, []byte(line+"\n")...) // Add "components:" line - commentLines = nil - } else { - if strings.HasPrefix(line, "#") || strings.HasPrefix(line, " - #") { - commentLines = append(commentLines, line) - } else { - if len(commentLines) > 0 { - commentText := strings.Join(commentLines, "\n") - updated = append(updated, []byte("\n"+commentText+"\n")...) - commentLines = nil - } - updated = append(updated, []byte(line+"\n")...) - } - } - } - - if err := scanner.Err(); err != nil { + formatted, err := utils.FormatZarfYAML(b) + if err != nil { return err } - return os.WriteFile(filepath.Join(dir, layout.ZarfYAML), updated, fi.Mode()) + return os.WriteFile(filepath.Join(dir, layout.ZarfYAML), formatted, fi.Mode()) }, } diff --git a/src/pkg/utils/yaml.go b/src/pkg/utils/yaml.go index fcf81c497c..980710310b 100644 --- a/src/pkg/utils/yaml.go +++ b/src/pkg/utils/yaml.go @@ -7,6 +7,7 @@ package utils // fork from https://github.com/goccy/go-yaml/blob/master/cmd/ycat/ycat.go import ( + "bufio" "bytes" "fmt" "io" @@ -233,3 +234,39 @@ func SplitYAMLToString(yamlData []byte) ([]string, error) { } return objs, nil } + +// FormatZarfYAML formats a zarf.yaml file w/ some opinionated formatting. +func FormatZarfYAML(b []byte) (formatted []byte, err error) { + scanner := bufio.NewScanner(bytes.NewReader(b)) + + var commentLines []string + + // Some opinionated formatting for the zarf.yaml + for scanner.Scan() { + line := scanner.Text() + + if strings.HasPrefix(line, "components:") || strings.HasPrefix(line, " - name: ") || strings.HasPrefix(line, " name: ") { + if len(commentLines) > 0 { + commentText := strings.Join(commentLines, "\n") + formatted = append(formatted, []byte("\n"+commentText+"\n")...) + } else { + formatted = append(formatted, []byte("\n")...) + } + formatted = append(formatted, []byte(line+"\n")...) // Add "components:" line + commentLines = nil + } else { + if strings.HasPrefix(line, "#") || strings.HasPrefix(line, " - #") { + commentLines = append(commentLines, line) + } else { + if len(commentLines) > 0 { + commentText := strings.Join(commentLines, "\n") + formatted = append(formatted, []byte("\n"+commentText+"\n")...) + commentLines = nil + } + formatted = append(formatted, []byte(line+"\n")...) + } + } + } + + return formatted, scanner.Err() +} From cd3a266902372853e8bfc06020e9a51abf1b4249 Mon Sep 17 00:00:00 2001 From: razzle Date: Mon, 5 Feb 2024 16:44:45 -0600 Subject: [PATCH 062/113] feedback Signed-off-by: razzle --- src/pkg/packager/deprecated/required_to_optional.go | 2 +- src/pkg/packager/filters/utils.go | 4 +--- src/pkg/packager/filters/utils_test.go | 2 +- src/pkg/utils/io.go | 5 +++-- src/types/component.go | 2 +- 5 files changed, 7 insertions(+), 8 deletions(-) diff --git a/src/pkg/packager/deprecated/required_to_optional.go b/src/pkg/packager/deprecated/required_to_optional.go index d3ff92435f..f961b48fd8 100644 --- a/src/pkg/packager/deprecated/required_to_optional.go +++ b/src/pkg/packager/deprecated/required_to_optional.go @@ -31,7 +31,7 @@ func (m RequiredToOptional) Clear(mc types.ZarfComponent) types.ZarfComponent { return mc } -// Run is a no-op for this migration. +// Run migrates required to optional. func (m RequiredToOptional) Run(c types.ZarfComponent) (types.ZarfComponent, string) { if c.DeprecatedRequired == nil { return c, "" diff --git a/src/pkg/packager/filters/utils.go b/src/pkg/packager/filters/utils.go index cf166b27f4..c87132703f 100644 --- a/src/pkg/packager/filters/utils.go +++ b/src/pkg/packager/filters/utils.go @@ -68,11 +68,9 @@ func isRequired(c types.ZarfComponent, useRequiredLogic bool) bool { return false } - // optional "wins" when defined + // follow inverse of optional when defined if optionalExists { return !optional - } else if requiredExists { - return required } // If neither required nor optional are set, then the component is required return true diff --git a/src/pkg/packager/filters/utils_test.go b/src/pkg/packager/filters/utils_test.go index 321aba8967..af60115c48 100644 --- a/src/pkg/packager/filters/utils_test.go +++ b/src/pkg/packager/filters/utils_test.go @@ -50,7 +50,7 @@ func Test_isRequired(t *testing.T) { DeprecatedRequired: helpers.BoolPtr(false), Optional: nil, }, - want: false, + want: true, }, { name: "Test when DeprecatedRequired is false and Optional is false", diff --git a/src/pkg/utils/io.go b/src/pkg/utils/io.go index ed9a39780c..d07f790f50 100755 --- a/src/pkg/utils/io.go +++ b/src/pkg/utils/io.go @@ -14,6 +14,7 @@ import ( "net/http" "os" "os/exec" + "path" "path/filepath" "regexp" "strings" @@ -462,8 +463,8 @@ func CreateReproducibleTarballFromDir(dirPath, dirPrefix, tarballPath string) er if err != nil { return fmt.Errorf("error getting relative path: %w", err) } - name = filepath.Join(dirPrefix, name) - name = filepath.ToSlash(name) + name = path.Join(dirPrefix, name) + // name = filepath.ToSlash(name) header.Name = name // Write the header to the tarball diff --git a/src/types/component.go b/src/types/component.go index 181735b46e..717354c80a 100644 --- a/src/types/component.go +++ b/src/types/component.go @@ -83,7 +83,7 @@ func (c ZarfComponent) RequiresCluster() bool { return false } -// IsEmpty returns if the components fields (other than the fields we were told to ignore) are empty or set to the types zero-value +// IsEmpty returns true if the components fields (other than the fields we were told to ignore) are empty or set to the types zero-value func (c ZarfComponent) IsEmpty(fieldsToIgnore []string) bool { // Make a map for the fields we are going to ignore ignoredFieldsMap := make(map[string]bool) From 8f0fb190e4395f185a7e079e8b60c39b82662367 Mon Sep 17 00:00:00 2001 From: razzle Date: Mon, 5 Feb 2024 16:48:48 -0600 Subject: [PATCH 063/113] update adr Signed-off-by: razzle --- adr/0023-required-optional.md | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/adr/0023-required-optional.md b/adr/0023-required-optional.md index 2d22645f07..2457c53522 100644 --- a/adr/0023-required-optional.md +++ b/adr/0023-required-optional.md @@ -4,7 +4,7 @@ Date: 2024-01-02 ## Status -Pending +Accepted ## Context @@ -24,8 +24,6 @@ A few ways to handle this: ## Decision -> The change that we're proposing or have agreed to implement. - Option 2: deprecate `required` and introduce an optional `optional` key, which defaults to _false_. Components are now **required** by default, instead of **optional**. @@ -36,4 +34,4 @@ Components are now **required** by default, instead of **optional**. Packages created w/ Zarf v0.33.0+ will have their implicit _required_ logic flipped from previous versions (implicit `required: false` --> implicit `optional: false`). -A `required-to-optional` migration. +A `required-to-optional` migration (both accomplished behind the scenes on a `zarf package create`, or available via **new** CLI migration: `zarf dev migrate --run required-to-optional`). From e19e3c50a6433a2ee67dedebbe0c86908bb21961 Mon Sep 17 00:00:00 2001 From: razzle Date: Mon, 5 Feb 2024 16:53:41 -0600 Subject: [PATCH 064/113] changes from review Signed-off-by: razzle --- packages/zarf-registry/zarf.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/zarf-registry/zarf.yaml b/packages/zarf-registry/zarf.yaml index e1b987f591..3570454333 100644 --- a/packages/zarf-registry/zarf.yaml +++ b/packages/zarf-registry/zarf.yaml @@ -95,7 +95,6 @@ components: executable: true - name: zarf-seed-registry - optional: true description: | Deploys the Zarf Registry using the registry image provided by the Zarf Injector. charts: From 6e6472c87ca6478efa31fba7f3e223dd18858dd7 Mon Sep 17 00:00:00 2001 From: razzle Date: Tue, 6 Feb 2024 10:45:11 -0600 Subject: [PATCH 065/113] revert Signed-off-by: razzle --- src/pkg/utils/io.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/pkg/utils/io.go b/src/pkg/utils/io.go index f441a5e7a3..7742a5a4b4 100755 --- a/src/pkg/utils/io.go +++ b/src/pkg/utils/io.go @@ -14,7 +14,6 @@ import ( "net/http" "os" "os/exec" - "path" "path/filepath" "regexp" "strings" @@ -477,8 +476,8 @@ func CreateReproducibleTarballFromDir(dirPath, dirPrefix, tarballPath string) er if err != nil { return fmt.Errorf("error getting relative path: %w", err) } - name = path.Join(dirPrefix, name) - // name = filepath.ToSlash(name) + name = filepath.Join(dirPrefix, name) + name = filepath.ToSlash(name) header.Name = name // Write the header to the tarball From cc99d8d964aa3bf4ee97c2c9f74f4c868577ec1f Mon Sep 17 00:00:00 2001 From: razzle Date: Tue, 6 Feb 2024 10:52:07 -0600 Subject: [PATCH 066/113] simpler filter logic Signed-off-by: razzle --- src/pkg/packager/filters/deploy.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/pkg/packager/filters/deploy.go b/src/pkg/packager/filters/deploy.go index b28168155f..4502e04e6a 100644 --- a/src/pkg/packager/filters/deploy.go +++ b/src/pkg/packager/filters/deploy.go @@ -177,14 +177,14 @@ func (f *DeploymentFilter) Apply(pkg types.ZarfPackage) ([]types.ZarfComponent, } else { component := groupedComponents[groupKey][0] - if isRequired(component, useRequiredLogic) { - selectedComponents = append(selectedComponents, component) - } else if component.Default && !f.isInteractive { - selectedComponents = append(selectedComponents, component) - } else { + if f.isInteractive { if selected := interactive.SelectOptionalComponent(component); selected { selectedComponents = append(selectedComponents, component) } + } else { + if isRequired(component, useRequiredLogic) || component.Default { + selectedComponents = append(selectedComponents, component) + } } } } From b8e49967f669db8bbd8f7efd30b13f5301d38fea Mon Sep 17 00:00:00 2001 From: razzle Date: Tue, 6 Feb 2024 16:37:54 -0600 Subject: [PATCH 067/113] migrate manifest-with-symlink Signed-off-by: razzle --- src/test/packages/34-manifest-with-symlink/zarf.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/src/test/packages/34-manifest-with-symlink/zarf.yaml b/src/test/packages/34-manifest-with-symlink/zarf.yaml index 41d21b48e6..d421f45088 100644 --- a/src/test/packages/34-manifest-with-symlink/zarf.yaml +++ b/src/test/packages/34-manifest-with-symlink/zarf.yaml @@ -6,7 +6,6 @@ metadata: components: - name: manifest-with-symlink - required: true files: - source: manifests target: temp/manifests From a0595437417b9f7b268adb16ace97538f05ce2fd Mon Sep 17 00:00:00 2001 From: razzle Date: Fri, 9 Feb 2024 11:54:43 -0600 Subject: [PATCH 068/113] feedback Signed-off-by: razzle --- src/cmd/common/utils.go | 10 +++------- src/cmd/dev.go | 6 +++--- src/cmd/package.go | 2 +- src/pkg/packager/filters/deploy.go | 11 +---------- 4 files changed, 8 insertions(+), 21 deletions(-) diff --git a/src/cmd/common/utils.go b/src/cmd/common/utils.go index 01a6b104d6..1c3cc0f9e1 100644 --- a/src/cmd/common/utils.go +++ b/src/cmd/common/utils.go @@ -4,15 +4,11 @@ // Package common handles command configuration across all commands package common -import ( - "github.com/defenseunicorns/zarf/src/types" -) - // SetBaseDirectory sets base directory on package config when given in args -func SetBaseDirectory(args []string, pkgConfig *types.PackagerConfig) { +func SetBaseDirectory(args []string) string { if len(args) > 0 { - pkgConfig.CreateOpts.BaseDir = args[0] + return args[0] } else { - pkgConfig.CreateOpts.BaseDir = "." + return "." } } diff --git a/src/cmd/dev.go b/src/cmd/dev.go index 00d483cf5c..2b5191aca5 100644 --- a/src/cmd/dev.go +++ b/src/cmd/dev.go @@ -46,7 +46,7 @@ var devDeployCmd = &cobra.Command{ Short: lang.CmdDevDeployShort, Long: lang.CmdDevDeployLong, Run: func(cmd *cobra.Command, args []string) { - common.SetBaseDirectory(args, &pkgConfig) + pkgConfig.CreateOpts.BaseDir = common.SetBaseDirectory(args) v := common.GetViper() pkgConfig.CreateOpts.SetVariables = helpers.TransformAndMergeMap( @@ -267,7 +267,7 @@ var devFindImagesCmd = &cobra.Command{ Long: lang.CmdDevFindImagesLong, Run: func(cmd *cobra.Command, args []string) { // If a directory was provided, use that as the base directory - common.SetBaseDirectory(args, &pkgConfig) + pkgConfig.CreateOpts.BaseDir = common.SetBaseDirectory(args) // Ensure uppercase keys from viper v := common.GetViper() @@ -313,7 +313,7 @@ var devLintCmd = &cobra.Command{ Short: lang.CmdDevLintShort, Long: lang.CmdDevLintLong, Run: func(cmd *cobra.Command, args []string) { - common.SetBaseDirectory(args, &pkgConfig) + pkgConfig.CreateOpts.BaseDir = common.SetBaseDirectory(args) v := common.GetViper() pkgConfig.CreateOpts.SetVariables = helpers.TransformAndMergeMap( v.GetStringMapString(common.VPkgCreateSet), pkgConfig.CreateOpts.SetVariables, strings.ToUpper) diff --git a/src/cmd/package.go b/src/cmd/package.go index 1ccd3e3061..b9d83453af 100644 --- a/src/cmd/package.go +++ b/src/cmd/package.go @@ -40,7 +40,7 @@ var packageCreateCmd = &cobra.Command{ Short: lang.CmdPackageCreateShort, Long: lang.CmdPackageCreateLong, Run: func(cmd *cobra.Command, args []string) { - common.SetBaseDirectory(args, &pkgConfig) + pkgConfig.CreateOpts.BaseDir = common.SetBaseDirectory(args) var isCleanPathRegex = regexp.MustCompile(`^[a-zA-Z0-9\_\-\/\.\~\\:]+$`) if !isCleanPathRegex.MatchString(config.CommonOptions.CachePath) { diff --git a/src/pkg/packager/filters/deploy.go b/src/pkg/packager/filters/deploy.go index 4502e04e6a..93ae8311af 100644 --- a/src/pkg/packager/filters/deploy.go +++ b/src/pkg/packager/filters/deploy.go @@ -128,7 +128,6 @@ func (f *DeploymentFilter) Apply(pkg types.ZarfPackage) ([]types.ZarfComponent, } // Check that we have matched against all requests - var err error for _, requestedComponent := range f.requestedComponents { if _, ok := matchedRequests[requestedComponent]; !ok { closeEnough := []string{} @@ -138,17 +137,9 @@ func (f *DeploymentFilter) Apply(pkg types.ZarfPackage) ([]types.ZarfComponent, closeEnough = append(closeEnough, c.Name) } } - failure := fmt.Errorf(lang.PkgDeployErrNoCompatibleComponentsForSelection, requestedComponent, strings.Join(closeEnough, ", ")) - if err != nil { - err = fmt.Errorf("%w, %w", err, failure) - } else { - err = failure - } + return nil, fmt.Errorf(lang.PkgDeployErrNoCompatibleComponentsForSelection, requestedComponent, strings.Join(closeEnough, ", ")) } } - if err != nil { - return []types.ZarfComponent{}, err - } } else { for _, groupKey := range orderedComponentGroups { group := groupedComponents[groupKey] From 7cf2339bc089f223d21e36484de1c1455e6ea875 Mon Sep 17 00:00:00 2001 From: razzle Date: Fri, 9 Feb 2024 11:55:41 -0600 Subject: [PATCH 069/113] feedback Signed-off-by: razzle --- src/cmd/dev.go | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/cmd/dev.go b/src/cmd/dev.go index 2b5191aca5..b14bab1e25 100644 --- a/src/cmd/dev.go +++ b/src/cmd/dev.go @@ -72,12 +72,7 @@ var devMigrateCmd = &cobra.Command{ Long: lang.CmdDevMigrateLong, Args: cobra.MaximumNArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - var dir string - if len(args) == 0 { - dir = "." - } else { - dir = args[0] - } + dir := common.SetBaseDirectory(args) var pkg types.ZarfPackage cm := goyaml.CommentMap{} From 41b9d2990f5f88b2b8da17ec60522a98f832dc1d Mon Sep 17 00:00:00 2001 From: razzle Date: Fri, 9 Feb 2024 13:46:35 -0600 Subject: [PATCH 070/113] comments Signed-off-by: razzle --- src/types/component.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/types/component.go b/src/types/component.go index 717354c80a..f3576823d9 100644 --- a/src/types/component.go +++ b/src/types/component.go @@ -24,6 +24,9 @@ type ZarfComponent struct { // DeprecatedRequired makes this component mandatory for package deployment DeprecatedRequired *bool `json:"required,omitempty" jsonschema:"description=[Deprecated] Do not prompt user to install this component, always install on package deploy. This will be removed in Zarf v1.0.0.,deprecated=true"` + // Optional controls if the component is required or not + // + // Components are required by default (implicit optional:false) Optional *bool `json:"optional,omitempty" jsonschema:"description=Prompt user to install this component on package deploy, defaults to false"` // Only include compatible components during package deployment From a7d9ccbcbab49315bd65a99e3d557bfa8ab655fe Mon Sep 17 00:00:00 2001 From: razzle Date: Sat, 17 Feb 2024 23:04:05 -0600 Subject: [PATCH 071/113] unit testing filters Signed-off-by: razzle --- src/pkg/packager/deploy.go | 4 +- src/pkg/packager/dev.go | 2 +- src/pkg/packager/filters/deploy.go | 16 +- src/pkg/packager/filters/deploy_test.go | 176 ++++++++++++++++++ src/pkg/packager/filters/empty.go | 12 +- src/pkg/packager/filters/empty_test.go | 32 ++++ src/pkg/packager/filters/included.go | 16 +- src/pkg/packager/filters/platform.go | 37 ++-- src/pkg/packager/filters/platform_test.go | 49 +++++ .../packager/filters/{common.go => strat.go} | 0 src/pkg/packager/mirror.go | 2 +- src/pkg/packager/publish.go | 2 +- src/pkg/packager/remove.go | 2 +- src/pkg/packager/yaml.go | 2 +- 14 files changed, 306 insertions(+), 46 deletions(-) create mode 100644 src/pkg/packager/filters/deploy_test.go create mode 100644 src/pkg/packager/filters/empty_test.go create mode 100644 src/pkg/packager/filters/platform_test.go rename src/pkg/packager/filters/{common.go => strat.go} (100%) diff --git a/src/pkg/packager/deploy.go b/src/pkg/packager/deploy.go index 40310dd872..78616624f2 100644 --- a/src/pkg/packager/deploy.go +++ b/src/pkg/packager/deploy.go @@ -41,7 +41,7 @@ func (p *Packager) resetRegistryHPA() { // Deploy attempts to deploy the given PackageConfig. func (p *Packager) Deploy() (err error) { - filter := filters.NewDeploymentFilter(p.cfg.PkgOpts.OptionalComponents, !config.CommonOptions.Confirm) + filter := filters.ForDeploy(p.cfg.PkgOpts.OptionalComponents, !config.CommonOptions.Confirm) if err = p.source.LoadPackage(p.layout, filter, true); err != nil { return fmt.Errorf("unable to load the package: %w", err) @@ -98,7 +98,7 @@ func (p *Packager) Deploy() (err error) { // deployComponents loops through a list of ZarfComponents and deploys them. func (p *Packager) deployComponents() (deployedComponents []types.DeployedComponent, err error) { - filter := filters.NewDeploymentFilter(p.cfg.PkgOpts.OptionalComponents, !config.CommonOptions.Confirm) + filter := filters.ForDeploy(p.cfg.PkgOpts.OptionalComponents, !config.CommonOptions.Confirm) componentsToDeploy, err := filter.Apply(p.cfg.Pkg) if err != nil { return deployedComponents, fmt.Errorf("unable to get selected components: %w", err) diff --git a/src/pkg/packager/dev.go b/src/pkg/packager/dev.go index aa8f99a188..2b9268975b 100644 --- a/src/pkg/packager/dev.go +++ b/src/pkg/packager/dev.go @@ -46,7 +46,7 @@ func (p *Packager) DevDeploy() error { // the user's selection and the component's `required`/`optional` field // This is also different from regular package creation, where we still assemble and package up // all components and their dependencies, regardless of whether they are required or not - filter := filters.NewDeploymentFilter(p.cfg.PkgOpts.OptionalComponents, false) + filter := filters.ForDeploy(p.cfg.PkgOpts.OptionalComponents, false) p.cfg.Pkg.Components, err = filter.Apply(p.cfg.Pkg) if err != nil { return err diff --git a/src/pkg/packager/filters/deploy.go b/src/pkg/packager/filters/deploy.go index 93ae8311af..4737f36695 100644 --- a/src/pkg/packager/filters/deploy.go +++ b/src/pkg/packager/filters/deploy.go @@ -18,28 +18,24 @@ import ( "github.com/defenseunicorns/zarf/src/types" ) -var ( - _ ComponentFilterStrategy = &DeploymentFilter{} -) - -// NewDeploymentFilter creates a new deployment filter. -func NewDeploymentFilter(optionalComponents string, isInteractive bool) *DeploymentFilter { +// ForDeploy creates a new deployment filter. +func ForDeploy(optionalComponents string, isInteractive bool) ComponentFilterStrategy { requested := helpers.StringToSlice(optionalComponents) - return &DeploymentFilter{ + return &deploymentFilter{ requested, isInteractive, } } -// DeploymentFilter is the default filter for deployments. -type DeploymentFilter struct { +// deploymentFilter is the default filter for deployments. +type deploymentFilter struct { requestedComponents []string isInteractive bool } // Apply applies the filter. -func (f *DeploymentFilter) Apply(pkg types.ZarfPackage) ([]types.ZarfComponent, error) { +func (f *deploymentFilter) Apply(pkg types.ZarfPackage) ([]types.ZarfComponent, error) { useRequiredLogic := false if pkg.Build.Version != config.UnsetCLIVersion { buildVersion, err := semver.NewVersion(pkg.Build.Version) diff --git a/src/pkg/packager/filters/deploy_test.go b/src/pkg/packager/filters/deploy_test.go new file mode 100644 index 0000000000..9f01df447a --- /dev/null +++ b/src/pkg/packager/filters/deploy_test.go @@ -0,0 +1,176 @@ +package filters + +import ( + "fmt" + "strings" + "testing" + + "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" + "github.com/defenseunicorns/zarf/src/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func cfq(t *testing.T, q string) types.ZarfComponent { + c := types.ZarfComponent{ + Name: q, + } + + // conditions := strings.Split(q[firstEq+1:], "&&") + conditions := strings.Split(q, "&&") + for _, cond := range conditions { + cond = strings.TrimSpace(cond) + switch cond { + case "default=true": + c.Default = true + case "default=false": + c.Default = false + case "optional=": + c.Optional = nil + case "optional=false": + c.Optional = helpers.BoolPtr(false) + case "optional=true": + c.Optional = helpers.BoolPtr(true) + case "required=": + c.DeprecatedRequired = nil + case "required=false": + c.DeprecatedRequired = helpers.BoolPtr(false) + case "required=true": + c.DeprecatedRequired = helpers.BoolPtr(true) + default: + if strings.HasPrefix(cond, "group=") { + c.DeprecatedGroup = cond[6:] + continue + } + require.Fail(t, "unknown condition", "unknown condition %q", cond) + } + } + + return c +} + +func componentMatrix(t *testing.T) []types.ZarfComponent { + var components []types.ZarfComponent + + defaultValues := []bool{true, false} + optionalValues := []interface{}{nil, true, false} + requiredValues := []interface{}{nil, true, false} + groupValues := []string{"", "foo", "foo", "bar", "bar"} + + // how many combinations of the above values are there? + // 2 * 3 * 3 * 5 = 90 + + for _, defaultValue := range defaultValues { + for _, optionalValue := range optionalValues { + for _, requiredValue := range requiredValues { + for _, groupValue := range groupValues { + c := types.ZarfComponent{ + Name: fmt.Sprintf("default=%t && optional=%v && required=%v && group=%s", defaultValue, optionalValue, requiredValue, groupValue), + Default: defaultValue, + DeprecatedGroup: groupValue, + } + + if optionalValue != nil { + c.Optional = helpers.BoolPtr(optionalValue.(bool)) + } + + if requiredValue != nil { + c.DeprecatedRequired = helpers.BoolPtr(requiredValue.(bool)) + } + + components = append(components, c) + } + } + } + } + + return components +} + +func TestDeployFilter_Apply(t *testing.T) { + testCases := map[string]struct { + pkg types.ZarfPackage + want []types.ZarfComponent + }{ + "Test when version is less than v0.33.0": { + pkg: types.ZarfPackage{ + Build: types.ZarfBuildData{ + Version: "v0.32.0", + }, + Components: componentMatrix(t), + }, + want: []types.ZarfComponent{ + cfq(t, "default=true && optional= && required= && group="), + cfq(t, "default=true && optional= && required= && group=foo"), + cfq(t, "default=true && optional= && required= && group=bar"), + cfq(t, "default=true && optional= && required=true && group="), + cfq(t, "default=true && optional= && required=false && group="), + cfq(t, "default=true && optional=true && required= && group="), + cfq(t, "default=true && optional=true && required=true && group="), + cfq(t, "default=true && optional=true && required=false && group="), + cfq(t, "default=true && optional=false && required= && group="), + cfq(t, "default=true && optional=false && required=true && group="), + cfq(t, "default=true && optional=false && required=false && group="), + cfq(t, "default=false && optional= && required=true && group="), + cfq(t, "default=false && optional=true && required=true && group="), + cfq(t, "default=false && optional=false && required=true && group="), + }, + }, + "Test when version is less than v0.33.0 and component is optional": { + pkg: types.ZarfPackage{ + Build: types.ZarfBuildData{ + Version: "v0.33.0", + }, + Components: componentMatrix(t), + }, + want: []types.ZarfComponent{ + cfq(t, "default=true && optional= && required= && group="), + cfq(t, "default=true && optional= && required= && group=foo"), + cfq(t, "default=true && optional= && required= && group=bar"), + cfq(t, "default=true && optional= && required=true && group="), + cfq(t, "default=true && optional= && required=false && group="), + cfq(t, "default=true && optional=true && required= && group="), + cfq(t, "default=true && optional=true && required=true && group="), + cfq(t, "default=true && optional=true && required=false && group="), + cfq(t, "default=true && optional=false && required= && group="), + cfq(t, "default=true && optional=false && required=true && group="), + cfq(t, "default=true && optional=false && required=false && group="), + cfq(t, "default=false && optional= && required= && group="), + cfq(t, "default=false && optional= && required=true && group="), + cfq(t, "default=false && optional= && required=false && group="), + cfq(t, "default=false && optional=false && required= && group="), + cfq(t, "default=false && optional=false && required=true && group="), + cfq(t, "default=false && optional=false && required=false && group="), + }, + }, + } + + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + filter := &deploymentFilter{ + // we dont currently unit test interactive CLI behavior + isInteractive: false, + } + + result, err := filter.Apply(tc.pkg) + require.NoError(t, err) + equal := assert.Equal(t, tc.want, result) + if !equal { + left := []string{} + right := []string{} + + for _, c := range tc.want { + left = append(left, c.Name) + } + + for _, c := range result { + right = append(right, c.Name) + fmt.Printf("cfq(t, \"%s\"),\n", strings.TrimSpace(c.Name)) + } + + // cause the test to fail + require.Equal(t, left, right) + } + }) + } +} diff --git a/src/pkg/packager/filters/empty.go b/src/pkg/packager/filters/empty.go index 6ecdb3f6be..2291f8653b 100644 --- a/src/pkg/packager/filters/empty.go +++ b/src/pkg/packager/filters/empty.go @@ -6,14 +6,14 @@ package filters import "github.com/defenseunicorns/zarf/src/types" -var ( - _ ComponentFilterStrategy = &EmptyFilter{} -) +func Empty() ComponentFilterStrategy { + return &emptyFilter{} +} -// EmptyFilter is a filter that does nothing. -type EmptyFilter struct{} +// emptyFilter is a filter that does nothing. +type emptyFilter struct{} // Apply returns the components unchanged. -func (f *EmptyFilter) Apply(pkg types.ZarfPackage) ([]types.ZarfComponent, error) { +func (f *emptyFilter) Apply(pkg types.ZarfPackage) ([]types.ZarfComponent, error) { return pkg.Components, nil } diff --git a/src/pkg/packager/filters/empty_test.go b/src/pkg/packager/filters/empty_test.go new file mode 100644 index 0000000000..8096219ad5 --- /dev/null +++ b/src/pkg/packager/filters/empty_test.go @@ -0,0 +1,32 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package filters contains core implementations of the ComponentFilterStrategy interface. +package filters + +import ( + "testing" + + "github.com/defenseunicorns/zarf/src/types" + "github.com/stretchr/testify/require" +) + +func TestEmptyFilter_Apply(t *testing.T) { + components := []types.ZarfComponent{ + { + Name: "component1", + }, + { + Name: "component2", + }, + } + pkg := types.ZarfPackage{ + Components: components, + } + filter := Empty() + + result, err := filter.Apply(pkg) + + require.NoError(t, err) + require.Equal(t, components, result) +} diff --git a/src/pkg/packager/filters/included.go b/src/pkg/packager/filters/included.go index 92776886ca..9aab444256 100644 --- a/src/pkg/packager/filters/included.go +++ b/src/pkg/packager/filters/included.go @@ -9,26 +9,22 @@ import ( "github.com/defenseunicorns/zarf/src/types" ) -var ( - _ ComponentFilterStrategy = &IncludedFilter{} -) - -// NewIncludedFilter creates a new simple included filter. -func NewIncludedFilter(optionalComponents string) *IncludedFilter { +// ByIncluded creates a new simple included filter. +func ByIncluded(optionalComponents string) ComponentFilterStrategy { requested := helpers.StringToSlice(optionalComponents) - return &IncludedFilter{ + return &includedFilter{ requested, } } -// IncludedFilter sorts based purely on the internal included state of the component. -type IncludedFilter struct { +// includedFilter sorts based purely on the internal included state of the component. +type includedFilter struct { requestedComponents []string } // Apply applies the filter. -func (f *IncludedFilter) Apply(pkg types.ZarfPackage) ([]types.ZarfComponent, error) { +func (f *includedFilter) Apply(pkg types.ZarfPackage) ([]types.ZarfComponent, error) { isPartial := len(f.requestedComponents) > 0 && f.requestedComponents[0] != "" result := []types.ZarfComponent{} diff --git a/src/pkg/packager/filters/platform.go b/src/pkg/packager/filters/platform.go index 49ad6a4d6d..ef9d4eec62 100644 --- a/src/pkg/packager/filters/platform.go +++ b/src/pkg/packager/filters/platform.go @@ -5,30 +5,41 @@ package filters import ( + "fmt" + "slices" + "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/types" ) -var ( - _ ComponentFilterStrategy = &ArchAndOSFilter{} -) - -// NewArchAndOSFilter creates an architecture and OS filter. -func NewArchAndOSFilter(arch string, os string) *ArchAndOSFilter { - return &ArchAndOSFilter{ - arch: arch, - os: os, - } +// ByArchAndOS creates an architecture and OS filter. +func ByArchAndOS(arch string, os string) ComponentFilterStrategy { + return &archAndOSFilter{arch, os} } -// ArchAndOSFilter filters components based on OS/architecture. -type ArchAndOSFilter struct { +// archAndOSFilter filters components based on OS/architecture. +type archAndOSFilter struct { arch string os string } +// Define allowed OS and architecture, an empty string means it is allowed on all platforms. +// same as enums on ZarfComponentOnlyTarget +var allowedOs = []string{"linux", "darwin", "windows", ""} + +// same as enums on ZarfComponentOnlyClusterTarget +var allowedArch = []string{"amd64", "arm64", ""} + // Apply applies the filter. -func (f *ArchAndOSFilter) Apply(pkg types.ZarfPackage) ([]types.ZarfComponent, error) { +func (f *archAndOSFilter) Apply(pkg types.ZarfPackage) ([]types.ZarfComponent, error) { + if !slices.Contains(allowedOs, f.os) { + return nil, fmt.Errorf("invalid OS: %s", f.os) + } + + if !slices.Contains(allowedArch, f.arch) { + return nil, fmt.Errorf("invalid architecture: %s", f.arch) + } + filtered := []types.ZarfComponent{} // Filter each component to only compatible platforms. for _, component := range pkg.Components { diff --git a/src/pkg/packager/filters/platform_test.go b/src/pkg/packager/filters/platform_test.go new file mode 100644 index 0000000000..d346b865e9 --- /dev/null +++ b/src/pkg/packager/filters/platform_test.go @@ -0,0 +1,49 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +package filters + +import ( + "testing" + + "github.com/defenseunicorns/zarf/src/types" + "github.com/stretchr/testify/require" +) + +func archOSMatrix() [][2]string { + // Create a matrix of all possible combinations + matrix := [][2]string{} + for _, os := range allowedOs { + for _, arch := range allowedArch { + matrix = append(matrix, [2]string{arch, os}) + } + } + return matrix +} + +func TestArchAndOSFilter(t *testing.T) { + // Create a test package + pkg := types.ZarfPackage{} + for _, platform := range archOSMatrix() { + pkg.Components = append(pkg.Components, types.ZarfComponent{ + Only: types.ZarfComponentOnlyTarget{ + LocalOS: platform[1], + Cluster: types.ZarfComponentOnlyCluster{Architecture: platform[0]}, + }, + }) + } + + for _, platform := range archOSMatrix() { + filter := ByArchAndOS(platform[0], platform[1]) + result, err := filter.Apply(pkg) + require.NoError(t, err) + for _, component := range result { + if component.Only.Cluster.Architecture != "" { + require.Equal(t, platform[0], component.Only.Cluster.Architecture) + } + if component.Only.LocalOS != "" { + require.Equal(t, platform[1], component.Only.LocalOS) + } + } + } +} diff --git a/src/pkg/packager/filters/common.go b/src/pkg/packager/filters/strat.go similarity index 100% rename from src/pkg/packager/filters/common.go rename to src/pkg/packager/filters/strat.go diff --git a/src/pkg/packager/mirror.go b/src/pkg/packager/mirror.go index 330a91ecb8..9fd5727403 100644 --- a/src/pkg/packager/mirror.go +++ b/src/pkg/packager/mirror.go @@ -19,7 +19,7 @@ func (p *Packager) Mirror() (err error) { spinner := message.NewProgressSpinner("Mirroring Zarf package %s", p.cfg.PkgOpts.PackageSource) defer spinner.Stop() - filter := filters.NewIncludedFilter(p.cfg.PkgOpts.OptionalComponents) + filter := filters.ByIncluded(p.cfg.PkgOpts.OptionalComponents) if err = p.source.LoadPackage(p.layout, filter, true); err != nil { return fmt.Errorf("unable to load the package: %w", err) diff --git a/src/pkg/packager/publish.go b/src/pkg/packager/publish.go index 80f44fa176..0fa33bbea3 100644 --- a/src/pkg/packager/publish.go +++ b/src/pkg/packager/publish.go @@ -99,7 +99,7 @@ func (p *Packager) Publish() (err error) { return err } } else { - filter := &filters.EmptyFilter{} + filter := filters.Empty() if err = p.source.LoadPackage(p.layout, filter, false); err != nil { return fmt.Errorf("unable to load the package: %w", err) } diff --git a/src/pkg/packager/remove.go b/src/pkg/packager/remove.go index 3f2e7063f3..7b058e6488 100644 --- a/src/pkg/packager/remove.go +++ b/src/pkg/packager/remove.go @@ -52,7 +52,7 @@ func (p *Packager) Remove() (err error) { packageRequiresCluster := false // If components were provided; just remove the things we were asked to remove - filter := filters.NewIncludedFilter(p.cfg.PkgOpts.OptionalComponents) + filter := filters.ByIncluded(p.cfg.PkgOpts.OptionalComponents) included, err := filter.Apply(p.cfg.Pkg) if err != nil { return err diff --git a/src/pkg/packager/yaml.go b/src/pkg/packager/yaml.go index 8c3fc35556..b3b68a15a6 100644 --- a/src/pkg/packager/yaml.go +++ b/src/pkg/packager/yaml.go @@ -43,7 +43,7 @@ func (p *Packager) readZarfYAML(path string) error { // filterComponentsByOSAndArch removes components not matching the current OS and architecture. func (p *Packager) filterComponentsByOSAndArch() (err error) { - p.cfg.Pkg.Components, err = filters.NewArchAndOSFilter(p.arch, runtime.GOOS).Apply(p.cfg.Pkg) + p.cfg.Pkg.Components, err = filters.ByArchAndOS(p.arch, runtime.GOOS).Apply(p.cfg.Pkg) return err } From d3e342ce4e46f61558d3a9cc23eb4d2f28e9f3cd Mon Sep 17 00:00:00 2001 From: razzle Date: Sun, 18 Feb 2024 00:48:09 -0600 Subject: [PATCH 072/113] comments Signed-off-by: razzle --- src/pkg/packager/filters/platform.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pkg/packager/filters/platform.go b/src/pkg/packager/filters/platform.go index ef9d4eec62..d6d4518a31 100644 --- a/src/pkg/packager/filters/platform.go +++ b/src/pkg/packager/filters/platform.go @@ -27,7 +27,7 @@ type archAndOSFilter struct { // same as enums on ZarfComponentOnlyTarget var allowedOs = []string{"linux", "darwin", "windows", ""} -// same as enums on ZarfComponentOnlyClusterTarget +// same as enums on ZarfComponentOnlyCluster var allowedArch = []string{"amd64", "arm64", ""} // Apply applies the filter. From 693b79427f9928a451f5016b075dfa3f23888b85 Mon Sep 17 00:00:00 2001 From: razzle Date: Sun, 18 Feb 2024 12:46:09 -0600 Subject: [PATCH 073/113] move stuff to a better spot Signed-off-by: razzle --- src/config/config.go | 16 ---------------- src/pkg/packager/sources/new.go | 2 +- src/pkg/packager/sources/utils.go | 24 ++++++++++++++++++++---- 3 files changed, 21 insertions(+), 21 deletions(-) diff --git a/src/config/config.go b/src/config/config.go index fa4495bd17..cd2656ec4b 100644 --- a/src/config/config.go +++ b/src/config/config.go @@ -168,22 +168,6 @@ func GetCraneAuthOption(username string, secret string) crane.Option { })) } -// GetValidPackageExtensions returns the valid package extensions. -func GetValidPackageExtensions() [2]string { - return [...]string{".tar.zst", ".tar"} -} - -// IsValidFileExtension returns true if the filename has a valid package extension. -func IsValidFileExtension(filename string) bool { - for _, extension := range GetValidPackageExtensions() { - if strings.HasSuffix(filename, extension) { - return true - } - } - - return false -} - // GetAbsCachePath gets the absolute cache path for images and git repos. func GetAbsCachePath() string { return GetAbsHomePath(CommonOptions.CachePath) diff --git a/src/pkg/packager/sources/new.go b/src/pkg/packager/sources/new.go index 414e25954e..466c899601 100644 --- a/src/pkg/packager/sources/new.go +++ b/src/pkg/packager/sources/new.go @@ -49,7 +49,7 @@ func Identify(pkgSrc string) string { return "split" } - if config.IsValidFileExtension(pkgSrc) { + if IsValidFileExtension(pkgSrc) { return "tarball" } diff --git a/src/pkg/packager/sources/utils.go b/src/pkg/packager/sources/utils.go index 383cc262cd..802d7e2fd2 100644 --- a/src/pkg/packager/sources/utils.go +++ b/src/pkg/packager/sources/utils.go @@ -19,14 +19,30 @@ import ( "github.com/mholt/archiver/v3" ) +// GetValidPackageExtensions returns the valid package extensions. +func GetValidPackageExtensions() [2]string { + return [...]string{".tar.zst", ".tar"} +} + +// IsValidFileExtension returns true if the filename has a valid package extension. +func IsValidFileExtension(filename string) bool { + for _, extension := range GetValidPackageExtensions() { + if strings.HasSuffix(filename, extension) { + return true + } + } + + return false +} + func identifyUnknownTarball(path string) (string, error) { if utils.InvalidPath(path) { return "", &os.PathError{Op: "open", Path: path, Err: os.ErrNotExist} } - if filepath.Ext(path) != "" && config.IsValidFileExtension(path) { + if filepath.Ext(path) != "" && IsValidFileExtension(path) { return path, nil - } else if filepath.Ext(path) != "" && !config.IsValidFileExtension(path) { - return "", fmt.Errorf("%s is not a supported tarball format (%+v)", path, config.GetValidPackageExtensions()) + } else if filepath.Ext(path) != "" && !IsValidFileExtension(path) { + return "", fmt.Errorf("%s is not a supported tarball format (%+v)", path, GetValidPackageExtensions()) } // rename to .tar.zst and check if it's a valid tar.zst @@ -57,7 +73,7 @@ func identifyUnknownTarball(path string) (string, error) { return tb, nil } - return "", fmt.Errorf("%s is not a supported tarball format (%+v)", path, config.GetValidPackageExtensions()) + return "", fmt.Errorf("%s is not a supported tarball format (%+v)", path, GetValidPackageExtensions()) } // RenameFromMetadata renames a tarball based on its metadata. From 3bc3f979c02acb8e8da554ea67acc5e23a520e88 Mon Sep 17 00:00:00 2001 From: razzle Date: Sun, 18 Feb 2024 12:51:26 -0600 Subject: [PATCH 074/113] remove multi from eks distro Signed-off-by: razzle --- .github/workflows/nightly-eks.yml | 4 ++-- .../docs/tutorials/package_deploy_wordpress_suggestions.html | 2 +- packages/distros/eks/zarf.yaml | 1 - src/test/e2e/00_use_cli_test.go | 2 +- 4 files changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/nightly-eks.yml b/.github/workflows/nightly-eks.yml index 5ec53f5a7a..c4bee3432f 100644 --- a/.github/workflows/nightly-eks.yml +++ b/.github/workflows/nightly-eks.yml @@ -48,7 +48,7 @@ jobs: - name: Deploy the eks package run: | - ./build/zarf package deploy build/zarf-package-distro-eks-multi-0.0.3.tar.zst \ + ./build/zarf package deploy build/zarf-package-distro-eks-amd64-0.0.3.tar.zst \ --components=deploy-eks-cluster \ --set=EKS_CLUSTER_NAME=${{ inputs.cluster_name || 'zarf-nightly-eks-e2e-test' }} \ --set=EKS_INSTANCE_TYPE=${{ inputs.instance_type || 't3.medium' }} \ @@ -60,7 +60,7 @@ jobs: - name: Teardown the cluster if: always() run: | - ./build/zarf package deploy build/zarf-package-distro-eks-multi-0.0.3.tar.zst \ + ./build/zarf package deploy build/zarf-package-distro-eks-amd64-0.0.3.tar.zst \ --components=teardown-eks-cluster \ --set=EKS_CLUSTER_NAME=${{ inputs.cluster_name || 'zarf-nightly-eks-e2e-test' }} \ --confirm diff --git a/docs-website/static/docs/tutorials/package_deploy_wordpress_suggestions.html b/docs-website/static/docs/tutorials/package_deploy_wordpress_suggestions.html index 5e37e51d12..f9326440b2 100644 --- a/docs-website/static/docs/tutorials/package_deploy_wordpress_suggestions.html +++ b/docs-website/static/docs/tutorials/package_deploy_wordpress_suggestions.html @@ -50,7 +50,7 @@ Saving log file to /var/folders/bk/rz1xx2sd5zn134c0_j1s2n5r0000gp/T/zarf-2023-03-23-13-18-54-4086179855.log ? Choose or type the package file zarf-package-helm-oci-chart-arm64-0.0.1.tar.zst[tab for suggestions] - zarf-package-distro-eks-multi.tar.zst + zarf-package-distro-eks-amd64.tar.zst > zarf-package-wordpress-amd64-16.0.4.tar.zst diff --git a/packages/distros/eks/zarf.yaml b/packages/distros/eks/zarf.yaml index fda011b742..593d2d7650 100644 --- a/packages/distros/eks/zarf.yaml +++ b/packages/distros/eks/zarf.yaml @@ -2,7 +2,6 @@ kind: ZarfPackageConfig metadata: name: distro-eks description: Deploy a EKS K8s cluster - architecture: multi version: 0.0.3 variables: diff --git a/src/test/e2e/00_use_cli_test.go b/src/test/e2e/00_use_cli_test.go index 2eaf6a6ed8..11b577e57a 100644 --- a/src/test/e2e/00_use_cli_test.go +++ b/src/test/e2e/00_use_cli_test.go @@ -155,7 +155,7 @@ func TestUseCLI(t *testing.T) { stdOut, stdErr, err := e2e.Zarf("package", "create", "packages/distros/eks", "--confirm") require.NoError(t, err, stdOut, stdErr) - path := "zarf-package-distro-eks-multi-0.0.3.tar.zst" + path := fmt.Sprintf("zarf-package-distro-eks-%s-0.0.3.tar.zst", e2e.Arch) stdOut, stdErr, err = e2e.Zarf("package", "deploy", path, "--confirm") require.NoError(t, err, stdOut, stdErr) From fe8c2339ef8be9d7b0530c0e7d1dffb15488334b Mon Sep 17 00:00:00 2001 From: razzle Date: Sun, 18 Feb 2024 13:07:20 -0600 Subject: [PATCH 075/113] md Signed-off-by: razzle --- packages/README.md | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/packages/README.md b/packages/README.md index 6c838cc8fd..cac31b8501 100644 --- a/packages/README.md +++ b/packages/README.md @@ -3,23 +3,25 @@ This folder contains packages maintained by the [Zarf team](https://github.com/defenseunicorns/zarf/graphs/contributors). Some of these packages are used by `zarf init` for new cluster initialization. **Packages** -- [distros](#distros) -- [gitea](#gitea) -- [logging-pgl](#logging-pgl) -- [zarf-agent](#zarf-agent) -- [zarf-registry](#zarf-registry) +- [Zarf Packages](#zarf-packages) + - [Distros](#distros) + - [Usage Examples](#usage-examples) + - [Gitea](#gitea) + - [Logging PGL](#logging-pgl) + - [Zarf Agent](#zarf-agent) + - [Zarf Registry](#zarf-registry) ### Distros The distros package adds optional capabilities for spinning up and tearing down clusters. Currently, the following distros are supported: -- [EKS](https://aws.amazon.com/eks/) - Zarf deploys and tears down using the `eksctl` binary under the hood. See how it's done in the EKS package's [`zarf.yaml`](./distros/eks/zarf.yaml) and checkout the [EKS package's config](./distros/eks/eks.yaml) for more information. +- [EKS](https://aws.amazon.com/eks/) - Zarf deploys and tears down using the `eksctl` binary under the hood. See how it's done in the EKS package's [`zarf.yaml`](./distros/eks/zarf.yaml) and checkout the [EKS package's config](./distros/eks/eks.yaml) for more information. -- [k3s](https://k3s.io/) - Zarf deploys and tears down using the `k3s` service under the hood. See how it's done in the k3s package's [`zarf.yaml`](./distros/k3s/common/zarf.yaml). +- [k3s](https://k3s.io/) - Zarf deploys and tears down using the `k3s` service under the hood. See how it's done in the k3s package's [`zarf.yaml`](./distros/k3s/common/zarf.yaml). #### Usage Examples - + **EKS** - Create/Deploy EKS cluster. > **Note** - requires `eksctl` credentials. @@ -27,7 +29,7 @@ The distros package adds optional capabilities for spinning up and tearing down ```bash zarf package create packages/distros/eks -o build --confirm -zarf package deploy build/zarf-package-distro-eks-multi-x.x.x.tar.zst --components=deploy-eks-cluster --set=CLUSTER_NAME='zarf-nightly-eks-e2e-test',INSTANCE_TYPE='t3.medium' --confirm +zarf package deploy build/zarf-package-distro-eks-amd64-x.x.x.tar.zst --components=deploy-eks-cluster --set=CLUSTER_NAME='zarf-nightly-eks-e2e-test',INSTANCE_TYPE='t3.medium' --confirm ``` See the [nightly-eks test](../.github/workflows/nightly-eks.yml) for another example. @@ -65,7 +67,7 @@ zarf init --components=logging The Zarf Agent is a mutating admission controller used to modify the image property within a PodSpec. The purpose is to redirect it to Zarf's configured registry instead of the the original registry (such as DockerHub, GHCR, or Quay). Additionally, the webhook attaches the appropriate `ImagePullSecret` for the seed registry to the pod. This configuration allows the pod to successfully retrieve the image from the seed registry, even when operating in an air-gapped environment. ```bash -$ zarf tools kubectl get deploy -n zarf agent-hook +$ zarf tools kubectl get deploy -n zarf agent-hook NAME READY UP-TO-DATE AVAILABLE AGE agent-hook 2/2 2 2 17m @@ -73,4 +75,4 @@ agent-hook 2/2 2 2 17m ### Zarf Registry -The Zarf internal registry is utilized to store container images for use in air-gapped environments. The registry is deployed as a `Deployment` with a single replica and a `PersistentVolumeClaim` to store the images. Credentials for basic authentication are autogenerated and stored within a secret in the `zarf` namespace. The internal registry is `HTTP` only. \ No newline at end of file +The Zarf internal registry is utilized to store container images for use in air-gapped environments. The registry is deployed as a `Deployment` with a single replica and a `PersistentVolumeClaim` to store the images. Credentials for basic authentication are autogenerated and stored within a secret in the `zarf` namespace. The internal registry is `HTTP` only. From 2e823b7185e9a3e62690eae9edbc6235e5558559 Mon Sep 17 00:00:00 2001 From: razzle Date: Mon, 19 Feb 2024 14:22:46 -0600 Subject: [PATCH 076/113] more unit tests Signed-off-by: razzle --- .../filters/{included.go => select.go} | 12 +-- src/pkg/packager/filters/select_test.go | 92 +++++++++++++++++++ src/pkg/packager/filters/utils_test.go | 76 +++++++++++++++ src/pkg/packager/mirror.go | 2 +- src/pkg/packager/remove.go | 2 +- 5 files changed, 176 insertions(+), 8 deletions(-) rename src/pkg/packager/filters/{included.go => select.go} (71%) create mode 100644 src/pkg/packager/filters/select_test.go diff --git a/src/pkg/packager/filters/included.go b/src/pkg/packager/filters/select.go similarity index 71% rename from src/pkg/packager/filters/included.go rename to src/pkg/packager/filters/select.go index 9aab444256..36ab641636 100644 --- a/src/pkg/packager/filters/included.go +++ b/src/pkg/packager/filters/select.go @@ -9,22 +9,22 @@ import ( "github.com/defenseunicorns/zarf/src/types" ) -// ByIncluded creates a new simple included filter. -func ByIncluded(optionalComponents string) ComponentFilterStrategy { +// BySelectState creates a new simple included filter. +func BySelectState(optionalComponents string) ComponentFilterStrategy { requested := helpers.StringToSlice(optionalComponents) - return &includedFilter{ + return &selectStateFilter{ requested, } } -// includedFilter sorts based purely on the internal included state of the component. -type includedFilter struct { +// selectStateFilter sorts based purely on the internal included state of the component. +type selectStateFilter struct { requestedComponents []string } // Apply applies the filter. -func (f *includedFilter) Apply(pkg types.ZarfPackage) ([]types.ZarfComponent, error) { +func (f *selectStateFilter) Apply(pkg types.ZarfPackage) ([]types.ZarfComponent, error) { isPartial := len(f.requestedComponents) > 0 && f.requestedComponents[0] != "" result := []types.ZarfComponent{} diff --git a/src/pkg/packager/filters/select_test.go b/src/pkg/packager/filters/select_test.go new file mode 100644 index 0000000000..aaff3bde3e --- /dev/null +++ b/src/pkg/packager/filters/select_test.go @@ -0,0 +1,92 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package filters contains core implementations of the ComponentFilterStrategy interface. +package filters + +import ( + "testing" + + "github.com/defenseunicorns/zarf/src/types" + "github.com/stretchr/testify/require" +) + +func Test_selectStateFilter_Apply(t *testing.T) { + tests := []struct { + name string + requestedComponents string + components []types.ZarfComponent + expectedResult []types.ZarfComponent + expectedError error + }{ + { + name: "Test when requestedComponents is empty", + requestedComponents: "", + components: []types.ZarfComponent{ + {Name: "component1"}, + {Name: "component2"}, + {Name: "component3"}, + }, + expectedResult: []types.ZarfComponent{ + {Name: "component1"}, + {Name: "component2"}, + {Name: "component3"}, + }, + expectedError: nil, + }, + { + name: "Test when requestedComponents contains a valid component name", + requestedComponents: "component2", + components: []types.ZarfComponent{ + {Name: "component1"}, + {Name: "component2"}, + {Name: "component3"}, + }, + expectedResult: []types.ZarfComponent{ + {Name: "component2"}, + }, + expectedError: nil, + }, + { + name: "Test when requestedComponents contains an excluded component name", + requestedComponents: "-component2", + components: []types.ZarfComponent{ + {Name: "component1"}, + {Name: "component2"}, + {Name: "component3"}, + }, + expectedResult: []types.ZarfComponent{ + {Name: "component1"}, + {Name: "component3"}, + }, + expectedError: nil, + }, + { + name: "Test when requestedComponents contains a glob pattern", + requestedComponents: "comp*", + components: []types.ZarfComponent{ + {Name: "component1"}, + {Name: "component2"}, + {Name: "other"}, + }, + expectedResult: []types.ZarfComponent{ + {Name: "component1"}, + {Name: "component2"}, + }, + expectedError: nil, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + filter := BySelectState(tc.requestedComponents) + + result, err := filter.Apply(types.ZarfPackage{ + Components: tc.components, + }) + + require.Equal(t, tc.expectedResult, result) + require.Equal(t, tc.expectedError, err) + }) + } +} diff --git a/src/pkg/packager/filters/utils_test.go b/src/pkg/packager/filters/utils_test.go index af60115c48..270c5e6615 100644 --- a/src/pkg/packager/filters/utils_test.go +++ b/src/pkg/packager/filters/utils_test.go @@ -142,3 +142,79 @@ func Test_isRequired(t *testing.T) { }) } } + +func Test_includedOrExcluded(t *testing.T) { + tests := []struct { + name string + componentName string + requestedComponentNames []string + wantState selectState + wantRequestedComponent string + }{ + { + name: "Test when component is excluded", + componentName: "example", + requestedComponentNames: []string{"-example"}, + wantState: excluded, + wantRequestedComponent: "-example", + }, + { + name: "Test when component is included", + componentName: "example", + requestedComponentNames: []string{"example"}, + wantState: included, + wantRequestedComponent: "example", + }, + { + name: "Test when component is not included or excluded", + componentName: "example", + requestedComponentNames: []string{"other"}, + wantState: unknown, + wantRequestedComponent: "", + }, + { + name: "Test when component is excluded and included", + componentName: "example", + requestedComponentNames: []string{"-example", "example"}, + wantState: excluded, + wantRequestedComponent: "-example", + }, + // interesting case, excluded wins + { + name: "Test when component is included and excluded", + componentName: "example", + requestedComponentNames: []string{"example", "-example"}, + wantState: excluded, + wantRequestedComponent: "-example", + }, + { + name: "Test when component is included via glob", + componentName: "example", + requestedComponentNames: []string{"ex*"}, + wantState: included, + wantRequestedComponent: "ex*", + }, + { + name: "Test when component is excluded via glob", + componentName: "example", + requestedComponentNames: []string{"-ex*"}, + wantState: excluded, + wantRequestedComponent: "-ex*", + }, + { + name: "Test when component is not found via glob", + componentName: "example", + requestedComponentNames: []string{"other*"}, + wantState: unknown, + wantRequestedComponent: "", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + gotState, gotRequestedComponent := includedOrExcluded(tc.componentName, tc.requestedComponentNames) + require.Equal(t, tc.wantState, gotState) + require.Equal(t, tc.wantRequestedComponent, gotRequestedComponent) + }) + } +} diff --git a/src/pkg/packager/mirror.go b/src/pkg/packager/mirror.go index 9fd5727403..ad343982a0 100644 --- a/src/pkg/packager/mirror.go +++ b/src/pkg/packager/mirror.go @@ -19,7 +19,7 @@ func (p *Packager) Mirror() (err error) { spinner := message.NewProgressSpinner("Mirroring Zarf package %s", p.cfg.PkgOpts.PackageSource) defer spinner.Stop() - filter := filters.ByIncluded(p.cfg.PkgOpts.OptionalComponents) + filter := filters.BySelectState(p.cfg.PkgOpts.OptionalComponents) if err = p.source.LoadPackage(p.layout, filter, true); err != nil { return fmt.Errorf("unable to load the package: %w", err) diff --git a/src/pkg/packager/remove.go b/src/pkg/packager/remove.go index 7b058e6488..a756801ccb 100644 --- a/src/pkg/packager/remove.go +++ b/src/pkg/packager/remove.go @@ -52,7 +52,7 @@ func (p *Packager) Remove() (err error) { packageRequiresCluster := false // If components were provided; just remove the things we were asked to remove - filter := filters.ByIncluded(p.cfg.PkgOpts.OptionalComponents) + filter := filters.BySelectState(p.cfg.PkgOpts.OptionalComponents) included, err := filter.Apply(p.cfg.Pkg) if err != nil { return err From 57b0bd8dcdf29dad533add1afac6ea21cc49e6d9 Mon Sep 17 00:00:00 2001 From: razzle Date: Mon, 19 Feb 2024 15:31:21 -0600 Subject: [PATCH 077/113] rename Signed-off-by: razzle --- src/pkg/packager/deploy.go | 2 +- src/pkg/packager/dev.go | 2 +- src/pkg/packager/mirror.go | 4 +--- src/pkg/packager/remove.go | 2 +- src/pkg/packager/yaml.go | 4 ++-- 5 files changed, 6 insertions(+), 8 deletions(-) diff --git a/src/pkg/packager/deploy.go b/src/pkg/packager/deploy.go index 78616624f2..0f65abefa8 100644 --- a/src/pkg/packager/deploy.go +++ b/src/pkg/packager/deploy.go @@ -75,7 +75,7 @@ func (p *Packager) Deploy() (err error) { defer p.resetRegistryHPA() // Filter out components that are not compatible with this system - if err := p.filterComponentsByOSAndArch(); err != nil { + if err := p.filterComponentsByArchAndOS(); err != nil { return err } diff --git a/src/pkg/packager/dev.go b/src/pkg/packager/dev.go index 2b9268975b..80a4b3d9ce 100644 --- a/src/pkg/packager/dev.go +++ b/src/pkg/packager/dev.go @@ -37,7 +37,7 @@ func (p *Packager) DevDeploy() error { p.cfg.Pkg.Build.Version = config.CLIVersion // Filter out components that are not compatible with this system - if err := p.filterComponentsByOSAndArch(); err != nil { + if err := p.filterComponentsByArchAndOS(); err != nil { return err } diff --git a/src/pkg/packager/mirror.go b/src/pkg/packager/mirror.go index ad343982a0..a7cdec88ec 100644 --- a/src/pkg/packager/mirror.go +++ b/src/pkg/packager/mirror.go @@ -43,12 +43,10 @@ func (p *Packager) Mirror() (err error) { } p.cfg.State = state - // Filter out components that are not compatible with this system if we have loaded from a tarball - if err := p.filterComponentsByOSAndArch(); err != nil { + if err := p.filterComponentsByArchAndOS(); err != nil { return err } - // Run mirror for each requested component included, err := filter.Apply(p.cfg.Pkg) if err != nil { return err diff --git a/src/pkg/packager/remove.go b/src/pkg/packager/remove.go index a756801ccb..9a5476a415 100644 --- a/src/pkg/packager/remove.go +++ b/src/pkg/packager/remove.go @@ -42,7 +42,7 @@ func (p *Packager) Remove() (err error) { if err = p.readZarfYAML(p.layout.ZarfYAML); err != nil { return err } - if err := p.filterComponentsByOSAndArch(); err != nil { + if err := p.filterComponentsByArchAndOS(); err != nil { return err } packageName = p.cfg.Pkg.Metadata.Name diff --git a/src/pkg/packager/yaml.go b/src/pkg/packager/yaml.go index b3b68a15a6..a998633ca2 100644 --- a/src/pkg/packager/yaml.go +++ b/src/pkg/packager/yaml.go @@ -41,8 +41,8 @@ func (p *Packager) readZarfYAML(path string) error { return nil } -// filterComponentsByOSAndArch removes components not matching the current OS and architecture. -func (p *Packager) filterComponentsByOSAndArch() (err error) { +// filterComponentsByArchAndOS removes components not matching the current OS and architecture. +func (p *Packager) filterComponentsByArchAndOS() (err error) { p.cfg.Pkg.Components, err = filters.ByArchAndOS(p.arch, runtime.GOOS).Apply(p.cfg.Pkg) return err } From f61a836b49f7d1e49d70e1f0323ad0c7e5c19e62 Mon Sep 17 00:00:00 2001 From: razzle Date: Mon, 19 Feb 2024 15:46:56 -0600 Subject: [PATCH 078/113] fix unit tests Signed-off-by: razzle --- src/pkg/packager/filters/select_test.go | 3 +-- src/pkg/packager/mirror.go | 11 +++++------ 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/src/pkg/packager/filters/select_test.go b/src/pkg/packager/filters/select_test.go index aaff3bde3e..527bf66265 100644 --- a/src/pkg/packager/filters/select_test.go +++ b/src/pkg/packager/filters/select_test.go @@ -49,7 +49,7 @@ func Test_selectStateFilter_Apply(t *testing.T) { }, { name: "Test when requestedComponents contains an excluded component name", - requestedComponents: "-component2", + requestedComponents: "component1, -component2", components: []types.ZarfComponent{ {Name: "component1"}, {Name: "component2"}, @@ -57,7 +57,6 @@ func Test_selectStateFilter_Apply(t *testing.T) { }, expectedResult: []types.ZarfComponent{ {Name: "component1"}, - {Name: "component3"}, }, expectedError: nil, }, diff --git a/src/pkg/packager/mirror.go b/src/pkg/packager/mirror.go index a7cdec88ec..48083f12b3 100644 --- a/src/pkg/packager/mirror.go +++ b/src/pkg/packager/mirror.go @@ -37,12 +37,6 @@ func (p *Packager) Mirror() (err error) { return fmt.Errorf("mirror cancelled") } - state := &types.ZarfState{ - RegistryInfo: p.cfg.InitOpts.RegistryInfo, - GitServer: p.cfg.InitOpts.GitServer, - } - p.cfg.State = state - if err := p.filterComponentsByArchAndOS(); err != nil { return err } @@ -52,6 +46,11 @@ func (p *Packager) Mirror() (err error) { return err } + p.cfg.State = &types.ZarfState{ + RegistryInfo: p.cfg.InitOpts.RegistryInfo, + GitServer: p.cfg.InitOpts.GitServer, + } + for _, component := range included { if err := p.mirrorComponent(component); err != nil { return err From 8873032ea3c689e96f01eb49ed4a9e069a61db2c Mon Sep 17 00:00:00 2001 From: razzle Date: Mon, 25 Mar 2024 00:29:46 -0500 Subject: [PATCH 079/113] fix compile errors Signed-off-by: razzle --- src/pkg/packager/creator/utils.go | 8 +-- src/pkg/packager/generate.go | 3 +- src/pkg/packager/yaml.go | 96 ------------------------------- 3 files changed, 5 insertions(+), 102 deletions(-) delete mode 100644 src/pkg/packager/yaml.go diff --git a/src/pkg/packager/creator/utils.go b/src/pkg/packager/creator/utils.go index 43ab469565..452e44c0fc 100644 --- a/src/pkg/packager/creator/utils.go +++ b/src/pkg/packager/creator/utils.go @@ -43,10 +43,10 @@ func recordPackageMetadata(pkg *types.ZarfPackage, createOpts types.ZarfCreateOp // Record the time of package creation. pkg.Build.Timestamp = now.Format(time.RFC1123Z) - // Record the migrations that will be ran on the package. - pkg.Build.Migrations = []string{ - deprecated.ScriptsToActionsMigrated, - deprecated.PluralizeSetVariable, + // Record the migrations that will run on the package. + pkg.Build.Migrations = []string{} + for _, m := range deprecated.Migrations() { + pkg.Build.Migrations = append(pkg.Build.Migrations, m.ID()) } // Record the flavor of Zarf used to build this package (if any). diff --git a/src/pkg/packager/generate.go b/src/pkg/packager/generate.go index 23535fc0c9..d2a8043614 100644 --- a/src/pkg/packager/generate.go +++ b/src/pkg/packager/generate.go @@ -37,8 +37,7 @@ func (p *Packager) Generate() (err error) { } generatedComponent := types.ZarfComponent{ - Name: p.cfg.GenerateOpts.Name, - Required: helpers.BoolPtr(true), + Name: p.cfg.GenerateOpts.Name, Charts: []types.ZarfChart{ { Name: p.cfg.GenerateOpts.Name, diff --git a/src/pkg/packager/yaml.go b/src/pkg/packager/yaml.go deleted file mode 100644 index a998633ca2..0000000000 --- a/src/pkg/packager/yaml.go +++ /dev/null @@ -1,96 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: 2021-Present The Zarf Authors - -// Package packager contains functions for interacting with, managing and deploying Zarf packages. -package packager - -import ( - "os" - "runtime" - "time" - - "github.com/defenseunicorns/zarf/src/config" - "github.com/defenseunicorns/zarf/src/pkg/packager/deprecated" - "github.com/defenseunicorns/zarf/src/pkg/packager/filters" - "github.com/defenseunicorns/zarf/src/pkg/utils" -) - -// readZarfYAML reads a Zarf YAML file. -func (p *Packager) readZarfYAML(path string) error { - var warnings []string - - if err := utils.ReadYaml(path, &p.cfg.Pkg); err != nil { - return err - } - - if p.layout.IsLegacyLayout() { - warning := "Detected deprecated package layout, migrating to new layout - support for this package will be dropped in v1.0.0" - p.warnings = append(p.warnings, warning) - } - - if len(p.cfg.Pkg.Build.Migrations) > 0 { - for idx, component := range p.cfg.Pkg.Components { - // Handle component configuration deprecations - p.cfg.Pkg.Components[idx], warnings = deprecated.MigrateComponent(p.cfg.Pkg.Build, component) - p.warnings = append(p.warnings, warnings...) - } - } - - p.arch = config.GetArch(p.cfg.Pkg.Metadata.Architecture, p.cfg.Pkg.Build.Architecture) - - return nil -} - -// filterComponentsByArchAndOS removes components not matching the current OS and architecture. -func (p *Packager) filterComponentsByArchAndOS() (err error) { - p.cfg.Pkg.Components, err = filters.ByArchAndOS(p.arch, runtime.GOOS).Apply(p.cfg.Pkg) - return err -} - -// writeYaml adds build information and writes the config to the temp directory. -func (p *Packager) writeYaml() error { - now := time.Now() - // Just use $USER env variable to avoid CGO issue. - // https://groups.google.com/g/golang-dev/c/ZFDDX3ZiJ84. - // Record the name of the user creating the package. - if runtime.GOOS == "windows" { - p.cfg.Pkg.Build.User = os.Getenv("USERNAME") - } else { - p.cfg.Pkg.Build.User = os.Getenv("USER") - } - hostname, hostErr := os.Hostname() - - // Normalize these for the package confirmation. - p.cfg.Pkg.Metadata.Architecture = p.arch - p.cfg.Pkg.Build.Architecture = p.arch - - if p.cfg.CreateOpts.IsSkeleton { - p.cfg.Pkg.Build.Architecture = "skeleton" - } - - // Record the time of package creation. - p.cfg.Pkg.Build.Timestamp = now.Format(time.RFC1123Z) - - // Record the Zarf Version the CLI was built with. - p.cfg.Pkg.Build.Version = config.CLIVersion - - if hostErr == nil { - // Record the hostname of the package creation terminal. - p.cfg.Pkg.Build.Terminal = hostname - } - - // Record the migrations that will be run on the package. - for _, m := range deprecated.Migrations() { - p.cfg.Pkg.Build.Migrations = append(p.cfg.Pkg.Build.Migrations, m.ID()) - } - - // Record the flavor of Zarf used to build this package (if any). - p.cfg.Pkg.Build.Flavor = p.cfg.CreateOpts.Flavor - - p.cfg.Pkg.Build.RegistryOverrides = p.cfg.CreateOpts.RegistryOverrides - - // Record the latest version of Zarf without breaking changes to the package structure. - p.cfg.Pkg.Build.LastNonBreakingVersion = deprecated.LastNonBreakingVersion - - return utils.WriteYaml(p.layout.ZarfYAML, p.cfg.Pkg, 0400) -} From 0cbd06fc34f691ed7e6e3169249e45e6551f37dc Mon Sep 17 00:00:00 2001 From: razzle Date: Tue, 26 Mar 2024 22:34:30 -0500 Subject: [PATCH 080/113] fix unit tests Signed-off-by: razzle --- src/internal/packager/validate/validate.go | 2 +- .../deprecated/required_to_optional.go | 2 +- src/pkg/packager/filters/deploy.go | 1 - src/pkg/packager/filters/deploy_test.go | 158 ++++++++++-------- src/pkg/packager/filters/platform.go | 69 -------- src/pkg/packager/filters/platform_test.go | 49 ------ src/pkg/packager/filters/utils_test.go | 2 +- 7 files changed, 88 insertions(+), 195 deletions(-) delete mode 100644 src/pkg/packager/filters/platform.go delete mode 100644 src/pkg/packager/filters/platform_test.go diff --git a/src/internal/packager/validate/validate.go b/src/internal/packager/validate/validate.go index 66822cb005..0eea4c0b3a 100644 --- a/src/internal/packager/validate/validate.go +++ b/src/internal/packager/validate/validate.go @@ -150,7 +150,7 @@ func validateComponent(pkg types.ZarfPackage, component types.ZarfComponent) err return fmt.Errorf(lang.PkgValidateErrComponentLocalOS, component.Name, component.Only.LocalOS, supportedOS) } - if component.Optional == nil || !*component.Optional { + if component.Optional != nil || !*component.Optional { if component.Default { return fmt.Errorf(lang.PkgValidateErrComponentReqDefault, component.Name) } diff --git a/src/pkg/packager/deprecated/required_to_optional.go b/src/pkg/packager/deprecated/required_to_optional.go index f961b48fd8..6436e9eda2 100644 --- a/src/pkg/packager/deprecated/required_to_optional.go +++ b/src/pkg/packager/deprecated/required_to_optional.go @@ -7,7 +7,7 @@ package deprecated import ( "fmt" - "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" + "github.com/defenseunicorns/pkg/helpers" "github.com/defenseunicorns/zarf/src/types" ) diff --git a/src/pkg/packager/filters/deploy.go b/src/pkg/packager/filters/deploy.go index 07e87169fd..272e376ff8 100644 --- a/src/pkg/packager/filters/deploy.go +++ b/src/pkg/packager/filters/deploy.go @@ -77,7 +77,6 @@ func (f *deploymentFilter) Apply(pkg types.ZarfPackage) ([]types.ZarfComponent, if isPartial { matchedRequests := map[string]bool{} - // NOTE: This does not use forIncludedComponents as it takes group, default and required status into account. for _, groupKey := range orderedComponentGroups { var groupDefault *types.ZarfComponent var groupSelected *types.ZarfComponent diff --git a/src/pkg/packager/filters/deploy_test.go b/src/pkg/packager/filters/deploy_test.go index 4d666494d6..b8ae18c6fa 100644 --- a/src/pkg/packager/filters/deploy_test.go +++ b/src/pkg/packager/filters/deploy_test.go @@ -55,67 +55,53 @@ func componentFromQuery(t *testing.T, q string) types.ZarfComponent { return c } -func componentMatrix(_ *testing.T) []types.ZarfComponent { +// componentMatrix generates all possible combinations of component values +// for testing the deploy filter. +// some of these combinations are invalid, but the deploy filter should +// handle them gracefully. +func componentMatrix(t *testing.T) []types.ZarfComponent { var components []types.ZarfComponent defaultValues := []bool{true, false} requiredValues := []interface{}{nil, true, false} - // TODO: add optional to matrix - // optionalValues := []interface{}{nil, true, false} - // the duplicate groups are intentional - // this is to test group membership + default filtering - groupValues := []string{"", "foo", "foo", "foo", "bar", "bar", "bar"} - - for idx, groupValue := range groupValues { - for _, defaultValue := range defaultValues { + optionalValues := []interface{}{nil, true, false} + + // all possible combinations of default, required, and optional + for _, defaultValue := range defaultValues { + for _, optionalValue := range optionalValues { for _, requiredValue := range requiredValues { - name := strings.Builder{} - // per validate rules, components in groups cannot be required - if requiredValue != nil && requiredValue.(bool) == true && groupValue != "" { + // per validate, components cannot be both default and required + if defaultValue == true && (requiredValue == true || optionalValue == false || optionalValue == nil) { continue } - name.WriteString(fmt.Sprintf("required=%v", requiredValue)) - - if groupValue != "" { - name.WriteString(fmt.Sprintf(" && group=%s && idx=%d && default=%t", groupValue, idx, defaultValue)) - } else if defaultValue { - name.WriteString(" && default=true") - } - - if groupValue != "" { - // if there already exists a component in this group that is default, then set the default to false - // otherwise the filter will error - defaultAlreadyExists := false - if defaultValue { - for _, c := range components { - if c.DeprecatedGroup == groupValue && c.Default { - defaultAlreadyExists = true - break - } - } - } - if defaultAlreadyExists { - defaultValue = false - } - } - - c := types.ZarfComponent{ - Name: name.String(), - Default: defaultValue, - DeprecatedGroup: groupValue, - } - - if requiredValue != nil { - c.Required = helpers.BoolPtr(requiredValue.(bool)) - } + query := fmt.Sprintf("default=%t && required=%v && optional=%v", defaultValue, requiredValue, optionalValue) + c := componentFromQuery(t, query) components = append(components, c) } } } + members := 3 + for _, group := range []string{"foo", "bar"} { + for i := 0; i < members; i++ { + var defaultValue bool + // ensure there is only one default per group + // this enforced on `zarf package create`'s validate + if i == 0 { + defaultValue = true + } + c := componentFromQuery(t, fmt.Sprintf("group=%s && idx=%d && default=%t", group, i, defaultValue)) + // due to validation on create, there will not be a case where + // c.Default == true && (c.Optional = false || c.DeprecatedRequired = true) + c.Optional = nil + c.DeprecatedRequired = nil + components = append(components, c) + } + } + return components } @@ -129,60 +115,65 @@ func TestDeployFilter_Apply(t *testing.T) { want []types.ZarfComponent expectedErr error }{ - "Test when version is less than v0.33.0 w/ no optional components selected": { + "[v0.32.6] Test when no optional components selected": { pkg: types.ZarfPackage{ Build: types.ZarfBuildData{ - Version: "v0.32.0", + Version: "v0.32.6", }, Components: possibilities, }, optionalComponents: "", want: []types.ZarfComponent{ - componentFromQuery(t, "required= && default=true"), - componentFromQuery(t, "required=true && default=true"), - componentFromQuery(t, "required=false && default=true"), - componentFromQuery(t, "required=true"), - componentFromQuery(t, "required= && group=foo && idx=1 && default=true"), - componentFromQuery(t, "required= && group=bar && idx=4 && default=true"), + componentFromQuery(t, "default=true && required= && optional=true"), + componentFromQuery(t, "default=true && required=false && optional=true"), + componentFromQuery(t, "default=false && required=true && optional="), + componentFromQuery(t, "default=false && required=true && optional=true"), + componentFromQuery(t, "default=false && required=true && optional=false"), + componentFromQuery(t, "group=foo && idx=0 && default=true"), + componentFromQuery(t, "group=bar && idx=0 && default=true"), }, }, - "Test when version is less than v0.33.0 w/ some optional components selected": { + "[v0.32.6] Test when some optional components selected": { pkg: types.ZarfPackage{ Build: types.ZarfBuildData{ - Version: "v0.32.0", + Version: "v0.32.6", }, Components: possibilities, }, - optionalComponents: strings.Join([]string{"required=false", "required= && group=bar && idx=5 && default=false", "-required=true"}, ","), + optionalComponents: strings.Join([]string{ + "default=false && required=false && optional=true", + "group=bar && idx=2 && default=false", + "-default=false && required=true && optional=false", + }, ","), want: []types.ZarfComponent{ - componentFromQuery(t, "required= && default=true"), - componentFromQuery(t, "required=true && default=true"), - componentFromQuery(t, "required=false && default=true"), - // while "required=true" was deselected, it is still required - // therefore it should be included - componentFromQuery(t, "required=true"), - componentFromQuery(t, "required=false"), - componentFromQuery(t, "required= && group=foo && idx=1 && default=true"), - componentFromQuery(t, "required= && group=bar && idx=5 && default=false"), + componentFromQuery(t, "default=true && required= && optional=true"), + componentFromQuery(t, "default=true && required=false && optional=true"), + componentFromQuery(t, "default=false && required=true && optional="), + componentFromQuery(t, "default=false && required=true && optional=true"), + componentFromQuery(t, "default=false && required=false && optional=true"), + componentFromQuery(t, "default=false && required=true && optional=false"), + componentFromQuery(t, "group=foo && idx=0 && default=true"), + componentFromQuery(t, "group=bar && idx=2 && default=false"), }, }, - "Test failing when group has no default and no selection was made": { + "[v0.32.6] Test failing when group has no default and no selection was made": { pkg: types.ZarfPackage{ Build: types.ZarfBuildData{ - Version: "v0.32.0", + Version: "v0.32.6", }, Components: []types.ZarfComponent{ + componentFromQuery(t, "group=foo && default=true"), componentFromQuery(t, "group=foo && default=false"), componentFromQuery(t, "group=foo && default=false"), }, }, - optionalComponents: "", + optionalComponents: "-group=foo && default=true", expectedErr: ErrNoDefaultOrSelection, }, - "Test failing when multiple are selected from the same group": { + "[v0.32.6] Test failing when multiple are selected from the same group": { pkg: types.ZarfPackage{ Build: types.ZarfBuildData{ - Version: "v0.32.0", + Version: "v0.32.6", }, Components: []types.ZarfComponent{ componentFromQuery(t, "group=foo && default=true"), @@ -192,16 +183,37 @@ func TestDeployFilter_Apply(t *testing.T) { optionalComponents: strings.Join([]string{"group=foo && default=false", "group=foo && default=true"}, ","), expectedErr: ErrMultipleSameGroup, }, - "Test failing when no components are found that match the query": { + "[v0.32.6] Test failing when no components are found that match the query": { pkg: types.ZarfPackage{ Build: types.ZarfBuildData{ - Version: "v0.32.0", + Version: "v0.32.6", }, Components: possibilities, }, optionalComponents: "nonexistent", expectedErr: ErrNotFound, }, + "[v0.33.0+] Test when no optional components selected": { + pkg: types.ZarfPackage{ + Build: types.ZarfBuildData{ + Version: "v0.33.0", + }, + Components: possibilities, + }, + optionalComponents: "", + want: []types.ZarfComponent{ + componentFromQuery(t, "default=true && required= && optional=true"), + componentFromQuery(t, "default=true && required=false && optional=true"), + componentFromQuery(t, "default=false && required= && optional="), + componentFromQuery(t, "default=false && required=true && optional="), + componentFromQuery(t, "default=false && required=false && optional="), + componentFromQuery(t, "default=false && required= && optional=false"), + componentFromQuery(t, "default=false && required=true && optional=false"), + componentFromQuery(t, "default=false && required=false && optional=false"), + componentFromQuery(t, "group=foo && idx=0 && default=true"), + componentFromQuery(t, "group=bar && idx=0 && default=true"), + }, + }, } for name, tc := range testCases { diff --git a/src/pkg/packager/filters/platform.go b/src/pkg/packager/filters/platform.go deleted file mode 100644 index d6d4518a31..0000000000 --- a/src/pkg/packager/filters/platform.go +++ /dev/null @@ -1,69 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: 2021-Present The Zarf Authors - -// Package filters contains core implementations of the ComponentFilterStrategy interface. -package filters - -import ( - "fmt" - "slices" - - "github.com/defenseunicorns/zarf/src/pkg/message" - "github.com/defenseunicorns/zarf/src/types" -) - -// ByArchAndOS creates an architecture and OS filter. -func ByArchAndOS(arch string, os string) ComponentFilterStrategy { - return &archAndOSFilter{arch, os} -} - -// archAndOSFilter filters components based on OS/architecture. -type archAndOSFilter struct { - arch string - os string -} - -// Define allowed OS and architecture, an empty string means it is allowed on all platforms. -// same as enums on ZarfComponentOnlyTarget -var allowedOs = []string{"linux", "darwin", "windows", ""} - -// same as enums on ZarfComponentOnlyCluster -var allowedArch = []string{"amd64", "arm64", ""} - -// Apply applies the filter. -func (f *archAndOSFilter) Apply(pkg types.ZarfPackage) ([]types.ZarfComponent, error) { - if !slices.Contains(allowedOs, f.os) { - return nil, fmt.Errorf("invalid OS: %s", f.os) - } - - if !slices.Contains(allowedArch, f.arch) { - return nil, fmt.Errorf("invalid architecture: %s", f.arch) - } - - filtered := []types.ZarfComponent{} - // Filter each component to only compatible platforms. - for _, component := range pkg.Components { - // Ignore only filters that are empty - var validArch, validOS bool - - // Test for valid architecture - if component.Only.Cluster.Architecture == "" || component.Only.Cluster.Architecture == f.arch { - validArch = true - } else { - message.Debugf("Skipping component %s, %s is not compatible with %s", component.Name, component.Only.Cluster.Architecture, f.arch) - } - - // Test for a valid OS - if component.Only.LocalOS == "" || component.Only.LocalOS == f.os { - validOS = true - } else { - message.Debugf("Skipping component %s, %s is not compatible with %s", component.Name, component.Only.LocalOS, f.os) - } - - // If both the OS and architecture are valid, add the component to the filtered list - if validArch && validOS { - filtered = append(filtered, component) - } - } - return filtered, nil -} diff --git a/src/pkg/packager/filters/platform_test.go b/src/pkg/packager/filters/platform_test.go deleted file mode 100644 index d346b865e9..0000000000 --- a/src/pkg/packager/filters/platform_test.go +++ /dev/null @@ -1,49 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: 2021-Present The Zarf Authors - -package filters - -import ( - "testing" - - "github.com/defenseunicorns/zarf/src/types" - "github.com/stretchr/testify/require" -) - -func archOSMatrix() [][2]string { - // Create a matrix of all possible combinations - matrix := [][2]string{} - for _, os := range allowedOs { - for _, arch := range allowedArch { - matrix = append(matrix, [2]string{arch, os}) - } - } - return matrix -} - -func TestArchAndOSFilter(t *testing.T) { - // Create a test package - pkg := types.ZarfPackage{} - for _, platform := range archOSMatrix() { - pkg.Components = append(pkg.Components, types.ZarfComponent{ - Only: types.ZarfComponentOnlyTarget{ - LocalOS: platform[1], - Cluster: types.ZarfComponentOnlyCluster{Architecture: platform[0]}, - }, - }) - } - - for _, platform := range archOSMatrix() { - filter := ByArchAndOS(platform[0], platform[1]) - result, err := filter.Apply(pkg) - require.NoError(t, err) - for _, component := range result { - if component.Only.Cluster.Architecture != "" { - require.Equal(t, platform[0], component.Only.Cluster.Architecture) - } - if component.Only.LocalOS != "" { - require.Equal(t, platform[1], component.Only.LocalOS) - } - } - } -} diff --git a/src/pkg/packager/filters/utils_test.go b/src/pkg/packager/filters/utils_test.go index 270c5e6615..be00a3065c 100644 --- a/src/pkg/packager/filters/utils_test.go +++ b/src/pkg/packager/filters/utils_test.go @@ -7,7 +7,7 @@ package filters import ( "testing" - "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" + "github.com/defenseunicorns/pkg/helpers" "github.com/defenseunicorns/zarf/src/types" "github.com/stretchr/testify/require" ) From 2a66b2930e3d21a8c13fde5ce30720160387a592 Mon Sep 17 00:00:00 2001 From: razzle Date: Tue, 2 Apr 2024 11:48:17 -0500 Subject: [PATCH 081/113] feature flag Signed-off-by: razzle --- src/config/config.go | 3 - src/config/lang/english.go | 6 +- src/internal/packager/validate/validate.go | 372 ------------------ src/pkg/packager/composer/list.go | 8 +- src/pkg/packager/composer/override.go | 3 +- src/pkg/packager/create.go | 3 +- src/pkg/packager/deprecated/common.go | 1 - .../deprecated/required_to_optional.go | 53 --- src/pkg/packager/dev.go | 3 +- src/pkg/packager/filters/deploy.go | 21 +- src/pkg/packager/filters/deploy_test.go | 103 ++--- src/pkg/packager/filters/os_test.go | 5 +- src/pkg/packager/filters/utils.go | 34 -- src/pkg/packager/filters/utils_test.go | 133 ------- src/pkg/packager/generate.go | 3 +- src/pkg/packager/sources/cluster.go | 3 +- src/types/component.go | 30 +- src/types/package.go | 42 +- src/types/validate.go | 368 +++++++++++++++++ zarf.yaml | 7 +- 20 files changed, 480 insertions(+), 721 deletions(-) delete mode 100644 src/internal/packager/validate/validate.go delete mode 100644 src/pkg/packager/deprecated/required_to_optional.go create mode 100644 src/types/validate.go diff --git a/src/config/config.go b/src/config/config.go index b8e3e2d386..01cf6d2c0c 100644 --- a/src/config/config.go +++ b/src/config/config.go @@ -25,9 +25,6 @@ import ( const ( GithubProject = "defenseunicorns/zarf" - // ZarfMaxChartNameLength limits helm chart name size to account for K8s/helm limits and zarf prefix - ZarfMaxChartNameLength = 40 - ZarfAgentHost = "agent-hook.zarf.svc" ZarfConnectLabelName = "zarf.dev/connect-name" diff --git a/src/config/lang/english.go b/src/config/lang/english.go index 2ce8951a21..84843ac553 100644 --- a/src/config/lang/english.go +++ b/src/config/lang/english.go @@ -480,7 +480,7 @@ $ zarf tools registry digest reg.example.com/stefanprodan/podinfo:6.4.0 CmdToolsGetGitPasswdShort = "[Deprecated] Returns the push user's password for the Git server" CmdToolsGetGitPasswdLong = "[Deprecated] Reads the password for a user with push access to the configured Git server in Zarf State. Note that this command has been replaced by 'zarf tools get-creds git' and will be removed in Zarf v1.0.0." CmdToolsGetGitPasswdDeprecation = "Deprecated: This command has been replaced by 'zarf tools get-creds git' and will be removed in Zarf v1.0.0." - CmdToolsYqExample = ` + CmdToolsYqExample = ` # yq defaults to 'eval' command if no command is specified. See "zarf tools yq eval --help" for more examples. # read the "stuff" node from "myfile.yml" @@ -687,10 +687,10 @@ const ( // Package validate const ( + PkgValidateErrBetaFeatureNotFound = "beta feature %q not found, available features: %s" PkgValidateTemplateDeprecation = "Package template %q is using the deprecated syntax ###ZARF_PKG_VAR_%s###. This will be removed in Zarf v1.0.0. Please update to ###ZARF_PKG_TMPL_%s###." PkgValidateMustBeUppercase = "variable name %q must be all uppercase and contain no special characters except _" PkgValidateErrAction = "invalid action: %w" - PkgValidateErrActionVariables = "component %q cannot contain setVariables outside of onDeploy in actions" PkgValidateErrActionCmdWait = "action %q cannot be both a command and wait action" PkgValidateErrActionClusterNetwork = "a single wait action must contain only one of cluster or network" PkgValidateErrChart = "invalid chart definition: %w" @@ -704,7 +704,7 @@ const ( PkgValidateErrComponentLocalOS = "component %q contains a localOS value that is not supported: %s (supported: %s)" PkgValidateErrComponentNameNotUnique = "component name %q is not unique" PkgValidateErrComponent = "invalid component %q: %w" - PkgValidateErrComponentRequired = "component %q contains deprecated usage of required. Please use `zarf dev migrate --run required-to-optional` to migrate your zarf.yaml" + PkgValidateErrComponentMissingGroup = "component %q cannot use default without a group" PkgValidateErrComponentReqDefault = "component %q cannot be both required and default" PkgValidateErrComponentReqGrouped = "component %q cannot be both required and grouped" PkgValidateErrComponentYOLO = "component %q incompatible with the online-only package flag (metadata.yolo): %w" diff --git a/src/internal/packager/validate/validate.go b/src/internal/packager/validate/validate.go deleted file mode 100644 index 0eea4c0b3a..0000000000 --- a/src/internal/packager/validate/validate.go +++ /dev/null @@ -1,372 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: 2021-Present The Zarf Authors - -// Package validate provides Zarf package validation functions. -package validate - -import ( - "fmt" - "path/filepath" - "regexp" - "slices" - - "github.com/defenseunicorns/pkg/helpers" - "github.com/defenseunicorns/zarf/src/config" - "github.com/defenseunicorns/zarf/src/config/lang" - "github.com/defenseunicorns/zarf/src/types" -) - -var ( - // IsLowercaseNumberHyphenNoStartHyphen is a regex for lowercase, numbers and hyphens that cannot start with a hyphen. - // https://regex101.com/r/FLdG9G/2 - IsLowercaseNumberHyphenNoStartHyphen = regexp.MustCompile(`^[a-z0-9][a-z0-9\-]*$`).MatchString - // IsUppercaseNumberUnderscore is a regex for uppercase, numbers and underscores. - // https://regex101.com/r/tfsEuZ/1 - IsUppercaseNumberUnderscore = regexp.MustCompile(`^[A-Z0-9_]+$`).MatchString - // Define allowed OS, an empty string means it is allowed on all operating systems - // same as enums on ZarfComponentOnlyTarget - supportedOS = []string{"linux", "darwin", "windows", ""} -) - -// SupportedOS returns the supported operating systems. -// -// The supported operating systems are: linux, darwin, windows. -// -// An empty string signifies no OS restrictions. -func SupportedOS() []string { - return supportedOS -} - -// Run performs config validations. -func Run(pkg types.ZarfPackage) error { - if pkg.Kind == types.ZarfInitConfig && pkg.Metadata.YOLO { - return fmt.Errorf(lang.PkgValidateErrInitNoYOLO) - } - - if err := validatePackageName(pkg.Metadata.Name); err != nil { - return fmt.Errorf(lang.PkgValidateErrName, err) - } - - for _, variable := range pkg.Variables { - if err := validatePackageVariable(variable); err != nil { - return fmt.Errorf(lang.PkgValidateErrVariable, err) - } - } - - for _, constant := range pkg.Constants { - if err := validatePackageConstant(constant); err != nil { - return fmt.Errorf(lang.PkgValidateErrConstant, err) - } - } - - uniqueComponentNames := make(map[string]bool) - groupDefault := make(map[string]string) - groupedComponents := make(map[string][]string) - - for _, component := range pkg.Components { - // ensure component name is unique - if _, ok := uniqueComponentNames[component.Name]; ok { - return fmt.Errorf(lang.PkgValidateErrComponentNameNotUnique, component.Name) - } - uniqueComponentNames[component.Name] = true - - if err := validateComponent(pkg, component); err != nil { - return fmt.Errorf(lang.PkgValidateErrComponent, component.Name, err) - } - - // ensure groups don't have multiple defaults or only one component - if component.DeprecatedGroup != "" { - if component.Default { - if _, ok := groupDefault[component.DeprecatedGroup]; ok { - return fmt.Errorf(lang.PkgValidateErrGroupMultipleDefaults, component.DeprecatedGroup, groupDefault[component.DeprecatedGroup], component.Name) - } - groupDefault[component.DeprecatedGroup] = component.Name - } - groupedComponents[component.DeprecatedGroup] = append(groupedComponents[component.DeprecatedGroup], component.Name) - } - } - - for groupKey, componentNames := range groupedComponents { - if len(componentNames) == 1 { - return fmt.Errorf(lang.PkgValidateErrGroupOneComponent, groupKey, componentNames[0]) - } - } - - return nil -} - -// ImportDefinition validates the component trying to be imported. -func ImportDefinition(component *types.ZarfComponent) error { - path := component.Import.Path - url := component.Import.URL - - // ensure path or url is provided - if path == "" && url == "" { - return fmt.Errorf(lang.PkgValidateErrImportDefinition, component.Name, "neither a path nor a URL was provided") - } - - // ensure path and url are not both provided - if path != "" && url != "" { - return fmt.Errorf(lang.PkgValidateErrImportDefinition, component.Name, "both a path and a URL were provided") - } - - // validation for path - if url == "" && path != "" { - // ensure path is not an absolute path - if filepath.IsAbs(path) { - return fmt.Errorf(lang.PkgValidateErrImportDefinition, component.Name, "path cannot be an absolute path") - } - } - - // validation for url - if url != "" && path == "" { - ok := helpers.IsOCIURL(url) - if !ok { - return fmt.Errorf(lang.PkgValidateErrImportDefinition, component.Name, "URL is not a valid OCI URL") - } - } - - return nil -} - -func oneIfNotEmpty(testString string) int { - if testString == "" { - return 0 - } - - return 1 -} - -func validateComponent(pkg types.ZarfPackage, component types.ZarfComponent) error { - if !IsLowercaseNumberHyphenNoStartHyphen(component.Name) { - return fmt.Errorf(lang.PkgValidateErrComponentName, component.Name) - } - - if component.DeprecatedRequired != nil { - return fmt.Errorf(lang.PkgValidateErrComponentRequired, component.Name) - } - - if !slices.Contains(supportedOS, component.Only.LocalOS) { - return fmt.Errorf(lang.PkgValidateErrComponentLocalOS, component.Name, component.Only.LocalOS, supportedOS) - } - - if component.Optional != nil || !*component.Optional { - if component.Default { - return fmt.Errorf(lang.PkgValidateErrComponentReqDefault, component.Name) - } - if component.DeprecatedGroup != "" { - return fmt.Errorf(lang.PkgValidateErrComponentReqGrouped, component.Name) - } - } - - uniqueChartNames := make(map[string]bool) - for _, chart := range component.Charts { - // ensure chart name is unique - if _, ok := uniqueChartNames[chart.Name]; ok { - return fmt.Errorf(lang.PkgValidateErrChartNameNotUnique, chart.Name) - } - uniqueChartNames[chart.Name] = true - - if err := validateChart(chart); err != nil { - return fmt.Errorf(lang.PkgValidateErrChart, err) - } - } - - uniqueManifestNames := make(map[string]bool) - for _, manifest := range component.Manifests { - // ensure manifest name is unique - if _, ok := uniqueManifestNames[manifest.Name]; ok { - return fmt.Errorf(lang.PkgValidateErrManifestNameNotUnique, manifest.Name) - } - uniqueManifestNames[manifest.Name] = true - - if err := validateManifest(manifest); err != nil { - return fmt.Errorf(lang.PkgValidateErrManifest, err) - } - } - - if pkg.Metadata.YOLO { - if err := validateYOLO(component); err != nil { - return fmt.Errorf(lang.PkgValidateErrComponentYOLO, component.Name, err) - } - } - - if containsVariables, err := validateActionset(component.Actions.OnCreate); err != nil { - return fmt.Errorf(lang.PkgValidateErrAction, err) - } else if containsVariables { - return fmt.Errorf(lang.PkgValidateErrActionVariables, component.Name) - } - - if _, err := validateActionset(component.Actions.OnDeploy); err != nil { - return fmt.Errorf(lang.PkgValidateErrAction, err) - } - - if containsVariables, err := validateActionset(component.Actions.OnRemove); err != nil { - return fmt.Errorf(lang.PkgValidateErrAction, err) - } else if containsVariables { - return fmt.Errorf(lang.PkgValidateErrActionVariables, component.Name) - } - - return nil -} - -func validateActionset(actions types.ZarfComponentActionSet) (bool, error) { - containsVariables := false - - validate := func(actions []types.ZarfComponentAction) error { - for _, action := range actions { - if cv, err := validateAction(action); err != nil { - return err - } else if cv { - containsVariables = true - } - } - - return nil - } - - if err := validate(actions.Before); err != nil { - return containsVariables, err - } - if err := validate(actions.After); err != nil { - return containsVariables, err - } - if err := validate(actions.OnSuccess); err != nil { - return containsVariables, err - } - if err := validate(actions.OnFailure); err != nil { - return containsVariables, err - } - - return containsVariables, nil -} - -func validateAction(action types.ZarfComponentAction) (bool, error) { - containsVariables := false - - // Validate SetVariable - for _, variable := range action.SetVariables { - if !IsUppercaseNumberUnderscore(variable.Name) { - return containsVariables, fmt.Errorf(lang.PkgValidateMustBeUppercase, variable.Name) - } - containsVariables = true - } - - if action.Wait != nil { - // Validate only cmd or wait, not both - if action.Cmd != "" { - return containsVariables, fmt.Errorf(lang.PkgValidateErrActionCmdWait, action.Cmd) - } - - // Validate only cluster or network, not both - if action.Wait.Cluster != nil && action.Wait.Network != nil { - return containsVariables, fmt.Errorf(lang.PkgValidateErrActionClusterNetwork) - } - - // Validate at least one of cluster or network - if action.Wait.Cluster == nil && action.Wait.Network == nil { - return containsVariables, fmt.Errorf(lang.PkgValidateErrActionClusterNetwork) - } - } - - return containsVariables, nil -} - -func validateYOLO(component types.ZarfComponent) error { - if len(component.Images) > 0 { - return fmt.Errorf(lang.PkgValidateErrYOLONoOCI) - } - - if len(component.Repos) > 0 { - return fmt.Errorf(lang.PkgValidateErrYOLONoGit) - } - - if component.Only.Cluster.Architecture != "" { - return fmt.Errorf(lang.PkgValidateErrYOLONoArch) - } - - if len(component.Only.Cluster.Distros) > 0 { - return fmt.Errorf(lang.PkgValidateErrYOLONoDistro) - } - - return nil -} - -func validatePackageName(subject string) error { - if !IsLowercaseNumberHyphenNoStartHyphen(subject) { - return fmt.Errorf(lang.PkgValidateErrPkgName, subject) - } - - return nil -} - -func validatePackageVariable(subject types.ZarfPackageVariable) error { - // ensure the variable name is only capitals and underscores - if !IsUppercaseNumberUnderscore(subject.Name) { - return fmt.Errorf(lang.PkgValidateMustBeUppercase, subject.Name) - } - - return nil -} - -func validatePackageConstant(subject types.ZarfPackageConstant) error { - // ensure the constant name is only capitals and underscores - if !IsUppercaseNumberUnderscore(subject.Name) { - return fmt.Errorf(lang.PkgValidateErrPkgConstantName, subject.Name) - } - - if !regexp.MustCompile(subject.Pattern).MatchString(subject.Value) { - return fmt.Errorf(lang.PkgValidateErrPkgConstantPattern, subject.Name, subject.Pattern) - } - - return nil -} - -func validateChart(chart types.ZarfChart) error { - // Don't allow empty names - if chart.Name == "" { - return fmt.Errorf(lang.PkgValidateErrChartNameMissing, chart.Name) - } - - // Helm max release name - if len(chart.Name) > config.ZarfMaxChartNameLength { - return fmt.Errorf(lang.PkgValidateErrChartName, chart.Name, config.ZarfMaxChartNameLength) - } - - // Must have a namespace - if chart.Namespace == "" { - return fmt.Errorf(lang.PkgValidateErrChartNamespaceMissing, chart.Name) - } - - // Must have a url or localPath (and not both) - count := oneIfNotEmpty(chart.URL) + oneIfNotEmpty(chart.LocalPath) - if count != 1 { - return fmt.Errorf(lang.PkgValidateErrChartURLOrPath, chart.Name) - } - - // Must have a version - if chart.Version == "" { - return fmt.Errorf(lang.PkgValidateErrChartVersion, chart.Name) - } - - return nil -} - -func validateManifest(manifest types.ZarfManifest) error { - // Don't allow empty names - if manifest.Name == "" { - return fmt.Errorf(lang.PkgValidateErrManifestNameMissing, manifest.Name) - } - - // Helm max release name - if len(manifest.Name) > config.ZarfMaxChartNameLength { - return fmt.Errorf(lang.PkgValidateErrManifestNameLength, manifest.Name, config.ZarfMaxChartNameLength) - } - - // Require files in manifest - if len(manifest.Files) < 1 && len(manifest.Kustomizations) < 1 { - return fmt.Errorf(lang.PkgValidateErrManifestFileOrKustomize, manifest.Name) - } - - return nil -} diff --git a/src/pkg/packager/composer/list.go b/src/pkg/packager/composer/list.go index 6b290203ea..e53f26a659 100644 --- a/src/pkg/packager/composer/list.go +++ b/src/pkg/packager/composer/list.go @@ -11,7 +11,6 @@ import ( "strings" "github.com/defenseunicorns/pkg/helpers" - "github.com/defenseunicorns/zarf/src/internal/packager/validate" "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/packager/deprecated" "github.com/defenseunicorns/zarf/src/pkg/utils" @@ -141,7 +140,7 @@ func NewImportChain(head types.ZarfComponent, index int, originalPackageName, ar } // TODO: stuff like this should also happen in linting - if err := validate.ImportDefinition(&node.ZarfComponent); err != nil { + if err := node.ZarfComponent.ValidateImportDefinition(); err != nil { return ic, err } @@ -291,6 +290,11 @@ func (ic *ImportChain) Compose() (composed *types.ZarfComponent, err error) { // start overriding with the tail node node := ic.tail for node != nil { + // if prev.required is nil, and node.required is != nil, then error + if node.prev != nil && node.prev.Required == nil && node.Required != nil { + return nil, fmt.Errorf("component %q: required cannot be unset during composition from %q to %q", node.ImportName(), node.ImportLocation(), node.prev.ImportLocation()) + } + fixPaths(&node.ZarfComponent, node.relativeToHead) // perform overrides here diff --git a/src/pkg/packager/composer/override.go b/src/pkg/packager/composer/override.go index 3ae12c0068..4a07a1d936 100644 --- a/src/pkg/packager/composer/override.go +++ b/src/pkg/packager/composer/override.go @@ -13,8 +13,7 @@ import ( func overrideMetadata(c *types.ZarfComponent, override types.ZarfComponent) error { c.Name = override.Name c.Default = override.Default - c.Optional = override.Optional - c.DeprecatedRequired = override.DeprecatedRequired + c.Required = override.Required // Override description if it was provided. if override.Description != "" { diff --git a/src/pkg/packager/create.go b/src/pkg/packager/create.go index 9ef4523fca..94e3cb94dc 100755 --- a/src/pkg/packager/create.go +++ b/src/pkg/packager/create.go @@ -10,7 +10,6 @@ import ( "github.com/defenseunicorns/pkg/helpers" "github.com/defenseunicorns/zarf/src/config" - "github.com/defenseunicorns/zarf/src/internal/packager/validate" "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/packager/creator" @@ -41,7 +40,7 @@ func (p *Packager) Create() (err error) { } // Perform early package validation. - if err := validate.Run(p.cfg.Pkg); err != nil { + if err := p.cfg.Pkg.Validate(); err != nil { return fmt.Errorf("unable to validate package: %w", err) } diff --git a/src/pkg/packager/deprecated/common.go b/src/pkg/packager/deprecated/common.go index c3f87d1bef..d57b666497 100644 --- a/src/pkg/packager/deprecated/common.go +++ b/src/pkg/packager/deprecated/common.go @@ -55,7 +55,6 @@ func Migrations() []Migration { return []Migration{ ScriptsToActions{}, SetVariableToSetVariables{}, - RequiredToOptional{}, } } diff --git a/src/pkg/packager/deprecated/required_to_optional.go b/src/pkg/packager/deprecated/required_to_optional.go deleted file mode 100644 index 6436e9eda2..0000000000 --- a/src/pkg/packager/deprecated/required_to_optional.go +++ /dev/null @@ -1,53 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: 2021-Present The Zarf Authors - -// Package deprecated handles package deprecations and migrations -package deprecated - -import ( - "fmt" - - "github.com/defenseunicorns/pkg/helpers" - "github.com/defenseunicorns/zarf/src/types" -) - -// RequiredToOptionalID is the ID of the RequiredToOptional migration -const RequiredToOptionalID = "required-to-optional" - -// RequiredToOptional migrates required to optional -type RequiredToOptional struct{} - -// ID returns the ID of the migration -func (m RequiredToOptional) ID() string { - return RequiredToOptionalID -} - -// Clear the deprecated required. -func (m RequiredToOptional) Clear(mc types.ZarfComponent) types.ZarfComponent { - mc.DeprecatedRequired = nil - if mc.Optional != nil && !*mc.Optional { - mc.Optional = nil - } - return mc -} - -// Run migrates required to optional. -func (m RequiredToOptional) Run(c types.ZarfComponent) (types.ZarfComponent, string) { - if c.DeprecatedRequired == nil { - return c, "" - } - - warning := fmt.Sprintf("Component %q is using \"required\" which will be removed in Zarf v1.0.0. Please migrate to \"optional\".", c.Name) - - // Make optional agree with required if not set - if c.Optional == nil && c.DeprecatedRequired != nil { - switch *c.DeprecatedRequired { - case true: - c.Optional = helpers.BoolPtr(false) - case false: - c.Optional = helpers.BoolPtr(true) - } - } - - return c, warning -} diff --git a/src/pkg/packager/dev.go b/src/pkg/packager/dev.go index 99b73fc1d6..bbf026c388 100644 --- a/src/pkg/packager/dev.go +++ b/src/pkg/packager/dev.go @@ -11,7 +11,6 @@ import ( "github.com/defenseunicorns/pkg/helpers" "github.com/defenseunicorns/zarf/src/config" - "github.com/defenseunicorns/zarf/src/internal/packager/validate" "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/packager/creator" @@ -62,7 +61,7 @@ func (p *Packager) DevDeploy() error { return err } - if err := validate.Run(p.cfg.Pkg); err != nil { + if err := p.cfg.Pkg.Validate(); err != nil { return fmt.Errorf("unable to validate package: %w", err) } diff --git a/src/pkg/packager/filters/deploy.go b/src/pkg/packager/filters/deploy.go index 272e376ff8..e791363cb9 100644 --- a/src/pkg/packager/filters/deploy.go +++ b/src/pkg/packager/filters/deploy.go @@ -9,10 +9,8 @@ import ( "slices" "strings" - "github.com/Masterminds/semver/v3" "github.com/agnivade/levenshtein" "github.com/defenseunicorns/pkg/helpers" - "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/pkg/interactive" "github.com/defenseunicorns/zarf/src/types" ) @@ -35,7 +33,7 @@ type deploymentFilter struct { // Errors for the deployment filter. var ( - ErrMultipleSameGroup = fmt.Errorf("cannot specify multiple components from the same group") + ErrMultipleSameGroup = fmt.Errorf("cannot specify multiple components") ErrNoDefaultOrSelection = fmt.Errorf("no default or selected component found") ErrNotFound = fmt.Errorf("no compatible components found") ErrSelectionCanceled = fmt.Errorf("selection canceled") @@ -43,17 +41,6 @@ var ( // Apply applies the filter. func (f *deploymentFilter) Apply(pkg types.ZarfPackage) ([]types.ZarfComponent, error) { - useRequiredLogic := false - if pkg.Build.Version != config.UnsetCLIVersion { - buildVersion, err := semver.NewVersion(pkg.Build.Version) - if err != nil { - return []types.ZarfComponent{}, fmt.Errorf("unable to parse package version %q: %w", pkg.Build.Version, err) - } - if buildVersion.LessThan(semver.MustParse("v0.33.0")) { - useRequiredLogic = true - } - } - var selectedComponents []types.ZarfComponent groupedComponents := map[string][]types.ZarfComponent{} orderedComponentGroups := []string{} @@ -87,7 +74,7 @@ func (f *deploymentFilter) Apply(pkg types.ZarfPackage) ([]types.ZarfComponent, selectState, matchedRequest := includedOrExcluded(component.Name, f.requestedComponents) - if !isRequired(component, useRequiredLogic) { + if !component.IsRequired(pkg.Metadata.BetaFeatures) { if selectState == excluded { // If the component was explicitly excluded, record the match and continue matchedRequests[matchedRequest] = true @@ -107,7 +94,7 @@ func (f *deploymentFilter) Apply(pkg types.ZarfPackage) ([]types.ZarfComponent, // Then check for already selected groups if groupSelected != nil { - return nil, fmt.Errorf("%w: group: %s selected: %s, %s", ErrMultipleSameGroup, component.DeprecatedGroup, groupSelected.Name, component.Name) + return nil, fmt.Errorf("%w: group: %q selected: %q, %s", ErrMultipleSameGroup, component.DeprecatedGroup, groupSelected.Name, component.Name) } // Then append to the final list @@ -173,7 +160,7 @@ func (f *deploymentFilter) Apply(pkg types.ZarfPackage) ([]types.ZarfComponent, } else { component := groupedComponents[groupKey][0] - if isRequired(component, useRequiredLogic) { + if component.IsRequired(pkg.Metadata.BetaFeatures) { selectedComponents = append(selectedComponents, component) continue } diff --git a/src/pkg/packager/filters/deploy_test.go b/src/pkg/packager/filters/deploy_test.go index b8ae18c6fa..aa1523c154 100644 --- a/src/pkg/packager/filters/deploy_test.go +++ b/src/pkg/packager/filters/deploy_test.go @@ -28,18 +28,12 @@ func componentFromQuery(t *testing.T, q string) types.ZarfComponent { c.Default = true case "default=false": c.Default = false - case "optional=": - c.Optional = nil - case "optional=false": - c.Optional = helpers.BoolPtr(false) - case "optional=true": - c.Optional = helpers.BoolPtr(true) case "required=": - c.DeprecatedRequired = nil + c.Required = nil case "required=false": - c.DeprecatedRequired = helpers.BoolPtr(false) + c.Required = helpers.BoolPtr(false) case "required=true": - c.DeprecatedRequired = helpers.BoolPtr(true) + c.Required = helpers.BoolPtr(true) default: if strings.HasPrefix(cond, "group=") { c.DeprecatedGroup = cond[6:] @@ -64,23 +58,20 @@ func componentMatrix(t *testing.T) []types.ZarfComponent { defaultValues := []bool{true, false} requiredValues := []interface{}{nil, true, false} - optionalValues := []interface{}{nil, true, false} // all possible combinations of default, required, and optional for _, defaultValue := range defaultValues { - for _, optionalValue := range optionalValues { - for _, requiredValue := range requiredValues { + for _, requiredValue := range requiredValues { - // per validate, components cannot be both default and required - if defaultValue == true && (requiredValue == true || optionalValue == false || optionalValue == nil) { - continue - } + // per validate, components cannot be both default and required + if defaultValue == true && requiredValue == true { + continue + } - query := fmt.Sprintf("default=%t && required=%v && optional=%v", defaultValue, requiredValue, optionalValue) + query := fmt.Sprintf("default=%t && required=%v", defaultValue, requiredValue) - c := componentFromQuery(t, query) - components = append(components, c) - } + c := componentFromQuery(t, query) + components = append(components, c) } } @@ -95,9 +86,8 @@ func componentMatrix(t *testing.T) []types.ZarfComponent { } c := componentFromQuery(t, fmt.Sprintf("group=%s && idx=%d && default=%t", group, i, defaultValue)) // due to validation on create, there will not be a case where - // c.Default == true && (c.Optional = false || c.DeprecatedRequired = true) - c.Optional = nil - c.DeprecatedRequired = nil + // c.Default == true && c.Required == true) + c.Required = nil components = append(components, c) } } @@ -115,52 +105,39 @@ func TestDeployFilter_Apply(t *testing.T) { want []types.ZarfComponent expectedErr error }{ - "[v0.32.6] Test when no optional components selected": { + "Test when no optional components selected": { pkg: types.ZarfPackage{ - Build: types.ZarfBuildData{ - Version: "v0.32.6", - }, Components: possibilities, }, optionalComponents: "", want: []types.ZarfComponent{ - componentFromQuery(t, "default=true && required= && optional=true"), - componentFromQuery(t, "default=true && required=false && optional=true"), - componentFromQuery(t, "default=false && required=true && optional="), - componentFromQuery(t, "default=false && required=true && optional=true"), - componentFromQuery(t, "default=false && required=true && optional=false"), + componentFromQuery(t, "default=true && required="), + componentFromQuery(t, "default=true && required=false"), + componentFromQuery(t, "default=false && required=true"), componentFromQuery(t, "group=foo && idx=0 && default=true"), componentFromQuery(t, "group=bar && idx=0 && default=true"), }, }, - "[v0.32.6] Test when some optional components selected": { + "Test when some optional components selected": { pkg: types.ZarfPackage{ - Build: types.ZarfBuildData{ - Version: "v0.32.6", - }, Components: possibilities, }, optionalComponents: strings.Join([]string{ - "default=false && required=false && optional=true", + "default=false && required=false", "group=bar && idx=2 && default=false", - "-default=false && required=true && optional=false", + "-default=false && required=true", }, ","), want: []types.ZarfComponent{ - componentFromQuery(t, "default=true && required= && optional=true"), - componentFromQuery(t, "default=true && required=false && optional=true"), - componentFromQuery(t, "default=false && required=true && optional="), - componentFromQuery(t, "default=false && required=true && optional=true"), - componentFromQuery(t, "default=false && required=false && optional=true"), - componentFromQuery(t, "default=false && required=true && optional=false"), + componentFromQuery(t, "default=true && required="), + componentFromQuery(t, "default=true && required=false"), + componentFromQuery(t, "default=false && required=true"), + componentFromQuery(t, "default=false && required=false"), componentFromQuery(t, "group=foo && idx=0 && default=true"), componentFromQuery(t, "group=bar && idx=2 && default=false"), }, }, - "[v0.32.6] Test failing when group has no default and no selection was made": { + "Test failing when group has no default and no selection was made": { pkg: types.ZarfPackage{ - Build: types.ZarfBuildData{ - Version: "v0.32.6", - }, Components: []types.ZarfComponent{ componentFromQuery(t, "group=foo && default=true"), componentFromQuery(t, "group=foo && default=false"), @@ -170,11 +147,8 @@ func TestDeployFilter_Apply(t *testing.T) { optionalComponents: "-group=foo && default=true", expectedErr: ErrNoDefaultOrSelection, }, - "[v0.32.6] Test failing when multiple are selected from the same group": { + "Test failing when multiple are selected from the same group": { pkg: types.ZarfPackage{ - Build: types.ZarfBuildData{ - Version: "v0.32.6", - }, Components: []types.ZarfComponent{ componentFromQuery(t, "group=foo && default=true"), componentFromQuery(t, "group=foo && default=false"), @@ -183,33 +157,28 @@ func TestDeployFilter_Apply(t *testing.T) { optionalComponents: strings.Join([]string{"group=foo && default=false", "group=foo && default=true"}, ","), expectedErr: ErrMultipleSameGroup, }, - "[v0.32.6] Test failing when no components are found that match the query": { + "Test failing when no components are found that match the query": { pkg: types.ZarfPackage{ - Build: types.ZarfBuildData{ - Version: "v0.32.6", - }, Components: possibilities, }, optionalComponents: "nonexistent", expectedErr: ErrNotFound, }, - "[v0.33.0+] Test when no optional components selected": { + "[default-required] Test when no optional components selected": { pkg: types.ZarfPackage{ - Build: types.ZarfBuildData{ - Version: "v0.33.0", + Metadata: types.ZarfMetadata{ + BetaFeatures: []types.FeatureFlag{ + types.DefaultRequired, + }, }, Components: possibilities, }, optionalComponents: "", want: []types.ZarfComponent{ - componentFromQuery(t, "default=true && required= && optional=true"), - componentFromQuery(t, "default=true && required=false && optional=true"), - componentFromQuery(t, "default=false && required= && optional="), - componentFromQuery(t, "default=false && required=true && optional="), - componentFromQuery(t, "default=false && required=false && optional="), - componentFromQuery(t, "default=false && required= && optional=false"), - componentFromQuery(t, "default=false && required=true && optional=false"), - componentFromQuery(t, "default=false && required=false && optional=false"), + componentFromQuery(t, "default=true && required="), + componentFromQuery(t, "default=true && required=false"), + componentFromQuery(t, "default=false && required="), + componentFromQuery(t, "default=false && required=true"), componentFromQuery(t, "group=foo && idx=0 && default=true"), componentFromQuery(t, "group=bar && idx=0 && default=true"), }, diff --git a/src/pkg/packager/filters/os_test.go b/src/pkg/packager/filters/os_test.go index 9e6b9e7722..7d54beecd3 100644 --- a/src/pkg/packager/filters/os_test.go +++ b/src/pkg/packager/filters/os_test.go @@ -7,7 +7,6 @@ package filters import ( "testing" - "github.com/defenseunicorns/zarf/src/internal/packager/validate" "github.com/defenseunicorns/zarf/src/types" "github.com/stretchr/testify/require" ) @@ -15,7 +14,7 @@ import ( func TestLocalOSFilter(t *testing.T) { pkg := types.ZarfPackage{} - for _, os := range validate.SupportedOS() { + for _, os := range types.SupportedOS() { pkg.Components = append(pkg.Components, types.ZarfComponent{ Only: types.ZarfComponentOnlyTarget{ LocalOS: os, @@ -23,7 +22,7 @@ func TestLocalOSFilter(t *testing.T) { }) } - for _, os := range validate.SupportedOS() { + for _, os := range types.SupportedOS() { filter := ByLocalOS(os) result, err := filter.Apply(pkg) if os == "" { diff --git a/src/pkg/packager/filters/utils.go b/src/pkg/packager/filters/utils.go index c87132703f..73b5174779 100644 --- a/src/pkg/packager/filters/utils.go +++ b/src/pkg/packager/filters/utils.go @@ -7,8 +7,6 @@ package filters import ( "path" "strings" - - "github.com/defenseunicorns/zarf/src/types" ) type selectState int @@ -43,35 +41,3 @@ func includedOrExcluded(componentName string, requestedComponentNames []string) return unknown, "" } -// isRequired returns if the component is required or not. -// -// The logic is as follows: -// -// 1. If "optional" is set, then the component follows the inverse of that value -// -// 2. If "required" is set, then the component follows that value -// -// 3. If neither "optional" nor "required" are set, then the component is required -// -// Note: "required" is deprecated and will be removed in Zarf v1.0.0 -func isRequired(c types.ZarfComponent, useRequiredLogic bool) bool { - requiredExists := c.DeprecatedRequired != nil - optionalExists := c.Optional != nil - required := requiredExists && *c.DeprecatedRequired - optional := optionalExists && *c.Optional - - if useRequiredLogic { - // old required logic does not know about optional - if requiredExists { - return required - } - return false - } - - // follow inverse of optional when defined - if optionalExists { - return !optional - } - // If neither required nor optional are set, then the component is required - return true -} diff --git a/src/pkg/packager/filters/utils_test.go b/src/pkg/packager/filters/utils_test.go index be00a3065c..f81b8e105f 100644 --- a/src/pkg/packager/filters/utils_test.go +++ b/src/pkg/packager/filters/utils_test.go @@ -7,142 +7,9 @@ package filters import ( "testing" - "github.com/defenseunicorns/pkg/helpers" - "github.com/defenseunicorns/zarf/src/types" "github.com/stretchr/testify/require" ) -func Test_isRequired(t *testing.T) { - tests := []struct { - name string - component types.ZarfComponent - useRequiredLogic bool - want bool - }{ - { - name: "Test when DeprecatedRequired is true and Optional is nil", - component: types.ZarfComponent{ - DeprecatedRequired: helpers.BoolPtr(true), - Optional: nil, - }, - want: true, - }, - { - name: "Test when DeprecatedRequired is true and Optional is false", - component: types.ZarfComponent{ - DeprecatedRequired: helpers.BoolPtr(true), - Optional: helpers.BoolPtr(false), - }, - want: true, - }, - { - name: "Test when DeprecatedRequired is true and Optional is true", - component: types.ZarfComponent{ - DeprecatedRequired: helpers.BoolPtr(true), - Optional: helpers.BoolPtr(true), - }, - // optional "wins" when defined - want: false, - }, - { - name: "Test when DeprecatedRequired is false and Optional is nil", - component: types.ZarfComponent{ - DeprecatedRequired: helpers.BoolPtr(false), - Optional: nil, - }, - want: true, - }, - { - name: "Test when DeprecatedRequired is false and Optional is false", - component: types.ZarfComponent{ - DeprecatedRequired: helpers.BoolPtr(false), - Optional: helpers.BoolPtr(false), - }, - // optional "wins" when defined - want: true, - }, - { - name: "Test when DeprecatedRequired is false and Optional is true", - component: types.ZarfComponent{ - DeprecatedRequired: helpers.BoolPtr(false), - Optional: helpers.BoolPtr(true), - }, - // optional "wins" when defined - want: false, - }, - { - name: "Test when DeprecatedRequired is nil and Optional is nil", - component: types.ZarfComponent{ - DeprecatedRequired: nil, - Optional: nil, - }, - // default is true (required: true || optional: false) - want: true, - }, - { - name: "Test when DeprecatedRequired is nil and Optional is false", - component: types.ZarfComponent{ - DeprecatedRequired: nil, - Optional: helpers.BoolPtr(false), - }, - // optional "wins" when defined - want: true, - }, - { - name: "Test when DeprecatedRequired is nil and Optional is true", - component: types.ZarfComponent{ - DeprecatedRequired: nil, - Optional: helpers.BoolPtr(true), - }, - // optional "wins" when defined - want: false, - }, - { - name: "Test when DeprecatedRequired is true, Optional is true and useRequiredLogic is true", - component: types.ZarfComponent{ - DeprecatedRequired: helpers.BoolPtr(true), - Optional: helpers.BoolPtr(true), - }, - useRequiredLogic: true, - want: true, - }, - { - name: "Test when DeprecatedRequired is true, Optional is false and useRequiredLogic is false", - component: types.ZarfComponent{ - DeprecatedRequired: helpers.BoolPtr(true), - Optional: helpers.BoolPtr(false), - }, - useRequiredLogic: false, - want: true, - }, - { - name: "Test when DeprecatedRequired is false, Optional is true and useRequiredLogic is true", - component: types.ZarfComponent{ - DeprecatedRequired: helpers.BoolPtr(false), - Optional: helpers.BoolPtr(true), - }, - useRequiredLogic: true, - want: false, - }, - { - name: "Test when DeprecatedRequired is false, Optional is false and useRequiredLogic is false", - component: types.ZarfComponent{ - DeprecatedRequired: helpers.BoolPtr(false), - Optional: helpers.BoolPtr(false), - }, - useRequiredLogic: false, - want: true, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got := isRequired(tt.component, tt.useRequiredLogic) - require.Equal(t, tt.want, got) - }) - } -} - func Test_includedOrExcluded(t *testing.T) { tests := []struct { name string diff --git a/src/pkg/packager/generate.go b/src/pkg/packager/generate.go index 5f091cb417..9f1bc80b03 100644 --- a/src/pkg/packager/generate.go +++ b/src/pkg/packager/generate.go @@ -12,7 +12,6 @@ import ( "github.com/defenseunicorns/pkg/helpers" "github.com/defenseunicorns/zarf/src/config" - "github.com/defenseunicorns/zarf/src/internal/packager/validate" "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/types" @@ -72,7 +71,7 @@ func (p *Packager) Generate() (err error) { p.cfg.Pkg.Components[i].Images = images[name] } - if err := validate.Run(p.cfg.Pkg); err != nil { + if err := p.cfg.Pkg.Validate(); err != nil { return err } diff --git a/src/pkg/packager/sources/cluster.go b/src/pkg/packager/sources/cluster.go index 1d506c6937..9fb74d6e81 100644 --- a/src/pkg/packager/sources/cluster.go +++ b/src/pkg/packager/sources/cluster.go @@ -8,7 +8,6 @@ import ( "fmt" "github.com/defenseunicorns/pkg/helpers" - "github.com/defenseunicorns/zarf/src/internal/packager/validate" "github.com/defenseunicorns/zarf/src/pkg/cluster" "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/packager/filters" @@ -23,7 +22,7 @@ var ( // NewClusterSource creates a new cluster source. func NewClusterSource(pkgOpts *types.ZarfPackageOptions) (PackageSource, error) { - if !validate.IsLowercaseNumberHyphenNoStartHyphen(pkgOpts.PackageSource) { + if !types.IsLowercaseNumberHyphenNoStartHyphen(pkgOpts.PackageSource) { return nil, fmt.Errorf("invalid package name %q", pkgOpts.PackageSource) } cluster, err := cluster.NewClusterWithWait(cluster.DefaultTimeout) diff --git a/src/types/component.go b/src/types/component.go index ee588e52e7..0e91e75a49 100644 --- a/src/types/component.go +++ b/src/types/component.go @@ -5,6 +5,8 @@ package types import ( + "slices" + "github.com/defenseunicorns/zarf/src/pkg/utils/exec" "github.com/defenseunicorns/zarf/src/types/extensions" ) @@ -20,13 +22,8 @@ type ZarfComponent struct { // Default changes the default option when deploying this component Default bool `json:"default,omitempty" jsonschema:"description=Determines the default Y/N state for installing this component on package deploy"` - // DeprecatedRequired makes this component mandatory for package deployment - DeprecatedRequired *bool `json:"required,omitempty" jsonschema:"description=[Deprecated] Do not prompt user to install this component, always install on package deploy. This will be removed in Zarf v1.0.0.,deprecated=true"` - - // Optional controls if the component is required or not - // - // Components are required by default (implicit optional:false) - Optional *bool `json:"optional,omitempty" jsonschema:"description=Prompt user to install this component on package deploy, defaults to false"` + // Required makes this component mandatory for package deployment + Required *bool `json:"required,omitempty" jsonschema:"description=Do not prompt user to install this component, always install on package deploy."` // Only include compatible components during package deployment Only ZarfComponentOnlyTarget `json:"only,omitempty" jsonschema:"description=Filter when this component is included in package creation or deployment"` @@ -85,6 +82,25 @@ func (c ZarfComponent) RequiresCluster() bool { return false } +// IsRequired returns if the component is required or not. +// +// If the `Required` field is set, it will return that value. +// +// If the `DefaultRequired` feature flag is present, it will return true. +// +// Otherwise, it will return false. +func (c ZarfComponent) IsRequired(ff []FeatureFlag) bool { + if c.Required != nil { + return *c.Required + } + + if slices.Contains(ff, DefaultRequired) { + return true + } + + return false +} + // ZarfComponentOnlyTarget filters a component to only show it for a given local OS and cluster. type ZarfComponentOnlyTarget struct { LocalOS string `json:"localOS,omitempty" jsonschema:"description=Only deploy component to specified OS,enum=linux,enum=darwin,enum=windows"` diff --git a/src/types/package.go b/src/types/package.go index 7a4e91a503..1f117784a2 100644 --- a/src/types/package.go +++ b/src/types/package.go @@ -7,6 +7,9 @@ package types // ZarfPackageKind is an enum of the different kinds of Zarf packages. type ZarfPackageKind string +// FeatureFlag is an enum of the different feature flags that can be set on a Zarf package. +type FeatureFlag string + const ( // ZarfInitConfig is the kind of Zarf package used during `zarf init`. ZarfInitConfig ZarfPackageKind = "ZarfInitConfig" @@ -14,6 +17,18 @@ const ( ZarfPackageConfig ZarfPackageKind = "ZarfPackageConfig" ) +const ( + // DefaultRequired changes the default state for all components in a package to be required. + DefaultRequired FeatureFlag = "default-required" +) + +// AllFeatureFlags returns a list of all available feature flags. +func AllFeatureFlags() []FeatureFlag { + return []FeatureFlag{ + DefaultRequired, + } +} + // ZarfPackage the top-level structure of a Zarf config file. type ZarfPackage struct { Kind ZarfPackageKind `json:"kind" jsonschema:"description=The kind of Zarf package,enum=ZarfInitConfig,enum=ZarfPackageConfig,default=ZarfPackageConfig"` @@ -41,19 +56,20 @@ func (pkg ZarfPackage) IsSBOMAble() bool { // ZarfMetadata lists information about the current ZarfPackage. type ZarfMetadata struct { - Name string `json:"name" jsonschema:"description=Name to identify this Zarf package,pattern=^[a-z0-9\\-]*[a-z0-9]$"` - Description string `json:"description,omitempty" jsonschema:"description=Additional information about this package"` - Version string `json:"version,omitempty" jsonschema:"description=Generic string set by a package author to track the package version (Note: ZarfInitConfigs will always be versioned to the CLIVersion they were created with)"` - URL string `json:"url,omitempty" jsonschema:"description=Link to package information when online"` - Image string `json:"image,omitempty" jsonschema:"description=An image URL to embed in this package (Reserved for future use in Zarf UI)"` - Uncompressed bool `json:"uncompressed,omitempty" jsonschema:"description=Disable compression of this package"` - Architecture string `json:"architecture,omitempty" jsonschema:"description=The target cluster architecture for this package,example=arm64,example=amd64"` - YOLO bool `json:"yolo,omitempty" jsonschema:"description=Yaml OnLy Online (YOLO): True enables deploying a Zarf package without first running zarf init against the cluster. This is ideal for connected environments where you want to use existing VCS and container registries."` - Authors string `json:"authors,omitempty" jsonschema:"description=Comma-separated list of package authors (including contact info),example=Doug <hello@defenseunicorns.com>, Pepr <hello@defenseunicorns.com>"` - Documentation string `json:"documentation,omitempty" jsonschema:"description=Link to package documentation when online"` - Source string `json:"source,omitempty" jsonschema:"description=Link to package source code when online"` - Vendor string `json:"vendor,omitempty" jsonschema_description:"Name of the distributing entity, organization or individual."` - AggregateChecksum string `json:"aggregateChecksum,omitempty" jsonschema:"description=Checksum of a checksums.txt file that contains checksums all the layers within the package."` + Name string `json:"name" jsonschema:"description=Name to identify this Zarf package,pattern=^[a-z0-9\\-]*[a-z0-9]$"` + Description string `json:"description,omitempty" jsonschema:"description=Additional information about this package"` + Version string `json:"version,omitempty" jsonschema:"description=Generic string set by a package author to track the package version (Note: ZarfInitConfigs will always be versioned to the CLIVersion they were created with)"` + URL string `json:"url,omitempty" jsonschema:"description=Link to package information when online"` + Image string `json:"image,omitempty" jsonschema:"description=An image URL to embed in this package (Reserved for future use in Zarf UI)"` + Uncompressed bool `json:"uncompressed,omitempty" jsonschema:"description=Disable compression of this package"` + Architecture string `json:"architecture,omitempty" jsonschema:"description=The target cluster architecture for this package,example=arm64,example=amd64"` + YOLO bool `json:"yolo,omitempty" jsonschema:"description=Yaml OnLy Online (YOLO): True enables deploying a Zarf package without first running zarf init against the cluster. This is ideal for connected environments where you want to use existing VCS and container registries."` + Authors string `json:"authors,omitempty" jsonschema:"description=Comma-separated list of package authors (including contact info),example=Doug <hello@defenseunicorns.com>, Pepr <hello@defenseunicorns.com>"` + Documentation string `json:"documentation,omitempty" jsonschema:"description=Link to package documentation when online"` + Source string `json:"source,omitempty" jsonschema:"description=Link to package source code when online"` + Vendor string `json:"vendor,omitempty" jsonschema_description:"Name of the distributing entity, organization or individual."` + AggregateChecksum string `json:"aggregateChecksum,omitempty" jsonschema:"description=Checksum of a checksums.txt file that contains checksums all the layers within the package."` + BetaFeatures []FeatureFlag `json:"betaFeatures,omitempty" jsonschema:"description=List of beta feature flags to enable for this package"` } // ZarfBuildData is written during the packager.Create() operation to track details of the created package. diff --git a/src/types/validate.go b/src/types/validate.go new file mode 100644 index 0000000000..426690aff9 --- /dev/null +++ b/src/types/validate.go @@ -0,0 +1,368 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +package types + +import ( + "fmt" + "path/filepath" + "regexp" + "slices" + + "github.com/defenseunicorns/pkg/helpers" + "github.com/defenseunicorns/zarf/src/config/lang" +) + +const ( + // ZarfMaxChartNameLength limits helm chart name size to account for K8s/helm limits and zarf prefix + ZarfMaxChartNameLength = 40 +) + +var ( + // IsLowercaseNumberHyphenNoStartHyphen is a regex for lowercase, numbers and hyphens that cannot start with a hyphen. + // https://regex101.com/r/FLdG9G/2 + IsLowercaseNumberHyphenNoStartHyphen = regexp.MustCompile(`^[a-z0-9][a-z0-9\-]*$`).MatchString + // IsUppercaseNumberUnderscore is a regex for uppercase, numbers and underscores. + // https://regex101.com/r/tfsEuZ/1 + IsUppercaseNumberUnderscore = regexp.MustCompile(`^[A-Z0-9_]+$`).MatchString + // Define allowed OS, an empty string means it is allowed on all operating systems + // same as enums on ZarfComponentOnlyTarget + supportedOS = []string{"linux", "darwin", "windows", ""} +) + +// SupportedOS returns the supported operating systems. +// +// The supported operating systems are: linux, darwin, windows. +// +// An empty string signifies no OS restrictions. +func SupportedOS() []string { + return supportedOS +} + +// Run performs config validations. +func (pkg ZarfPackage) Validate() error { + if pkg.Kind == ZarfInitConfig && pkg.Metadata.YOLO { + return fmt.Errorf(lang.PkgValidateErrInitNoYOLO) + } + + allFF := AllFeatureFlags() + for _, ff := range pkg.Metadata.BetaFeatures { + if !slices.Contains(allFF, ff) { + return fmt.Errorf(lang.PkgValidateErrBetaFeatureNotFound, ff, allFF) + } + } + + if !IsLowercaseNumberHyphenNoStartHyphen(pkg.Metadata.Name) { + return fmt.Errorf(lang.PkgValidateErrPkgName, pkg.Metadata.Name) + } + + for _, variable := range pkg.Variables { + if err := variable.Validate(); err != nil { + return fmt.Errorf(lang.PkgValidateErrVariable, err) + } + } + + for _, constant := range pkg.Constants { + if err := constant.Validate(); err != nil { + return fmt.Errorf(lang.PkgValidateErrConstant, err) + } + } + + uniqueComponentNames := make(map[string]bool) + groupDefault := make(map[string]string) + groupedComponents := make(map[string][]string) + + if pkg.Metadata.YOLO { + for _, component := range pkg.Components { + if len(component.Images) > 0 { + return fmt.Errorf(lang.PkgValidateErrYOLONoOCI) + } + + if len(component.Repos) > 0 { + return fmt.Errorf(lang.PkgValidateErrYOLONoGit) + } + + if component.Only.Cluster.Architecture != "" { + return fmt.Errorf(lang.PkgValidateErrYOLONoArch) + } + + if len(component.Only.Cluster.Distros) > 0 { + return fmt.Errorf(lang.PkgValidateErrYOLONoDistro) + } + } + } + + for _, component := range pkg.Components { + // ensure component name is unique + if _, ok := uniqueComponentNames[component.Name]; ok { + return fmt.Errorf(lang.PkgValidateErrComponentNameNotUnique, component.Name) + } + uniqueComponentNames[component.Name] = true + + if !IsLowercaseNumberHyphenNoStartHyphen(component.Name) { + return fmt.Errorf(lang.PkgValidateErrComponentName, component.Name) + } + + if !slices.Contains(supportedOS, component.Only.LocalOS) { + return fmt.Errorf(lang.PkgValidateErrComponentLocalOS, component.Name, component.Only.LocalOS, supportedOS) + } + + if component.IsRequired(pkg.Metadata.BetaFeatures) { + if component.Default { + return fmt.Errorf(lang.PkgValidateErrComponentReqDefault, component.Name) + } + if component.DeprecatedGroup != "" { + return fmt.Errorf(lang.PkgValidateErrComponentReqGrouped, component.Name) + } + } + + // progressive enhancement: "required" behavior can only follow "required" key + // if using `DefaultRequired` behavior + if slices.Contains(pkg.Metadata.BetaFeatures, DefaultRequired) && component.Default && component.DeprecatedGroup == "" { + return fmt.Errorf(lang.PkgValidateErrComponentMissingGroup, component.Name) + } + + uniqueChartNames := make(map[string]bool) + for _, chart := range component.Charts { + // ensure chart name is unique + if _, ok := uniqueChartNames[chart.Name]; ok { + return fmt.Errorf(lang.PkgValidateErrChartNameNotUnique, chart.Name) + } + uniqueChartNames[chart.Name] = true + + if err := chart.Validate(); err != nil { + return fmt.Errorf(lang.PkgValidateErrChart, err) + } + } + + uniqueManifestNames := make(map[string]bool) + for _, manifest := range component.Manifests { + // ensure manifest name is unique + if _, ok := uniqueManifestNames[manifest.Name]; ok { + return fmt.Errorf(lang.PkgValidateErrManifestNameNotUnique, manifest.Name) + } + uniqueManifestNames[manifest.Name] = true + + if err := manifest.Validate(); err != nil { + return fmt.Errorf(lang.PkgValidateErrManifest, err) + } + } + + if err := component.Actions.Validate(); err != nil { + return fmt.Errorf("%q: %w", component.Name, err) + } + + // ensure groups don't have multiple defaults or only one component + if component.DeprecatedGroup != "" { + if component.Default { + if _, ok := groupDefault[component.DeprecatedGroup]; ok { + return fmt.Errorf(lang.PkgValidateErrGroupMultipleDefaults, component.DeprecatedGroup, groupDefault[component.DeprecatedGroup], component.Name) + } + groupDefault[component.DeprecatedGroup] = component.Name + } + groupedComponents[component.DeprecatedGroup] = append(groupedComponents[component.DeprecatedGroup], component.Name) + } + } + + for groupKey, componentNames := range groupedComponents { + if len(componentNames) == 1 { + return fmt.Errorf(lang.PkgValidateErrGroupOneComponent, groupKey, componentNames[0]) + } + } + + return nil +} + +func (a ZarfComponentActions) Validate() error { + if err := a.OnCreate.Validate(); err != nil { + return fmt.Errorf(lang.PkgValidateErrAction, err) + } + + if a.OnCreate.HasSetVariables() { + return fmt.Errorf("cannot contain setVariables outside of onDeploy in actions") + } + + if err := a.OnDeploy.Validate(); err != nil { + return fmt.Errorf(lang.PkgValidateErrAction, err) + } + + if a.OnRemove.HasSetVariables() { + return fmt.Errorf("cannot contain setVariables outside of onDeploy in actions") + } + + return nil +} + +// ImportDefinition validates the component trying to be imported. +func (c ZarfComponent) ValidateImportDefinition() error { + path := c.Import.Path + url := c.Import.URL + + // ensure path or url is provided + if path == "" && url == "" { + return fmt.Errorf(lang.PkgValidateErrImportDefinition, c.Name, "neither a path nor a URL was provided") + } + + // ensure path and url are not both provided + if path != "" && url != "" { + return fmt.Errorf(lang.PkgValidateErrImportDefinition, c.Name, "both a path and a URL were provided") + } + + // validation for path + if url == "" && path != "" { + // ensure path is not an absolute path + if filepath.IsAbs(path) { + return fmt.Errorf(lang.PkgValidateErrImportDefinition, c.Name, "path cannot be an absolute path") + } + } + + // validation for url + if url != "" && path == "" { + ok := helpers.IsOCIURL(url) + if !ok { + return fmt.Errorf(lang.PkgValidateErrImportDefinition, c.Name, "URL is not a valid OCI URL") + } + } + + return nil +} + +func (as ZarfComponentActionSet) HasSetVariables() bool { + check := func(actions []ZarfComponentAction) bool { + for _, action := range actions { + if len(action.SetVariables) > 0 { + return true + } + } + return false + } + + return check(as.Before) || check(as.After) || check(as.OnSuccess) || check(as.OnFailure) +} + +func (as ZarfComponentActionSet) Validate() error { + validate := func(actions []ZarfComponentAction) error { + for _, action := range actions { + return action.Validate() + } + + return nil + } + + if err := validate(as.Before); err != nil { + return err + } + if err := validate(as.After); err != nil { + return err + } + if err := validate(as.OnSuccess); err != nil { + return err + } + if err := validate(as.OnFailure); err != nil { + return err + } + + return nil +} + +func (action ZarfComponentAction) Validate() error { + // Validate SetVariable + for _, variable := range action.SetVariables { + if !IsUppercaseNumberUnderscore(variable.Name) { + return fmt.Errorf(lang.PkgValidateMustBeUppercase, variable.Name) + } + } + + if action.Wait != nil { + // Validate only cmd or wait, not both + if action.Cmd != "" { + return fmt.Errorf(lang.PkgValidateErrActionCmdWait, action.Cmd) + } + + // Validate only cluster or network, not both + if action.Wait.Cluster != nil && action.Wait.Network != nil { + return fmt.Errorf(lang.PkgValidateErrActionClusterNetwork) + } + + // Validate at least one of cluster or network + if action.Wait.Cluster == nil && action.Wait.Network == nil { + return fmt.Errorf(lang.PkgValidateErrActionClusterNetwork) + } + } + + return nil +} + +func (variable ZarfPackageVariable) Validate() error { + if !IsUppercaseNumberUnderscore(variable.Name) { + return fmt.Errorf(lang.PkgValidateMustBeUppercase, variable.Name) + } + + return nil +} + +func (constant ZarfPackageConstant) Validate() error { + // ensure the constant name is only capitals and underscores + if !IsUppercaseNumberUnderscore(constant.Name) { + return fmt.Errorf(lang.PkgValidateErrPkgConstantName, constant.Name) + } + + if !regexp.MustCompile(constant.Pattern).MatchString(constant.Value) { + return fmt.Errorf(lang.PkgValidateErrPkgConstantPattern, constant.Name, constant.Pattern) + } + + return nil +} + +func (chart ZarfChart) Validate() error { + // Don't allow empty names + if chart.Name == "" { + return fmt.Errorf(lang.PkgValidateErrChartNameMissing, chart.Name) + } + + // Helm max release name + if len(chart.Name) > ZarfMaxChartNameLength { + return fmt.Errorf(lang.PkgValidateErrChartName, chart.Name, ZarfMaxChartNameLength) + } + + // Must have a namespace + if chart.Namespace == "" { + return fmt.Errorf(lang.PkgValidateErrChartNamespaceMissing, chart.Name) + } + + // Must have a url or localPath (and not both) + if chart.URL != "" && chart.LocalPath != "" { + return fmt.Errorf(lang.PkgValidateErrChartURLOrPath, chart.Name) + } + + // Must have a url or localPath (and not both) + if chart.URL == "" && chart.LocalPath == "" { + return fmt.Errorf(lang.PkgValidateErrChartURLOrPath, chart.Name) + } + + // Must have a version + if chart.Version == "" { + return fmt.Errorf(lang.PkgValidateErrChartVersion, chart.Name) + } + + return nil +} + +func (manifest ZarfManifest) Validate() error { + // Don't allow empty names + if manifest.Name == "" { + return fmt.Errorf(lang.PkgValidateErrManifestNameMissing, manifest.Name) + } + + // Helm max release name + if len(manifest.Name) > ZarfMaxChartNameLength { + return fmt.Errorf(lang.PkgValidateErrManifestNameLength, manifest.Name, ZarfMaxChartNameLength) + } + + // Require files in manifest + if len(manifest.Files) < 1 && len(manifest.Kustomizations) < 1 { + return fmt.Errorf(lang.PkgValidateErrManifestFileOrKustomize, manifest.Name) + } + + return nil +} diff --git a/zarf.yaml b/zarf.yaml index fc40c32283..e5c3366857 100644 --- a/zarf.yaml +++ b/zarf.yaml @@ -2,10 +2,11 @@ kind: ZarfInitConfig metadata: name: init description: Used to establish a new Zarf cluster + betaFeatures: [default-required] components: - name: k3s - optional: true + required: false import: path: packages/distros/k3s @@ -31,12 +32,12 @@ components: # (Optional) Adds logging to the cluster - name: logging - optional: true + required: false import: path: packages/logging-pgl # (Optional) Adds a git server to the cluster - name: git-server - optional: true + required: false import: path: packages/gitea From 2433771585db7e858fb1c15c2f7dd1cf872b09d0 Mon Sep 17 00:00:00 2001 From: razzle Date: Tue, 2 Apr 2024 11:48:39 -0500 Subject: [PATCH 082/113] feature flag Signed-off-by: razzle --- src/config/lang/english.go | 8 ++++---- src/pkg/packager/filters/utils.go | 1 - 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/src/config/lang/english.go b/src/config/lang/english.go index 84843ac553..fb3f47a054 100644 --- a/src/config/lang/english.go +++ b/src/config/lang/english.go @@ -508,10 +508,10 @@ cat file2.yml | zarf tools yq ea '.a.b' file1.yml - file3.yml ` CmdToolsYqEvalExample = ` # Reads field under the given path for each file -zarf tools yq e '.a.b' f1.yml f2.yml +zarf tools yq e '.a.b' f1.yml f2.yml # Prints out the file -zarf tools yq e sample.yaml +zarf tools yq e sample.yaml # Pipe from STDIN ## use '-' as a filename to pipe from STDIN @@ -519,10 +519,10 @@ cat file2.yml | zarf tools yq e '.a.b' file1.yml - file3.yml # Creates a new yaml document ## Note that editing an empty file does not work. -zarf tools yq e -n '.a.b.c = "cat"' +zarf tools yq e -n '.a.b.c = "cat"' # Update a file inplace -zarf tools yq e '.a.b = "cool"' -i file.yaml +zarf tools yq e '.a.b = "cool"' -i file.yaml ` CmdToolsMonitorShort = "Launches a terminal UI to monitor the connected cluster using K9s." diff --git a/src/pkg/packager/filters/utils.go b/src/pkg/packager/filters/utils.go index 73b5174779..3b33f9b594 100644 --- a/src/pkg/packager/filters/utils.go +++ b/src/pkg/packager/filters/utils.go @@ -40,4 +40,3 @@ func includedOrExcluded(componentName string, requestedComponentNames []string) // All other cases we don't know if we should include or exclude yet return unknown, "" } - From bfc150a66cc30a5e5fe6f761dc18edd4dd7abd95 Mon Sep 17 00:00:00 2001 From: razzle Date: Tue, 2 Apr 2024 11:49:52 -0500 Subject: [PATCH 083/113] docs and schema Signed-off-by: razzle --- .../100-cli-commands/zarf_dev_migrate.md | 2 +- .../100-cli-commands/zarf_tools_yq_eval.md | 8 +- docs/3-create-a-zarf-package/4-zarf-schema.md | 81 ++++++++++++------- zarf.schema.json | 13 +-- 4 files changed, 67 insertions(+), 37 deletions(-) diff --git a/docs/2-the-zarf-cli/100-cli-commands/zarf_dev_migrate.md b/docs/2-the-zarf-cli/100-cli-commands/zarf_dev_migrate.md index 92d451e079..25fa11a6b0 100644 --- a/docs/2-the-zarf-cli/100-cli-commands/zarf_dev_migrate.md +++ b/docs/2-the-zarf-cli/100-cli-commands/zarf_dev_migrate.md @@ -17,7 +17,7 @@ zarf dev migrate [flags] ``` -h, --help help for migrate - --run stringArray migrations to run (default: all, available: scripts-to-actions, pluralize-set-variable, required-to-optional) + --run stringArray migrations to run (default: all, available: scripts-to-actions, pluralize-set-variable) ``` ## Options inherited from parent commands diff --git a/docs/2-the-zarf-cli/100-cli-commands/zarf_tools_yq_eval.md b/docs/2-the-zarf-cli/100-cli-commands/zarf_tools_yq_eval.md index 234ca9957e..616f81c27b 100644 --- a/docs/2-the-zarf-cli/100-cli-commands/zarf_tools_yq_eval.md +++ b/docs/2-the-zarf-cli/100-cli-commands/zarf_tools_yq_eval.md @@ -21,10 +21,10 @@ zarf tools yq eval [expression] [yaml_file1]... [flags] ``` # Reads field under the given path for each file -zarf tools yq e '.a.b' f1.yml f2.yml +zarf tools yq e '.a.b' f1.yml f2.yml # Prints out the file -zarf tools yq e sample.yaml +zarf tools yq e sample.yaml # Pipe from STDIN # use '-' as a filename to pipe from STDIN @@ -32,10 +32,10 @@ cat file2.yml | zarf tools yq e '.a.b' file1.yml - file3.yml # Creates a new yaml document # Note that editing an empty file does not work. -zarf tools yq e -n '.a.b.c = "cat"' +zarf tools yq e -n '.a.b.c = "cat"' # Update a file inplace -zarf tools yq e '.a.b = "cool"' -i file.yaml +zarf tools yq e '.a.b = "cool"' -i file.yaml ``` diff --git a/docs/3-create-a-zarf-package/4-zarf-schema.md b/docs/3-create-a-zarf-package/4-zarf-schema.md index 77a5ddcf68..ad5e247567 100644 --- a/docs/3-create-a-zarf-package/4-zarf-schema.md +++ b/docs/3-create-a-zarf-package/4-zarf-schema.md @@ -272,6 +272,33 @@ Must be one of:
+
+ + betaFeatures + +  +
+ +**Description:** List of beta feature flags to enable for this package + +| | | +| -------- | ----------------- | +| **Type** | `array of string` | + +![Min Items: N/A](https://img.shields.io/badge/Min%20Items%3A%20N/A-gold) +![Max Items: N/A](https://img.shields.io/badge/Max%20Items%3A%20N/A-gold) +![Item unicity: False](https://img.shields.io/badge/Item%20unicity%3A%20False-gold) +![Additional items: N/A](https://img.shields.io/badge/Additional%20items%3A%20N/A-gold) + + ### betaFeatures items + +| | | +| -------- | -------- | +| **Type** | `string` | + +
+
+
@@ -400,7 +427,7 @@ Must be one of: ![Item unicity: False](https://img.shields.io/badge/Item%20unicity%3A%20False-gold) ![Additional items: N/A](https://img.shields.io/badge/Additional%20items%3A%20N/A-gold) - ### migrations items + ### migrations items | | | | -------- | -------- | @@ -478,7 +505,7 @@ Must be one of: ![Item unicity: False](https://img.shields.io/badge/Item%20unicity%3A%20False-gold) ![Additional items: N/A](https://img.shields.io/badge/Additional%20items%3A%20N/A-gold) - ### differentialMissing items + ### differentialMissing items | | | | -------- | -------- | @@ -543,7 +570,7 @@ Must be one of: ![Item unicity: False](https://img.shields.io/badge/Item%20unicity%3A%20False-gold) ![Additional items: N/A](https://img.shields.io/badge/Additional%20items%3A%20N/A-gold) - ### ZarfComponent + ### ZarfComponent | | | | ------------------------- | -------------------------------------------------------------------------------------------------------- | @@ -607,12 +634,12 @@ Must be one of:
- optional + required  
-**Description:** Prompt user to install this component on package deploy +**Description:** Do not prompt user to install this component | | | | -------- | --------- | @@ -718,7 +745,7 @@ Must be one of: ![Item unicity: False](https://img.shields.io/badge/Item%20unicity%3A%20False-gold) ![Additional items: N/A](https://img.shields.io/badge/Additional%20items%3A%20N/A-gold) - ### distros items + ### distros items | | | | -------- | -------- | @@ -841,7 +868,7 @@ Must be one of: ![Item unicity: False](https://img.shields.io/badge/Item%20unicity%3A%20False-gold) ![Additional items: N/A](https://img.shields.io/badge/Additional%20items%3A%20N/A-gold) - ### ZarfManifest + ### ZarfManifest | | | | ------------------------- | -------------------------------------------------------------------------------------------------------- | @@ -901,7 +928,7 @@ Must be one of: ![Item unicity: False](https://img.shields.io/badge/Item%20unicity%3A%20False-gold) ![Additional items: N/A](https://img.shields.io/badge/Additional%20items%3A%20N/A-gold) - ### files items + ### files items | | | | -------- | -------- | @@ -944,7 +971,7 @@ Must be one of: ![Item unicity: False](https://img.shields.io/badge/Item%20unicity%3A%20False-gold) ![Additional items: N/A](https://img.shields.io/badge/Additional%20items%3A%20N/A-gold) - ### kustomizations items + ### kustomizations items | | | | -------- | -------- | @@ -992,7 +1019,7 @@ Must be one of: ![Item unicity: False](https://img.shields.io/badge/Item%20unicity%3A%20False-gold) ![Additional items: N/A](https://img.shields.io/badge/Additional%20items%3A%20N/A-gold) - ### ZarfChart + ### ZarfChart | | | | ------------------------- | -------------------------------------------------------------------------------------------------------- | @@ -1176,7 +1203,7 @@ Must be one of: ![Item unicity: False](https://img.shields.io/badge/Item%20unicity%3A%20False-gold) ![Additional items: N/A](https://img.shields.io/badge/Additional%20items%3A%20N/A-gold) - ### valuesFiles items + ### valuesFiles items | | | | -------- | -------- | @@ -1208,7 +1235,7 @@ Must be one of: ![Item unicity: False](https://img.shields.io/badge/Item%20unicity%3A%20False-gold) ![Additional items: N/A](https://img.shields.io/badge/Additional%20items%3A%20N/A-gold) - ### ZarfDataInjection + ### ZarfDataInjection | | | | ------------------------- | -------------------------------------------------------------------------------------------------------- | @@ -1371,7 +1398,7 @@ Must be one of: ![Item unicity: False](https://img.shields.io/badge/Item%20unicity%3A%20False-gold) ![Additional items: N/A](https://img.shields.io/badge/Additional%20items%3A%20N/A-gold) - ### ZarfFile + ### ZarfFile | | | | ------------------------- | -------------------------------------------------------------------------------------------------------- | @@ -1465,7 +1492,7 @@ Must be one of: ![Item unicity: False](https://img.shields.io/badge/Item%20unicity%3A%20False-gold) ![Additional items: N/A](https://img.shields.io/badge/Additional%20items%3A%20N/A-gold) - ### symlinks items + ### symlinks items | | | | -------- | -------- | @@ -1511,7 +1538,7 @@ Must be one of: ![Item unicity: False](https://img.shields.io/badge/Item%20unicity%3A%20False-gold) ![Additional items: N/A](https://img.shields.io/badge/Additional%20items%3A%20N/A-gold) - ### images items + ### images items | | | | -------- | -------- | @@ -1538,7 +1565,7 @@ Must be one of: ![Item unicity: False](https://img.shields.io/badge/Item%20unicity%3A%20False-gold) ![Additional items: N/A](https://img.shields.io/badge/Additional%20items%3A%20N/A-gold) - ### repos items + ### repos items | | | | -------- | -------- | @@ -1633,7 +1660,7 @@ Must be one of: ![Item unicity: False](https://img.shields.io/badge/Item%20unicity%3A%20False-gold) ![Additional items: N/A](https://img.shields.io/badge/Additional%20items%3A%20N/A-gold) - ### valuesFiles items + ### valuesFiles items | | | | -------- | -------- | @@ -1676,7 +1703,7 @@ Must be one of: ![Item unicity: False](https://img.shields.io/badge/Item%20unicity%3A%20False-gold) ![Additional items: N/A](https://img.shields.io/badge/Additional%20items%3A%20N/A-gold) - ### fluxPatchFiles items + ### fluxPatchFiles items | | | | -------- | -------- | @@ -1824,7 +1851,7 @@ Must be one of: ![Item unicity: False](https://img.shields.io/badge/Item%20unicity%3A%20False-gold) ![Additional items: N/A](https://img.shields.io/badge/Additional%20items%3A%20N/A-gold) - ### env items + ### env items | | | | -------- | -------- | @@ -1939,7 +1966,7 @@ Must be one of: ![Item unicity: False](https://img.shields.io/badge/Item%20unicity%3A%20False-gold) ![Additional items: N/A](https://img.shields.io/badge/Additional%20items%3A%20N/A-gold) - ### ZarfComponentAction + ### ZarfComponentAction | | | | ------------------------- | -------------------------------------------------------------------------------------------------------- | @@ -2029,7 +2056,7 @@ Must be one of: ![Item unicity: False](https://img.shields.io/badge/Item%20unicity%3A%20False-gold) ![Additional items: N/A](https://img.shields.io/badge/Additional%20items%3A%20N/A-gold) - ### env items + ### env items | | | | -------- | -------- | @@ -2094,7 +2121,7 @@ Must be one of: ![Item unicity: False](https://img.shields.io/badge/Item%20unicity%3A%20False-gold) ![Additional items: N/A](https://img.shields.io/badge/Additional%20items%3A%20N/A-gold) - ### ZarfComponentActionSetVariable + ### ZarfComponentActionSetVariable | | | | ------------------------- | -------------------------------------------------------------------------------------------------------- | @@ -2448,7 +2475,7 @@ Must be one of: ![Item unicity: False](https://img.shields.io/badge/Item%20unicity%3A%20False-gold) ![Additional items: N/A](https://img.shields.io/badge/Additional%20items%3A%20N/A-gold) - ### ZarfComponentAction + ### ZarfComponentAction | | | | ------------------------- | -------------------------------------------------------------------------------------------------------- | @@ -2479,7 +2506,7 @@ Must be one of: ![Item unicity: False](https://img.shields.io/badge/Item%20unicity%3A%20False-gold) ![Additional items: N/A](https://img.shields.io/badge/Additional%20items%3A%20N/A-gold) - ### ZarfComponentAction + ### ZarfComponentAction | | | | ------------------------- | -------------------------------------------------------------------------------------------------------- | @@ -2510,7 +2537,7 @@ Must be one of: ![Item unicity: False](https://img.shields.io/badge/Item%20unicity%3A%20False-gold) ![Additional items: N/A](https://img.shields.io/badge/Additional%20items%3A%20N/A-gold) - ### ZarfComponentAction + ### ZarfComponentAction | | | | ------------------------- | -------------------------------------------------------------------------------------------------------- | @@ -2590,7 +2617,7 @@ Must be one of: ![Item unicity: False](https://img.shields.io/badge/Item%20unicity%3A%20False-gold) ![Additional items: N/A](https://img.shields.io/badge/Additional%20items%3A%20N/A-gold) - ### ZarfPackageConstant + ### ZarfPackageConstant | | | | ------------------------- | -------------------------------------------------------------------------------------------------------- | @@ -2709,7 +2736,7 @@ Must be one of: ![Item unicity: False](https://img.shields.io/badge/Item%20unicity%3A%20False-gold) ![Additional items: N/A](https://img.shields.io/badge/Additional%20items%3A%20N/A-gold) - ### ZarfPackageVariable + ### ZarfPackageVariable | | | | ------------------------- | -------------------------------------------------------------------------------------------------------- | diff --git a/zarf.schema.json b/zarf.schema.json index e25dd3ee66..e348b8382c 100644 --- a/zarf.schema.json +++ b/zarf.schema.json @@ -284,11 +284,7 @@ }, "required": { "type": "boolean", - "description": "[Deprecated] Do not prompt user to install this component" - }, - "optional": { - "type": "boolean", - "description": "Prompt user to install this component on package deploy" + "description": "Do not prompt user to install this component" }, "only": { "$schema": "http://json-schema.org/draft-04/schema#", @@ -968,6 +964,13 @@ "aggregateChecksum": { "type": "string", "description": "Checksum of a checksums.txt file that contains checksums all the layers within the package." + }, + "betaFeatures": { + "items": { + "type": "string" + }, + "type": "array", + "description": "List of beta feature flags to enable for this package" } }, "additionalProperties": false, From 0f8fa6db2611b3c2c51f4ac87e010e3642a574bb Mon Sep 17 00:00:00 2001 From: razzle Date: Tue, 2 Apr 2024 11:59:32 -0500 Subject: [PATCH 084/113] restore examples and packages Signed-off-by: razzle --- examples/argocd/zarf.yaml | 2 ++ examples/big-bang/yolo/zarf.yaml | 2 ++ examples/big-bang/zarf.yaml | 2 +- examples/component-actions/zarf.yaml | 12 ------------ examples/component-choice/zarf.yaml | 4 ---- examples/component-webhooks/zarf.yaml | 1 + examples/composable-packages/zarf.yaml | 8 ++++---- examples/config-file/zarf.yaml | 3 +-- examples/dos-games/zarf.yaml | 1 + examples/helm-charts/zarf.yaml | 1 + examples/kiwix/zarf.yaml | 1 + examples/longhorn/zarf.yaml | 2 ++ examples/manifests/zarf.yaml | 3 +++ examples/package-flavors/zarf.yaml | 5 +++++ examples/podinfo-flux/zarf.yaml | 2 ++ examples/variables/README.md | 1 + examples/variables/zarf.yaml | 2 ++ examples/wordpress/zarf.yaml | 1 + examples/yolo/zarf.yaml | 1 + packages/distros/eks/zarf.yaml | 3 +-- packages/zarf-agent/zarf.yaml | 1 + packages/zarf-registry/zarf.yaml | 2 ++ 22 files changed, 35 insertions(+), 25 deletions(-) diff --git a/examples/argocd/zarf.yaml b/examples/argocd/zarf.yaml index c6f4681349..6d893d8639 100644 --- a/examples/argocd/zarf.yaml +++ b/examples/argocd/zarf.yaml @@ -5,6 +5,7 @@ metadata: components: - name: argocd-helm-chart + required: true charts: - name: argo-cd version: 5.54.0 @@ -20,6 +21,7 @@ components: - quay.io/argoproj/argocd:sha256-2dafd800fb617ba5b16ae429e388ca140f66f88171463d23d158b372bb2fae08.sig - quay.io/argoproj/argocd:sha256-2dafd800fb617ba5b16ae429e388ca140f66f88171463d23d158b372bb2fae08.att - name: argocd-apps + required: true charts: - name: argocd-apps version: 1.6.1 diff --git a/examples/big-bang/yolo/zarf.yaml b/examples/big-bang/yolo/zarf.yaml index 7edd28bd5a..80caf3bfd1 100644 --- a/examples/big-bang/yolo/zarf.yaml +++ b/examples/big-bang/yolo/zarf.yaml @@ -22,12 +22,14 @@ variables: components: - name: flux-private-registry + required: true manifests: - name: private-registry namespace: flux-system files: - private-registry.yaml - name: bigbang + required: true extensions: bigbang: # renovate: datasource=gitlab-releases depName=big-bang/bigbang versioning=semver registryUrl=https://repo1.dso.mil/ diff --git a/examples/big-bang/zarf.yaml b/examples/big-bang/zarf.yaml index 0e876e409f..ba1dda61b3 100644 --- a/examples/big-bang/zarf.yaml +++ b/examples/big-bang/zarf.yaml @@ -15,6 +15,7 @@ variables: components: - name: bigbang + required: true actions: onRemove: before: @@ -56,7 +57,6 @@ components: # Values are merged in order, so this would override the above and disable everything if uncommented # - config/disable-all.yaml - name: gitea-virtual-service - optional: true description: > Expose the internal Zarf Gitea server through the Big Bang Istio deployment via a virtual service. (only applies if you are using the Zarf-provided Gitea deployment - not an externally configured git host) diff --git a/examples/component-actions/zarf.yaml b/examples/component-actions/zarf.yaml index 92184ec5f6..b3a2d5a53f 100644 --- a/examples/component-actions/zarf.yaml +++ b/examples/component-actions/zarf.yaml @@ -10,7 +10,6 @@ variables: # See the example README.md in this folder or /adrs/0010-scripts-actions.md for more info. components: - name: on-create - optional: true actions: # runs during "zarf package create" onCreate: @@ -64,7 +63,6 @@ components: sleep 0.5 - name: on-deploy-and-remove - optional: true actions: # runs during "zarf package deploy" onDeploy: @@ -84,7 +82,6 @@ components: - cmd: rm test-deploy-after.txt - name: on-deploy-with-variable - optional: true actions: # runs during "zarf package deploy" onDeploy: @@ -93,7 +90,6 @@ components: - cmd: echo "the dog says ${ZARF_VAR_DOG_SOUND}" - name: on-deploy-with-dynamic-variable - optional: true actions: # runs during "zarf package deploy" onDeploy: @@ -108,7 +104,6 @@ components: - cmd: echo "the cat says ${ZARF_VAR_CAT_SOUND}" - name: on-deploy-with-multiple-variables - optional: true actions: # runs during "zarf package deploy" onDeploy: @@ -138,7 +133,6 @@ components: - cmd: echo "with a TF_VAR, the snake also says ${TF_VAR_snake_sound}" - name: on-deploy-with-template-use-of-variable - optional: true files: # this file will be copied to the target location and the cat, dog, and snake sounds will be replaced with their values # requires the on-deploy-with-dynamic-variable and on-deploy-with-multiple-variables components @@ -147,7 +141,6 @@ components: shasum: 3c0404e0c767ace905c361fadded6c4b91fdb88aa07d5c42d2a220a87564836d - name: on-deploy-with-timeout - optional: true description: This component will fail after 1 second actions: # runs during "zarf package deploy" @@ -163,7 +156,6 @@ components: - cmd: echo "😭😭😭 this action failed because it took too long to run 😭😭😭" - name: on-remove - optional: true # A manifest that we expect to be removed by Zarf manifests: - name: test-configmap @@ -180,7 +172,6 @@ components: - cmd: ./zarf tools kubectl get configmap -n zarf remove-test-configmap || echo "Not Found" - name: on-deploy-with-env-var - optional: true actions: onDeploy: before: @@ -190,7 +181,6 @@ components: - ZARF_VAR_TEST_FILENAME=test-filename-from-env.txt - name: on-create-with-network-wait-action - optional: true description: This component will wait for 15 seconds for a network resource to be available actions: onCreate: @@ -205,7 +195,6 @@ components: code: 200 - name: on-deploy-with-wait-action - optional: true description: This component will wait for 5 seconds for the test-configmap to be exist manifests: - name: test-configmap @@ -224,7 +213,6 @@ components: namespace: zarf - name: on-deploy-immediate-failure - optional: true description: This component will fail on the first error instead of continuing execution # the default for multi-line commands is set -e actions: diff --git a/examples/component-choice/zarf.yaml b/examples/component-choice/zarf.yaml index fd422430b7..9c263ca7ba 100644 --- a/examples/component-choice/zarf.yaml +++ b/examples/component-choice/zarf.yaml @@ -5,16 +5,12 @@ metadata: components: - name: first-choice - # if using a group, all members of the group must be optional - optional: true group: example-choice files: - source: blank-file.txt target: first-choice-file.txt - name: second-choice - # if using a group, all members of the group must be optional - optional: true group: example-choice default: true files: diff --git a/examples/component-webhooks/zarf.yaml b/examples/component-webhooks/zarf.yaml index 5547a983d3..1baa3ab50d 100644 --- a/examples/component-webhooks/zarf.yaml +++ b/examples/component-webhooks/zarf.yaml @@ -6,6 +6,7 @@ metadata: version: 0.0.1 components: - name: module + required: true manifests: - name: module namespace: pepr-system diff --git a/examples/composable-packages/zarf.yaml b/examples/composable-packages/zarf.yaml index d2aa599f98..efd16ce6d9 100644 --- a/examples/composable-packages/zarf.yaml +++ b/examples/composable-packages/zarf.yaml @@ -5,8 +5,8 @@ metadata: components: - name: local-games-path - # The component logic keys ('optional', 'group', and 'default') always override those of the imported package - optional: false + # The component logic keys ('required', 'group', and 'default') always override those of the imported package + required: true # group: "" # the initial value overrides the child component # default: false # the initial value overrides the child component description: "Example of a local composed package with a unique description for this component" @@ -25,8 +25,8 @@ components: - quake-service.yaml - name: oci-games-url - # The component logic keys ('optional', 'group', and 'default') always override those of the imported package - # optional: false # the initial value overrides the child component + # The component logic keys ('required', 'group', and 'default') always override those of the imported package + # required: false # the initial value overrides the child component # group: "" # the initial value overrides the child component # default: false # the initial value overrides the child component import: diff --git a/examples/config-file/zarf.yaml b/examples/config-file/zarf.yaml index 979e8963a3..d18cf80d73 100644 --- a/examples/config-file/zarf.yaml +++ b/examples/config-file/zarf.yaml @@ -12,16 +12,15 @@ variables: components: - name: lion description: By default, only this component will be deployed when using the config file + required: true manifests: - name: config-example-configmap files: - simple-configmap.yaml - name: zebra - optional: true # Notice this string is wrapped in quotes, as Zarf variables use #### as a delimiter, which is also used in YAML for comments description: "This is a zebra and they have ###ZARF_PKG_TMPL_ZEBRA###" - name: leopard - optional: true description: "This is a leopard and they have ###ZARF_PKG_TMPL_LEOPARD###" diff --git a/examples/dos-games/zarf.yaml b/examples/dos-games/zarf.yaml index f01f19ffae..f6dc923b75 100644 --- a/examples/dos-games/zarf.yaml +++ b/examples/dos-games/zarf.yaml @@ -6,6 +6,7 @@ metadata: components: - name: baseline + required: true manifests: - name: multi-games namespace: dos-games diff --git a/examples/helm-charts/zarf.yaml b/examples/helm-charts/zarf.yaml index f84f1310f3..d6481b334d 100644 --- a/examples/helm-charts/zarf.yaml +++ b/examples/helm-charts/zarf.yaml @@ -6,6 +6,7 @@ metadata: components: - name: demo-helm-charts + required: true charts: # Charts are organized in a list with unique chart names per component - note that a Zarf chart name does not need to match the chart name in a Chart.yaml - name: podinfo-local diff --git a/examples/kiwix/zarf.yaml b/examples/kiwix/zarf.yaml index 45f52ee902..8f3a300312 100644 --- a/examples/kiwix/zarf.yaml +++ b/examples/kiwix/zarf.yaml @@ -8,6 +8,7 @@ metadata: components: - name: kiwix-serve + required: true manifests: - name: kiwix-serve namespace: kiwix diff --git a/examples/longhorn/zarf.yaml b/examples/longhorn/zarf.yaml index f7146ba148..b447e8debe 100644 --- a/examples/longhorn/zarf.yaml +++ b/examples/longhorn/zarf.yaml @@ -5,6 +5,7 @@ metadata: version: 1.4.0 components: - name: longhorn-environment-check + required: true files: - source: https://raw.githubusercontent.com/longhorn/longhorn/v1.4.0/scripts/environment_check.sh target: environment_check.sh @@ -25,6 +26,7 @@ components: chmod +x ./environment_check.sh ./environment_check.sh - name: longhorn + required: true description: Deploy Longhorn into a Kubernetes cluster. https://longhorn.io actions: # Set the delete confirmation flag for Longhorn diff --git a/examples/manifests/zarf.yaml b/examples/manifests/zarf.yaml index 116eb5c913..cb363312c0 100644 --- a/examples/manifests/zarf.yaml +++ b/examples/manifests/zarf.yaml @@ -5,6 +5,7 @@ metadata: components: - name: httpd-local + required: true manifests: - name: simple-httpd-deployment namespace: httpd @@ -27,6 +28,7 @@ components: images: - httpd:alpine3.18 - name: nginx-remote + required: true manifests: - name: simple-nginx-deployment namespace: nginx @@ -52,6 +54,7 @@ components: images: - nginx:1.14.2 - name: podinfo-kustomize + required: true manifests: - name: simple-podinfo-deployment namespace: podinfo diff --git a/examples/package-flavors/zarf.yaml b/examples/package-flavors/zarf.yaml index c92fe9cca3..873b36f493 100644 --- a/examples/package-flavors/zarf.yaml +++ b/examples/package-flavors/zarf.yaml @@ -6,6 +6,7 @@ metadata: components: - name: image + required: true description: "Sets the Enterprise Linux flavor to Rocky Linux" only: flavor: rocky-road @@ -19,6 +20,7 @@ components: - name: IMAGE - name: image + required: true description: "Sets the Enterprise Linux flavor to Oracle Linux" only: flavor: oracle-cookie-crunch @@ -32,6 +34,7 @@ components: - name: IMAGE - name: image + required: true description: "Sets the Enterprise Linux flavor to Alma Linux" only: flavor: vanilla-alma-nd @@ -45,6 +48,7 @@ components: - name: IMAGE - name: image + required: true description: "Sets the Enterprise Linux flavor to OpenSUSE" only: flavor: strawberry-suse @@ -59,6 +63,7 @@ components: - name: pod description: "The pod that runs the specified flavor of Enterprise Linux" + required: true manifests: - name: enterprise-linux namespace: enterprise-linux diff --git a/examples/podinfo-flux/zarf.yaml b/examples/podinfo-flux/zarf.yaml index 428eec17c6..0f0e4b3336 100644 --- a/examples/podinfo-flux/zarf.yaml +++ b/examples/podinfo-flux/zarf.yaml @@ -6,6 +6,7 @@ metadata: components: - name: flux description: Installs the flux CRDs / controllers to use flux-based deployments in the cluster + required: true manifests: - name: flux-crds namespace: flux @@ -17,6 +18,7 @@ components: - name: podinfo-via-flux description: Example deployment via flux using the famous podinfo example + required: true manifests: - name: podinfo-via-flux namespace: podinfo diff --git a/examples/variables/README.md b/examples/variables/README.md index f8e549cbf7..fbb616aff6 100644 --- a/examples/variables/README.md +++ b/examples/variables/README.md @@ -133,6 +133,7 @@ constants: components: - name: zarf-prompt-image + required: true images: - '###ZARF_PKG_TMPL_PROMPT_ON_CREATE###' ``` diff --git a/examples/variables/zarf.yaml b/examples/variables/zarf.yaml index b709d05348..f1a3564b5a 100644 --- a/examples/variables/zarf.yaml +++ b/examples/variables/zarf.yaml @@ -50,6 +50,7 @@ components: # NOTE: this component does not actually execute this file in this example - name: variables-with-terraform description: Change a value in a regular file with a Zarf variable. Set AWS_REGION variable to modify the file. + required: true files: - source: simple-terraform.tf target: modified-terraform.tf @@ -70,6 +71,7 @@ components: # The following component deploys nginx to the cluster using the defined variables - name: variables-with-nginx description: "This component deploys nginx version ###ZARF_PKG_TMPL_NGINX_VERSION### to the cluster" + required: true images: # This sets the nginx image tag to the same PKG_TMPL used for the constant above to keep the zarf.yaml and nginx-deployment.yaml in sync - "nginx:###ZARF_PKG_TMPL_NGINX_VERSION###" diff --git a/examples/wordpress/zarf.yaml b/examples/wordpress/zarf.yaml index 96fa0d4700..9900bbd0e0 100644 --- a/examples/wordpress/zarf.yaml +++ b/examples/wordpress/zarf.yaml @@ -40,6 +40,7 @@ components: - name: wordpress # specifies the name of our component and should be unique and unchanging through updates description: | # (optional) a human-readable description of the component you are defining Deploys the Bitnami-packaged WordPress chart into the cluster + required: true # (optional) sets the component as 'required' so that it is always deployed charts: - name: wordpress url: oci://registry-1.docker.io/bitnamicharts/wordpress diff --git a/examples/yolo/zarf.yaml b/examples/yolo/zarf.yaml index fa2e7c2a1f..2f7770c498 100644 --- a/examples/yolo/zarf.yaml +++ b/examples/yolo/zarf.yaml @@ -6,6 +6,7 @@ metadata: components: - name: yolo-games + required: true manifests: - name: multi-games namespace: zarf-yolo-example diff --git a/packages/distros/eks/zarf.yaml b/packages/distros/eks/zarf.yaml index 593d2d7650..8172dfdf60 100644 --- a/packages/distros/eks/zarf.yaml +++ b/packages/distros/eks/zarf.yaml @@ -27,6 +27,7 @@ variables: components: - name: load-eksctl + required: true files: - source: eks.yaml target: eks.yaml @@ -47,7 +48,6 @@ components: extractPath: eksctl - name: deploy-eks-cluster - optional: true description: Create an EKS cluster! actions: onDeploy: @@ -58,7 +58,6 @@ components: - cmd: ./binaries/eksctl_$(uname -s)_$(uname -m) utils write-kubeconfig -c ${ZARF_VAR_EKS_CLUSTER_NAME} - name: teardown-eks-cluster - optional: true description: Delete the EKS cluster that this package was used to create. actions: onDeploy: diff --git a/packages/zarf-agent/zarf.yaml b/packages/zarf-agent/zarf.yaml index 4d2c3d2c5e..86799ff03f 100644 --- a/packages/zarf-agent/zarf.yaml +++ b/packages/zarf-agent/zarf.yaml @@ -16,6 +16,7 @@ components: images and git repository references in Kubernetes manifests. This prevents the need to manually update URLs from their original sources to the Zarf-managed docker registry and git server. + required: true images: - "###ZARF_PKG_TMPL_AGENT_IMAGE_DOMAIN######ZARF_PKG_TMPL_AGENT_IMAGE###:###ZARF_PKG_TMPL_AGENT_IMAGE_TAG###" manifests: diff --git a/packages/zarf-registry/zarf.yaml b/packages/zarf-registry/zarf.yaml index 84b11372fe..38a13006c1 100644 --- a/packages/zarf-registry/zarf.yaml +++ b/packages/zarf-registry/zarf.yaml @@ -70,6 +70,7 @@ components: description: | Bootstraps a Kubernetes cluster by cloning a running pod in the cluster and hosting the registry image. Removed and destroyed after the Zarf Registry is self-hosting the registry image. + required: true only: cluster: architecture: amd64 @@ -84,6 +85,7 @@ components: description: | Bootstraps a Kubernetes cluster by cloning a running pod in the cluster and hosting the registry image. Removed and destroyed after the Zarf Registry is self-hosting the registry image. + required: true only: cluster: architecture: arm64 From e521098ea52934b6553dd6a1dacff1b2eef3888a Mon Sep 17 00:00:00 2001 From: razzle Date: Tue, 2 Apr 2024 11:59:50 -0500 Subject: [PATCH 085/113] restore upgrade test Signed-off-by: razzle --- .github/workflows/test-upgrade.yml | 4 ++-- Makefile | 2 +- src/test/upgrade/pre-0.33/zarf.yaml | 11 ----------- src/test/upgrade/zarf.yaml | 1 + 4 files changed, 4 insertions(+), 14 deletions(-) delete mode 100644 src/test/upgrade/pre-0.33/zarf.yaml diff --git a/.github/workflows/test-upgrade.yml b/.github/workflows/test-upgrade.yml index d83ecaa31a..af686448d7 100644 --- a/.github/workflows/test-upgrade.yml +++ b/.github/workflows/test-upgrade.yml @@ -81,8 +81,8 @@ jobs: # NOTE: "PATH=$PATH" preserves the default user $PATH. This is needed to maintain the version of zarf installed # in a previous step. This test run will use the current release to create a K3s cluster. run: | - zarf package create src/test/upgrade/pre-0.33 --set PODINFO_VERSION=6.3.3 --confirm - sudo env "PATH=$PATH" CI=true zarf package deploy zarf-package-test-upgrade-package-amd64-6.3.3.tar.zst --components="test-upgrade-package" --confirm + zarf package create src/test/upgrade --set PODINFO_VERSION=6.3.3 --confirm + sudo env "PATH=$PATH" CI=true zarf package deploy zarf-package-test-upgrade-package-amd64-6.3.3.tar.zst --confirm sudo env "PATH=$PATH" CI=true zarf tools kubectl describe deployments -n=podinfo-upgrade sudo env "PATH=$PATH" CI=true zarf tools kubectl describe pods -n=podinfo-upgrade diff --git a/Makefile b/Makefile index 55febc7e3e..a1e0a552da 100644 --- a/Makefile +++ b/Makefile @@ -203,7 +203,7 @@ test-upgrade: ## Run the Zarf CLI E2E tests for an external registry and cluster @test -s $(ZARF_BIN) || $(MAKE) build-cli [ -n "$(shell zarf version)" ] || (echo "Zarf must be installed prior to the upgrade test" && exit 1) [ -n "$(shell zarf package list 2>&1 | grep test-upgrade-package)" ] || (echo "Zarf must be initialized and have the 6.3.3 upgrade-test package installed prior to the upgrade test" && exit 1) - @test -s ./zarf-package-test-upgrade-package-$(ARCH)-6.3.4.tar.zst || zarf package create src/test/upgrade/pre-0.33 --set PODINFO_VERSION=6.3.4 --confirm + @test -s ./zarf-package-test-upgrade-package-$(ARCH)-6.3.4.tar.zst || zarf package create src/test/upgrade --set PODINFO_VERSION=6.3.4 --confirm cd src/test/upgrade && go test -failfast -v -timeout 30m .PHONY: test-unit diff --git a/src/test/upgrade/pre-0.33/zarf.yaml b/src/test/upgrade/pre-0.33/zarf.yaml deleted file mode 100644 index 114e101ad3..0000000000 --- a/src/test/upgrade/pre-0.33/zarf.yaml +++ /dev/null @@ -1,11 +0,0 @@ -kind: ZarfPackageConfig -metadata: - name: test-upgrade-package - description: Deploy podinfo using a Helm OCI chart - version: "###ZARF_PKG_VAR_PODINFO_VERSION###" - -components: - - name: test-upgrade-package - required: true - import: - path: .. diff --git a/src/test/upgrade/zarf.yaml b/src/test/upgrade/zarf.yaml index 3539fce943..34453ce6bc 100644 --- a/src/test/upgrade/zarf.yaml +++ b/src/test/upgrade/zarf.yaml @@ -6,6 +6,7 @@ metadata: components: - name: test-upgrade-package + required: true description: A semi-contrived example that deploys podinfo using many Zarf primitives and lets us upgrade it a few times. charts: - name: podinfo-upgrade From 8d2b5bab7f93c9c66bae170baa0d1056b8096568 Mon Sep 17 00:00:00 2001 From: razzle Date: Tue, 2 Apr 2024 12:07:19 -0500 Subject: [PATCH 086/113] restore test packages Signed-off-by: razzle --- src/test/e2e/09_component_compose_test.go | 1 + src/test/packages/00-helm-annotations/zarf.yaml | 1 + src/test/packages/00-kube-version-override/zarf.yaml | 1 + src/test/packages/00-no-components/zarf.yaml | 2 -- src/test/packages/00-remote-pull-fail/zarf.yaml | 1 + .../packages/03-deprecated-component-scripts/zarf.yaml | 3 --- .../packages/04-file-folders-templating-sbom/zarf.yaml | 2 ++ src/test/packages/05-multi-part/zarf.yaml | 1 + src/test/packages/08-differential-package/zarf.yaml | 2 ++ .../09-composable-packages/bad-local-os/zarf.yaml | 1 + src/test/packages/09-composable-packages/zarf.yaml | 1 + src/test/packages/22-git-data/zarf.yaml | 4 ++++ src/test/packages/24-evil-variables/zarf.yaml | 1 + src/test/packages/25-evil-chart-deps/zarf.yaml | 2 ++ src/test/packages/25-evil-chart-lookup/zarf.yaml | 2 +- src/test/packages/25-evil-dos-games/zarf.yaml | 1 + src/test/packages/25-evil-templates/zarf.yaml | 1 + src/test/packages/25-local-tgz-chart/zarf.yaml | 2 +- src/test/packages/26-image-dos-games/zarf.yaml | 1 + src/test/packages/28-helm-no-wait/zarf.yaml | 1 + .../packages/31-component-actions-edgecases/zarf.yaml | 1 + src/test/packages/34-manifest-with-symlink/zarf.yaml | 1 + .../51-import-everything/big-bang-min/zarf.yaml | 1 + .../packages/51-import-everything/inception/zarf.yaml | 5 +++++ .../packages/51-import-everything/oci-import/zarf.yaml | 2 +- src/test/packages/51-import-everything/zarf.yaml | 10 +++++----- src/test/packages/external-helm-auth/zarf.yaml | 1 + 27 files changed, 39 insertions(+), 13 deletions(-) diff --git a/src/test/e2e/09_component_compose_test.go b/src/test/e2e/09_component_compose_test.go index 2c7f8f950c..0d5acf7c0e 100644 --- a/src/test/e2e/09_component_compose_test.go +++ b/src/test/e2e/09_component_compose_test.go @@ -80,6 +80,7 @@ func (suite *CompositionSuite) Test_1_FullComposability() { suite.Contains(stdErr, ` - name: test-compose-package description: A contrived example for podinfo using many Zarf primitives for compose testing + required: true only: localOS: linux `) diff --git a/src/test/packages/00-helm-annotations/zarf.yaml b/src/test/packages/00-helm-annotations/zarf.yaml index b9742919a4..c48384748a 100644 --- a/src/test/packages/00-helm-annotations/zarf.yaml +++ b/src/test/packages/00-helm-annotations/zarf.yaml @@ -5,6 +5,7 @@ metadata: components: - name: istio-repo1-component description: Helm chart for the Istio Control Plane + required: true charts: - name: istio version: 1.17.2-bb.2 diff --git a/src/test/packages/00-kube-version-override/zarf.yaml b/src/test/packages/00-kube-version-override/zarf.yaml index c889bbb910..62521c1df8 100644 --- a/src/test/packages/00-kube-version-override/zarf.yaml +++ b/src/test/packages/00-kube-version-override/zarf.yaml @@ -5,6 +5,7 @@ metadata: components: - name: cert-manager-component description: Cloud native certificate management + required: true charts: - name: cert-manager version: v1.11.1 diff --git a/src/test/packages/00-no-components/zarf.yaml b/src/test/packages/00-no-components/zarf.yaml index 33fd9a27b8..78d89b650f 100644 --- a/src/test/packages/00-no-components/zarf.yaml +++ b/src/test/packages/00-no-components/zarf.yaml @@ -4,8 +4,6 @@ metadata: components: - name: deselect-me - optional: true default: true - name: optional - optional: true diff --git a/src/test/packages/00-remote-pull-fail/zarf.yaml b/src/test/packages/00-remote-pull-fail/zarf.yaml index 4ef5d94112..f44d99d4b1 100644 --- a/src/test/packages/00-remote-pull-fail/zarf.yaml +++ b/src/test/packages/00-remote-pull-fail/zarf.yaml @@ -3,5 +3,6 @@ metadata: name: doesnotexist components: - name: doesnotexist-docker + required: true images: - ghcr.io/defenseunicorns/doesnotexist:1.3.3.7 diff --git a/src/test/packages/03-deprecated-component-scripts/zarf.yaml b/src/test/packages/03-deprecated-component-scripts/zarf.yaml index e7845b5a3e..57a7c8f838 100644 --- a/src/test/packages/03-deprecated-component-scripts/zarf.yaml +++ b/src/test/packages/03-deprecated-component-scripts/zarf.yaml @@ -6,7 +6,6 @@ metadata: components: # Test that prepare scripts become onCreate actions - name: 1-test-deprecated-prepare-scripts - optional: true scripts: prepare: # on Windows, touch is replaced with New-Item @@ -14,7 +13,6 @@ components: # Test that deploy scripts become onCreate actions - name: 2-test-deprecated-deploy-scripts - optional: true scripts: before: - touch test-deprecated-deploy-before-hook.txt @@ -23,7 +21,6 @@ components: # Test that script timeouts still get set - name: 3-test-deprecated-timeout-scripts - optional: true scripts: timeoutSeconds: 1 before: diff --git a/src/test/packages/04-file-folders-templating-sbom/zarf.yaml b/src/test/packages/04-file-folders-templating-sbom/zarf.yaml index b251d00300..5aa4cb4ddf 100644 --- a/src/test/packages/04-file-folders-templating-sbom/zarf.yaml +++ b/src/test/packages/04-file-folders-templating-sbom/zarf.yaml @@ -9,6 +9,7 @@ variables: components: - name: folders + required: true files: - source: include-files target: temp/include-files @@ -19,6 +20,7 @@ components: - cmd: cat temp/include-files/something.yaml - cmd: rm -r temp - name: files + required: true files: - source: requirements.txt target: temp/requirements.txt diff --git a/src/test/packages/05-multi-part/zarf.yaml b/src/test/packages/05-multi-part/zarf.yaml index 8a2d816fae..0fb4a75652 100644 --- a/src/test/packages/05-multi-part/zarf.yaml +++ b/src/test/packages/05-multi-part/zarf.yaml @@ -5,6 +5,7 @@ metadata: components: - name: big-ol-file + required: true description: Include a 50 MB file needed to demonstrate a multi-part package actions: onCreate: diff --git a/src/test/packages/08-differential-package/zarf.yaml b/src/test/packages/08-differential-package/zarf.yaml index a2639b786e..9d911a66d0 100644 --- a/src/test/packages/08-differential-package/zarf.yaml +++ b/src/test/packages/08-differential-package/zarf.yaml @@ -6,6 +6,7 @@ metadata: components: - name: versioned-assets + required: true images: - ghcr.io/stefanprodan/podinfo:6.0.0 - ghcr.io/defenseunicorns/zarf/agent:###ZARF_PKG_TMPL_PACKAGE_VERSION### @@ -14,6 +15,7 @@ components: - https://github.com/defenseunicorns/zarf.git@refs/tags/###ZARF_PKG_TMPL_PACKAGE_VERSION### - name: generalized-assets + required: true images: - ghcr.io/stefanprodan/podinfo:latest repos: diff --git a/src/test/packages/09-composable-packages/bad-local-os/zarf.yaml b/src/test/packages/09-composable-packages/bad-local-os/zarf.yaml index d2607a55d2..d0772be32e 100644 --- a/src/test/packages/09-composable-packages/bad-local-os/zarf.yaml +++ b/src/test/packages/09-composable-packages/bad-local-os/zarf.yaml @@ -8,6 +8,7 @@ components: description: A contrived example for localOS compose testing only: localOS: windows + required: true import: path: ../sub-package name: test-compose-sub-package diff --git a/src/test/packages/09-composable-packages/zarf.yaml b/src/test/packages/09-composable-packages/zarf.yaml index 0605e72a85..cf121120e6 100644 --- a/src/test/packages/09-composable-packages/zarf.yaml +++ b/src/test/packages/09-composable-packages/zarf.yaml @@ -7,6 +7,7 @@ metadata: components: - name: test-compose-package description: A contrived example for podinfo using many Zarf primitives for compose testing + required: true import: path: sub-package name: test-compose-sub-package diff --git a/src/test/packages/22-git-data/zarf.yaml b/src/test/packages/22-git-data/zarf.yaml index c16bc55693..2e3b56d75c 100644 --- a/src/test/packages/22-git-data/zarf.yaml +++ b/src/test/packages/22-git-data/zarf.yaml @@ -6,6 +6,7 @@ metadata: components: - name: full-repo + required: true repos: # Do a full Git Repo Mirror - https://github.com/defenseunicorns/zarf-public-test.git @@ -15,6 +16,7 @@ components: - https://github.com/defenseunicorns/golang-tekton-hello-world.git - name: specific-tag + required: true repos: # The following performs a tag Git Repo Mirror with `go-git` (internal to Zarf) - https://github.com/defenseunicorns/zarf-public-test.git@v0.0.1 @@ -31,6 +33,7 @@ components: maxRetries: 3 - name: specific-branch + required: true repos: # The following performs a branch Git Repo Mirror with `go-git` (internal to Zarf) - https://github.com/defenseunicorns/zarf-public-test.git@refs/heads/dragons @@ -45,6 +48,7 @@ components: maxRetries: 3 - name: specific-hash + required: true repos: # The following performs a SHA Git Repo Mirror with `go-git` (internal to Zarf) - https://github.com/defenseunicorns/zarf-public-test.git@01a23218923f24194133b5eb11268cf8d73ff1bb diff --git a/src/test/packages/24-evil-variables/zarf.yaml b/src/test/packages/24-evil-variables/zarf.yaml index 6b4a15b505..906bced7c5 100644 --- a/src/test/packages/24-evil-variables/zarf.yaml +++ b/src/test/packages/24-evil-variables/zarf.yaml @@ -6,6 +6,7 @@ metadata: components: - name: bad-set-variables-pattern description: "###ZARF_PKG_TMPL_NUMB3R5###" + required: true actions: onDeploy: before: diff --git a/src/test/packages/25-evil-chart-deps/zarf.yaml b/src/test/packages/25-evil-chart-deps/zarf.yaml index 75afea1d1e..1199c46b66 100644 --- a/src/test/packages/25-evil-chart-deps/zarf.yaml +++ b/src/test/packages/25-evil-chart-deps/zarf.yaml @@ -5,6 +5,7 @@ metadata: components: - name: good + required: true charts: - name: dogs localPath: good-chart @@ -12,6 +13,7 @@ components: namespace: good - name: bad + required: true charts: - name: cats localPath: bad-chart diff --git a/src/test/packages/25-evil-chart-lookup/zarf.yaml b/src/test/packages/25-evil-chart-lookup/zarf.yaml index ee12ec8866..b9f8a0441f 100644 --- a/src/test/packages/25-evil-chart-lookup/zarf.yaml +++ b/src/test/packages/25-evil-chart-lookup/zarf.yaml @@ -6,7 +6,7 @@ metadata: components: - name: bad-helm-repository-name - optional: true + required: false charts: - name: asdf version: 6.4.0 diff --git a/src/test/packages/25-evil-dos-games/zarf.yaml b/src/test/packages/25-evil-dos-games/zarf.yaml index f8188cf543..41ecd44716 100644 --- a/src/test/packages/25-evil-dos-games/zarf.yaml +++ b/src/test/packages/25-evil-dos-games/zarf.yaml @@ -5,6 +5,7 @@ metadata: components: - name: baseline + required: true manifests: - name: multi-games namespace: dos-games diff --git a/src/test/packages/25-evil-templates/zarf.yaml b/src/test/packages/25-evil-templates/zarf.yaml index b431ddc843..66dbc71c30 100644 --- a/src/test/packages/25-evil-templates/zarf.yaml +++ b/src/test/packages/25-evil-templates/zarf.yaml @@ -4,6 +4,7 @@ metadata: components: - name: test + required: true manifests: - name: configmap files: diff --git a/src/test/packages/25-local-tgz-chart/zarf.yaml b/src/test/packages/25-local-tgz-chart/zarf.yaml index 8d884a567c..42158bc7be 100644 --- a/src/test/packages/25-local-tgz-chart/zarf.yaml +++ b/src/test/packages/25-local-tgz-chart/zarf.yaml @@ -6,7 +6,7 @@ metadata: components: - name: demo-helm-local-chart - optional: true + required: false charts: - name: podinfo version: 6.4.0 diff --git a/src/test/packages/26-image-dos-games/zarf.yaml b/src/test/packages/26-image-dos-games/zarf.yaml index c64ec1ca78..06131de09b 100644 --- a/src/test/packages/26-image-dos-games/zarf.yaml +++ b/src/test/packages/26-image-dos-games/zarf.yaml @@ -5,6 +5,7 @@ metadata: components: - name: baseline + required: true images: - defenseunicorns/zarf-game:multi-tile-dark - defenseunicorns/zarf-game:multi-tile-dark@sha256:f78e442f0f3eb3e9459b5ae6b1a8fda62f8dfe818112e7d130a4e8ae72b3cbff diff --git a/src/test/packages/28-helm-no-wait/zarf.yaml b/src/test/packages/28-helm-no-wait/zarf.yaml index 44b2157f13..177fe6792a 100644 --- a/src/test/packages/28-helm-no-wait/zarf.yaml +++ b/src/test/packages/28-helm-no-wait/zarf.yaml @@ -5,6 +5,7 @@ metadata: components: - name: helm-no-wait + required: true manifests: - name: never-ready namespace: no-wait diff --git a/src/test/packages/31-component-actions-edgecases/zarf.yaml b/src/test/packages/31-component-actions-edgecases/zarf.yaml index 2d22ef34e9..3f8e08264f 100644 --- a/src/test/packages/31-component-actions-edgecases/zarf.yaml +++ b/src/test/packages/31-component-actions-edgecases/zarf.yaml @@ -5,6 +5,7 @@ metadata: components: - name: on-deploy + required: true actions: # runs during "zarf package remove" onDeploy: diff --git a/src/test/packages/34-manifest-with-symlink/zarf.yaml b/src/test/packages/34-manifest-with-symlink/zarf.yaml index d421f45088..41d21b48e6 100644 --- a/src/test/packages/34-manifest-with-symlink/zarf.yaml +++ b/src/test/packages/34-manifest-with-symlink/zarf.yaml @@ -6,6 +6,7 @@ metadata: components: - name: manifest-with-symlink + required: true files: - source: manifests target: temp/manifests diff --git a/src/test/packages/51-import-everything/big-bang-min/zarf.yaml b/src/test/packages/51-import-everything/big-bang-min/zarf.yaml index e1540e9b57..6fd483c848 100644 --- a/src/test/packages/51-import-everything/big-bang-min/zarf.yaml +++ b/src/test/packages/51-import-everything/big-bang-min/zarf.yaml @@ -14,6 +14,7 @@ variables: components: - name: bigbang + required: true extensions: bigbang: repo: https://github.com/DoD-Platform-One/big-bang.git diff --git a/src/test/packages/51-import-everything/inception/zarf.yaml b/src/test/packages/51-import-everything/inception/zarf.yaml index 78f5337ee2..7de7d43d81 100644 --- a/src/test/packages/51-import-everything/inception/zarf.yaml +++ b/src/test/packages/51-import-everything/inception/zarf.yaml @@ -6,21 +6,26 @@ metadata: components: - name: import-component-local + required: true import: url: oci://localhost:555/import-everything:0.0.1 - name: import-component-oci + required: true import: url: oci://localhost:555/import-everything:0.0.1 - name: import-big-bang + required: true import: url: oci://localhost:555/import-everything:0.0.1 - name: file-imports + required: true import: url: oci://localhost:555/import-everything:0.0.1 - name: local-chart-import + required: true import: url: oci://localhost:555/import-everything:0.0.1 diff --git a/src/test/packages/51-import-everything/oci-import/zarf.yaml b/src/test/packages/51-import-everything/oci-import/zarf.yaml index b544186e69..42d4b79901 100644 --- a/src/test/packages/51-import-everything/oci-import/zarf.yaml +++ b/src/test/packages/51-import-everything/oci-import/zarf.yaml @@ -7,7 +7,7 @@ metadata: components: - name: import-component-oci description: "import-component-oci == ###ZARF_COMPONENT_NAME###" - optional: true + required: false import: name: demo-helm-charts url: oci://localhost:555/helm-charts:0.0.1 diff --git a/src/test/packages/51-import-everything/zarf.yaml b/src/test/packages/51-import-everything/zarf.yaml index 653d2ff1e6..b69ef73086 100644 --- a/src/test/packages/51-import-everything/zarf.yaml +++ b/src/test/packages/51-import-everything/zarf.yaml @@ -8,7 +8,7 @@ components: # Test every simple primitive that Zarf has through a local import - name: import-component-local description: "import-component-local == ###ZARF_COMPONENT_NAME###" - optional: true + required: false import: path: ../09-composable-packages name: test-compose-package @@ -16,7 +16,7 @@ components: # Test nested local to oci imports - name: import-component-oci description: "import-component-oci == ###ZARF_COMPONENT_NAME###" - optional: true + required: false import: name: import-component-oci path: oci-import @@ -24,7 +24,7 @@ components: # Test big bang extension files - name: import-big-bang description: "import-big-bang == ###ZARF_COMPONENT_NAME###" - optional: true + required: false import: name: bigbang url: oci://localhost:555/big-bang-min:2.10.0 @@ -32,7 +32,7 @@ components: # Test file imports including cosignKeyPath - name: file-imports description: "file-imports == ###ZARF_COMPONENT_NAME###" - optional: true + required: false cosignKeyPath: ../../../../cosign.pub files: # Import of a local file @@ -57,7 +57,7 @@ components: # Test local charts (for skeletons) - name: local-chart-import description: "local-chart-import == ###ZARF_COMPONENT_NAME###" - optional: true + required: false charts: - name: podinfo localPath: ../../../../examples/helm-charts/chart diff --git a/src/test/packages/external-helm-auth/zarf.yaml b/src/test/packages/external-helm-auth/zarf.yaml index 26e5ba2782..d373670825 100644 --- a/src/test/packages/external-helm-auth/zarf.yaml +++ b/src/test/packages/external-helm-auth/zarf.yaml @@ -4,6 +4,7 @@ metadata: version: 0.0.1 components: - name: private-chart + required: true charts: - name: podinfo version: 6.4.0 From 9b25338fe5ee548b5f2f70aec71bd0ff75ebc2e4 Mon Sep 17 00:00:00 2001 From: razzle Date: Tue, 2 Apr 2024 12:12:52 -0500 Subject: [PATCH 087/113] restore test packages Signed-off-by: razzle --- src/extensions/bigbang/test/package/zarf.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/src/extensions/bigbang/test/package/zarf.yaml b/src/extensions/bigbang/test/package/zarf.yaml index 478e3fc01c..426fed8f54 100644 --- a/src/extensions/bigbang/test/package/zarf.yaml +++ b/src/extensions/bigbang/test/package/zarf.yaml @@ -14,6 +14,7 @@ variables: components: - name: bigbang + required: true extensions: bigbang: version: "###ZARF_PKG_TMPL_BB_VERSION###" From cc2d72ade81186537578c17e73e01dc44f534f08 Mon Sep 17 00:00:00 2001 From: razzle Date: Wed, 3 Apr 2024 02:17:32 -0500 Subject: [PATCH 088/113] ok, i might be a little crazy Signed-off-by: razzle --- src/cmd/dev.go | 87 ++++++++++++------- src/pkg/layout/package.go | 13 +-- src/pkg/packager/common.go | 4 +- src/pkg/packager/composer/list.go | 25 ++++-- src/pkg/packager/creator/compose.go | 2 +- src/pkg/packager/creator/utils.go | 6 +- .../{deprecated => migrations}/common.go | 61 ++++++------- .../packager/migrations/default_required.go | 35 ++++++++ .../pluralize_set_variable.go | 9 +- .../scripts_to_actions.go | 9 +- src/types/beta.go | 20 +++++ src/types/package.go | 17 +--- src/types/validate.go | 18 ++-- zarf.yaml | 3 +- 14 files changed, 189 insertions(+), 120 deletions(-) rename src/pkg/packager/{deprecated => migrations}/common.go (72%) create mode 100644 src/pkg/packager/migrations/default_required.go rename src/pkg/packager/{deprecated => migrations}/pluralize_set_variable.go (91%) rename src/pkg/packager/{deprecated => migrations}/scripts_to_actions.go (89%) create mode 100644 src/types/beta.go diff --git a/src/cmd/dev.go b/src/cmd/dev.go index f19910b309..fe2ceaf71a 100644 --- a/src/cmd/dev.go +++ b/src/cmd/dev.go @@ -9,6 +9,7 @@ import ( "io" "os" "path/filepath" + "slices" "strings" goyaml "github.com/goccy/go-yaml" @@ -21,8 +22,8 @@ import ( "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/packager" - "github.com/defenseunicorns/zarf/src/pkg/packager/deprecated" "github.com/defenseunicorns/zarf/src/pkg/packager/lint" + "github.com/defenseunicorns/zarf/src/pkg/packager/migrations" "github.com/defenseunicorns/zarf/src/pkg/transform" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/types" @@ -32,7 +33,8 @@ import ( ) var extractPath string -var migrationsToRun []string +var deprecatedMigrationsToRun []string +var betaFeatureMigrationsToRun []string var devCmd = &cobra.Command{ Use: "dev", @@ -71,7 +73,7 @@ var devMigrateCmd = &cobra.Command{ Short: lang.CmdDevMigrateShort, Long: lang.CmdDevMigrateLong, Args: cobra.MaximumNArgs(1), - RunE: func(cmd *cobra.Command, args []string) error { + RunE: func(_ *cobra.Command, args []string) error { dir := common.SetBaseDirectory(args) var pkg types.ZarfPackage cm := goyaml.CommentMap{} @@ -90,36 +92,34 @@ var devMigrateCmd = &cobra.Command{ return err } - all := deprecated.Migrations() + deprecatedMigrationsToRun = slices.Compact(deprecatedMigrationsToRun) - migrations := []deprecated.Migration{} + for _, m := range migrations.DeprecatedComponentMigrations() { + if !slices.Contains(deprecatedMigrationsToRun, m.ID()) && deprecatedMigrationsToRun != nil { + continue + } + + // Migrate the package definition + for idx, component := range pkg.Components { + ran := []string{} + ran = append(ran, m.ID()) + c, _ := m.Run(component) + c = m.Clear(c) + pkg.Components[idx] = c - // Only run the specified migrations - for _, migrationToRun := range migrationsToRun { - for _, migration := range all { - if migration.ID() == migrationToRun { - migrations = append(migrations, migration) + if len(ran) > 0 { + message.Successf("Ran %s on %q", strings.Join(ran, ", "), component.Name) } } } - if len(migrations) == 0 { - // Run all migrations - migrations = all - } - - // Migrate the package definition - for idx, component := range pkg.Components { - ran := []string{} - for _, migration := range migrations { - ran = append(ran, migration.ID()) - c, _ := migration.Run(component) - c = migration.Clear(c) - pkg.Components[idx] = c - } - if len(ran) > 0 { - message.Successf("Ran %s on %q", strings.Join(ran, ", "), component.Name) + betaFeatureMigrationsToRun = slices.Compact(betaFeatureMigrationsToRun) + for _, m := range migrations.BetaFeatureMigrations() { + if !slices.Contains(betaFeatureMigrationsToRun, m.ID()) && betaFeatureMigrationsToRun != nil { + continue } + pkg = m.Run(pkg) + message.Successf("Enabled [BETA FEATURE FLAG] %s", m.ID()) } b, err = goyaml.MarshalWithOptions(pkg, goyaml.WithComment(cm), goyaml.IndentSequence(true), goyaml.UseSingleQuote(false)) @@ -362,13 +362,36 @@ func init() { bindDevDeployFlags(v) bindDevGenerateFlags(v) - allMigrations := []string{} - for _, migration := range deprecated.Migrations() { - allMigrations = append(allMigrations, migration.ID()) + dcm := []string{} + for _, m := range migrations.DeprecatedComponentMigrations() { + dcm = append(dcm, m.ID()) } - devMigrateCmd.Flags().StringArrayVar(&migrationsToRun, "run", []string{}, fmt.Sprintf("migrations to run (default: all, available: %s)", strings.Join(allMigrations, ", "))) - devMigrateCmd.RegisterFlagCompletionFunc("run", func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - return allMigrations, cobra.ShellCompDirectiveNoFileComp + devMigrateCmd.Flags().StringArrayVar(&deprecatedMigrationsToRun, "run", []string{}, fmt.Sprintf("migrations to run (default: all, available: %s)", strings.Join(dcm, ", "))) + devMigrateCmd.RegisterFlagCompletionFunc("run", func(_ *cobra.Command, _ []string, _ string) ([]string, cobra.ShellCompDirective) { + ids := []string{} + for _, migration := range migrations.DeprecatedComponentMigrations() { + if slices.Contains(deprecatedMigrationsToRun, migration.ID()) { + continue + } + ids = append(ids, migration.ID()) + } + return ids, cobra.ShellCompDirectiveNoFileComp + }) + + bfm := []string{} + for _, m := range migrations.BetaFeatureMigrations() { + bfm = append(bfm, m.ID()) + } + devMigrateCmd.Flags().StringArrayVar(&betaFeatureMigrationsToRun, "enable-beta-feature", []string{}, fmt.Sprintf("beta migrations to run and enable (default: all, available: %s)", strings.Join(bfm, ", "))) + devMigrateCmd.RegisterFlagCompletionFunc("enable-beta-feature", func(_ *cobra.Command, _ []string, _ string) ([]string, cobra.ShellCompDirective) { + ids := []string{} + for _, migration := range migrations.BetaFeatureMigrations() { + if slices.Contains(betaFeatureMigrationsToRun, migration.ID()) { + continue + } + ids = append(ids, migration.ID()) + } + return ids, cobra.ShellCompDirectiveNoFileComp }) devSha256SumCmd.Flags().StringVarP(&extractPath, "extract-path", "e", "", lang.CmdDevFlagExtractPath) diff --git a/src/pkg/layout/package.go b/src/pkg/layout/package.go index 3949f4fcc2..8f6ea15363 100644 --- a/src/pkg/layout/package.go +++ b/src/pkg/layout/package.go @@ -15,7 +15,7 @@ import ( "github.com/defenseunicorns/pkg/helpers" "github.com/defenseunicorns/zarf/src/pkg/interactive" "github.com/defenseunicorns/zarf/src/pkg/message" - "github.com/defenseunicorns/zarf/src/pkg/packager/deprecated" + "github.com/defenseunicorns/zarf/src/pkg/packager/migrations" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/types" "github.com/google/go-containerregistry/pkg/crane" @@ -69,11 +69,14 @@ func (pp *PackagePaths) ReadZarfYAML() (pkg types.ZarfPackage, warnings []string } if len(pkg.Build.Migrations) > 0 { - var componentWarnings []string for idx, component := range pkg.Components { - // Handle component configuration deprecations - pkg.Components[idx], componentWarnings = deprecated.MigrateComponent(pkg.Build, component) - warnings = append(warnings, componentWarnings...) + // Clear out component configuration migrations + for _, m := range migrations.DeprecatedComponentMigrations() { + if slices.Contains(pkg.Build.Migrations, m.ID()) { + mc := m.Clear(component) + pkg.Components[idx] = mc + } + } } } diff --git a/src/pkg/packager/common.go b/src/pkg/packager/common.go index ba1335852e..7bfa7bebe2 100644 --- a/src/pkg/packager/common.go +++ b/src/pkg/packager/common.go @@ -22,7 +22,7 @@ import ( "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/message" - "github.com/defenseunicorns/zarf/src/pkg/packager/deprecated" + "github.com/defenseunicorns/zarf/src/pkg/packager/migrations" "github.com/defenseunicorns/zarf/src/pkg/packager/sources" "github.com/defenseunicorns/zarf/src/pkg/utils" ) @@ -207,7 +207,7 @@ func (p *Packager) attemptClusterChecks() (err error) { // Check for any breaking changes between the initialized Zarf version and this CLI if existingInitPackage, _ := p.cluster.GetDeployedPackage("init"); existingInitPackage != nil { // Use the build version instead of the metadata since this will support older Zarf versions - deprecated.PrintBreakingChanges(existingInitPackage.Data.Build.Version) + migrations.PrintBreakingChanges(existingInitPackage.Data.Build.Version) } spinner.Success() diff --git a/src/pkg/packager/composer/list.go b/src/pkg/packager/composer/list.go index e53f26a659..fd0b35fb37 100644 --- a/src/pkg/packager/composer/list.go +++ b/src/pkg/packager/composer/list.go @@ -12,7 +12,7 @@ import ( "github.com/defenseunicorns/pkg/helpers" "github.com/defenseunicorns/zarf/src/pkg/layout" - "github.com/defenseunicorns/zarf/src/pkg/packager/deprecated" + "github.com/defenseunicorns/zarf/src/pkg/packager/migrations" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/pkg/zoci" "github.com/defenseunicorns/zarf/src/types" @@ -255,12 +255,21 @@ func (ic *ImportChain) String() string { } // Migrate performs migrations on the import chain -func (ic *ImportChain) Migrate(build types.ZarfBuildData) (warnings []string) { +func (ic *ImportChain) Migrate() (warnings []string) { node := ic.head for node != nil { - migrated, w := deprecated.MigrateComponent(build, node.ZarfComponent) - node.ZarfComponent = migrated - warnings = append(warnings, w...) + for _, m := range migrations.DeprecatedComponentMigrations() { + migrated, warning := m.Run(node.ZarfComponent) + node.ZarfComponent = migrated + if warning != "" { + warnings = append(warnings, warning) + } + } + + // Show a warning if the component contains a group as that has been deprecated and will be removed. + if node.DeprecatedGroup != "" { + warnings = append(warnings, fmt.Sprintf("Component %s is using group which has been deprecated and will be removed in v1.0.0. Please migrate to another solution.", node.Name)) + } node = node.next } if len(warnings) > 0 { @@ -291,9 +300,9 @@ func (ic *ImportChain) Compose() (composed *types.ZarfComponent, err error) { node := ic.tail for node != nil { // if prev.required is nil, and node.required is != nil, then error - if node.prev != nil && node.prev.Required == nil && node.Required != nil { - return nil, fmt.Errorf("component %q: required cannot be unset during composition from %q to %q", node.ImportName(), node.ImportLocation(), node.prev.ImportLocation()) - } + // if node.prev != nil && node.prev.Required == nil && node.Required != nil { + // return nil, fmt.Errorf("component %q: required cannot be unset during composition from %q to %q", node.ImportName(), node.ImportLocation(), node.prev.ImportLocation()) + // } fixPaths(&node.ZarfComponent, node.relativeToHead) diff --git a/src/pkg/packager/creator/compose.go b/src/pkg/packager/creator/compose.go index dd2e429731..cc1c93e390 100644 --- a/src/pkg/packager/creator/compose.go +++ b/src/pkg/packager/creator/compose.go @@ -38,7 +38,7 @@ func ComposeComponents(pkg types.ZarfPackage, flavor string) (types.ZarfPackage, message.Debugf("%s", chain) // migrate any deprecated component configurations now - warning := chain.Migrate(pkg.Build) + warning := chain.Migrate() warnings = append(warnings, warning...) // get the composed component diff --git a/src/pkg/packager/creator/utils.go b/src/pkg/packager/creator/utils.go index 452e44c0fc..43d7619f48 100644 --- a/src/pkg/packager/creator/utils.go +++ b/src/pkg/packager/creator/utils.go @@ -10,7 +10,7 @@ import ( "time" "github.com/defenseunicorns/zarf/src/config" - "github.com/defenseunicorns/zarf/src/pkg/packager/deprecated" + "github.com/defenseunicorns/zarf/src/pkg/packager/migrations" "github.com/defenseunicorns/zarf/src/types" ) @@ -45,7 +45,7 @@ func recordPackageMetadata(pkg *types.ZarfPackage, createOpts types.ZarfCreateOp // Record the migrations that will run on the package. pkg.Build.Migrations = []string{} - for _, m := range deprecated.Migrations() { + for _, m := range migrations.DeprecatedComponentMigrations() { pkg.Build.Migrations = append(pkg.Build.Migrations, m.ID()) } @@ -55,7 +55,7 @@ func recordPackageMetadata(pkg *types.ZarfPackage, createOpts types.ZarfCreateOp pkg.Build.RegistryOverrides = createOpts.RegistryOverrides // Record the latest version of Zarf without breaking changes to the package structure. - pkg.Build.LastNonBreakingVersion = deprecated.LastNonBreakingVersion + pkg.Build.LastNonBreakingVersion = migrations.LastNonBreakingVersion return nil } diff --git a/src/pkg/packager/deprecated/common.go b/src/pkg/packager/migrations/common.go similarity index 72% rename from src/pkg/packager/deprecated/common.go rename to src/pkg/packager/migrations/common.go index d57b666497..ad20b09b68 100644 --- a/src/pkg/packager/deprecated/common.go +++ b/src/pkg/packager/migrations/common.go @@ -1,15 +1,12 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: 2021-Present The Zarf Authors -// Package deprecated handles package deprecations and migrations -package deprecated +// Package migrations handles component deprecations and package migrations +package migrations import ( - "fmt" "strings" - "slices" - "github.com/Masterminds/semver/v3" "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/pkg/message" @@ -42,47 +39,41 @@ var breakingChanges = []breakingChange{ type Migration interface { // ID returns the ID of the migration ID() string - // Clear clears the deprecated configuration from the component - Clear(mc types.ZarfComponent) types.ZarfComponent +} + +// DeprecatedComponentMigration represents a migration that can be run on a component. +// +// DeprecatedComponentMigrations are migrations that seamlessly migrate deprecated component definitions. +type DeprecatedComponentMigration interface { + Migration // Run runs the migration on the component Run(c types.ZarfComponent) (types.ZarfComponent, string) + // Clear clears the deprecated configuration from the component + Clear(mc types.ZarfComponent) types.ZarfComponent } -// Migrations returns a list of all current migrations. -// -// This function operates as the source of truth for consuming migrations. -func Migrations() []Migration { - return []Migration{ +// DeprecatedComponentMigrations returns a list of all current deprecated component-level migrations. +func DeprecatedComponentMigrations() []DeprecatedComponentMigration { + return []DeprecatedComponentMigration{ ScriptsToActions{}, SetVariableToSetVariables{}, } } -// MigrateComponent runs all migrations on a component. -// Build should be empty on package create, but include just in case someone copied a zarf.yaml from a zarf package. -func MigrateComponent(build types.ZarfBuildData, component types.ZarfComponent) (migratedComponent types.ZarfComponent, warnings []string) { - migratedComponent = component - - // Run all migrations - for _, m := range Migrations() { - if slices.Contains(build.Migrations, m.ID()) { - migratedComponent = m.Clear(migratedComponent) - } else { - // Otherwise, run the migration. - var warning string - if migratedComponent, warning = m.Run(migratedComponent); warning != "" { - warnings = append(warnings, warning) - } - } - } +// BetaFeatureMigration represents a migration that can be run on a package. +// +// Every migration is mapped to a specific beta feature, and the beta feature is added to the package metadata. +type BetaFeatureMigration interface { + Migration + // Run runs the beta migration on the package + Run(pkg types.ZarfPackage) types.ZarfPackage +} - // Show a warning if the component contains a group as that has been deprecated and will be removed. - if component.DeprecatedGroup != "" { - warnings = append(warnings, fmt.Sprintf("Component %s is using group which has been deprecated and will be removed in v1.0.0. Please migrate to another solution.", component.Name)) +// BetaFeatureMigrations returns a list of all current beta feature migrations. +func BetaFeatureMigrations() []BetaFeatureMigration { + return []BetaFeatureMigration{ + DefaultRequired{}, } - - // Future migrations here. - return migratedComponent, warnings } // PrintBreakingChanges prints the breaking changes between the provided version and the current CLIVersion diff --git a/src/pkg/packager/migrations/default_required.go b/src/pkg/packager/migrations/default_required.go new file mode 100644 index 0000000000..efa6e62b58 --- /dev/null +++ b/src/pkg/packager/migrations/default_required.go @@ -0,0 +1,35 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +package migrations + +import ( + "slices" + + "github.com/defenseunicorns/zarf/src/types" +) + +// DefaultRequired migrates the package to change components to be required by default +type DefaultRequired struct{} + +// ID returns the ID of the migration +func (DefaultRequired) ID() string { + return string(types.DefaultRequired) +} + +// Run sets all components to be required by default +// +// and cleanly migrates components explicitly marked as required to be nil +func (DefaultRequired) Run(pkg types.ZarfPackage) types.ZarfPackage { + for idx, component := range pkg.Components { + if component.Required != nil && *component.Required { + pkg.Components[idx].Required = nil + } + } + + if !slices.Contains(pkg.Metadata.BetaFeatures, types.DefaultRequired) { + pkg.Metadata.BetaFeatures = append(pkg.Metadata.BetaFeatures, types.DefaultRequired) + } + + return pkg +} diff --git a/src/pkg/packager/deprecated/pluralize_set_variable.go b/src/pkg/packager/migrations/pluralize_set_variable.go similarity index 91% rename from src/pkg/packager/deprecated/pluralize_set_variable.go rename to src/pkg/packager/migrations/pluralize_set_variable.go index 3513422a4e..6824a18227 100644 --- a/src/pkg/packager/deprecated/pluralize_set_variable.go +++ b/src/pkg/packager/migrations/pluralize_set_variable.go @@ -1,8 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: 2021-Present The Zarf Authors -// Package deprecated handles package deprecations and migrations -package deprecated +package migrations import ( "fmt" @@ -17,12 +16,12 @@ const PluralizeSetVariableID = "pluralize-set-variable" type SetVariableToSetVariables struct{} // ID returns the ID of the migration -func (m SetVariableToSetVariables) ID() string { +func (SetVariableToSetVariables) ID() string { return PluralizeSetVariableID } // Clear the deprecated setVariable. -func (m SetVariableToSetVariables) Clear(mc types.ZarfComponent) types.ZarfComponent { +func (SetVariableToSetVariables) Clear(mc types.ZarfComponent) types.ZarfComponent { clear := func(actions []types.ZarfComponentAction) []types.ZarfComponentAction { for i := range actions { actions[i].DeprecatedSetVariable = "" @@ -53,7 +52,7 @@ func (m SetVariableToSetVariables) Clear(mc types.ZarfComponent) types.ZarfCompo } // Run coverts the deprecated setVariable to the new setVariables -func (m SetVariableToSetVariables) Run(c types.ZarfComponent) (types.ZarfComponent, string) { +func (SetVariableToSetVariables) Run(c types.ZarfComponent) (types.ZarfComponent, string) { hasSetVariable := false migrate := func(actions []types.ZarfComponentAction) []types.ZarfComponentAction { diff --git a/src/pkg/packager/deprecated/scripts_to_actions.go b/src/pkg/packager/migrations/scripts_to_actions.go similarity index 89% rename from src/pkg/packager/deprecated/scripts_to_actions.go rename to src/pkg/packager/migrations/scripts_to_actions.go index beba4e919b..c46088f6de 100644 --- a/src/pkg/packager/deprecated/scripts_to_actions.go +++ b/src/pkg/packager/migrations/scripts_to_actions.go @@ -1,8 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: 2021-Present The Zarf Authors -// Package deprecated handles package deprecations and migrations -package deprecated +package migrations import ( "fmt" @@ -18,12 +17,12 @@ const ScriptsToActionsID = "scripts-to-actions" type ScriptsToActions struct{} // ID returns the ID of the migration -func (m ScriptsToActions) ID() string { +func (ScriptsToActions) ID() string { return ScriptsToActionsID } // Clear the deprecated scripts. -func (m ScriptsToActions) Clear(mc types.ZarfComponent) types.ZarfComponent { +func (ScriptsToActions) Clear(mc types.ZarfComponent) types.ZarfComponent { mc.DeprecatedScripts = types.DeprecatedZarfComponentScripts{} return mc } @@ -35,7 +34,7 @@ func (m ScriptsToActions) Clear(mc types.ZarfComponent) types.ZarfComponent { // - Actions.*.OnSuccess // - Actions.*.OnFailure // - Actions.*.*.Env -func (m ScriptsToActions) Run(c types.ZarfComponent) (types.ZarfComponent, string) { +func (ScriptsToActions) Run(c types.ZarfComponent) (types.ZarfComponent, string) { var hasScripts bool // Convert a script configs to action defaults. diff --git a/src/types/beta.go b/src/types/beta.go new file mode 100644 index 0000000000..0c3acd5fd7 --- /dev/null +++ b/src/types/beta.go @@ -0,0 +1,20 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package types contains all the types used by Zarf. +package types + +// FeatureFlag is an enum of the different feature flags that can be set on a Zarf package. +type FeatureFlag string + +const ( + // DefaultRequired changes the default state for all components in a package to be required. + DefaultRequired FeatureFlag = "default-required" +) + +// AllFeatureFlags returns a list of all available feature flags. +func AllFeatureFlags() []FeatureFlag { + return []FeatureFlag{ + DefaultRequired, + } +} diff --git a/src/types/package.go b/src/types/package.go index 1f117784a2..b1ad07e132 100644 --- a/src/types/package.go +++ b/src/types/package.go @@ -7,9 +7,6 @@ package types // ZarfPackageKind is an enum of the different kinds of Zarf packages. type ZarfPackageKind string -// FeatureFlag is an enum of the different feature flags that can be set on a Zarf package. -type FeatureFlag string - const ( // ZarfInitConfig is the kind of Zarf package used during `zarf init`. ZarfInitConfig ZarfPackageKind = "ZarfInitConfig" @@ -17,23 +14,11 @@ const ( ZarfPackageConfig ZarfPackageKind = "ZarfPackageConfig" ) -const ( - // DefaultRequired changes the default state for all components in a package to be required. - DefaultRequired FeatureFlag = "default-required" -) - -// AllFeatureFlags returns a list of all available feature flags. -func AllFeatureFlags() []FeatureFlag { - return []FeatureFlag{ - DefaultRequired, - } -} - // ZarfPackage the top-level structure of a Zarf config file. type ZarfPackage struct { Kind ZarfPackageKind `json:"kind" jsonschema:"description=The kind of Zarf package,enum=ZarfInitConfig,enum=ZarfPackageConfig,default=ZarfPackageConfig"` Metadata ZarfMetadata `json:"metadata,omitempty" jsonschema:"description=Package metadata"` - Build ZarfBuildData `json:"build,omitempty" jsonschema:"description=Zarf-generated package build data"` + Build ZarfBuildData `json:"build,omitempty" jsonschema:"-"` Components []ZarfComponent `json:"components" jsonschema:"description=List of components to deploy in this package"` Constants []ZarfPackageConstant `json:"constants,omitempty" jsonschema:"description=Constant template values applied on deploy for K8s resources"` Variables []ZarfPackageVariable `json:"variables,omitempty" jsonschema:"description=Variable template values applied on deploy for K8s resources"` diff --git a/src/types/validate.go b/src/types/validate.go index 426690aff9..71ae710aa2 100644 --- a/src/types/validate.go +++ b/src/types/validate.go @@ -39,7 +39,7 @@ func SupportedOS() []string { return supportedOS } -// Run performs config validations. +// Validate runs all validation checks on the package. func (pkg ZarfPackage) Validate() error { if pkg.Kind == ZarfInitConfig && pkg.Metadata.YOLO { return fmt.Errorf(lang.PkgValidateErrInitNoYOLO) @@ -173,6 +173,7 @@ func (pkg ZarfPackage) Validate() error { return nil } +// Validate runs all validation checks on component actions. func (a ZarfComponentActions) Validate() error { if err := a.OnCreate.Validate(); err != nil { return fmt.Errorf(lang.PkgValidateErrAction, err) @@ -193,7 +194,7 @@ func (a ZarfComponentActions) Validate() error { return nil } -// ImportDefinition validates the component trying to be imported. +// ValidateImportDefinition validates the component trying to be imported. func (c ZarfComponent) ValidateImportDefinition() error { path := c.Import.Path url := c.Import.URL @@ -227,6 +228,7 @@ func (c ZarfComponent) ValidateImportDefinition() error { return nil } +// HasSetVariables returns true if any of the actions contain setVariables. func (as ZarfComponentActionSet) HasSetVariables() bool { check := func(actions []ZarfComponentAction) bool { for _, action := range actions { @@ -240,6 +242,7 @@ func (as ZarfComponentActionSet) HasSetVariables() bool { return check(as.Before) || check(as.After) || check(as.OnSuccess) || check(as.OnFailure) } +// Validate runs all validation checks on component action sets. func (as ZarfComponentActionSet) Validate() error { validate := func(actions []ZarfComponentAction) error { for _, action := range actions { @@ -258,13 +261,10 @@ func (as ZarfComponentActionSet) Validate() error { if err := validate(as.OnSuccess); err != nil { return err } - if err := validate(as.OnFailure); err != nil { - return err - } - - return nil + return validate(as.OnFailure) } +// Validate runs all validation checks on an action. func (action ZarfComponentAction) Validate() error { // Validate SetVariable for _, variable := range action.SetVariables { @@ -293,6 +293,7 @@ func (action ZarfComponentAction) Validate() error { return nil } +// Validate runs all validation checks on a package variable. func (variable ZarfPackageVariable) Validate() error { if !IsUppercaseNumberUnderscore(variable.Name) { return fmt.Errorf(lang.PkgValidateMustBeUppercase, variable.Name) @@ -301,6 +302,7 @@ func (variable ZarfPackageVariable) Validate() error { return nil } +// Validate runs all validation checks on a package constant. func (constant ZarfPackageConstant) Validate() error { // ensure the constant name is only capitals and underscores if !IsUppercaseNumberUnderscore(constant.Name) { @@ -314,6 +316,7 @@ func (constant ZarfPackageConstant) Validate() error { return nil } +// Validate runs all validation checks on a chart. func (chart ZarfChart) Validate() error { // Don't allow empty names if chart.Name == "" { @@ -348,6 +351,7 @@ func (chart ZarfChart) Validate() error { return nil } +// Validate runs all validation checks on a manifest. func (manifest ZarfManifest) Validate() error { // Don't allow empty names if manifest.Name == "" { diff --git a/zarf.yaml b/zarf.yaml index e5c3366857..cf3d0d14cb 100644 --- a/zarf.yaml +++ b/zarf.yaml @@ -2,7 +2,8 @@ kind: ZarfInitConfig metadata: name: init description: Used to establish a new Zarf cluster - betaFeatures: [default-required] + betaFeatures: + - default-required components: - name: k3s From 7f17d63e3408ff09ae743a89c22439824f789d9d Mon Sep 17 00:00:00 2001 From: razzle Date: Wed, 3 Apr 2024 02:18:05 -0500 Subject: [PATCH 089/113] docs and schema Signed-off-by: razzle --- .../100-cli-commands/zarf_dev_migrate.md | 5 +- docs/3-create-a-zarf-package/4-zarf-schema.md | 293 ++---------------- zarf.schema.json | 80 ----- 3 files changed, 26 insertions(+), 352 deletions(-) diff --git a/docs/2-the-zarf-cli/100-cli-commands/zarf_dev_migrate.md b/docs/2-the-zarf-cli/100-cli-commands/zarf_dev_migrate.md index 25fa11a6b0..8518a57dfd 100644 --- a/docs/2-the-zarf-cli/100-cli-commands/zarf_dev_migrate.md +++ b/docs/2-the-zarf-cli/100-cli-commands/zarf_dev_migrate.md @@ -16,8 +16,9 @@ zarf dev migrate [flags] ## Options ``` - -h, --help help for migrate - --run stringArray migrations to run (default: all, available: scripts-to-actions, pluralize-set-variable) + --enable-beta-feature stringArray beta migrations to run and enable (default: all, available: default-required) + -h, --help help for migrate + --run stringArray migrations to run (default: all, available: scripts-to-actions, pluralize-set-variable) ``` ## Options inherited from parent commands diff --git a/docs/3-create-a-zarf-package/4-zarf-schema.md b/docs/3-create-a-zarf-package/4-zarf-schema.md index ad5e247567..f3a4f03816 100644 --- a/docs/3-create-a-zarf-package/4-zarf-schema.md +++ b/docs/3-create-a-zarf-package/4-zarf-schema.md @@ -302,253 +302,6 @@ Must be one of:
-
- - build - -  -
- - ## build - -**Description:** Zarf-generated package build data - -| | | -| ------------------------- | -------------------------------------------------------------------------------------------------------- | -| **Type** | `object` | -| **Additional properties** | [![Not allowed](https://img.shields.io/badge/Not%20allowed-red)](# "Additional Properties not allowed.") | -| **Defined in** | #/definitions/ZarfBuildData | - -
- - terminal * - -  -
- -![Required](https://img.shields.io/badge/Required-red) - -**Description:** The machine name that created this package - -| | | -| -------- | -------- | -| **Type** | `string` | - -
-
- -
- - user * - -  -
- -![Required](https://img.shields.io/badge/Required-red) - -**Description:** The username who created this package - -| | | -| -------- | -------- | -| **Type** | `string` | - -
-
- -
- - architecture * - -  -
- -![Required](https://img.shields.io/badge/Required-red) - -**Description:** The architecture this package was created on - -| | | -| -------- | -------- | -| **Type** | `string` | - -
-
- -
- - timestamp * - -  -
- -![Required](https://img.shields.io/badge/Required-red) - -**Description:** The timestamp when this package was created - -| | | -| -------- | -------- | -| **Type** | `string` | - -
-
- -
- - version * - -  -
- -![Required](https://img.shields.io/badge/Required-red) - -**Description:** The version of Zarf used to build this package - -| | | -| -------- | -------- | -| **Type** | `string` | - -
-
- -
- - migrations - -  -
- -**Description:** Any migrations that have been run on this package - -| | | -| -------- | ----------------- | -| **Type** | `array of string` | - -![Min Items: N/A](https://img.shields.io/badge/Min%20Items%3A%20N/A-gold) -![Max Items: N/A](https://img.shields.io/badge/Max%20Items%3A%20N/A-gold) -![Item unicity: False](https://img.shields.io/badge/Item%20unicity%3A%20False-gold) -![Additional items: N/A](https://img.shields.io/badge/Additional%20items%3A%20N/A-gold) - - ### migrations items - -| | | -| -------- | -------- | -| **Type** | `string` | - -
-
- -
- - registryOverrides - -  -
- - ## build > registryOverrides - -**Description:** Any registry domains that were overridden on package create when pulling images - -| | | -| ------------------------- | --------------------------------------------------------------------------------------------------------------------------------- | -| **Type** | `object` | -| **Additional properties** | [![Any type: allowed](https://img.shields.io/badge/Any%20type-allowed-green)](# "Additional Properties of any type are allowed.") | - -
-
- -
- - differential - -  -
- -**Description:** Whether this package was created with differential components - -| | | -| -------- | --------- | -| **Type** | `boolean` | - -
-
- -
- - differentialPackageVersion - -  -
- -**Description:** Version of a previously built package used as the basis for creating this differential package - -| | | -| -------- | -------- | -| **Type** | `string` | - -
-
- -
- - differentialMissing - -  -
- -**Description:** List of components that were not included in this package due to differential packaging - -| | | -| -------- | ----------------- | -| **Type** | `array of string` | - -![Min Items: N/A](https://img.shields.io/badge/Min%20Items%3A%20N/A-gold) -![Max Items: N/A](https://img.shields.io/badge/Max%20Items%3A%20N/A-gold) -![Item unicity: False](https://img.shields.io/badge/Item%20unicity%3A%20False-gold) -![Additional items: N/A](https://img.shields.io/badge/Additional%20items%3A%20N/A-gold) - - ### differentialMissing items - -| | | -| -------- | -------- | -| **Type** | `string` | - -
-
- -
- - lastNonBreakingVersion - -  -
- -**Description:** The minimum version of Zarf that does not have breaking package structure changes - -| | | -| -------- | -------- | -| **Type** | `string` | - -
-
- -
- - flavor - -  -
- -**Description:** The flavor of Zarf used to build this package - -| | | -| -------- | -------- | -| **Type** | `string` | - -
-
- -
-
-
components * @@ -570,7 +323,7 @@ Must be one of: ![Item unicity: False](https://img.shields.io/badge/Item%20unicity%3A%20False-gold) ![Additional items: N/A](https://img.shields.io/badge/Additional%20items%3A%20N/A-gold) - ### ZarfComponent + ### ZarfComponent | | | | ------------------------- | -------------------------------------------------------------------------------------------------------- | @@ -745,7 +498,7 @@ Must be one of: ![Item unicity: False](https://img.shields.io/badge/Item%20unicity%3A%20False-gold) ![Additional items: N/A](https://img.shields.io/badge/Additional%20items%3A%20N/A-gold) - ### distros items + ### distros items | | | | -------- | -------- | @@ -868,7 +621,7 @@ Must be one of: ![Item unicity: False](https://img.shields.io/badge/Item%20unicity%3A%20False-gold) ![Additional items: N/A](https://img.shields.io/badge/Additional%20items%3A%20N/A-gold) - ### ZarfManifest + ### ZarfManifest | | | | ------------------------- | -------------------------------------------------------------------------------------------------------- | @@ -928,7 +681,7 @@ Must be one of: ![Item unicity: False](https://img.shields.io/badge/Item%20unicity%3A%20False-gold) ![Additional items: N/A](https://img.shields.io/badge/Additional%20items%3A%20N/A-gold) - ### files items + ### files items | | | | -------- | -------- | @@ -971,7 +724,7 @@ Must be one of: ![Item unicity: False](https://img.shields.io/badge/Item%20unicity%3A%20False-gold) ![Additional items: N/A](https://img.shields.io/badge/Additional%20items%3A%20N/A-gold) - ### kustomizations items + ### kustomizations items | | | | -------- | -------- | @@ -1019,7 +772,7 @@ Must be one of: ![Item unicity: False](https://img.shields.io/badge/Item%20unicity%3A%20False-gold) ![Additional items: N/A](https://img.shields.io/badge/Additional%20items%3A%20N/A-gold) - ### ZarfChart + ### ZarfChart | | | | ------------------------- | -------------------------------------------------------------------------------------------------------- | @@ -1203,7 +956,7 @@ Must be one of: ![Item unicity: False](https://img.shields.io/badge/Item%20unicity%3A%20False-gold) ![Additional items: N/A](https://img.shields.io/badge/Additional%20items%3A%20N/A-gold) - ### valuesFiles items + ### valuesFiles items | | | | -------- | -------- | @@ -1235,7 +988,7 @@ Must be one of: ![Item unicity: False](https://img.shields.io/badge/Item%20unicity%3A%20False-gold) ![Additional items: N/A](https://img.shields.io/badge/Additional%20items%3A%20N/A-gold) - ### ZarfDataInjection + ### ZarfDataInjection | | | | ------------------------- | -------------------------------------------------------------------------------------------------------- | @@ -1398,7 +1151,7 @@ Must be one of: ![Item unicity: False](https://img.shields.io/badge/Item%20unicity%3A%20False-gold) ![Additional items: N/A](https://img.shields.io/badge/Additional%20items%3A%20N/A-gold) - ### ZarfFile + ### ZarfFile | | | | ------------------------- | -------------------------------------------------------------------------------------------------------- | @@ -1492,7 +1245,7 @@ Must be one of: ![Item unicity: False](https://img.shields.io/badge/Item%20unicity%3A%20False-gold) ![Additional items: N/A](https://img.shields.io/badge/Additional%20items%3A%20N/A-gold) - ### symlinks items + ### symlinks items | | | | -------- | -------- | @@ -1538,7 +1291,7 @@ Must be one of: ![Item unicity: False](https://img.shields.io/badge/Item%20unicity%3A%20False-gold) ![Additional items: N/A](https://img.shields.io/badge/Additional%20items%3A%20N/A-gold) - ### images items + ### images items | | | | -------- | -------- | @@ -1565,7 +1318,7 @@ Must be one of: ![Item unicity: False](https://img.shields.io/badge/Item%20unicity%3A%20False-gold) ![Additional items: N/A](https://img.shields.io/badge/Additional%20items%3A%20N/A-gold) - ### repos items + ### repos items | | | | -------- | -------- | @@ -1660,7 +1413,7 @@ Must be one of: ![Item unicity: False](https://img.shields.io/badge/Item%20unicity%3A%20False-gold) ![Additional items: N/A](https://img.shields.io/badge/Additional%20items%3A%20N/A-gold) - ### valuesFiles items + ### valuesFiles items | | | | -------- | -------- | @@ -1703,7 +1456,7 @@ Must be one of: ![Item unicity: False](https://img.shields.io/badge/Item%20unicity%3A%20False-gold) ![Additional items: N/A](https://img.shields.io/badge/Additional%20items%3A%20N/A-gold) - ### fluxPatchFiles items + ### fluxPatchFiles items | | | | -------- | -------- | @@ -1851,7 +1604,7 @@ Must be one of: ![Item unicity: False](https://img.shields.io/badge/Item%20unicity%3A%20False-gold) ![Additional items: N/A](https://img.shields.io/badge/Additional%20items%3A%20N/A-gold) - ### env items + ### env items | | | | -------- | -------- | @@ -1966,7 +1719,7 @@ Must be one of: ![Item unicity: False](https://img.shields.io/badge/Item%20unicity%3A%20False-gold) ![Additional items: N/A](https://img.shields.io/badge/Additional%20items%3A%20N/A-gold) - ### ZarfComponentAction + ### ZarfComponentAction | | | | ------------------------- | -------------------------------------------------------------------------------------------------------- | @@ -2056,7 +1809,7 @@ Must be one of: ![Item unicity: False](https://img.shields.io/badge/Item%20unicity%3A%20False-gold) ![Additional items: N/A](https://img.shields.io/badge/Additional%20items%3A%20N/A-gold) - ### env items + ### env items | | | | -------- | -------- | @@ -2121,7 +1874,7 @@ Must be one of: ![Item unicity: False](https://img.shields.io/badge/Item%20unicity%3A%20False-gold) ![Additional items: N/A](https://img.shields.io/badge/Additional%20items%3A%20N/A-gold) - ### ZarfComponentActionSetVariable + ### ZarfComponentActionSetVariable | | | | ------------------------- | -------------------------------------------------------------------------------------------------------- | @@ -2475,7 +2228,7 @@ Must be one of: ![Item unicity: False](https://img.shields.io/badge/Item%20unicity%3A%20False-gold) ![Additional items: N/A](https://img.shields.io/badge/Additional%20items%3A%20N/A-gold) - ### ZarfComponentAction + ### ZarfComponentAction | | | | ------------------------- | -------------------------------------------------------------------------------------------------------- | @@ -2506,7 +2259,7 @@ Must be one of: ![Item unicity: False](https://img.shields.io/badge/Item%20unicity%3A%20False-gold) ![Additional items: N/A](https://img.shields.io/badge/Additional%20items%3A%20N/A-gold) - ### ZarfComponentAction + ### ZarfComponentAction | | | | ------------------------- | -------------------------------------------------------------------------------------------------------- | @@ -2537,7 +2290,7 @@ Must be one of: ![Item unicity: False](https://img.shields.io/badge/Item%20unicity%3A%20False-gold) ![Additional items: N/A](https://img.shields.io/badge/Additional%20items%3A%20N/A-gold) - ### ZarfComponentAction + ### ZarfComponentAction | | | | ------------------------- | -------------------------------------------------------------------------------------------------------- | @@ -2617,7 +2370,7 @@ Must be one of: ![Item unicity: False](https://img.shields.io/badge/Item%20unicity%3A%20False-gold) ![Additional items: N/A](https://img.shields.io/badge/Additional%20items%3A%20N/A-gold) - ### ZarfPackageConstant + ### ZarfPackageConstant | | | | ------------------------- | -------------------------------------------------------------------------------------------------------- | @@ -2736,7 +2489,7 @@ Must be one of: ![Item unicity: False](https://img.shields.io/badge/Item%20unicity%3A%20False-gold) ![Additional items: N/A](https://img.shields.io/badge/Additional%20items%3A%20N/A-gold) - ### ZarfPackageVariable + ### ZarfPackageVariable | | | | ------------------------- | -------------------------------------------------------------------------------------------------------- | diff --git a/zarf.schema.json b/zarf.schema.json index e348b8382c..a0622206f8 100644 --- a/zarf.schema.json +++ b/zarf.schema.json @@ -125,81 +125,6 @@ "^x-": {} } }, - "ZarfBuildData": { - "required": [ - "terminal", - "user", - "architecture", - "timestamp", - "version" - ], - "properties": { - "terminal": { - "type": "string", - "description": "The machine name that created this package" - }, - "user": { - "type": "string", - "description": "The username who created this package" - }, - "architecture": { - "type": "string", - "description": "The architecture this package was created on" - }, - "timestamp": { - "type": "string", - "description": "The timestamp when this package was created" - }, - "version": { - "type": "string", - "description": "The version of Zarf used to build this package" - }, - "migrations": { - "items": { - "type": "string" - }, - "type": "array", - "description": "Any migrations that have been run on this package" - }, - "registryOverrides": { - "patternProperties": { - ".*": { - "type": "string" - } - }, - "type": "object", - "description": "Any registry domains that were overridden on package create when pulling images" - }, - "differential": { - "type": "boolean", - "description": "Whether this package was created with differential components" - }, - "differentialPackageVersion": { - "type": "string", - "description": "Version of a previously built package used as the basis for creating this differential package" - }, - "differentialMissing": { - "items": { - "type": "string" - }, - "type": "array", - "description": "List of components that were not included in this package due to differential packaging" - }, - "lastNonBreakingVersion": { - "type": "string", - "description": "The minimum version of Zarf that does not have breaking package structure changes" - }, - "flavor": { - "type": "string", - "description": "The flavor of Zarf used to build this package" - } - }, - "additionalProperties": false, - "type": "object", - "patternProperties": { - "^x-": {} - } - }, "ZarfChart": { "required": [ "name", @@ -999,11 +924,6 @@ "$ref": "#/definitions/ZarfMetadata", "description": "Package metadata" }, - "build": { - "$schema": "http://json-schema.org/draft-04/schema#", - "$ref": "#/definitions/ZarfBuildData", - "description": "Zarf-generated package build data" - }, "components": { "items": { "$schema": "http://json-schema.org/draft-04/schema#", From 6bd2941133079a2281ab75aeb2af45fb89dd32f2 Mon Sep 17 00:00:00 2001 From: razzle Date: Wed, 3 Apr 2024 03:19:01 -0500 Subject: [PATCH 090/113] don't format and don't mutate, use stdout Signed-off-by: razzle --- src/cmd/dev.go | 67 +++++++++++++++++++++++++++++-------------- src/pkg/utils/yaml.go | 37 ------------------------ 2 files changed, 46 insertions(+), 58 deletions(-) diff --git a/src/cmd/dev.go b/src/cmd/dev.go index fe2ceaf71a..ecca00e776 100644 --- a/src/cmd/dev.go +++ b/src/cmd/dev.go @@ -13,6 +13,8 @@ import ( "strings" goyaml "github.com/goccy/go-yaml" + "github.com/pterm/pterm" + "github.com/sergi/go-diff/diffmatchpatch" "github.com/AlecAivazis/survey/v2" "github.com/defenseunicorns/pkg/helpers" @@ -68,6 +70,24 @@ var devDeployCmd = &cobra.Command{ }, } +func diffYAML[T any](old, new T) string { + oldYaml, _ := goyaml.MarshalWithOptions(old, goyaml.IndentSequence(true), goyaml.UseSingleQuote(false)) + newYaml, _ := goyaml.MarshalWithOptions(new, goyaml.IndentSequence(true), goyaml.UseSingleQuote(false)) + + if string(oldYaml) == string(newYaml) { + return "" + } + + dmp := diffmatchpatch.New() + + diffs := dmp.DiffMain(string(oldYaml), string(newYaml), true) + diffs = dmp.DiffCleanupSemantic(diffs) + diffs = dmp.DiffCleanupEfficiency(diffs) + s := dmp.DiffPrettyText(diffs) + + return s +} + var devMigrateCmd = &cobra.Command{ Use: "migrate", Short: lang.CmdDevMigrateShort, @@ -78,17 +98,12 @@ var devMigrateCmd = &cobra.Command{ var pkg types.ZarfPackage cm := goyaml.CommentMap{} - fi, err := os.Stat(filepath.Join(dir, layout.ZarfYAML)) + before, err := os.ReadFile(filepath.Join(dir, layout.ZarfYAML)) if err != nil { return err } - b, err := os.ReadFile(filepath.Join(dir, layout.ZarfYAML)) - if err != nil { - return err - } - - if err := goyaml.UnmarshalWithOptions(b, &pkg, goyaml.CommentToMap(cm)); err != nil { + if err := goyaml.UnmarshalWithOptions(before, &pkg, goyaml.CommentToMap(cm)); err != nil { return err } @@ -99,18 +114,22 @@ var devMigrateCmd = &cobra.Command{ continue } - // Migrate the package definition + message.HeaderInfof("🔨 Running %q", m.ID()) + + ran := false for idx, component := range pkg.Components { - ran := []string{} - ran = append(ran, m.ID()) c, _ := m.Run(component) c = m.Clear(c) pkg.Components[idx] = c - - if len(ran) > 0 { - message.Successf("Ran %s on %q", strings.Join(ran, ", "), component.Name) + diff := diffYAML(component, c) + if diff != "" { + ran = true + pterm.Println(diff) } } + if !ran { + message.Successf("No changes made by migration %q", m.ID()) + } } betaFeatureMigrationsToRun = slices.Compact(betaFeatureMigrationsToRun) @@ -118,21 +137,27 @@ var devMigrateCmd = &cobra.Command{ if !slices.Contains(betaFeatureMigrationsToRun, m.ID()) && betaFeatureMigrationsToRun != nil { continue } - pkg = m.Run(pkg) - message.Successf("Enabled [BETA FEATURE FLAG] %s", m.ID()) + message.HeaderInfof("🚀 Enabling %q", m.ID()) + new := m.Run(pkg) + diff := diffYAML(pkg, new) + if diff != "" { + pterm.Println(diff) + } else { + message.Successf("No changes made by enabling %q", m.ID()) + } + pkg = new } - b, err = goyaml.MarshalWithOptions(pkg, goyaml.WithComment(cm), goyaml.IndentSequence(true), goyaml.UseSingleQuote(false)) + after, err := goyaml.MarshalWithOptions(pkg, goyaml.WithComment(cm), goyaml.IndentSequence(true), goyaml.UseSingleQuote(false)) if err != nil { return err } - formatted, err := utils.FormatZarfYAML(b) - if err != nil { - return err - } + message.HeaderInfof("📦 Final zarf.yaml") + + fmt.Println(string(after)) - return os.WriteFile(filepath.Join(dir, layout.ZarfYAML), formatted, fi.Mode()) + return nil }, } diff --git a/src/pkg/utils/yaml.go b/src/pkg/utils/yaml.go index 711350da0d..f2c73d5f77 100644 --- a/src/pkg/utils/yaml.go +++ b/src/pkg/utils/yaml.go @@ -7,7 +7,6 @@ package utils // fork from https://github.com/goccy/go-yaml/blob/master/cmd/ycat/ycat.go import ( - "bufio" "bytes" "fmt" "io" @@ -232,39 +231,3 @@ func SplitYAMLToString(yamlData []byte) ([]string, error) { } return objs, nil } - -// FormatZarfYAML formats a zarf.yaml file w/ some opinionated formatting. -func FormatZarfYAML(b []byte) (formatted []byte, err error) { - scanner := bufio.NewScanner(bytes.NewReader(b)) - - var commentLines []string - - // Some opinionated formatting for the zarf.yaml - for scanner.Scan() { - line := scanner.Text() - - if strings.HasPrefix(line, "components:") || strings.HasPrefix(line, " - name: ") || strings.HasPrefix(line, " name: ") { - if len(commentLines) > 0 { - commentText := strings.Join(commentLines, "\n") - formatted = append(formatted, []byte("\n"+commentText+"\n")...) - } else { - formatted = append(formatted, []byte("\n")...) - } - formatted = append(formatted, []byte(line+"\n")...) // Add "components:" line - commentLines = nil - } else { - if strings.HasPrefix(line, "#") || strings.HasPrefix(line, " - #") { - commentLines = append(commentLines, line) - } else { - if len(commentLines) > 0 { - commentText := strings.Join(commentLines, "\n") - formatted = append(formatted, []byte("\n"+commentText+"\n")...) - commentLines = nil - } - formatted = append(formatted, []byte(line+"\n")...) - } - } - } - - return formatted, scanner.Err() -} From 2701990d2e6481324f8ff14b919aa8b590baf1dc Mon Sep 17 00:00:00 2001 From: razzle Date: Wed, 3 Apr 2024 11:29:30 -0500 Subject: [PATCH 091/113] cleanup Signed-off-by: razzle --- src/cmd/dev.go | 14 +++++----- src/pkg/packager/filters/deploy_test.go | 34 ++++++++++++++----------- 2 files changed, 26 insertions(+), 22 deletions(-) diff --git a/src/cmd/dev.go b/src/cmd/dev.go index ecca00e776..7b88c39084 100644 --- a/src/cmd/dev.go +++ b/src/cmd/dev.go @@ -118,10 +118,10 @@ var devMigrateCmd = &cobra.Command{ ran := false for idx, component := range pkg.Components { - c, _ := m.Run(component) - c = m.Clear(c) - pkg.Components[idx] = c - diff := diffYAML(component, c) + mc, _ := m.Run(component) + mc = m.Clear(mc) + pkg.Components[idx] = mc + diff := diffYAML(component, mc) if diff != "" { ran = true pterm.Println(diff) @@ -138,14 +138,14 @@ var devMigrateCmd = &cobra.Command{ continue } message.HeaderInfof("🚀 Enabling %q", m.ID()) - new := m.Run(pkg) - diff := diffYAML(pkg, new) + pkgWithBeta := m.Run(pkg) + diff := diffYAML(pkg, pkgWithBeta) if diff != "" { pterm.Println(diff) } else { message.Successf("No changes made by enabling %q", m.ID()) } - pkg = new + pkg = pkgWithBeta } after, err := goyaml.MarshalWithOptions(pkg, goyaml.WithComment(cm), goyaml.IndentSequence(true), goyaml.UseSingleQuote(false)) diff --git a/src/pkg/packager/filters/deploy_test.go b/src/pkg/packager/filters/deploy_test.go index aa1523c154..983a4d823f 100644 --- a/src/pkg/packager/filters/deploy_test.go +++ b/src/pkg/packager/filters/deploy_test.go @@ -68,7 +68,11 @@ func componentMatrix(t *testing.T) []types.ZarfComponent { continue } - query := fmt.Sprintf("default=%t && required=%v", defaultValue, requiredValue) + query := fmt.Sprintf("required=%v", requiredValue) + + if defaultValue { + query = fmt.Sprintf("%s && default=true", query) + } c := componentFromQuery(t, query) components = append(components, c) @@ -111,9 +115,9 @@ func TestDeployFilter_Apply(t *testing.T) { }, optionalComponents: "", want: []types.ZarfComponent{ - componentFromQuery(t, "default=true && required="), - componentFromQuery(t, "default=true && required=false"), - componentFromQuery(t, "default=false && required=true"), + componentFromQuery(t, "required= && default=true"), + componentFromQuery(t, "required=false && default=true"), + componentFromQuery(t, "required=true"), componentFromQuery(t, "group=foo && idx=0 && default=true"), componentFromQuery(t, "group=bar && idx=0 && default=true"), }, @@ -123,17 +127,17 @@ func TestDeployFilter_Apply(t *testing.T) { Components: possibilities, }, optionalComponents: strings.Join([]string{ - "default=false && required=false", + "required=false", "group=bar && idx=2 && default=false", - "-default=false && required=true", + "-required=true", }, ","), want: []types.ZarfComponent{ - componentFromQuery(t, "default=true && required="), - componentFromQuery(t, "default=true && required=false"), - componentFromQuery(t, "default=false && required=true"), - componentFromQuery(t, "default=false && required=false"), + componentFromQuery(t, "required= && default=true"), + componentFromQuery(t, "required=false && default=true"), + componentFromQuery(t, "required=true"), // required components cannot be deselected + componentFromQuery(t, "required=false"), // optional components can be selected componentFromQuery(t, "group=foo && idx=0 && default=true"), - componentFromQuery(t, "group=bar && idx=2 && default=false"), + componentFromQuery(t, "group=bar && idx=2 && default=false"), // components within a group can be selected, the default is not selected }, }, "Test failing when group has no default and no selection was made": { @@ -175,10 +179,10 @@ func TestDeployFilter_Apply(t *testing.T) { }, optionalComponents: "", want: []types.ZarfComponent{ - componentFromQuery(t, "default=true && required="), - componentFromQuery(t, "default=true && required=false"), - componentFromQuery(t, "default=false && required="), - componentFromQuery(t, "default=false && required=true"), + componentFromQuery(t, "required= && default=true"), + componentFromQuery(t, "required=false && default=true"), + componentFromQuery(t, "required="), + componentFromQuery(t, "required=true"), componentFromQuery(t, "group=foo && idx=0 && default=true"), componentFromQuery(t, "group=bar && idx=0 && default=true"), }, From 16d6bcf5818f2e52f49ce02686c9532829d64ebe Mon Sep 17 00:00:00 2001 From: razzle Date: Thu, 4 Apr 2024 00:01:22 -0500 Subject: [PATCH 092/113] ux Signed-off-by: razzle --- src/cmd/dev.go | 70 +++++++++---------- .../packager/migrations/default_required.go | 14 ++-- 2 files changed, 43 insertions(+), 41 deletions(-) diff --git a/src/cmd/dev.go b/src/cmd/dev.go index 7b88c39084..c9904a364f 100644 --- a/src/cmd/dev.go +++ b/src/cmd/dev.go @@ -9,12 +9,12 @@ import ( "io" "os" "path/filepath" + "reflect" "slices" "strings" goyaml "github.com/goccy/go-yaml" "github.com/pterm/pterm" - "github.com/sergi/go-diff/diffmatchpatch" "github.com/AlecAivazis/survey/v2" "github.com/defenseunicorns/pkg/helpers" @@ -70,24 +70,6 @@ var devDeployCmd = &cobra.Command{ }, } -func diffYAML[T any](old, new T) string { - oldYaml, _ := goyaml.MarshalWithOptions(old, goyaml.IndentSequence(true), goyaml.UseSingleQuote(false)) - newYaml, _ := goyaml.MarshalWithOptions(new, goyaml.IndentSequence(true), goyaml.UseSingleQuote(false)) - - if string(oldYaml) == string(newYaml) { - return "" - } - - dmp := diffmatchpatch.New() - - diffs := dmp.DiffMain(string(oldYaml), string(newYaml), true) - diffs = dmp.DiffCleanupSemantic(diffs) - diffs = dmp.DiffCleanupEfficiency(diffs) - s := dmp.DiffPrettyText(diffs) - - return s -} - var devMigrateCmd = &cobra.Command{ Use: "migrate", Short: lang.CmdDevMigrateShort, @@ -107,28 +89,32 @@ var devMigrateCmd = &cobra.Command{ return err } - deprecatedMigrationsToRun = slices.Compact(deprecatedMigrationsToRun) + data := [][]string{} + deprecatedMigrationsToRun = slices.Compact(deprecatedMigrationsToRun) for _, m := range migrations.DeprecatedComponentMigrations() { if !slices.Contains(deprecatedMigrationsToRun, m.ID()) && deprecatedMigrationsToRun != nil { continue } - message.HeaderInfof("🔨 Running %q", m.ID()) + entry := []string{ + m.ID(), + "deprecation", + "", + } - ran := false for idx, component := range pkg.Components { mc, _ := m.Run(component) mc = m.Clear(mc) - pkg.Components[idx] = mc - diff := diffYAML(component, mc) - if diff != "" { - ran = true - pterm.Println(diff) + if !reflect.DeepEqual(mc, component) { + entry[2] = fmt.Sprintf("%s, %s", entry[2], component.Name) } + pkg.Components[idx] = mc } - if !ran { - message.Successf("No changes made by migration %q", m.ID()) + + if entry[2] != "" { + entry[2] = strings.TrimSuffix(entry[2], ", ") + data = append(data, entry) } } @@ -137,13 +123,13 @@ var devMigrateCmd = &cobra.Command{ if !slices.Contains(betaFeatureMigrationsToRun, m.ID()) && betaFeatureMigrationsToRun != nil { continue } - message.HeaderInfof("🚀 Enabling %q", m.ID()) pkgWithBeta := m.Run(pkg) - diff := diffYAML(pkg, pkgWithBeta) - if diff != "" { - pterm.Println(diff) - } else { - message.Successf("No changes made by enabling %q", m.ID()) + if !reflect.DeepEqual(pkgWithBeta, pkg) { + data = append(data, []string{ + m.ID(), + "beta-feature", + "entire package", + }) } pkg = pkgWithBeta } @@ -153,9 +139,19 @@ var devMigrateCmd = &cobra.Command{ return err } - message.HeaderInfof("📦 Final zarf.yaml") + header := []string{ + "Migration", + "Type", + "Affected", + } - fmt.Println(string(after)) + if len(data) == 0 { + message.Warn("No changes made") + } else { + pterm.Println() + fmt.Println(string(after)) + message.Table(header, data) + } return nil }, diff --git a/src/pkg/packager/migrations/default_required.go b/src/pkg/packager/migrations/default_required.go index efa6e62b58..97d641b50e 100644 --- a/src/pkg/packager/migrations/default_required.go +++ b/src/pkg/packager/migrations/default_required.go @@ -6,6 +6,7 @@ package migrations import ( "slices" + "github.com/defenseunicorns/pkg/helpers" "github.com/defenseunicorns/zarf/src/types" ) @@ -21,14 +22,19 @@ func (DefaultRequired) ID() string { // // and cleanly migrates components explicitly marked as required to be nil func (DefaultRequired) Run(pkg types.ZarfPackage) types.ZarfPackage { + if slices.Contains(pkg.Metadata.BetaFeatures, types.DefaultRequired) { + return pkg + } else { + pkg.Metadata.BetaFeatures = append(pkg.Metadata.BetaFeatures, types.DefaultRequired) + } + for idx, component := range pkg.Components { if component.Required != nil && *component.Required { pkg.Components[idx].Required = nil } - } - - if !slices.Contains(pkg.Metadata.BetaFeatures, types.DefaultRequired) { - pkg.Metadata.BetaFeatures = append(pkg.Metadata.BetaFeatures, types.DefaultRequired) + if component.Required == nil { + pkg.Components[idx].Required = helpers.BoolPtr(false) + } } return pkg From cdd4f9e65f16b21c363c3293368154193c1ab934 Mon Sep 17 00:00:00 2001 From: razzle Date: Thu, 4 Apr 2024 00:01:32 -0500 Subject: [PATCH 093/113] update ADR Signed-off-by: razzle --- adr/0023-required-optional.md | 104 +++++++++++++++++++++++++++++++--- 1 file changed, 97 insertions(+), 7 deletions(-) diff --git a/adr/0023-required-optional.md b/adr/0023-required-optional.md index 2457c53522..e89bf95ad6 100644 --- a/adr/0023-required-optional.md +++ b/adr/0023-required-optional.md @@ -1,4 +1,4 @@ -# 23. Required -> Optional +# 23. Components can be required by default Date: 2024-01-02 @@ -10,7 +10,7 @@ Accepted > Feature request: -Currently, all Zarf components default to being optional due to the `required` key being _optional_ in the yaml. This leads to package authors needing to ensure that they annotate this key for each component, and since nothing in the current validations prompts them about this they may be confused about the "all things are optional" default state. +Currently, all Zarf components default to being optional due to the `required` key being _optional_ in the YAML. This leads to package authors needing to ensure that they annotate this key for each component, and since nothing in the current validations prompts them about this they may be confused about the "all things are optional" default state. When Zarf was first created, we didn't really know how it would evolve and this key was introduced in those very early days. At this point it would be better to require all components by default--especially with the introduction of composability and the OCI skeleton work, there is plenty of flexibility in the API to compose bespoke packages assembled from other packages. @@ -22,16 +22,106 @@ A few ways to handle this: 3. Do something more significant like combine various condition-based things such as `only`, `optional` (instead of `required`), or `default`. +4. Introduce `betaFeature` flags to allow for certain schema behavior to be configurable to the user. + ## Decision -Option 2: deprecate `required` and introduce an optional `optional` key, which defaults to _false_. +Option 4. Introduce `.metadata.betaFeatures` flags to `zarf.yaml` to allow for certain schema behavior to be configurable to the user. + +The `betaFeatures` key will be added to the `metadata` section of the package schema. This key will be an array of strings, where each string is the name of a beta feature that can be enabled. To enable a beta feature, the user will need to add the name of the feature to this array and edit the package schema accordingly. + +> Such beta feature migrations can also be accomplished using `zarf dev migrate`, see the [Consequences](#consequences) section for more information. + +e.g. + +```diff +kind: ZarfInitConfig +metadata: + name: init + description: Used to establish a new Zarf cluster ++ betaFeatures: ++ - default-required + +components: + - name: k3s ++ required: false + import: + path: packages/distros/k3s -Components are now **required** by default, instead of **optional**. + # This package moves the injector & registries binaries + - name: zarf-injector +- required: true + import: + path: packages/zarf-registry +``` ## Consequences -`zarf package create` will fail if any usage of `required` is detected in the `zarf.yaml`, resulting in some thrash for package creators. +The introduction of feature flags will allow Zarf to introduce new features and schema behavior without breaking existing packages and schemas, but also introduces more complexity. This will require more documentation and user education to ensure that users understand how to use these flags. + +Beta feature flags will become the default behavior of Zarf upon the next major release, and will _not_ be configurable by the user at that time. This will allow for a more consistent experience across all Zarf packages. + +There will be a flag added to the `zarf dev migrate` command `--enable-beta-feature ` to allow users to enable beta features on a per-package basis. This will allow users to test new features in a controlled environment before they are enabled by default. + +> Tab autocompletion for the `--enable-beta-feature` flag is enabled for the `zarf dev migrate` command. + +e.g. (some output omitted for brevity) + +```bash +$ zarf dev migrate --enable-beta-feature default-required > migrated-zarf.yaml + + NOTE Using config file ... + + NOTE Saving log file to ... + + + Migration | Type | Affected + default-required | beta-feature | entire package + +``` + +```diff +$ git diff --no-index zarf.yaml migrated-zarf.yaml + +kind: ZarfInitConfig + metadata: + name: init + description: Used to establish a new Zarf cluster ++ betaFeatures: ++ - default-required + components: + - name: k3s ++ required: false + import: + path: packages/distros/k3s + + - name: zarf-injector +- required: true + import: + path: packages/zarf-registry + + - name: zarf-seed-registry +- required: true + import: + path: packages/zarf-registry + + - name: zarf-registry +- required: true + import: + path: packages/zarf-registry + + - name: zarf-agent +- required: true + import: + path: packages/zarf-agent -Packages created w/ Zarf v0.33.0+ will have their implicit _required_ logic flipped from previous versions (implicit `required: false` --> implicit `optional: false`). + - name: logging ++ required: false + import: + path: packages/logging-pgl -A `required-to-optional` migration (both accomplished behind the scenes on a `zarf package create`, or available via **new** CLI migration: `zarf dev migrate --run required-to-optional`). + - name: git-server ++ required: false + import: + path: packages/gitea +``` From 1f5a7194a8541d58437338db26f9ba4c86dada3d Mon Sep 17 00:00:00 2001 From: razzle Date: Thu, 4 Apr 2024 00:09:53 -0500 Subject: [PATCH 094/113] cleanup Signed-off-by: razzle --- src/cmd/dev.go | 24 +++++++++---------- src/pkg/layout/package.go | 2 +- src/pkg/packager/creator/utils.go | 2 +- src/pkg/packager/migrations/common.go | 11 +++------ .../packager/migrations/default_required.go | 4 ++-- .../migrations/pluralize_set_variable.go | 4 ++-- .../packager/migrations/scripts_to_actions.go | 4 ++-- 7 files changed, 23 insertions(+), 28 deletions(-) diff --git a/src/cmd/dev.go b/src/cmd/dev.go index c9904a364f..9d9b0c3976 100644 --- a/src/cmd/dev.go +++ b/src/cmd/dev.go @@ -93,12 +93,12 @@ var devMigrateCmd = &cobra.Command{ deprecatedMigrationsToRun = slices.Compact(deprecatedMigrationsToRun) for _, m := range migrations.DeprecatedComponentMigrations() { - if !slices.Contains(deprecatedMigrationsToRun, m.ID()) && deprecatedMigrationsToRun != nil { + if !slices.Contains(deprecatedMigrationsToRun, m.String()) && deprecatedMigrationsToRun != nil { continue } entry := []string{ - m.ID(), + m.String(), "deprecation", "", } @@ -120,13 +120,13 @@ var devMigrateCmd = &cobra.Command{ betaFeatureMigrationsToRun = slices.Compact(betaFeatureMigrationsToRun) for _, m := range migrations.BetaFeatureMigrations() { - if !slices.Contains(betaFeatureMigrationsToRun, m.ID()) && betaFeatureMigrationsToRun != nil { + if !slices.Contains(betaFeatureMigrationsToRun, m.String()) && betaFeatureMigrationsToRun != nil { continue } pkgWithBeta := m.Run(pkg) if !reflect.DeepEqual(pkgWithBeta, pkg) { data = append(data, []string{ - m.ID(), + m.String(), "beta-feature", "entire package", }) @@ -385,32 +385,32 @@ func init() { dcm := []string{} for _, m := range migrations.DeprecatedComponentMigrations() { - dcm = append(dcm, m.ID()) + dcm = append(dcm, m.String()) } devMigrateCmd.Flags().StringArrayVar(&deprecatedMigrationsToRun, "run", []string{}, fmt.Sprintf("migrations to run (default: all, available: %s)", strings.Join(dcm, ", "))) devMigrateCmd.RegisterFlagCompletionFunc("run", func(_ *cobra.Command, _ []string, _ string) ([]string, cobra.ShellCompDirective) { ids := []string{} - for _, migration := range migrations.DeprecatedComponentMigrations() { - if slices.Contains(deprecatedMigrationsToRun, migration.ID()) { + for _, m := range migrations.DeprecatedComponentMigrations() { + if slices.Contains(deprecatedMigrationsToRun, m.String()) { continue } - ids = append(ids, migration.ID()) + ids = append(ids, m.String()) } return ids, cobra.ShellCompDirectiveNoFileComp }) bfm := []string{} for _, m := range migrations.BetaFeatureMigrations() { - bfm = append(bfm, m.ID()) + bfm = append(bfm, m.String()) } devMigrateCmd.Flags().StringArrayVar(&betaFeatureMigrationsToRun, "enable-beta-feature", []string{}, fmt.Sprintf("beta migrations to run and enable (default: all, available: %s)", strings.Join(bfm, ", "))) devMigrateCmd.RegisterFlagCompletionFunc("enable-beta-feature", func(_ *cobra.Command, _ []string, _ string) ([]string, cobra.ShellCompDirective) { ids := []string{} - for _, migration := range migrations.BetaFeatureMigrations() { - if slices.Contains(betaFeatureMigrationsToRun, migration.ID()) { + for _, m := range migrations.BetaFeatureMigrations() { + if slices.Contains(betaFeatureMigrationsToRun, m.String()) { continue } - ids = append(ids, migration.ID()) + ids = append(ids, m.String()) } return ids, cobra.ShellCompDirectiveNoFileComp }) diff --git a/src/pkg/layout/package.go b/src/pkg/layout/package.go index 8f6ea15363..ef61a20cbd 100644 --- a/src/pkg/layout/package.go +++ b/src/pkg/layout/package.go @@ -72,7 +72,7 @@ func (pp *PackagePaths) ReadZarfYAML() (pkg types.ZarfPackage, warnings []string for idx, component := range pkg.Components { // Clear out component configuration migrations for _, m := range migrations.DeprecatedComponentMigrations() { - if slices.Contains(pkg.Build.Migrations, m.ID()) { + if slices.Contains(pkg.Build.Migrations, m.String()) { mc := m.Clear(component) pkg.Components[idx] = mc } diff --git a/src/pkg/packager/creator/utils.go b/src/pkg/packager/creator/utils.go index 43d7619f48..7ad4b6c8b0 100644 --- a/src/pkg/packager/creator/utils.go +++ b/src/pkg/packager/creator/utils.go @@ -46,7 +46,7 @@ func recordPackageMetadata(pkg *types.ZarfPackage, createOpts types.ZarfCreateOp // Record the migrations that will run on the package. pkg.Build.Migrations = []string{} for _, m := range migrations.DeprecatedComponentMigrations() { - pkg.Build.Migrations = append(pkg.Build.Migrations, m.ID()) + pkg.Build.Migrations = append(pkg.Build.Migrations, m.String()) } // Record the flavor of Zarf used to build this package (if any). diff --git a/src/pkg/packager/migrations/common.go b/src/pkg/packager/migrations/common.go index ad20b09b68..80a81c5e9c 100644 --- a/src/pkg/packager/migrations/common.go +++ b/src/pkg/packager/migrations/common.go @@ -5,6 +5,7 @@ package migrations import ( + "fmt" "strings" "github.com/Masterminds/semver/v3" @@ -35,17 +36,11 @@ var breakingChanges = []breakingChange{ }, } -// Migration represents a migration that can be run on a component. -type Migration interface { - // ID returns the ID of the migration - ID() string -} - // DeprecatedComponentMigration represents a migration that can be run on a component. // // DeprecatedComponentMigrations are migrations that seamlessly migrate deprecated component definitions. type DeprecatedComponentMigration interface { - Migration + fmt.Stringer // Run runs the migration on the component Run(c types.ZarfComponent) (types.ZarfComponent, string) // Clear clears the deprecated configuration from the component @@ -64,7 +59,7 @@ func DeprecatedComponentMigrations() []DeprecatedComponentMigration { // // Every migration is mapped to a specific beta feature, and the beta feature is added to the package metadata. type BetaFeatureMigration interface { - Migration + fmt.Stringer // Run runs the beta migration on the package Run(pkg types.ZarfPackage) types.ZarfPackage } diff --git a/src/pkg/packager/migrations/default_required.go b/src/pkg/packager/migrations/default_required.go index 97d641b50e..a2ab22f938 100644 --- a/src/pkg/packager/migrations/default_required.go +++ b/src/pkg/packager/migrations/default_required.go @@ -13,8 +13,8 @@ import ( // DefaultRequired migrates the package to change components to be required by default type DefaultRequired struct{} -// ID returns the ID of the migration -func (DefaultRequired) ID() string { +// String returns the name of the migration +func (DefaultRequired) String() string { return string(types.DefaultRequired) } diff --git a/src/pkg/packager/migrations/pluralize_set_variable.go b/src/pkg/packager/migrations/pluralize_set_variable.go index 6824a18227..93c0e11fb4 100644 --- a/src/pkg/packager/migrations/pluralize_set_variable.go +++ b/src/pkg/packager/migrations/pluralize_set_variable.go @@ -15,8 +15,8 @@ const PluralizeSetVariableID = "pluralize-set-variable" // SetVariableToSetVariables migrates setVariable to setVariables type SetVariableToSetVariables struct{} -// ID returns the ID of the migration -func (SetVariableToSetVariables) ID() string { +// String returns the name of the migration +func (SetVariableToSetVariables) String() string { return PluralizeSetVariableID } diff --git a/src/pkg/packager/migrations/scripts_to_actions.go b/src/pkg/packager/migrations/scripts_to_actions.go index c46088f6de..bf620e4b51 100644 --- a/src/pkg/packager/migrations/scripts_to_actions.go +++ b/src/pkg/packager/migrations/scripts_to_actions.go @@ -16,8 +16,8 @@ const ScriptsToActionsID = "scripts-to-actions" // ScriptsToActions migrates scripts to actions type ScriptsToActions struct{} -// ID returns the ID of the migration -func (ScriptsToActions) ID() string { +// String returns the name of the migration +func (ScriptsToActions) String() string { return ScriptsToActionsID } From c310870be8cf6bbb38c4542aadba6f9d04fa9aaa Mon Sep 17 00:00:00 2001 From: razzle Date: Thu, 4 Apr 2024 00:15:44 -0500 Subject: [PATCH 095/113] cleanup Signed-off-by: razzle --- src/pkg/packager/dev.go | 8 -------- src/pkg/packager/filters/deploy_test.go | 4 ---- src/pkg/packager/migrations/default_required.go | 4 ++-- 3 files changed, 2 insertions(+), 14 deletions(-) diff --git a/src/pkg/packager/dev.go b/src/pkg/packager/dev.go index bbf026c388..d94bf85a14 100644 --- a/src/pkg/packager/dev.go +++ b/src/pkg/packager/dev.go @@ -44,14 +44,6 @@ func (p *Packager) DevDeploy() error { return err } - // Set the package version to the CLI version - p.cfg.Pkg.Build.Version = config.CLIVersion - - // Also filter out components that are not required, nor requested via --components - // This is different from the above filter, as it is not based on the system, but rather - // the user's selection and the component's `required`/`optional` field - // This is also different from regular package creation, where we still assemble and package up - // all components and their dependencies, regardless of whether they are required or not filter := filters.Combine( filters.ByLocalOS(runtime.GOOS), filters.ForDeploy(p.cfg.PkgOpts.OptionalComponents, false), diff --git a/src/pkg/packager/filters/deploy_test.go b/src/pkg/packager/filters/deploy_test.go index 983a4d823f..4183c68d72 100644 --- a/src/pkg/packager/filters/deploy_test.go +++ b/src/pkg/packager/filters/deploy_test.go @@ -49,10 +49,6 @@ func componentFromQuery(t *testing.T, q string) types.ZarfComponent { return c } -// componentMatrix generates all possible combinations of component values -// for testing the deploy filter. -// some of these combinations are invalid, but the deploy filter should -// handle them gracefully. func componentMatrix(t *testing.T) []types.ZarfComponent { var components []types.ZarfComponent diff --git a/src/pkg/packager/migrations/default_required.go b/src/pkg/packager/migrations/default_required.go index a2ab22f938..8566d0a1c3 100644 --- a/src/pkg/packager/migrations/default_required.go +++ b/src/pkg/packager/migrations/default_required.go @@ -24,10 +24,10 @@ func (DefaultRequired) String() string { func (DefaultRequired) Run(pkg types.ZarfPackage) types.ZarfPackage { if slices.Contains(pkg.Metadata.BetaFeatures, types.DefaultRequired) { return pkg - } else { - pkg.Metadata.BetaFeatures = append(pkg.Metadata.BetaFeatures, types.DefaultRequired) } + pkg.Metadata.BetaFeatures = append(pkg.Metadata.BetaFeatures, types.DefaultRequired) + for idx, component := range pkg.Components { if component.Required != nil && *component.Required { pkg.Components[idx].Required = nil From d91590b979463bc37f2d27d683649e7e5dbfb642 Mon Sep 17 00:00:00 2001 From: razzle Date: Thu, 4 Apr 2024 23:53:51 -0500 Subject: [PATCH 096/113] cleanup Signed-off-by: razzle --- src/pkg/packager/generate.go | 3 ++- src/test/upgrade/zarf.yaml | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/pkg/packager/generate.go b/src/pkg/packager/generate.go index 9f1bc80b03..64bb824285 100644 --- a/src/pkg/packager/generate.go +++ b/src/pkg/packager/generate.go @@ -36,7 +36,8 @@ func (p *Packager) Generate() (err error) { } generatedComponent := types.ZarfComponent{ - Name: p.cfg.GenerateOpts.Name, + Name: p.cfg.GenerateOpts.Name, + Required: helpers.BoolPtr(true), Charts: []types.ZarfChart{ { Name: p.cfg.GenerateOpts.Name, diff --git a/src/test/upgrade/zarf.yaml b/src/test/upgrade/zarf.yaml index 34453ce6bc..5052a2bc55 100644 --- a/src/test/upgrade/zarf.yaml +++ b/src/test/upgrade/zarf.yaml @@ -6,8 +6,8 @@ metadata: components: - name: test-upgrade-package - required: true description: A semi-contrived example that deploys podinfo using many Zarf primitives and lets us upgrade it a few times. + required: true charts: - name: podinfo-upgrade version: "###ZARF_PKG_VAR_PODINFO_VERSION###" From 1e2404dd1a108ccd82466cda98db1af949a62d72 Mon Sep 17 00:00:00 2001 From: razzle Date: Tue, 9 Apr 2024 04:32:14 -0500 Subject: [PATCH 097/113] rename from beta features to just features Signed-off-by: razzle --- adr/0023-required-optional.md | 20 +++---- src/cmd/dev.go | 53 +++++++++---------- src/pkg/packager/filters/deploy.go | 4 +- src/pkg/packager/filters/deploy_test.go | 2 +- src/pkg/packager/migrations/common.go | 14 ++--- .../packager/migrations/default_required.go | 4 +- src/types/{beta.go => feature_flags.go} | 0 src/types/package.go | 2 +- src/types/validate.go | 6 +-- 9 files changed, 51 insertions(+), 54 deletions(-) rename src/types/{beta.go => feature_flags.go} (100%) diff --git a/adr/0023-required-optional.md b/adr/0023-required-optional.md index e89bf95ad6..1f6d11f894 100644 --- a/adr/0023-required-optional.md +++ b/adr/0023-required-optional.md @@ -22,15 +22,15 @@ A few ways to handle this: 3. Do something more significant like combine various condition-based things such as `only`, `optional` (instead of `required`), or `default`. -4. Introduce `betaFeature` flags to allow for certain schema behavior to be configurable to the user. +4. Introduce `feature` flags to allow for certain schema behavior to be configurable to the user. ## Decision -Option 4. Introduce `.metadata.betaFeatures` flags to `zarf.yaml` to allow for certain schema behavior to be configurable to the user. +Option 4. Introduce `.metadata.features` flags to `zarf.yaml` to allow for certain schema behavior to be configurable to the user. -The `betaFeatures` key will be added to the `metadata` section of the package schema. This key will be an array of strings, where each string is the name of a beta feature that can be enabled. To enable a beta feature, the user will need to add the name of the feature to this array and edit the package schema accordingly. +The `features` key will be added to the `metadata` section of the package schema. This key will be an array of strings, where each string is the name of a beta feature that can be enabled. To enable a feature, the user will need to add the name of the feature to this array and edit the package schema accordingly. -> Such beta feature migrations can also be accomplished using `zarf dev migrate`, see the [Consequences](#consequences) section for more information. +> Such feature migrations can also be accomplished using `zarf dev migrate`, see the [Consequences](#consequences) section for more information. e.g. @@ -39,7 +39,7 @@ kind: ZarfInitConfig metadata: name: init description: Used to establish a new Zarf cluster -+ betaFeatures: ++ features: + - default-required components: @@ -61,14 +61,14 @@ The introduction of feature flags will allow Zarf to introduce new features and Beta feature flags will become the default behavior of Zarf upon the next major release, and will _not_ be configurable by the user at that time. This will allow for a more consistent experience across all Zarf packages. -There will be a flag added to the `zarf dev migrate` command `--enable-beta-feature ` to allow users to enable beta features on a per-package basis. This will allow users to test new features in a controlled environment before they are enabled by default. +There will be a flag added to the `zarf dev migrate` command `--enable-feature ` to allow users to enable features on a per-package basis. This will allow users to test new features in a controlled environment before they are enabled by default. -> Tab autocompletion for the `--enable-beta-feature` flag is enabled for the `zarf dev migrate` command. +> Tab autocompletion for the `--enable-feature` flag is enabled for the `zarf dev migrate` command. e.g. (some output omitted for brevity) ```bash -$ zarf dev migrate --enable-beta-feature default-required > migrated-zarf.yaml +$ zarf dev migrate --enable-feature default-required > migrated-zarf.yaml NOTE Using config file ... @@ -76,7 +76,7 @@ $ zarf dev migrate --enable-beta-feature default-required > migrated-zarf.yaml Migration | Type | Affected - default-required | beta-feature | entire package + default-required | feature | . ``` @@ -87,7 +87,7 @@ kind: ZarfInitConfig metadata: name: init description: Used to establish a new Zarf cluster -+ betaFeatures: ++ features: + - default-required components: - name: k3s diff --git a/src/cmd/dev.go b/src/cmd/dev.go index 9d9b0c3976..20bae4d418 100644 --- a/src/cmd/dev.go +++ b/src/cmd/dev.go @@ -13,6 +13,7 @@ import ( "slices" "strings" + "github.com/fatih/color" goyaml "github.com/goccy/go-yaml" "github.com/pterm/pterm" @@ -36,7 +37,7 @@ import ( var extractPath string var deprecatedMigrationsToRun []string -var betaFeatureMigrationsToRun []string +var featureMigrationsToRun []string var devCmd = &cobra.Command{ Use: "dev", @@ -93,13 +94,13 @@ var devMigrateCmd = &cobra.Command{ deprecatedMigrationsToRun = slices.Compact(deprecatedMigrationsToRun) for _, m := range migrations.DeprecatedComponentMigrations() { - if !slices.Contains(deprecatedMigrationsToRun, m.String()) && deprecatedMigrationsToRun != nil { + if !slices.Contains(deprecatedMigrationsToRun, m.String()) && len(deprecatedMigrationsToRun) != 0 { continue } entry := []string{ m.String(), - "deprecation", + message.ColorWrap("deprecated", color.FgYellow), "", } @@ -107,31 +108,27 @@ var devMigrateCmd = &cobra.Command{ mc, _ := m.Run(component) mc = m.Clear(mc) if !reflect.DeepEqual(mc, component) { - entry[2] = fmt.Sprintf("%s, %s", entry[2], component.Name) + entry[2] = fmt.Sprintf(".components.[%d]", idx) + data = append(data, entry) } pkg.Components[idx] = mc } - - if entry[2] != "" { - entry[2] = strings.TrimSuffix(entry[2], ", ") - data = append(data, entry) - } } - betaFeatureMigrationsToRun = slices.Compact(betaFeatureMigrationsToRun) - for _, m := range migrations.BetaFeatureMigrations() { - if !slices.Contains(betaFeatureMigrationsToRun, m.String()) && betaFeatureMigrationsToRun != nil { + featureMigrationsToRun = slices.Compact(featureMigrationsToRun) + for _, m := range migrations.FeatureMigrations() { + if !slices.Contains(featureMigrationsToRun, m.String()) { continue } - pkgWithBeta := m.Run(pkg) - if !reflect.DeepEqual(pkgWithBeta, pkg) { + pkgWithFeature := m.Run(pkg) + if !reflect.DeepEqual(pkgWithFeature, pkg) { data = append(data, []string{ m.String(), - "beta-feature", - "entire package", + message.ColorWrap("feature", color.FgMagenta), + ".", }) } - pkg = pkgWithBeta + pkg = pkgWithFeature } after, err := goyaml.MarshalWithOptions(pkg, goyaml.WithComment(cm), goyaml.IndentSequence(true), goyaml.UseSingleQuote(false)) @@ -142,7 +139,7 @@ var devMigrateCmd = &cobra.Command{ header := []string{ "Migration", "Type", - "Affected", + "Affected Path(s)", } if len(data) == 0 { @@ -383,11 +380,11 @@ func init() { bindDevDeployFlags(v) bindDevGenerateFlags(v) - dcm := []string{} + cm := []string{} for _, m := range migrations.DeprecatedComponentMigrations() { - dcm = append(dcm, m.String()) + cm = append(cm, m.String()) } - devMigrateCmd.Flags().StringArrayVar(&deprecatedMigrationsToRun, "run", []string{}, fmt.Sprintf("migrations to run (default: all, available: %s)", strings.Join(dcm, ", "))) + devMigrateCmd.Flags().StringArrayVar(&deprecatedMigrationsToRun, "run", []string{}, fmt.Sprintf("migrations of deprecated features to run (default: all, available: %s)", strings.Join(cm, ", "))) devMigrateCmd.RegisterFlagCompletionFunc("run", func(_ *cobra.Command, _ []string, _ string) ([]string, cobra.ShellCompDirective) { ids := []string{} for _, m := range migrations.DeprecatedComponentMigrations() { @@ -399,15 +396,15 @@ func init() { return ids, cobra.ShellCompDirectiveNoFileComp }) - bfm := []string{} - for _, m := range migrations.BetaFeatureMigrations() { - bfm = append(bfm, m.String()) + fm := []string{} + for _, m := range migrations.FeatureMigrations() { + fm = append(fm, m.String()) } - devMigrateCmd.Flags().StringArrayVar(&betaFeatureMigrationsToRun, "enable-beta-feature", []string{}, fmt.Sprintf("beta migrations to run and enable (default: all, available: %s)", strings.Join(bfm, ", "))) - devMigrateCmd.RegisterFlagCompletionFunc("enable-beta-feature", func(_ *cobra.Command, _ []string, _ string) ([]string, cobra.ShellCompDirective) { + devMigrateCmd.Flags().StringArrayVar(&featureMigrationsToRun, "enable-feature", []string{}, fmt.Sprintf("feature migrations to run and enable (available: %s)", strings.Join(fm, ", "))) + devMigrateCmd.RegisterFlagCompletionFunc("enable-feature", func(_ *cobra.Command, _ []string, _ string) ([]string, cobra.ShellCompDirective) { ids := []string{} - for _, m := range migrations.BetaFeatureMigrations() { - if slices.Contains(betaFeatureMigrationsToRun, m.String()) { + for _, m := range migrations.FeatureMigrations() { + if slices.Contains(featureMigrationsToRun, m.String()) { continue } ids = append(ids, m.String()) diff --git a/src/pkg/packager/filters/deploy.go b/src/pkg/packager/filters/deploy.go index e791363cb9..ecac84efe5 100644 --- a/src/pkg/packager/filters/deploy.go +++ b/src/pkg/packager/filters/deploy.go @@ -74,7 +74,7 @@ func (f *deploymentFilter) Apply(pkg types.ZarfPackage) ([]types.ZarfComponent, selectState, matchedRequest := includedOrExcluded(component.Name, f.requestedComponents) - if !component.IsRequired(pkg.Metadata.BetaFeatures) { + if !component.IsRequired(pkg.Metadata.Features) { if selectState == excluded { // If the component was explicitly excluded, record the match and continue matchedRequests[matchedRequest] = true @@ -160,7 +160,7 @@ func (f *deploymentFilter) Apply(pkg types.ZarfPackage) ([]types.ZarfComponent, } else { component := groupedComponents[groupKey][0] - if component.IsRequired(pkg.Metadata.BetaFeatures) { + if component.IsRequired(pkg.Metadata.Features) { selectedComponents = append(selectedComponents, component) continue } diff --git a/src/pkg/packager/filters/deploy_test.go b/src/pkg/packager/filters/deploy_test.go index 4183c68d72..cc3a96e01e 100644 --- a/src/pkg/packager/filters/deploy_test.go +++ b/src/pkg/packager/filters/deploy_test.go @@ -167,7 +167,7 @@ func TestDeployFilter_Apply(t *testing.T) { "[default-required] Test when no optional components selected": { pkg: types.ZarfPackage{ Metadata: types.ZarfMetadata{ - BetaFeatures: []types.FeatureFlag{ + Features: []types.FeatureFlag{ types.DefaultRequired, }, }, diff --git a/src/pkg/packager/migrations/common.go b/src/pkg/packager/migrations/common.go index 80a81c5e9c..3d64a00741 100644 --- a/src/pkg/packager/migrations/common.go +++ b/src/pkg/packager/migrations/common.go @@ -55,18 +55,18 @@ func DeprecatedComponentMigrations() []DeprecatedComponentMigration { } } -// BetaFeatureMigration represents a migration that can be run on a package. +// FeatureMigration represents a feature migration that can be run on a package. // -// Every migration is mapped to a specific beta feature, and the beta feature is added to the package metadata. -type BetaFeatureMigration interface { +// Every migration is mapped to a specific feature, and the feature's identifier is added to the package metadata. +type FeatureMigration interface { fmt.Stringer - // Run runs the beta migration on the package + // Run runs the feature migration on the package Run(pkg types.ZarfPackage) types.ZarfPackage } -// BetaFeatureMigrations returns a list of all current beta feature migrations. -func BetaFeatureMigrations() []BetaFeatureMigration { - return []BetaFeatureMigration{ +// FeatureMigrations returns a list of all current feature migrations. +func FeatureMigrations() []FeatureMigration { + return []FeatureMigration{ DefaultRequired{}, } } diff --git a/src/pkg/packager/migrations/default_required.go b/src/pkg/packager/migrations/default_required.go index 8566d0a1c3..dd754de71e 100644 --- a/src/pkg/packager/migrations/default_required.go +++ b/src/pkg/packager/migrations/default_required.go @@ -22,11 +22,11 @@ func (DefaultRequired) String() string { // // and cleanly migrates components explicitly marked as required to be nil func (DefaultRequired) Run(pkg types.ZarfPackage) types.ZarfPackage { - if slices.Contains(pkg.Metadata.BetaFeatures, types.DefaultRequired) { + if slices.Contains(pkg.Metadata.Features, types.DefaultRequired) { return pkg } - pkg.Metadata.BetaFeatures = append(pkg.Metadata.BetaFeatures, types.DefaultRequired) + pkg.Metadata.Features = append(pkg.Metadata.Features, types.DefaultRequired) for idx, component := range pkg.Components { if component.Required != nil && *component.Required { diff --git a/src/types/beta.go b/src/types/feature_flags.go similarity index 100% rename from src/types/beta.go rename to src/types/feature_flags.go diff --git a/src/types/package.go b/src/types/package.go index b1ad07e132..ce15b11efb 100644 --- a/src/types/package.go +++ b/src/types/package.go @@ -54,7 +54,7 @@ type ZarfMetadata struct { Source string `json:"source,omitempty" jsonschema:"description=Link to package source code when online"` Vendor string `json:"vendor,omitempty" jsonschema_description:"Name of the distributing entity, organization or individual."` AggregateChecksum string `json:"aggregateChecksum,omitempty" jsonschema:"description=Checksum of a checksums.txt file that contains checksums all the layers within the package."` - BetaFeatures []FeatureFlag `json:"betaFeatures,omitempty" jsonschema:"description=List of beta feature flags to enable for this package"` + Features []FeatureFlag `json:"features,omitempty" jsonschema:"description=List of feature flags to enable for this package"` } // ZarfBuildData is written during the packager.Create() operation to track details of the created package. diff --git a/src/types/validate.go b/src/types/validate.go index 71ae710aa2..e5466a2a6c 100644 --- a/src/types/validate.go +++ b/src/types/validate.go @@ -46,7 +46,7 @@ func (pkg ZarfPackage) Validate() error { } allFF := AllFeatureFlags() - for _, ff := range pkg.Metadata.BetaFeatures { + for _, ff := range pkg.Metadata.Features { if !slices.Contains(allFF, ff) { return fmt.Errorf(lang.PkgValidateErrBetaFeatureNotFound, ff, allFF) } @@ -107,7 +107,7 @@ func (pkg ZarfPackage) Validate() error { return fmt.Errorf(lang.PkgValidateErrComponentLocalOS, component.Name, component.Only.LocalOS, supportedOS) } - if component.IsRequired(pkg.Metadata.BetaFeatures) { + if component.IsRequired(pkg.Metadata.Features) { if component.Default { return fmt.Errorf(lang.PkgValidateErrComponentReqDefault, component.Name) } @@ -118,7 +118,7 @@ func (pkg ZarfPackage) Validate() error { // progressive enhancement: "required" behavior can only follow "required" key // if using `DefaultRequired` behavior - if slices.Contains(pkg.Metadata.BetaFeatures, DefaultRequired) && component.Default && component.DeprecatedGroup == "" { + if slices.Contains(pkg.Metadata.Features, DefaultRequired) && component.Default && component.DeprecatedGroup == "" { return fmt.Errorf(lang.PkgValidateErrComponentMissingGroup, component.Name) } From d4841d11d7f5753350cd452c7d4913ddbb1927b3 Mon Sep 17 00:00:00 2001 From: razzle Date: Tue, 9 Apr 2024 04:33:52 -0500 Subject: [PATCH 098/113] docs and schema Signed-off-by: razzle --- docs/2-the-zarf-cli/100-cli-commands/zarf_dev_migrate.md | 6 +++--- docs/3-create-a-zarf-package/4-zarf-schema.md | 6 +++--- zarf.schema.json | 4 ++-- zarf.yaml | 2 +- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/2-the-zarf-cli/100-cli-commands/zarf_dev_migrate.md b/docs/2-the-zarf-cli/100-cli-commands/zarf_dev_migrate.md index 8518a57dfd..24fafb44b3 100644 --- a/docs/2-the-zarf-cli/100-cli-commands/zarf_dev_migrate.md +++ b/docs/2-the-zarf-cli/100-cli-commands/zarf_dev_migrate.md @@ -16,9 +16,9 @@ zarf dev migrate [flags] ## Options ``` - --enable-beta-feature stringArray beta migrations to run and enable (default: all, available: default-required) - -h, --help help for migrate - --run stringArray migrations to run (default: all, available: scripts-to-actions, pluralize-set-variable) + --enable-feature stringArray feature migrations to run and enable (available: default-required) + -h, --help help for migrate + --run stringArray migrations of deprecated features to run (default: all, available: scripts-to-actions, pluralize-set-variable) ``` ## Options inherited from parent commands diff --git a/docs/3-create-a-zarf-package/4-zarf-schema.md b/docs/3-create-a-zarf-package/4-zarf-schema.md index f3a4f03816..fb86a957dc 100644 --- a/docs/3-create-a-zarf-package/4-zarf-schema.md +++ b/docs/3-create-a-zarf-package/4-zarf-schema.md @@ -274,12 +274,12 @@ Must be one of:
- betaFeatures + features  
-**Description:** List of beta feature flags to enable for this package +**Description:** List of feature flags to enable for this package | | | | -------- | ----------------- | @@ -290,7 +290,7 @@ Must be one of: ![Item unicity: False](https://img.shields.io/badge/Item%20unicity%3A%20False-gold) ![Additional items: N/A](https://img.shields.io/badge/Additional%20items%3A%20N/A-gold) - ### betaFeatures items + ### features items | | | | -------- | -------- | diff --git a/zarf.schema.json b/zarf.schema.json index a0622206f8..82474ebb2e 100644 --- a/zarf.schema.json +++ b/zarf.schema.json @@ -890,12 +890,12 @@ "type": "string", "description": "Checksum of a checksums.txt file that contains checksums all the layers within the package." }, - "betaFeatures": { + "features": { "items": { "type": "string" }, "type": "array", - "description": "List of beta feature flags to enable for this package" + "description": "List of feature flags to enable for this package" } }, "additionalProperties": false, diff --git a/zarf.yaml b/zarf.yaml index cf3d0d14cb..e79b4493e2 100644 --- a/zarf.yaml +++ b/zarf.yaml @@ -2,7 +2,7 @@ kind: ZarfInitConfig metadata: name: init description: Used to establish a new Zarf cluster - betaFeatures: + features: - default-required components: From 14786e272636abb97e2efc8a9d64cc49e6882d89 Mon Sep 17 00:00:00 2001 From: razzle Date: Tue, 9 Apr 2024 04:55:55 -0500 Subject: [PATCH 099/113] docs and schema Signed-off-by: razzle --- docs/2-the-zarf-cli/100-cli-commands/zarf_dev.md | 2 +- .../100-cli-commands/zarf_dev_migrate.md | 8 +------- src/cmd/dev.go | 1 - src/config/lang/english.go | 16 ++++++++++++---- 4 files changed, 14 insertions(+), 13 deletions(-) diff --git a/docs/2-the-zarf-cli/100-cli-commands/zarf_dev.md b/docs/2-the-zarf-cli/100-cli-commands/zarf_dev.md index 0517af7e22..166fe4a35c 100644 --- a/docs/2-the-zarf-cli/100-cli-commands/zarf_dev.md +++ b/docs/2-the-zarf-cli/100-cli-commands/zarf_dev.md @@ -30,7 +30,7 @@ Commands useful for developing packages * [zarf dev generate](zarf_dev_generate.md) - [alpha] Creates a zarf.yaml automatically from a given remote (git) Helm chart * [zarf dev generate-config](zarf_dev_generate-config.md) - Generates a config file for Zarf * [zarf dev lint](zarf_dev_lint.md) - Lints the given package for valid schema and recommended practices -* [zarf dev migrate](zarf_dev_migrate.md) - [beta] Migrates the zarf.yaml in a given directory to the latest version of the zarf.yaml format +* [zarf dev migrate](zarf_dev_migrate.md) - [alpha] Migrates the zarf.yaml in a given directory to the latest version of the zarf.yaml format * [zarf dev patch-git](zarf_dev_patch-git.md) - Converts all .git URLs to the specified Zarf HOST and with the Zarf URL pattern in a given FILE. NOTE: This should only be used for manifests that are not mutated by the Zarf Agent Mutating Webhook. * [zarf dev sha256sum](zarf_dev_sha256sum.md) - Generates a SHA256SUM for the given file diff --git a/docs/2-the-zarf-cli/100-cli-commands/zarf_dev_migrate.md b/docs/2-the-zarf-cli/100-cli-commands/zarf_dev_migrate.md index 24fafb44b3..4354df4a1e 100644 --- a/docs/2-the-zarf-cli/100-cli-commands/zarf_dev_migrate.md +++ b/docs/2-the-zarf-cli/100-cli-commands/zarf_dev_migrate.md @@ -1,13 +1,7 @@ # zarf dev migrate -[beta] Migrates the zarf.yaml in a given directory to the latest version of the zarf.yaml format - -## Synopsis - -[beta] Migrates the zarf.yaml in a given directory to the latest version of the zarf.yaml format. - -This command modifies the original zarf.yaml file, performs a best effort attempt to preserve comments, and will format the contents in an opinionated manner. +[alpha] Migrates the zarf.yaml in a given directory to the latest version of the zarf.yaml format ``` zarf dev migrate [flags] diff --git a/src/cmd/dev.go b/src/cmd/dev.go index 20bae4d418..9c618d99ff 100644 --- a/src/cmd/dev.go +++ b/src/cmd/dev.go @@ -74,7 +74,6 @@ var devDeployCmd = &cobra.Command{ var devMigrateCmd = &cobra.Command{ Use: "migrate", Short: lang.CmdDevMigrateShort, - Long: lang.CmdDevMigrateLong, Args: cobra.MaximumNArgs(1), RunE: func(_ *cobra.Command, args []string) error { dir := common.SetBaseDirectory(args) diff --git a/src/config/lang/english.go b/src/config/lang/english.go index fb3f47a054..9f41369424 100644 --- a/src/config/lang/english.go +++ b/src/config/lang/english.go @@ -359,9 +359,17 @@ $ zarf package pull oci://ghcr.io/defenseunicorns/packages/dos-games:1.0.0 -a sk CmdDevDeployFlagNoYolo = "Disable the YOLO mode default override and create / deploy the package as-defined" CmdDevDeployErr = "Failed to dev deploy: %s" - CmdDevMigrateShort = "[beta] Migrates the zarf.yaml in a given directory to the latest version of the zarf.yaml format" - CmdDevMigrateLong = "[beta] Migrates the zarf.yaml in a given directory to the latest version of the zarf.yaml format.\n\n" + - "This command modifies the original zarf.yaml file, performs a best effort attempt to preserve comments, and will format the contents in an opinionated manner." + CmdDevMigrateShort = "[alpha] Migrates the zarf.yaml in a given directory to the latest version of the zarf.yaml format" + CmdDevMigrateExample = ` +# Migrate the zarf.yaml in the current directory +$ zarf dev migrate . + +# Run specific migrations +$ zarf dev migrate --run scripts-to-actions --run pluralize-set-variable . + +# Enable feature flag(s) and run migrations +$ zarf dev migrate --enable-feature default-required . +` CmdDevGenerateShort = "[alpha] Creates a zarf.yaml automatically from a given remote (git) Helm chart" CmdDevGenerateExample = "zarf dev generate podinfo --url https://github.com/stefanprodan/podinfo.git --version 6.4.0 --gitPath charts/podinfo" @@ -687,7 +695,7 @@ const ( // Package validate const ( - PkgValidateErrBetaFeatureNotFound = "beta feature %q not found, available features: %s" + PkgValidateErrBetaFeatureNotFound = "feature %q not found, available: %s" PkgValidateTemplateDeprecation = "Package template %q is using the deprecated syntax ###ZARF_PKG_VAR_%s###. This will be removed in Zarf v1.0.0. Please update to ###ZARF_PKG_TMPL_%s###." PkgValidateMustBeUppercase = "variable name %q must be all uppercase and contain no special characters except _" PkgValidateErrAction = "invalid action: %w" From b882fe74ff21ff26c85a905153b31206eaffe7b7 Mon Sep 17 00:00:00 2001 From: razzle Date: Tue, 9 Apr 2024 05:15:57 -0500 Subject: [PATCH 100/113] validate changes broken into separate PR Signed-off-by: razzle --- src/config/config.go | 3 + src/config/lang/english.go | 5 +- src/internal/packager/validate/validate.go | 368 ++++++++++++++++++++ src/pkg/packager/composer/list.go | 3 +- src/pkg/packager/create.go | 3 +- src/pkg/packager/dev.go | 3 +- src/pkg/packager/filters/os_test.go | 5 +- src/pkg/packager/generate.go | 3 +- src/pkg/packager/sources/cluster.go | 3 +- src/types/validate.go | 372 --------------------- 10 files changed, 387 insertions(+), 381 deletions(-) create mode 100644 src/internal/packager/validate/validate.go delete mode 100644 src/types/validate.go diff --git a/src/config/config.go b/src/config/config.go index 01cf6d2c0c..b8e3e2d386 100644 --- a/src/config/config.go +++ b/src/config/config.go @@ -25,6 +25,9 @@ import ( const ( GithubProject = "defenseunicorns/zarf" + // ZarfMaxChartNameLength limits helm chart name size to account for K8s/helm limits and zarf prefix + ZarfMaxChartNameLength = 40 + ZarfAgentHost = "agent-hook.zarf.svc" ZarfConnectLabelName = "zarf.dev/connect-name" diff --git a/src/config/lang/english.go b/src/config/lang/english.go index 9f41369424..c112dd6caf 100644 --- a/src/config/lang/english.go +++ b/src/config/lang/english.go @@ -359,8 +359,8 @@ $ zarf package pull oci://ghcr.io/defenseunicorns/packages/dos-games:1.0.0 -a sk CmdDevDeployFlagNoYolo = "Disable the YOLO mode default override and create / deploy the package as-defined" CmdDevDeployErr = "Failed to dev deploy: %s" - CmdDevMigrateShort = "[alpha] Migrates the zarf.yaml in a given directory to the latest version of the zarf.yaml format" - CmdDevMigrateExample = ` + CmdDevMigrateShort = "[alpha] Migrates the zarf.yaml in a given directory to the latest version of the zarf.yaml format" + CmdDevMigrateExample = ` # Migrate the zarf.yaml in the current directory $ zarf dev migrate . @@ -699,6 +699,7 @@ const ( PkgValidateTemplateDeprecation = "Package template %q is using the deprecated syntax ###ZARF_PKG_VAR_%s###. This will be removed in Zarf v1.0.0. Please update to ###ZARF_PKG_TMPL_%s###." PkgValidateMustBeUppercase = "variable name %q must be all uppercase and contain no special characters except _" PkgValidateErrAction = "invalid action: %w" + PkgValidateErrActionVariables = "component %q cannot contain setVariables outside of onDeploy in actions" PkgValidateErrActionCmdWait = "action %q cannot be both a command and wait action" PkgValidateErrActionClusterNetwork = "a single wait action must contain only one of cluster or network" PkgValidateErrChart = "invalid chart definition: %w" diff --git a/src/internal/packager/validate/validate.go b/src/internal/packager/validate/validate.go new file mode 100644 index 0000000000..83eba2dd07 --- /dev/null +++ b/src/internal/packager/validate/validate.go @@ -0,0 +1,368 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package validate provides Zarf package validation functions. +package validate + +import ( + "fmt" + "path/filepath" + "regexp" + "slices" + + "github.com/defenseunicorns/pkg/helpers" + "github.com/defenseunicorns/zarf/src/config" + "github.com/defenseunicorns/zarf/src/config/lang" + "github.com/defenseunicorns/zarf/src/types" +) + +var ( + // IsLowercaseNumberHyphenNoStartHyphen is a regex for lowercase, numbers and hyphens that cannot start with a hyphen. + // https://regex101.com/r/FLdG9G/2 + IsLowercaseNumberHyphenNoStartHyphen = regexp.MustCompile(`^[a-z0-9][a-z0-9\-]*$`).MatchString + // IsUppercaseNumberUnderscore is a regex for uppercase, numbers and underscores. + // https://regex101.com/r/tfsEuZ/1 + IsUppercaseNumberUnderscore = regexp.MustCompile(`^[A-Z0-9_]+$`).MatchString + // Define allowed OS, an empty string means it is allowed on all operating systems + // same as enums on ZarfComponentOnlyTarget + supportedOS = []string{"linux", "darwin", "windows", ""} +) + +// SupportedOS returns the supported operating systems. +// +// The supported operating systems are: linux, darwin, windows. +// +// An empty string signifies no OS restrictions. +func SupportedOS() []string { + return supportedOS +} + +// Run performs config validations. +func Run(pkg types.ZarfPackage) error { + if pkg.Kind == types.ZarfInitConfig && pkg.Metadata.YOLO { + return fmt.Errorf(lang.PkgValidateErrInitNoYOLO) + } + + if err := validatePackageName(pkg.Metadata.Name); err != nil { + return fmt.Errorf(lang.PkgValidateErrName, err) + } + + for _, variable := range pkg.Variables { + if err := validatePackageVariable(variable); err != nil { + return fmt.Errorf(lang.PkgValidateErrVariable, err) + } + } + + for _, constant := range pkg.Constants { + if err := validatePackageConstant(constant); err != nil { + return fmt.Errorf(lang.PkgValidateErrConstant, err) + } + } + + uniqueComponentNames := make(map[string]bool) + groupDefault := make(map[string]string) + groupedComponents := make(map[string][]string) + + for _, component := range pkg.Components { + // ensure component name is unique + if _, ok := uniqueComponentNames[component.Name]; ok { + return fmt.Errorf(lang.PkgValidateErrComponentNameNotUnique, component.Name) + } + uniqueComponentNames[component.Name] = true + + if err := validateComponent(pkg, component); err != nil { + return fmt.Errorf(lang.PkgValidateErrComponent, component.Name, err) + } + + // ensure groups don't have multiple defaults or only one component + if component.DeprecatedGroup != "" { + if component.Default { + if _, ok := groupDefault[component.DeprecatedGroup]; ok { + return fmt.Errorf(lang.PkgValidateErrGroupMultipleDefaults, component.DeprecatedGroup, groupDefault[component.DeprecatedGroup], component.Name) + } + groupDefault[component.DeprecatedGroup] = component.Name + } + groupedComponents[component.DeprecatedGroup] = append(groupedComponents[component.DeprecatedGroup], component.Name) + } + } + + for groupKey, componentNames := range groupedComponents { + if len(componentNames) == 1 { + return fmt.Errorf(lang.PkgValidateErrGroupOneComponent, groupKey, componentNames[0]) + } + } + + return nil +} + +// ImportDefinition validates the component trying to be imported. +func ImportDefinition(component *types.ZarfComponent) error { + path := component.Import.Path + url := component.Import.URL + + // ensure path or url is provided + if path == "" && url == "" { + return fmt.Errorf(lang.PkgValidateErrImportDefinition, component.Name, "neither a path nor a URL was provided") + } + + // ensure path and url are not both provided + if path != "" && url != "" { + return fmt.Errorf(lang.PkgValidateErrImportDefinition, component.Name, "both a path and a URL were provided") + } + + // validation for path + if url == "" && path != "" { + // ensure path is not an absolute path + if filepath.IsAbs(path) { + return fmt.Errorf(lang.PkgValidateErrImportDefinition, component.Name, "path cannot be an absolute path") + } + } + + // validation for url + if url != "" && path == "" { + ok := helpers.IsOCIURL(url) + if !ok { + return fmt.Errorf(lang.PkgValidateErrImportDefinition, component.Name, "URL is not a valid OCI URL") + } + } + + return nil +} + +func oneIfNotEmpty(testString string) int { + if testString == "" { + return 0 + } + + return 1 +} + +func validateComponent(pkg types.ZarfPackage, component types.ZarfComponent) error { + if !IsLowercaseNumberHyphenNoStartHyphen(component.Name) { + return fmt.Errorf(lang.PkgValidateErrComponentName, component.Name) + } + + if !slices.Contains(supportedOS, component.Only.LocalOS) { + return fmt.Errorf(lang.PkgValidateErrComponentLocalOS, component.Name, component.Only.LocalOS, supportedOS) + } + + if component.Required != nil && *component.Required { + if component.Default { + return fmt.Errorf(lang.PkgValidateErrComponentReqDefault, component.Name) + } + if component.DeprecatedGroup != "" { + return fmt.Errorf(lang.PkgValidateErrComponentReqGrouped, component.Name) + } + } + + uniqueChartNames := make(map[string]bool) + for _, chart := range component.Charts { + // ensure chart name is unique + if _, ok := uniqueChartNames[chart.Name]; ok { + return fmt.Errorf(lang.PkgValidateErrChartNameNotUnique, chart.Name) + } + uniqueChartNames[chart.Name] = true + + if err := validateChart(chart); err != nil { + return fmt.Errorf(lang.PkgValidateErrChart, err) + } + } + + uniqueManifestNames := make(map[string]bool) + for _, manifest := range component.Manifests { + // ensure manifest name is unique + if _, ok := uniqueManifestNames[manifest.Name]; ok { + return fmt.Errorf(lang.PkgValidateErrManifestNameNotUnique, manifest.Name) + } + uniqueManifestNames[manifest.Name] = true + + if err := validateManifest(manifest); err != nil { + return fmt.Errorf(lang.PkgValidateErrManifest, err) + } + } + + if pkg.Metadata.YOLO { + if err := validateYOLO(component); err != nil { + return fmt.Errorf(lang.PkgValidateErrComponentYOLO, component.Name, err) + } + } + + if containsVariables, err := validateActionset(component.Actions.OnCreate); err != nil { + return fmt.Errorf(lang.PkgValidateErrAction, err) + } else if containsVariables { + return fmt.Errorf(lang.PkgValidateErrActionVariables, component.Name) + } + + if _, err := validateActionset(component.Actions.OnDeploy); err != nil { + return fmt.Errorf(lang.PkgValidateErrAction, err) + } + + if containsVariables, err := validateActionset(component.Actions.OnRemove); err != nil { + return fmt.Errorf(lang.PkgValidateErrAction, err) + } else if containsVariables { + return fmt.Errorf(lang.PkgValidateErrActionVariables, component.Name) + } + + return nil +} + +func validateActionset(actions types.ZarfComponentActionSet) (bool, error) { + containsVariables := false + + validate := func(actions []types.ZarfComponentAction) error { + for _, action := range actions { + if cv, err := validateAction(action); err != nil { + return err + } else if cv { + containsVariables = true + } + } + + return nil + } + + if err := validate(actions.Before); err != nil { + return containsVariables, err + } + if err := validate(actions.After); err != nil { + return containsVariables, err + } + if err := validate(actions.OnSuccess); err != nil { + return containsVariables, err + } + if err := validate(actions.OnFailure); err != nil { + return containsVariables, err + } + + return containsVariables, nil +} + +func validateAction(action types.ZarfComponentAction) (bool, error) { + containsVariables := false + + // Validate SetVariable + for _, variable := range action.SetVariables { + if !IsUppercaseNumberUnderscore(variable.Name) { + return containsVariables, fmt.Errorf(lang.PkgValidateMustBeUppercase, variable.Name) + } + containsVariables = true + } + + if action.Wait != nil { + // Validate only cmd or wait, not both + if action.Cmd != "" { + return containsVariables, fmt.Errorf(lang.PkgValidateErrActionCmdWait, action.Cmd) + } + + // Validate only cluster or network, not both + if action.Wait.Cluster != nil && action.Wait.Network != nil { + return containsVariables, fmt.Errorf(lang.PkgValidateErrActionClusterNetwork) + } + + // Validate at least one of cluster or network + if action.Wait.Cluster == nil && action.Wait.Network == nil { + return containsVariables, fmt.Errorf(lang.PkgValidateErrActionClusterNetwork) + } + } + + return containsVariables, nil +} + +func validateYOLO(component types.ZarfComponent) error { + if len(component.Images) > 0 { + return fmt.Errorf(lang.PkgValidateErrYOLONoOCI) + } + + if len(component.Repos) > 0 { + return fmt.Errorf(lang.PkgValidateErrYOLONoGit) + } + + if component.Only.Cluster.Architecture != "" { + return fmt.Errorf(lang.PkgValidateErrYOLONoArch) + } + + if len(component.Only.Cluster.Distros) > 0 { + return fmt.Errorf(lang.PkgValidateErrYOLONoDistro) + } + + return nil +} + +func validatePackageName(subject string) error { + if !IsLowercaseNumberHyphenNoStartHyphen(subject) { + return fmt.Errorf(lang.PkgValidateErrPkgName, subject) + } + + return nil +} + +func validatePackageVariable(subject types.ZarfPackageVariable) error { + // ensure the variable name is only capitals and underscores + if !IsUppercaseNumberUnderscore(subject.Name) { + return fmt.Errorf(lang.PkgValidateMustBeUppercase, subject.Name) + } + + return nil +} + +func validatePackageConstant(subject types.ZarfPackageConstant) error { + // ensure the constant name is only capitals and underscores + if !IsUppercaseNumberUnderscore(subject.Name) { + return fmt.Errorf(lang.PkgValidateErrPkgConstantName, subject.Name) + } + + if !regexp.MustCompile(subject.Pattern).MatchString(subject.Value) { + return fmt.Errorf(lang.PkgValidateErrPkgConstantPattern, subject.Name, subject.Pattern) + } + + return nil +} + +func validateChart(chart types.ZarfChart) error { + // Don't allow empty names + if chart.Name == "" { + return fmt.Errorf(lang.PkgValidateErrChartNameMissing, chart.Name) + } + + // Helm max release name + if len(chart.Name) > config.ZarfMaxChartNameLength { + return fmt.Errorf(lang.PkgValidateErrChartName, chart.Name, config.ZarfMaxChartNameLength) + } + + // Must have a namespace + if chart.Namespace == "" { + return fmt.Errorf(lang.PkgValidateErrChartNamespaceMissing, chart.Name) + } + + // Must have a url or localPath (and not both) + count := oneIfNotEmpty(chart.URL) + oneIfNotEmpty(chart.LocalPath) + if count != 1 { + return fmt.Errorf(lang.PkgValidateErrChartURLOrPath, chart.Name) + } + + // Must have a version + if chart.Version == "" { + return fmt.Errorf(lang.PkgValidateErrChartVersion, chart.Name) + } + + return nil +} + +func validateManifest(manifest types.ZarfManifest) error { + // Don't allow empty names + if manifest.Name == "" { + return fmt.Errorf(lang.PkgValidateErrManifestNameMissing, manifest.Name) + } + + // Helm max release name + if len(manifest.Name) > config.ZarfMaxChartNameLength { + return fmt.Errorf(lang.PkgValidateErrManifestNameLength, manifest.Name, config.ZarfMaxChartNameLength) + } + + // Require files in manifest + if len(manifest.Files) < 1 && len(manifest.Kustomizations) < 1 { + return fmt.Errorf(lang.PkgValidateErrManifestFileOrKustomize, manifest.Name) + } + + return nil +} diff --git a/src/pkg/packager/composer/list.go b/src/pkg/packager/composer/list.go index fd0b35fb37..32632a8696 100644 --- a/src/pkg/packager/composer/list.go +++ b/src/pkg/packager/composer/list.go @@ -11,6 +11,7 @@ import ( "strings" "github.com/defenseunicorns/pkg/helpers" + "github.com/defenseunicorns/zarf/src/internal/packager/validate" "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/packager/migrations" "github.com/defenseunicorns/zarf/src/pkg/utils" @@ -140,7 +141,7 @@ func NewImportChain(head types.ZarfComponent, index int, originalPackageName, ar } // TODO: stuff like this should also happen in linting - if err := node.ZarfComponent.ValidateImportDefinition(); err != nil { + if err := validate.ImportDefinition(&node.ZarfComponent); err != nil { return ic, err } diff --git a/src/pkg/packager/create.go b/src/pkg/packager/create.go index 94e3cb94dc..9ef4523fca 100755 --- a/src/pkg/packager/create.go +++ b/src/pkg/packager/create.go @@ -10,6 +10,7 @@ import ( "github.com/defenseunicorns/pkg/helpers" "github.com/defenseunicorns/zarf/src/config" + "github.com/defenseunicorns/zarf/src/internal/packager/validate" "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/packager/creator" @@ -40,7 +41,7 @@ func (p *Packager) Create() (err error) { } // Perform early package validation. - if err := p.cfg.Pkg.Validate(); err != nil { + if err := validate.Run(p.cfg.Pkg); err != nil { return fmt.Errorf("unable to validate package: %w", err) } diff --git a/src/pkg/packager/dev.go b/src/pkg/packager/dev.go index d94bf85a14..1679933cfe 100644 --- a/src/pkg/packager/dev.go +++ b/src/pkg/packager/dev.go @@ -11,6 +11,7 @@ import ( "github.com/defenseunicorns/pkg/helpers" "github.com/defenseunicorns/zarf/src/config" + "github.com/defenseunicorns/zarf/src/internal/packager/validate" "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/packager/creator" @@ -53,7 +54,7 @@ func (p *Packager) DevDeploy() error { return err } - if err := p.cfg.Pkg.Validate(); err != nil { + if err := validate.Run(p.cfg.Pkg); err != nil { return fmt.Errorf("unable to validate package: %w", err) } diff --git a/src/pkg/packager/filters/os_test.go b/src/pkg/packager/filters/os_test.go index 7d54beecd3..9e6b9e7722 100644 --- a/src/pkg/packager/filters/os_test.go +++ b/src/pkg/packager/filters/os_test.go @@ -7,6 +7,7 @@ package filters import ( "testing" + "github.com/defenseunicorns/zarf/src/internal/packager/validate" "github.com/defenseunicorns/zarf/src/types" "github.com/stretchr/testify/require" ) @@ -14,7 +15,7 @@ import ( func TestLocalOSFilter(t *testing.T) { pkg := types.ZarfPackage{} - for _, os := range types.SupportedOS() { + for _, os := range validate.SupportedOS() { pkg.Components = append(pkg.Components, types.ZarfComponent{ Only: types.ZarfComponentOnlyTarget{ LocalOS: os, @@ -22,7 +23,7 @@ func TestLocalOSFilter(t *testing.T) { }) } - for _, os := range types.SupportedOS() { + for _, os := range validate.SupportedOS() { filter := ByLocalOS(os) result, err := filter.Apply(pkg) if os == "" { diff --git a/src/pkg/packager/generate.go b/src/pkg/packager/generate.go index 64bb824285..072c1f566b 100644 --- a/src/pkg/packager/generate.go +++ b/src/pkg/packager/generate.go @@ -12,6 +12,7 @@ import ( "github.com/defenseunicorns/pkg/helpers" "github.com/defenseunicorns/zarf/src/config" + "github.com/defenseunicorns/zarf/src/internal/packager/validate" "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/types" @@ -72,7 +73,7 @@ func (p *Packager) Generate() (err error) { p.cfg.Pkg.Components[i].Images = images[name] } - if err := p.cfg.Pkg.Validate(); err != nil { + if err := validate.Run(p.cfg.Pkg); err != nil { return err } diff --git a/src/pkg/packager/sources/cluster.go b/src/pkg/packager/sources/cluster.go index 9fb74d6e81..1d506c6937 100644 --- a/src/pkg/packager/sources/cluster.go +++ b/src/pkg/packager/sources/cluster.go @@ -8,6 +8,7 @@ import ( "fmt" "github.com/defenseunicorns/pkg/helpers" + "github.com/defenseunicorns/zarf/src/internal/packager/validate" "github.com/defenseunicorns/zarf/src/pkg/cluster" "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/packager/filters" @@ -22,7 +23,7 @@ var ( // NewClusterSource creates a new cluster source. func NewClusterSource(pkgOpts *types.ZarfPackageOptions) (PackageSource, error) { - if !types.IsLowercaseNumberHyphenNoStartHyphen(pkgOpts.PackageSource) { + if !validate.IsLowercaseNumberHyphenNoStartHyphen(pkgOpts.PackageSource) { return nil, fmt.Errorf("invalid package name %q", pkgOpts.PackageSource) } cluster, err := cluster.NewClusterWithWait(cluster.DefaultTimeout) diff --git a/src/types/validate.go b/src/types/validate.go deleted file mode 100644 index e5466a2a6c..0000000000 --- a/src/types/validate.go +++ /dev/null @@ -1,372 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: 2021-Present The Zarf Authors - -package types - -import ( - "fmt" - "path/filepath" - "regexp" - "slices" - - "github.com/defenseunicorns/pkg/helpers" - "github.com/defenseunicorns/zarf/src/config/lang" -) - -const ( - // ZarfMaxChartNameLength limits helm chart name size to account for K8s/helm limits and zarf prefix - ZarfMaxChartNameLength = 40 -) - -var ( - // IsLowercaseNumberHyphenNoStartHyphen is a regex for lowercase, numbers and hyphens that cannot start with a hyphen. - // https://regex101.com/r/FLdG9G/2 - IsLowercaseNumberHyphenNoStartHyphen = regexp.MustCompile(`^[a-z0-9][a-z0-9\-]*$`).MatchString - // IsUppercaseNumberUnderscore is a regex for uppercase, numbers and underscores. - // https://regex101.com/r/tfsEuZ/1 - IsUppercaseNumberUnderscore = regexp.MustCompile(`^[A-Z0-9_]+$`).MatchString - // Define allowed OS, an empty string means it is allowed on all operating systems - // same as enums on ZarfComponentOnlyTarget - supportedOS = []string{"linux", "darwin", "windows", ""} -) - -// SupportedOS returns the supported operating systems. -// -// The supported operating systems are: linux, darwin, windows. -// -// An empty string signifies no OS restrictions. -func SupportedOS() []string { - return supportedOS -} - -// Validate runs all validation checks on the package. -func (pkg ZarfPackage) Validate() error { - if pkg.Kind == ZarfInitConfig && pkg.Metadata.YOLO { - return fmt.Errorf(lang.PkgValidateErrInitNoYOLO) - } - - allFF := AllFeatureFlags() - for _, ff := range pkg.Metadata.Features { - if !slices.Contains(allFF, ff) { - return fmt.Errorf(lang.PkgValidateErrBetaFeatureNotFound, ff, allFF) - } - } - - if !IsLowercaseNumberHyphenNoStartHyphen(pkg.Metadata.Name) { - return fmt.Errorf(lang.PkgValidateErrPkgName, pkg.Metadata.Name) - } - - for _, variable := range pkg.Variables { - if err := variable.Validate(); err != nil { - return fmt.Errorf(lang.PkgValidateErrVariable, err) - } - } - - for _, constant := range pkg.Constants { - if err := constant.Validate(); err != nil { - return fmt.Errorf(lang.PkgValidateErrConstant, err) - } - } - - uniqueComponentNames := make(map[string]bool) - groupDefault := make(map[string]string) - groupedComponents := make(map[string][]string) - - if pkg.Metadata.YOLO { - for _, component := range pkg.Components { - if len(component.Images) > 0 { - return fmt.Errorf(lang.PkgValidateErrYOLONoOCI) - } - - if len(component.Repos) > 0 { - return fmt.Errorf(lang.PkgValidateErrYOLONoGit) - } - - if component.Only.Cluster.Architecture != "" { - return fmt.Errorf(lang.PkgValidateErrYOLONoArch) - } - - if len(component.Only.Cluster.Distros) > 0 { - return fmt.Errorf(lang.PkgValidateErrYOLONoDistro) - } - } - } - - for _, component := range pkg.Components { - // ensure component name is unique - if _, ok := uniqueComponentNames[component.Name]; ok { - return fmt.Errorf(lang.PkgValidateErrComponentNameNotUnique, component.Name) - } - uniqueComponentNames[component.Name] = true - - if !IsLowercaseNumberHyphenNoStartHyphen(component.Name) { - return fmt.Errorf(lang.PkgValidateErrComponentName, component.Name) - } - - if !slices.Contains(supportedOS, component.Only.LocalOS) { - return fmt.Errorf(lang.PkgValidateErrComponentLocalOS, component.Name, component.Only.LocalOS, supportedOS) - } - - if component.IsRequired(pkg.Metadata.Features) { - if component.Default { - return fmt.Errorf(lang.PkgValidateErrComponentReqDefault, component.Name) - } - if component.DeprecatedGroup != "" { - return fmt.Errorf(lang.PkgValidateErrComponentReqGrouped, component.Name) - } - } - - // progressive enhancement: "required" behavior can only follow "required" key - // if using `DefaultRequired` behavior - if slices.Contains(pkg.Metadata.Features, DefaultRequired) && component.Default && component.DeprecatedGroup == "" { - return fmt.Errorf(lang.PkgValidateErrComponentMissingGroup, component.Name) - } - - uniqueChartNames := make(map[string]bool) - for _, chart := range component.Charts { - // ensure chart name is unique - if _, ok := uniqueChartNames[chart.Name]; ok { - return fmt.Errorf(lang.PkgValidateErrChartNameNotUnique, chart.Name) - } - uniqueChartNames[chart.Name] = true - - if err := chart.Validate(); err != nil { - return fmt.Errorf(lang.PkgValidateErrChart, err) - } - } - - uniqueManifestNames := make(map[string]bool) - for _, manifest := range component.Manifests { - // ensure manifest name is unique - if _, ok := uniqueManifestNames[manifest.Name]; ok { - return fmt.Errorf(lang.PkgValidateErrManifestNameNotUnique, manifest.Name) - } - uniqueManifestNames[manifest.Name] = true - - if err := manifest.Validate(); err != nil { - return fmt.Errorf(lang.PkgValidateErrManifest, err) - } - } - - if err := component.Actions.Validate(); err != nil { - return fmt.Errorf("%q: %w", component.Name, err) - } - - // ensure groups don't have multiple defaults or only one component - if component.DeprecatedGroup != "" { - if component.Default { - if _, ok := groupDefault[component.DeprecatedGroup]; ok { - return fmt.Errorf(lang.PkgValidateErrGroupMultipleDefaults, component.DeprecatedGroup, groupDefault[component.DeprecatedGroup], component.Name) - } - groupDefault[component.DeprecatedGroup] = component.Name - } - groupedComponents[component.DeprecatedGroup] = append(groupedComponents[component.DeprecatedGroup], component.Name) - } - } - - for groupKey, componentNames := range groupedComponents { - if len(componentNames) == 1 { - return fmt.Errorf(lang.PkgValidateErrGroupOneComponent, groupKey, componentNames[0]) - } - } - - return nil -} - -// Validate runs all validation checks on component actions. -func (a ZarfComponentActions) Validate() error { - if err := a.OnCreate.Validate(); err != nil { - return fmt.Errorf(lang.PkgValidateErrAction, err) - } - - if a.OnCreate.HasSetVariables() { - return fmt.Errorf("cannot contain setVariables outside of onDeploy in actions") - } - - if err := a.OnDeploy.Validate(); err != nil { - return fmt.Errorf(lang.PkgValidateErrAction, err) - } - - if a.OnRemove.HasSetVariables() { - return fmt.Errorf("cannot contain setVariables outside of onDeploy in actions") - } - - return nil -} - -// ValidateImportDefinition validates the component trying to be imported. -func (c ZarfComponent) ValidateImportDefinition() error { - path := c.Import.Path - url := c.Import.URL - - // ensure path or url is provided - if path == "" && url == "" { - return fmt.Errorf(lang.PkgValidateErrImportDefinition, c.Name, "neither a path nor a URL was provided") - } - - // ensure path and url are not both provided - if path != "" && url != "" { - return fmt.Errorf(lang.PkgValidateErrImportDefinition, c.Name, "both a path and a URL were provided") - } - - // validation for path - if url == "" && path != "" { - // ensure path is not an absolute path - if filepath.IsAbs(path) { - return fmt.Errorf(lang.PkgValidateErrImportDefinition, c.Name, "path cannot be an absolute path") - } - } - - // validation for url - if url != "" && path == "" { - ok := helpers.IsOCIURL(url) - if !ok { - return fmt.Errorf(lang.PkgValidateErrImportDefinition, c.Name, "URL is not a valid OCI URL") - } - } - - return nil -} - -// HasSetVariables returns true if any of the actions contain setVariables. -func (as ZarfComponentActionSet) HasSetVariables() bool { - check := func(actions []ZarfComponentAction) bool { - for _, action := range actions { - if len(action.SetVariables) > 0 { - return true - } - } - return false - } - - return check(as.Before) || check(as.After) || check(as.OnSuccess) || check(as.OnFailure) -} - -// Validate runs all validation checks on component action sets. -func (as ZarfComponentActionSet) Validate() error { - validate := func(actions []ZarfComponentAction) error { - for _, action := range actions { - return action.Validate() - } - - return nil - } - - if err := validate(as.Before); err != nil { - return err - } - if err := validate(as.After); err != nil { - return err - } - if err := validate(as.OnSuccess); err != nil { - return err - } - return validate(as.OnFailure) -} - -// Validate runs all validation checks on an action. -func (action ZarfComponentAction) Validate() error { - // Validate SetVariable - for _, variable := range action.SetVariables { - if !IsUppercaseNumberUnderscore(variable.Name) { - return fmt.Errorf(lang.PkgValidateMustBeUppercase, variable.Name) - } - } - - if action.Wait != nil { - // Validate only cmd or wait, not both - if action.Cmd != "" { - return fmt.Errorf(lang.PkgValidateErrActionCmdWait, action.Cmd) - } - - // Validate only cluster or network, not both - if action.Wait.Cluster != nil && action.Wait.Network != nil { - return fmt.Errorf(lang.PkgValidateErrActionClusterNetwork) - } - - // Validate at least one of cluster or network - if action.Wait.Cluster == nil && action.Wait.Network == nil { - return fmt.Errorf(lang.PkgValidateErrActionClusterNetwork) - } - } - - return nil -} - -// Validate runs all validation checks on a package variable. -func (variable ZarfPackageVariable) Validate() error { - if !IsUppercaseNumberUnderscore(variable.Name) { - return fmt.Errorf(lang.PkgValidateMustBeUppercase, variable.Name) - } - - return nil -} - -// Validate runs all validation checks on a package constant. -func (constant ZarfPackageConstant) Validate() error { - // ensure the constant name is only capitals and underscores - if !IsUppercaseNumberUnderscore(constant.Name) { - return fmt.Errorf(lang.PkgValidateErrPkgConstantName, constant.Name) - } - - if !regexp.MustCompile(constant.Pattern).MatchString(constant.Value) { - return fmt.Errorf(lang.PkgValidateErrPkgConstantPattern, constant.Name, constant.Pattern) - } - - return nil -} - -// Validate runs all validation checks on a chart. -func (chart ZarfChart) Validate() error { - // Don't allow empty names - if chart.Name == "" { - return fmt.Errorf(lang.PkgValidateErrChartNameMissing, chart.Name) - } - - // Helm max release name - if len(chart.Name) > ZarfMaxChartNameLength { - return fmt.Errorf(lang.PkgValidateErrChartName, chart.Name, ZarfMaxChartNameLength) - } - - // Must have a namespace - if chart.Namespace == "" { - return fmt.Errorf(lang.PkgValidateErrChartNamespaceMissing, chart.Name) - } - - // Must have a url or localPath (and not both) - if chart.URL != "" && chart.LocalPath != "" { - return fmt.Errorf(lang.PkgValidateErrChartURLOrPath, chart.Name) - } - - // Must have a url or localPath (and not both) - if chart.URL == "" && chart.LocalPath == "" { - return fmt.Errorf(lang.PkgValidateErrChartURLOrPath, chart.Name) - } - - // Must have a version - if chart.Version == "" { - return fmt.Errorf(lang.PkgValidateErrChartVersion, chart.Name) - } - - return nil -} - -// Validate runs all validation checks on a manifest. -func (manifest ZarfManifest) Validate() error { - // Don't allow empty names - if manifest.Name == "" { - return fmt.Errorf(lang.PkgValidateErrManifestNameMissing, manifest.Name) - } - - // Helm max release name - if len(manifest.Name) > ZarfMaxChartNameLength { - return fmt.Errorf(lang.PkgValidateErrManifestNameLength, manifest.Name, ZarfMaxChartNameLength) - } - - // Require files in manifest - if len(manifest.Files) < 1 && len(manifest.Kustomizations) < 1 { - return fmt.Errorf(lang.PkgValidateErrManifestFileOrKustomize, manifest.Name) - } - - return nil -} From baf3368171db3b978886ad9d6fa5382b26389581 Mon Sep 17 00:00:00 2001 From: razzle Date: Tue, 9 Apr 2024 05:23:09 -0500 Subject: [PATCH 101/113] use Run instead of RunE for consistency Signed-off-by: razzle --- src/cmd/dev.go | 10 ++++------ src/config/lang/english.go | 2 ++ 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/cmd/dev.go b/src/cmd/dev.go index 9c618d99ff..afa157755b 100644 --- a/src/cmd/dev.go +++ b/src/cmd/dev.go @@ -75,18 +75,18 @@ var devMigrateCmd = &cobra.Command{ Use: "migrate", Short: lang.CmdDevMigrateShort, Args: cobra.MaximumNArgs(1), - RunE: func(_ *cobra.Command, args []string) error { + Run: func(_ *cobra.Command, args []string) { dir := common.SetBaseDirectory(args) var pkg types.ZarfPackage cm := goyaml.CommentMap{} before, err := os.ReadFile(filepath.Join(dir, layout.ZarfYAML)) if err != nil { - return err + message.Fatalf(err, lang.CmdDevMigrateErr, err.Error()) } if err := goyaml.UnmarshalWithOptions(before, &pkg, goyaml.CommentToMap(cm)); err != nil { - return err + message.Fatalf(err, lang.CmdDevMigrateErr, err.Error()) } data := [][]string{} @@ -132,7 +132,7 @@ var devMigrateCmd = &cobra.Command{ after, err := goyaml.MarshalWithOptions(pkg, goyaml.WithComment(cm), goyaml.IndentSequence(true), goyaml.UseSingleQuote(false)) if err != nil { - return err + message.Fatalf(err, lang.CmdDevMigrateErr, err.Error()) } header := []string{ @@ -148,8 +148,6 @@ var devMigrateCmd = &cobra.Command{ fmt.Println(string(after)) message.Table(header, data) } - - return nil }, } diff --git a/src/config/lang/english.go b/src/config/lang/english.go index c112dd6caf..ba3ee213c2 100644 --- a/src/config/lang/english.go +++ b/src/config/lang/english.go @@ -370,6 +370,8 @@ $ zarf dev migrate --run scripts-to-actions --run pluralize-set-variable . # Enable feature flag(s) and run migrations $ zarf dev migrate --enable-feature default-required . ` + CmdDevMigrateErr = "Failed to migrate: %s" + CmdDevGenerateShort = "[alpha] Creates a zarf.yaml automatically from a given remote (git) Helm chart" CmdDevGenerateExample = "zarf dev generate podinfo --url https://github.com/stefanprodan/podinfo.git --version 6.4.0 --gitPath charts/podinfo" From d492e1797efe44891cf322f7c6fd201c01911f3c Mon Sep 17 00:00:00 2001 From: razzle Date: Tue, 9 Apr 2024 05:37:08 -0500 Subject: [PATCH 102/113] re-update validate Signed-off-by: razzle --- src/internal/packager/validate/validate.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/internal/packager/validate/validate.go b/src/internal/packager/validate/validate.go index 83eba2dd07..0b452bc19a 100644 --- a/src/internal/packager/validate/validate.go +++ b/src/internal/packager/validate/validate.go @@ -146,7 +146,7 @@ func validateComponent(pkg types.ZarfPackage, component types.ZarfComponent) err return fmt.Errorf(lang.PkgValidateErrComponentLocalOS, component.Name, component.Only.LocalOS, supportedOS) } - if component.Required != nil && *component.Required { + if component.IsRequired(pkg.Metadata.Features) { if component.Default { return fmt.Errorf(lang.PkgValidateErrComponentReqDefault, component.Name) } From 7d7b3f8c5dd7df6a090043e54e0e7cf1f45517c9 Mon Sep 17 00:00:00 2001 From: razzle Date: Tue, 9 Apr 2024 17:59:07 -0500 Subject: [PATCH 103/113] revert Signed-off-by: razzle --- src/pkg/packager/migrations/common.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pkg/packager/migrations/common.go b/src/pkg/packager/migrations/common.go index 3d64a00741..1cd09ae796 100644 --- a/src/pkg/packager/migrations/common.go +++ b/src/pkg/packager/migrations/common.go @@ -25,7 +25,7 @@ type breakingChange struct { // LastNonBreakingVersion is the last version that did not have any breaking changes // // This should be updated when a breaking change is introduced to the Zarf package structure. See: https://github.com/defenseunicorns/zarf/releases/tag/v0.32.2 -const LastNonBreakingVersion = "v0.32.2" +const LastNonBreakingVersion = "v0.27.0" // List of breaking changes to warn the user of. var breakingChanges = []breakingChange{ From 9af072b23e0b9ddcfda50a0ab03fe5521bf30d2f Mon Sep 17 00:00:00 2001 From: razzle Date: Tue, 9 Apr 2024 23:01:30 -0500 Subject: [PATCH 104/113] docs and schema Signed-off-by: razzle --- site/src/content/docs/commands/zarf_dev.md | 1 + .../content/docs/commands/zarf_dev_migrate.md | 20 +++++++++++++------ .../docs/commands/zarf_tools_yq_eval.md | 7 ++++--- 3 files changed, 19 insertions(+), 9 deletions(-) diff --git a/site/src/content/docs/commands/zarf_dev.md b/site/src/content/docs/commands/zarf_dev.md index a12090183d..c226b12b70 100644 --- a/site/src/content/docs/commands/zarf_dev.md +++ b/site/src/content/docs/commands/zarf_dev.md @@ -37,6 +37,7 @@ Commands useful for developing packages * [zarf dev generate](/commands/zarf_dev_generate/) - [alpha] Creates a zarf.yaml automatically from a given remote (git) Helm chart * [zarf dev generate-config](/commands/zarf_dev_generate-config/) - Generates a config file for Zarf * [zarf dev lint](/commands/zarf_dev_lint/) - Lints the given package for valid schema and recommended practices +* [zarf dev migrate](/commands/zarf_dev_migrate/) - [alpha] Migrates the zarf.yaml in a given directory to the latest version of the zarf.yaml format * [zarf dev patch-git](/commands/zarf_dev_patch-git/) - Converts all .git URLs to the specified Zarf HOST and with the Zarf URL pattern in a given FILE. NOTE: This should only be used for manifests that are not mutated by the Zarf Agent Mutating Webhook. * [zarf dev sha256sum](/commands/zarf_dev_sha256sum/) - Generates a SHA256SUM for the given file diff --git a/site/src/content/docs/commands/zarf_dev_migrate.md b/site/src/content/docs/commands/zarf_dev_migrate.md index 4354df4a1e..189f52e6f9 100644 --- a/site/src/content/docs/commands/zarf_dev_migrate.md +++ b/site/src/content/docs/commands/zarf_dev_migrate.md @@ -1,5 +1,12 @@ -# zarf dev migrate - +--- +title: zarf dev migrate +description: Zarf CLI command reference for zarf dev migrate. +tableOfContents: false +--- + + + +## zarf dev migrate [alpha] Migrates the zarf.yaml in a given directory to the latest version of the zarf.yaml format @@ -7,7 +14,7 @@ zarf dev migrate [flags] ``` -## Options +### Options ``` --enable-feature stringArray feature migrations to run and enable (available: default-required) @@ -15,7 +22,7 @@ zarf dev migrate [flags] --run stringArray migrations of deprecated features to run (default: all, available: scripts-to-actions, pluralize-set-variable) ``` -## Options inherited from parent commands +### Options inherited from parent commands ``` -a, --architecture string Architecture for OCI images and Zarf packages @@ -28,6 +35,7 @@ zarf dev migrate [flags] --zarf-cache string Specify the location of the Zarf cache directory (default "~/.zarf-cache") ``` -## SEE ALSO +### SEE ALSO + +* [zarf dev](/commands/zarf_dev/) - Commands useful for developing packages -* [zarf dev](zarf_dev.md) - Commands useful for developing packages diff --git a/site/src/content/docs/commands/zarf_tools_yq_eval.md b/site/src/content/docs/commands/zarf_tools_yq_eval.md index b65933dd9d..58921184ab 100644 --- a/site/src/content/docs/commands/zarf_tools_yq_eval.md +++ b/site/src/content/docs/commands/zarf_tools_yq_eval.md @@ -12,11 +12,11 @@ tableOfContents: false ### Synopsis -yq is a portable command-line data file processor (https://github.com/mikefarah/yq/) +yq is a portable command-line data file processor (https://github.com/mikefarah/yq/) See https://mikefarah.gitbook.io/yq/ for detailed documentation and examples. ## Evaluate Sequence ## -This command iterates over each yaml document from each given file, applies the +This command iterates over each yaml document from each given file, applies the expression and prints the result in sequence. ``` @@ -38,7 +38,7 @@ zarf tools yq e sample.yaml cat file2.yml | zarf tools yq e '.a.b' file1.yml - file3.yml # Creates a new yaml document -# Note that editing an empty file does not work. +## Note that editing an empty file does not work. zarf tools yq e -n '.a.b.c = "cat"' # Update a file inplace @@ -98,3 +98,4 @@ zarf tools yq e '.a.b = "cool"' -i file.yaml ### SEE ALSO * [zarf tools yq](/commands/zarf_tools_yq/) - yq is a lightweight and portable command-line data file processor. + From 2916c45651b306d8ef81aa49306096144f0ee0f4 Mon Sep 17 00:00:00 2001 From: razzle Date: Thu, 25 Apr 2024 05:20:38 -0500 Subject: [PATCH 105/113] docs and schema Signed-off-by: razzle --- zarf.schema.json | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/zarf.schema.json b/zarf.schema.json index 62db62d197..71d9a86672 100644 --- a/zarf.schema.json +++ b/zarf.schema.json @@ -1095,6 +1095,11 @@ "$ref": "#/definitions/ZarfMetadata", "description": "Package metadata" }, + "build": { + "$schema": "http://json-schema.org/draft-04/schema#", + "$ref": "#/definitions/ZarfBuildData", + "description": "Zarf-generated package build data" + }, "components": { "items": { "$schema": "http://json-schema.org/draft-04/schema#", From b6789e87d8ea2e41c33637a35dc2b54a3351ae64 Mon Sep 17 00:00:00 2001 From: razzle Date: Mon, 29 Apr 2024 13:29:04 -0500 Subject: [PATCH 106/113] move group deprecation from migration to read Signed-off-by: razzle --- src/pkg/layout/package.go | 7 +++++++ src/pkg/packager/composer/list.go | 5 ----- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/pkg/layout/package.go b/src/pkg/layout/package.go index ef61a20cbd..6a4266d53e 100644 --- a/src/pkg/layout/package.go +++ b/src/pkg/layout/package.go @@ -80,6 +80,13 @@ func (pp *PackagePaths) ReadZarfYAML() (pkg types.ZarfPackage, warnings []string } } + for _, component := range pkg.Components { + // Show a warning if the component contains a group as that has been deprecated and will be removed. + if component.DeprecatedGroup != "" { + warnings = append(warnings, fmt.Sprintf("Component %s is using group which has been deprecated and will be removed in v1.0.0. Please migrate to another solution.", component.Name)) + } + } + return pkg, warnings, nil } diff --git a/src/pkg/packager/composer/list.go b/src/pkg/packager/composer/list.go index ed0218c2fa..e7e3715a31 100644 --- a/src/pkg/packager/composer/list.go +++ b/src/pkg/packager/composer/list.go @@ -267,11 +267,6 @@ func (ic *ImportChain) Migrate() (warnings []string) { warnings = append(warnings, warning) } } - - // Show a warning if the component contains a group as that has been deprecated and will be removed. - if node.DeprecatedGroup != "" { - warnings = append(warnings, fmt.Sprintf("Component %s is using group which has been deprecated and will be removed in v1.0.0. Please migrate to another solution.", node.Name)) - } node = node.next } if len(warnings) > 0 { From be2f2d83fd30f80855534d1c339d2803cc70e54b Mon Sep 17 00:00:00 2001 From: razzle Date: Mon, 29 Apr 2024 14:06:01 -0500 Subject: [PATCH 107/113] update ADR Signed-off-by: razzle --- adr/{0023-required-optional.md => 0025-required-optional.md} | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) rename adr/{0023-required-optional.md => 0025-required-optional.md} (97%) diff --git a/adr/0023-required-optional.md b/adr/0025-required-optional.md similarity index 97% rename from adr/0023-required-optional.md rename to adr/0025-required-optional.md index 1f6d11f894..5b82cca96e 100644 --- a/adr/0023-required-optional.md +++ b/adr/0025-required-optional.md @@ -1,6 +1,6 @@ -# 23. Components can be required by default +# 25. Components can be required by default + introduction of feature flags -Date: 2024-01-02 +Date: 2024-04-29 ## Status From 3d33cd48848c8c5e056d603d2bd5cb70ad1ec3f0 Mon Sep 17 00:00:00 2001 From: razzle Date: Wed, 1 May 2024 06:15:56 -0500 Subject: [PATCH 108/113] better use of generics and composition Signed-off-by: razzle --- src/cmd/dev.go | 5 ++- src/pkg/packager/migrations/common.go | 39 +++++++++---------- .../packager/migrations/default_required.go | 7 ++-- 3 files changed, 26 insertions(+), 25 deletions(-) diff --git a/src/cmd/dev.go b/src/cmd/dev.go index 508c3d7c62..e151eebb8d 100644 --- a/src/cmd/dev.go +++ b/src/cmd/dev.go @@ -119,7 +119,10 @@ var devMigrateCmd = &cobra.Command{ if !slices.Contains(featureMigrationsToRun, m.String()) { continue } - pkgWithFeature := m.Run(pkg) + pkgWithFeature, warning := m.Run(pkg) + if warning != "" { + message.Warn(warning) + } if !reflect.DeepEqual(pkgWithFeature, pkg) { data = append(data, []string{ m.String(), diff --git a/src/pkg/packager/migrations/common.go b/src/pkg/packager/migrations/common.go index 1cd09ae796..328a042593 100644 --- a/src/pkg/packager/migrations/common.go +++ b/src/pkg/packager/migrations/common.go @@ -36,37 +36,34 @@ var breakingChanges = []breakingChange{ }, } -// DeprecatedComponentMigration represents a migration that can be run on a component. -// -// DeprecatedComponentMigrations are migrations that seamlessly migrate deprecated component definitions. -type DeprecatedComponentMigration interface { +// PackageOrComponent is a type that can be either a ZarfComponent or a ZarfPackage +type PackageOrComponent interface { + types.ZarfComponent | types.ZarfPackage +} + +// Migration represents a migration on types satisfying the PackageOrComponent interface. +type Migration[T PackageOrComponent] interface { fmt.Stringer - // Run runs the migration on the component - Run(c types.ZarfComponent) (types.ZarfComponent, string) - // Clear clears the deprecated configuration from the component - Clear(mc types.ZarfComponent) types.ZarfComponent + Run(T) (T, string) +} + +// DeprecatedMigration represents a migration on types satisfying the PackageOrComponent interface. +type DeprecatedMigration[T PackageOrComponent] interface { + Migration[T] + Clear(T) T } // DeprecatedComponentMigrations returns a list of all current deprecated component-level migrations. -func DeprecatedComponentMigrations() []DeprecatedComponentMigration { - return []DeprecatedComponentMigration{ +func DeprecatedComponentMigrations() []DeprecatedMigration[types.ZarfComponent] { + return []DeprecatedMigration[types.ZarfComponent]{ ScriptsToActions{}, SetVariableToSetVariables{}, } } -// FeatureMigration represents a feature migration that can be run on a package. -// -// Every migration is mapped to a specific feature, and the feature's identifier is added to the package metadata. -type FeatureMigration interface { - fmt.Stringer - // Run runs the feature migration on the package - Run(pkg types.ZarfPackage) types.ZarfPackage -} - // FeatureMigrations returns a list of all current feature migrations. -func FeatureMigrations() []FeatureMigration { - return []FeatureMigration{ +func FeatureMigrations() []Migration[types.ZarfPackage] { + return []Migration[types.ZarfPackage]{ DefaultRequired{}, } } diff --git a/src/pkg/packager/migrations/default_required.go b/src/pkg/packager/migrations/default_required.go index dd754de71e..97048dc959 100644 --- a/src/pkg/packager/migrations/default_required.go +++ b/src/pkg/packager/migrations/default_required.go @@ -4,6 +4,7 @@ package migrations import ( + "fmt" "slices" "github.com/defenseunicorns/pkg/helpers" @@ -21,9 +22,9 @@ func (DefaultRequired) String() string { // Run sets all components to be required by default // // and cleanly migrates components explicitly marked as required to be nil -func (DefaultRequired) Run(pkg types.ZarfPackage) types.ZarfPackage { +func (DefaultRequired) Run(pkg types.ZarfPackage) (types.ZarfPackage, string) { if slices.Contains(pkg.Metadata.Features, types.DefaultRequired) { - return pkg + return pkg, fmt.Sprintf("%s feature flag already enabled", types.DefaultRequired) } pkg.Metadata.Features = append(pkg.Metadata.Features, types.DefaultRequired) @@ -37,5 +38,5 @@ func (DefaultRequired) Run(pkg types.ZarfPackage) types.ZarfPackage { } } - return pkg + return pkg, "" } From 9519392bf5e54c212693af594a58dfe024ff4c28 Mon Sep 17 00:00:00 2001 From: razzle Date: Fri, 10 May 2024 03:19:47 -0500 Subject: [PATCH 109/113] restore docs due to non-breaking Signed-off-by: razzle --- adr/0017-zarf-bundle.md | 1 + .../tutorials/differential_package_edit.html | 2 + .../publish_and_deploy_manifest.html | 1 + ...ubleshoot_version_required_no_version.html | 1 + .../tutorials/0-creating-a-zarf-package.mdx | 43 ++++++++++--------- .../src/content/docs/tutorials/6-big-bang.mdx | 2 + 6 files changed, 29 insertions(+), 21 deletions(-) diff --git a/adr/0017-zarf-bundle.md b/adr/0017-zarf-bundle.md index f51a114788..50c0e6a2b8 100644 --- a/adr/0017-zarf-bundle.md +++ b/adr/0017-zarf-bundle.md @@ -14,6 +14,7 @@ Currently there is no official way to enable the deployment, publishing, pulling ```yaml - name: init + required: true files: - source: zarf-init-amd64-v0.27.0.tar.zst target: zarf-init-amd64-v0.27.0.tar.zst diff --git a/site/public/tutorials/differential_package_edit.html b/site/public/tutorials/differential_package_edit.html index cba5e373a0..a6393041c1 100644 --- a/site/public/tutorials/differential_package_edit.html +++ b/site/public/tutorials/differential_package_edit.html @@ -59,6 +59,7 @@ version: "1.4.2" components: - name: longhorn-environment-check + required: true files: - source: https://raw.githubusercontent.com/longhorn/longhorn/v1.4.2/scripts/environment_check.sh target: environment_check.sh @@ -77,6 +78,7 @@ chmod +x ./environment_check.sh ./environment_check.sh - name: longhorn + required: true description: "Deploy Longhorn into a Kubernetes cluster. https://longhorn.io" actions: # Set the delete confirmation flag for Longhorn diff --git a/site/public/tutorials/publish_and_deploy_manifest.html b/site/public/tutorials/publish_and_deploy_manifest.html index 4d58b2ab98..74e8928a4a 100644 --- a/site/public/tutorials/publish_and_deploy_manifest.html +++ b/site/public/tutorials/publish_and_deploy_manifest.html @@ -62,6 +62,7 @@ components: - name: helm-oci-chart + required: true charts: - name: podinfo version: 6.3.3 diff --git a/site/public/tutorials/troubleshoot_version_required_no_version.html b/site/public/tutorials/troubleshoot_version_required_no_version.html index e2b048154f..d2b396a540 100644 --- a/site/public/tutorials/troubleshoot_version_required_no_version.html +++ b/site/public/tutorials/troubleshoot_version_required_no_version.html @@ -57,6 +57,7 @@ components: - name: helm-oci-chart + required: true charts: - name: podinfo version: 6.3.3 diff --git a/site/src/content/docs/tutorials/0-creating-a-zarf-package.mdx b/site/src/content/docs/tutorials/0-creating-a-zarf-package.mdx index 41bb6e3d4d..859c479056 100644 --- a/site/src/content/docs/tutorials/0-creating-a-zarf-package.mdx +++ b/site/src/content/docs/tutorials/0-creating-a-zarf-package.mdx @@ -23,7 +23,7 @@ Before beginning this tutorial you will need the following: ## Putting Together a Zarf Package -In order to create a Zarf package you first need to have an idea of what application(s) you want to package. In this example we will be using the [WordPress chart from Bitnami](https://artifacthub.io/packages/helm/bitnami/wordpress) but the steps and tools used below are very similar for other applications. +In order to create a Zarf package you first need to have an idea of what application(s) you want to package. In this example we will be using the [WordPress chart from Bitnami](https://artifacthub.io/packages/helm/bitnami/wordpress) but the steps and tools used below are very similar for other applications. ### Creating the Package Definition @@ -32,9 +32,9 @@ A `zarf.yaml` file follows the [Zarf Package Schema](https://github.com/defenseu ```yaml kind: ZarfPackageConfig # ZarfPackageConfig is the package kind for most normal zarf packages metadata: - name: wordpress # specifies the name of our package and should be unique and unchanging through updates - version: 16.0.4 # (optional) a version we can track as we release updates or publish to a registry - description: | # (optional) a human-readable description of the package that you are creating + name: wordpress # specifies the name of our package and should be unique and unchanging through updates + version: 16.0.4 # (optional) a version we can track as we release updates or publish to a registry + description: | # (optional) a human-readable description of the package that you are creating "A Zarf Package that deploys the WordPress blogging and content management platform" ``` @@ -50,9 +50,10 @@ Components are the unit of Zarf Packages that define an application stack. Thes ```yaml components: - - name: wordpress # specifies the name of our component and should be unique and unchanging through updates - description: | # (optional) a human-readable description of the component you are defining + - name: wordpress # specifies the name of our component and should be unique and unchanging through updates + description: | # (optional) a human-readable description of the component you are defining "Deploys the Bitnami-packaged WordPress chart into the cluster" + required: true # (optional) sets the component as 'required' so that it is always deployed charts: - name: wordpress url: oci://registry-1.docker.io/bitnamicharts/wordpress @@ -62,7 +63,7 @@ components: - wordpress-values.yaml ``` -In addition to this component definition, we also need to create the `valuesFiles` we have specified. In this case we need to create a file named `wordpress-values.yaml` in the same directory as our `zarf.yaml` with the following contents: +In addition to this component definition, we also need to create the `valuesFiles` we have specified. In this case we need to create a file named `wordpress-values.yaml` in the same directory as our `zarf.yaml` with the following contents: ```yaml # We are hard-coding these for now but will make them dynamic in Setting up Variables. @@ -97,7 +98,7 @@ Note that we are explicitly defining the `wordpress` namespace for this deployme ### Finding the Images -Once you have the above defined we can now work on setting the images that we will need to bring with us into the air gap. For this, Zarf has a helper command you can run with `zarf dev find-images`. Running this command in the directory of your zarf.yaml will result in the following output: +Once you have the above defined we can now work on setting the images that we will need to bring with us into the air gap. For this, Zarf has a helper command you can run with `zarf dev find-images`. Running this command in the directory of your zarf.yaml will result in the following output: @@ -105,7 +106,7 @@ From here you can copy the `images` key and array of images into the `wordpress` :::note -Due to the way some applications are deployed, Zarf might not be able to find all of the images in this way (particularly with operators). For this you can look at the upstream charts or manifests and find them manually. +Due to the way some applications are deployed, Zarf might not be able to find all of the images in this way (particularly with operators). For this you can look at the upstream charts or manifests and find them manually. ::: @@ -117,13 +118,13 @@ Zarf has more `dev` commands you can learn about on the [dev CLI docs page](/ref ### Setting up Variables -We now have a deployable package definition, but it is currently not very configurable and might not fit every environment we want to deploy it to. If we deployed it as-is we would always have a Zarf Blog and a `zarf` user with an autogenerated password. +We now have a deployable package definition, but it is currently not very configurable and might not fit every environment we want to deploy it to. If we deployed it as-is we would always have a Zarf Blog and a `zarf` user with an autogenerated password. To resolve this, we can add configuration options with [Zarf Variables](/ref/examples/values/#variables-zarf_var_). For this package we will add a `variables` section to our `zarf.yaml` above `components` that will allow us to setup the user and the blog. ```yaml variables: - # The unique name of the variable corresponding to the ###ZARF_VAR_### template + # The unique name of the variable corresponding to the ###ZARF_VAR_### template - name: WORDPRESS_USERNAME # A human-readable description of the variable shown during prompting description: The username that is used to login to the WordPress admin account @@ -154,7 +155,7 @@ variables: prompt: true ``` -To use these variables in our chart we must add their corresponding templates to our `wordpress-values.yaml` file. Zarf can template chart values, manifests, included text files and more. +To use these variables in our chart we must add their corresponding templates to our `wordpress-values.yaml` file. Zarf can template chart values, manifests, included text files and more. ```yaml wordpressUsername: ###ZARF_VAR_WORDPRESS_USERNAME### @@ -167,15 +168,15 @@ wordpressBlogName: ###ZARF_VAR_WORDPRESS_BLOG_NAME### :::caution -When dealing with `sensitive` values in Zarf it is strongly recommended to not include them directly inside of a Zarf Package and to only define them at deploy-time. You should also be aware of where you are using these values as they may be printed in `actions` you create or `files` that you place on disk. +When dealing with `sensitive` values in Zarf it is strongly recommended to not include them directly inside of a Zarf Package and to only define them at deploy-time. You should also be aware of where you are using these values as they may be printed in `actions` you create or `files` that you place on disk. ::: ### Setting up a Zarf Connect Service -As-is, our package could be configured to interface with an ingress provider to provide access to our blog, but this may not be desired for every service, particularly those that provide a backend for other frontend services. To help with debugging, Zarf allows you to specify Zarf Connect Services that will be displayed after package deployment to quickly connect into our deployed application. +As-is, our package could be configured to interface with an ingress provider to provide access to our blog, but this may not be desired for every service, particularly those that provide a backend for other frontend services. To help with debugging, Zarf allows you to specify Zarf Connect Services that will be displayed after package deployment to quickly connect into our deployed application. -For this package we will define two services, one for the blog and the other for the admin panel. These are normal Kubernetes services with special labels and annotations that Zarf watches out for, and to defined them create a `connect-services.yaml` with the following contents: +For this package we will define two services, one for the blog and the other for the admin panel. These are normal Kubernetes services with special labels and annotations that Zarf watches out for, and to defined them create a `connect-services.yaml` with the following contents: ```yaml apiVersion: v1 @@ -222,18 +223,18 @@ spec: To add this to our `zarf.yaml` we can simply specify it under our `wordpress` component using the `manifests` key: ```yaml -manifests: - - name: connect-services - namespace: wordpress - files: - - connect-services.yaml + manifests: + - name: connect-services + namespace: wordpress + files: + - connect-services.yaml ``` ### Creating the Package Once you have followed the above you should now have a `zarf.yaml` file that matches the one found on the [WordPress example page](/ref/examples/wordpress/#zarfyaml). -Creating this package is as simple as running the `zarf package create` command with the directory containing our `zarf.yaml`. Zarf will show us the `zarf.yaml` one last time asking if we would like to build the package, and upon confirmation Zarf will pull down all of the resources and bundle them into a package tarball. +Creating this package is as simple as running the `zarf package create` command with the directory containing our `zarf.yaml`. Zarf will show us the `zarf.yaml` one last time asking if we would like to build the package, and upon confirmation Zarf will pull down all of the resources and bundle them into a package tarball. ```bash zarf package create . diff --git a/site/src/content/docs/tutorials/6-big-bang.mdx b/site/src/content/docs/tutorials/6-big-bang.mdx index 0a126d0cb6..b609c1668c 100644 --- a/site/src/content/docs/tutorials/6-big-bang.mdx +++ b/site/src/content/docs/tutorials/6-big-bang.mdx @@ -17,6 +17,7 @@ The current version of this extension requires Big Bang version `1.54.0` or late ```yaml components: - name: bigbang + required: true images: - registry1.dso.mil/ironbank/gitlab/gitlab/gitlab-exporter:15.9.2 extensions: @@ -91,6 +92,7 @@ Within the `examples/big-bang` folder you will see a `zarf.yaml` that has the fo ```yaml components: - name: bigbang + required: true extensions: bigbang: version: 1.54.0 From 2375770e8794bb7dd25c0c61d9bc0fc070101718 Mon Sep 17 00:00:00 2001 From: razzle Date: Fri, 10 May 2024 13:38:41 -0500 Subject: [PATCH 110/113] breaking change, will need to discuss Signed-off-by: razzle --- packages/distros/k3s/zarf.yaml | 5 ++++- packages/gitea/zarf.yaml | 4 +++- packages/logging-pgl/zarf.yaml | 6 +++++- packages/zarf-agent/zarf.yaml | 8 +++++--- packages/zarf-registry/zarf.yaml | 9 ++++++--- src/pkg/packager/composer/list.go | 9 +++++---- 6 files changed, 28 insertions(+), 13 deletions(-) diff --git a/packages/distros/k3s/zarf.yaml b/packages/distros/k3s/zarf.yaml index 0813b1ee38..d1980ab59b 100644 --- a/packages/distros/k3s/zarf.yaml +++ b/packages/distros/k3s/zarf.yaml @@ -1,7 +1,10 @@ kind: ZarfInitConfig metadata: name: distro-k3s - description: Used to establish a new Zarf cluster + description: | + Used to establish a new Zarf cluster + + This package is NOT meant to be used as a standalone package. It is meant to be used as a dependency of an 'init' package. components: # AMD-64 version of the K3s stack diff --git a/packages/gitea/zarf.yaml b/packages/gitea/zarf.yaml index 2f59bebdba..ef2ed54d02 100644 --- a/packages/gitea/zarf.yaml +++ b/packages/gitea/zarf.yaml @@ -1,6 +1,8 @@ kind: ZarfPackageConfig metadata: - name: init-package-git-server + name: git-server + description: | + This package is NOT meant to be used as a standalone package. It is meant to be used as a dependency of an 'init' package. variables: - name: GIT_SERVER_EXISTING_PVC diff --git a/packages/logging-pgl/zarf.yaml b/packages/logging-pgl/zarf.yaml index d5fc213dc3..6fd8edc5a8 100644 --- a/packages/logging-pgl/zarf.yaml +++ b/packages/logging-pgl/zarf.yaml @@ -1,6 +1,10 @@ kind: ZarfPackageConfig metadata: - name: init-package-logging + name: logging + description: | + Deploys the Promtail Grafana & Loki (PGL) stack. + + This package is NOT meant to be used as a standalone package. It is meant to be used as a dependency of an 'init' package. components: - name: logging diff --git a/packages/zarf-agent/zarf.yaml b/packages/zarf-agent/zarf.yaml index 86799ff03f..ce8b9d4ac1 100644 --- a/packages/zarf-agent/zarf.yaml +++ b/packages/zarf-agent/zarf.yaml @@ -1,7 +1,10 @@ kind: ZarfPackageConfig metadata: - name: init-package-zarf-agent - description: Install the zarf agent mutating webhook on a new cluster + name: zarf-agent + description: | + Install the zarf agent mutating webhook on a new cluster + + This package is NOT meant to be used as a standalone package. It is meant to be used as a dependency of an 'init' package. constants: - name: AGENT_IMAGE @@ -16,7 +19,6 @@ components: images and git repository references in Kubernetes manifests. This prevents the need to manually update URLs from their original sources to the Zarf-managed docker registry and git server. - required: true images: - "###ZARF_PKG_TMPL_AGENT_IMAGE_DOMAIN######ZARF_PKG_TMPL_AGENT_IMAGE###:###ZARF_PKG_TMPL_AGENT_IMAGE_TAG###" manifests: diff --git a/packages/zarf-registry/zarf.yaml b/packages/zarf-registry/zarf.yaml index 2da03ccda6..34a4b67450 100644 --- a/packages/zarf-registry/zarf.yaml +++ b/packages/zarf-registry/zarf.yaml @@ -1,6 +1,11 @@ kind: ZarfPackageConfig metadata: - name: init-package-zarf-registry + name: zarf-registry + description: | + Initializes the Zarf Registry by bootstrapping a Kubernetes cluster with a running pod and hosting the registry image. + The Zarf Registry is then updated to use the self-hosted registry image. + + This package is NOT meant to be used as a standalone package. It is meant to be used as a dependency of an 'init' package. variables: - name: REGISTRY_EXISTING_PVC @@ -105,7 +110,6 @@ components: description: | Bootstraps a Kubernetes cluster by cloning a running pod in the cluster and hosting the registry image. Removed and destroyed after the Zarf Registry is self-hosting the registry image. - required: true only: cluster: architecture: amd64 @@ -120,7 +124,6 @@ components: description: | Bootstraps a Kubernetes cluster by cloning a running pod in the cluster and hosting the registry image. Removed and destroyed after the Zarf Registry is self-hosting the registry image. - required: true only: cluster: architecture: arm64 diff --git a/src/pkg/packager/composer/list.go b/src/pkg/packager/composer/list.go index e7e3715a31..3f78f82b0c 100644 --- a/src/pkg/packager/composer/list.go +++ b/src/pkg/packager/composer/list.go @@ -296,10 +296,11 @@ func (ic *ImportChain) Compose() (composed *types.ZarfComponent, err error) { // start overriding with the tail node node := ic.tail for node != nil { - // if prev.required is nil, and node.required is != nil, then error - // if node.prev != nil && node.prev.Required == nil && node.Required != nil { - // return nil, fmt.Errorf("component %q: required cannot be unset during composition from %q to %q", node.ImportName(), node.ImportLocation(), node.prev.ImportLocation()) - // } + // required can go from explicit X --> explicit Y + // but cannot go from explicit X --> implicit Y + if node.prev != nil && node.prev.Required == nil && node.Required != nil { + return nil, fmt.Errorf("component %q: required cannot be unset during composition from %q to %q", node.ImportName(), node.ImportLocation(), node.prev.ImportLocation()) + } fixPaths(&node.ZarfComponent, node.relativeToHead) From be6d51d14b6f22107d0433a5a1a5f8ce888473a3 Mon Sep 17 00:00:00 2001 From: razzle Date: Tue, 14 May 2024 14:06:56 -0500 Subject: [PATCH 111/113] cleanup Signed-off-by: razzle --- src/types/validate.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/types/validate.go b/src/types/validate.go index ae6236227f..6d5d7458fc 100644 --- a/src/types/validate.go +++ b/src/types/validate.go @@ -98,7 +98,7 @@ func (pkg ZarfPackage) Validate() error { return fmt.Errorf(lang.PkgValidateErrComponentLocalOS, component.Name, component.Only.LocalOS, supportedOS) } - if component.IsRequired() { + if component.IsRequired(pkg.Metadata.Features) { if component.Default { return fmt.Errorf(lang.PkgValidateErrComponentReqDefault, component.Name) } From 0e1ef9b3a6feb3521fd971d8ad19dbbac18bba47 Mon Sep 17 00:00:00 2001 From: razzle Date: Wed, 15 May 2024 12:15:26 -0500 Subject: [PATCH 112/113] rm validate Signed-off-by: razzle --- src/internal/packager/validate/validate.go | 369 --------------------- 1 file changed, 369 deletions(-) delete mode 100644 src/internal/packager/validate/validate.go diff --git a/src/internal/packager/validate/validate.go b/src/internal/packager/validate/validate.go deleted file mode 100644 index 5d4c6e1d60..0000000000 --- a/src/internal/packager/validate/validate.go +++ /dev/null @@ -1,369 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: 2021-Present The Zarf Authors - -// Package validate provides Zarf package validation functions. -package validate - -import ( - "fmt" - "path/filepath" - "regexp" - "slices" - - "github.com/defenseunicorns/pkg/helpers" - "github.com/defenseunicorns/zarf/src/config" - "github.com/defenseunicorns/zarf/src/config/lang" - "github.com/defenseunicorns/zarf/src/pkg/variables" - "github.com/defenseunicorns/zarf/src/types" -) - -var ( - // IsLowercaseNumberHyphenNoStartHyphen is a regex for lowercase, numbers and hyphens that cannot start with a hyphen. - // https://regex101.com/r/FLdG9G/2 - IsLowercaseNumberHyphenNoStartHyphen = regexp.MustCompile(`^[a-z0-9][a-z0-9\-]*$`).MatchString - // IsUppercaseNumberUnderscore is a regex for uppercase, numbers and underscores. - // https://regex101.com/r/tfsEuZ/1 - IsUppercaseNumberUnderscore = regexp.MustCompile(`^[A-Z0-9_]+$`).MatchString - // Define allowed OS, an empty string means it is allowed on all operating systems - // same as enums on ZarfComponentOnlyTarget - supportedOS = []string{"linux", "darwin", "windows", ""} -) - -// SupportedOS returns the supported operating systems. -// -// The supported operating systems are: linux, darwin, windows. -// -// An empty string signifies no OS restrictions. -func SupportedOS() []string { - return supportedOS -} - -// Run performs config validations. -func Run(pkg types.ZarfPackage) error { - if pkg.Kind == types.ZarfInitConfig && pkg.Metadata.YOLO { - return fmt.Errorf(lang.PkgValidateErrInitNoYOLO) - } - - if err := validatePackageName(pkg.Metadata.Name); err != nil { - return fmt.Errorf(lang.PkgValidateErrName, err) - } - - for _, variable := range pkg.Variables { - if err := validatePackageVariable(variable); err != nil { - return fmt.Errorf(lang.PkgValidateErrVariable, err) - } - } - - for _, constant := range pkg.Constants { - if err := validatePackageConstant(constant); err != nil { - return fmt.Errorf(lang.PkgValidateErrConstant, err) - } - } - - uniqueComponentNames := make(map[string]bool) - groupDefault := make(map[string]string) - groupedComponents := make(map[string][]string) - - for _, component := range pkg.Components { - // ensure component name is unique - if _, ok := uniqueComponentNames[component.Name]; ok { - return fmt.Errorf(lang.PkgValidateErrComponentNameNotUnique, component.Name) - } - uniqueComponentNames[component.Name] = true - - if err := validateComponent(pkg, component); err != nil { - return fmt.Errorf(lang.PkgValidateErrComponent, component.Name, err) - } - - // ensure groups don't have multiple defaults or only one component - if component.DeprecatedGroup != "" { - if component.Default { - if _, ok := groupDefault[component.DeprecatedGroup]; ok { - return fmt.Errorf(lang.PkgValidateErrGroupMultipleDefaults, component.DeprecatedGroup, groupDefault[component.DeprecatedGroup], component.Name) - } - groupDefault[component.DeprecatedGroup] = component.Name - } - groupedComponents[component.DeprecatedGroup] = append(groupedComponents[component.DeprecatedGroup], component.Name) - } - } - - for groupKey, componentNames := range groupedComponents { - if len(componentNames) == 1 { - return fmt.Errorf(lang.PkgValidateErrGroupOneComponent, groupKey, componentNames[0]) - } - } - - return nil -} - -// ImportDefinition validates the component trying to be imported. -func ImportDefinition(component *types.ZarfComponent) error { - path := component.Import.Path - url := component.Import.URL - - // ensure path or url is provided - if path == "" && url == "" { - return fmt.Errorf(lang.PkgValidateErrImportDefinition, component.Name, "neither a path nor a URL was provided") - } - - // ensure path and url are not both provided - if path != "" && url != "" { - return fmt.Errorf(lang.PkgValidateErrImportDefinition, component.Name, "both a path and a URL were provided") - } - - // validation for path - if url == "" && path != "" { - // ensure path is not an absolute path - if filepath.IsAbs(path) { - return fmt.Errorf(lang.PkgValidateErrImportDefinition, component.Name, "path cannot be an absolute path") - } - } - - // validation for url - if url != "" && path == "" { - ok := helpers.IsOCIURL(url) - if !ok { - return fmt.Errorf(lang.PkgValidateErrImportDefinition, component.Name, "URL is not a valid OCI URL") - } - } - - return nil -} - -func oneIfNotEmpty(testString string) int { - if testString == "" { - return 0 - } - - return 1 -} - -func validateComponent(pkg types.ZarfPackage, component types.ZarfComponent) error { - if !IsLowercaseNumberHyphenNoStartHyphen(component.Name) { - return fmt.Errorf(lang.PkgValidateErrComponentName, component.Name) - } - - if !slices.Contains(supportedOS, component.Only.LocalOS) { - return fmt.Errorf(lang.PkgValidateErrComponentLocalOS, component.Name, component.Only.LocalOS, supportedOS) - } - - if component.IsRequired(pkg.Metadata.Features) { - if component.Default { - return fmt.Errorf(lang.PkgValidateErrComponentReqDefault, component.Name) - } - if component.DeprecatedGroup != "" { - return fmt.Errorf(lang.PkgValidateErrComponentReqGrouped, component.Name) - } - } - - uniqueChartNames := make(map[string]bool) - for _, chart := range component.Charts { - // ensure chart name is unique - if _, ok := uniqueChartNames[chart.Name]; ok { - return fmt.Errorf(lang.PkgValidateErrChartNameNotUnique, chart.Name) - } - uniqueChartNames[chart.Name] = true - - if err := validateChart(chart); err != nil { - return fmt.Errorf(lang.PkgValidateErrChart, err) - } - } - - uniqueManifestNames := make(map[string]bool) - for _, manifest := range component.Manifests { - // ensure manifest name is unique - if _, ok := uniqueManifestNames[manifest.Name]; ok { - return fmt.Errorf(lang.PkgValidateErrManifestNameNotUnique, manifest.Name) - } - uniqueManifestNames[manifest.Name] = true - - if err := validateManifest(manifest); err != nil { - return fmt.Errorf(lang.PkgValidateErrManifest, err) - } - } - - if pkg.Metadata.YOLO { - if err := validateYOLO(component); err != nil { - return fmt.Errorf(lang.PkgValidateErrComponentYOLO, component.Name, err) - } - } - - if containsVariables, err := validateActionset(component.Actions.OnCreate); err != nil { - return fmt.Errorf(lang.PkgValidateErrAction, err) - } else if containsVariables { - return fmt.Errorf(lang.PkgValidateErrActionVariables, component.Name) - } - - if _, err := validateActionset(component.Actions.OnDeploy); err != nil { - return fmt.Errorf(lang.PkgValidateErrAction, err) - } - - if containsVariables, err := validateActionset(component.Actions.OnRemove); err != nil { - return fmt.Errorf(lang.PkgValidateErrAction, err) - } else if containsVariables { - return fmt.Errorf(lang.PkgValidateErrActionVariables, component.Name) - } - - return nil -} - -func validateActionset(actions types.ZarfComponentActionSet) (bool, error) { - containsVariables := false - - validate := func(actions []types.ZarfComponentAction) error { - for _, action := range actions { - if cv, err := validateAction(action); err != nil { - return err - } else if cv { - containsVariables = true - } - } - - return nil - } - - if err := validate(actions.Before); err != nil { - return containsVariables, err - } - if err := validate(actions.After); err != nil { - return containsVariables, err - } - if err := validate(actions.OnSuccess); err != nil { - return containsVariables, err - } - if err := validate(actions.OnFailure); err != nil { - return containsVariables, err - } - - return containsVariables, nil -} - -func validateAction(action types.ZarfComponentAction) (bool, error) { - containsVariables := false - - // Validate SetVariable - for _, variable := range action.SetVariables { - if !IsUppercaseNumberUnderscore(variable.Name) { - return containsVariables, fmt.Errorf(lang.PkgValidateMustBeUppercase, variable.Name) - } - containsVariables = true - } - - if action.Wait != nil { - // Validate only cmd or wait, not both - if action.Cmd != "" { - return containsVariables, fmt.Errorf(lang.PkgValidateErrActionCmdWait, action.Cmd) - } - - // Validate only cluster or network, not both - if action.Wait.Cluster != nil && action.Wait.Network != nil { - return containsVariables, fmt.Errorf(lang.PkgValidateErrActionClusterNetwork) - } - - // Validate at least one of cluster or network - if action.Wait.Cluster == nil && action.Wait.Network == nil { - return containsVariables, fmt.Errorf(lang.PkgValidateErrActionClusterNetwork) - } - } - - return containsVariables, nil -} - -func validateYOLO(component types.ZarfComponent) error { - if len(component.Images) > 0 { - return fmt.Errorf(lang.PkgValidateErrYOLONoOCI) - } - - if len(component.Repos) > 0 { - return fmt.Errorf(lang.PkgValidateErrYOLONoGit) - } - - if component.Only.Cluster.Architecture != "" { - return fmt.Errorf(lang.PkgValidateErrYOLONoArch) - } - - if len(component.Only.Cluster.Distros) > 0 { - return fmt.Errorf(lang.PkgValidateErrYOLONoDistro) - } - - return nil -} - -func validatePackageName(subject string) error { - if !IsLowercaseNumberHyphenNoStartHyphen(subject) { - return fmt.Errorf(lang.PkgValidateErrPkgName, subject) - } - - return nil -} - -func validatePackageVariable(subject variables.InteractiveVariable) error { - // ensure the variable name is only capitals and underscores - if !IsUppercaseNumberUnderscore(subject.Name) { - return fmt.Errorf(lang.PkgValidateMustBeUppercase, subject.Name) - } - - return nil -} - -func validatePackageConstant(subject variables.Constant) error { - // ensure the constant name is only capitals and underscores - if !IsUppercaseNumberUnderscore(subject.Name) { - return fmt.Errorf(lang.PkgValidateErrPkgConstantName, subject.Name) - } - - if !regexp.MustCompile(subject.Pattern).MatchString(subject.Value) { - return fmt.Errorf(lang.PkgValidateErrPkgConstantPattern, subject.Name, subject.Pattern) - } - - return nil -} - -func validateChart(chart types.ZarfChart) error { - // Don't allow empty names - if chart.Name == "" { - return fmt.Errorf(lang.PkgValidateErrChartNameMissing, chart.Name) - } - - // Helm max release name - if len(chart.Name) > config.ZarfMaxChartNameLength { - return fmt.Errorf(lang.PkgValidateErrChartName, chart.Name, config.ZarfMaxChartNameLength) - } - - // Must have a namespace - if chart.Namespace == "" { - return fmt.Errorf(lang.PkgValidateErrChartNamespaceMissing, chart.Name) - } - - // Must have a url or localPath (and not both) - count := oneIfNotEmpty(chart.URL) + oneIfNotEmpty(chart.LocalPath) - if count != 1 { - return fmt.Errorf(lang.PkgValidateErrChartURLOrPath, chart.Name) - } - - // Must have a version - if chart.Version == "" { - return fmt.Errorf(lang.PkgValidateErrChartVersion, chart.Name) - } - - return nil -} - -func validateManifest(manifest types.ZarfManifest) error { - // Don't allow empty names - if manifest.Name == "" { - return fmt.Errorf(lang.PkgValidateErrManifestNameMissing, manifest.Name) - } - - // Helm max release name - if len(manifest.Name) > config.ZarfMaxChartNameLength { - return fmt.Errorf(lang.PkgValidateErrManifestNameLength, manifest.Name, config.ZarfMaxChartNameLength) - } - - // Require files in manifest - if len(manifest.Files) < 1 && len(manifest.Kustomizations) < 1 { - return fmt.Errorf(lang.PkgValidateErrManifestFileOrKustomize, manifest.Name) - } - - return nil -} From ece00d1de96606801fb641df6f035c3607ede56f Mon Sep 17 00:00:00 2001 From: razzle Date: Tue, 21 May 2024 16:22:55 -0400 Subject: [PATCH 113/113] revert behavior Signed-off-by: razzle --- src/pkg/packager/composer/list.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/pkg/packager/composer/list.go b/src/pkg/packager/composer/list.go index 1374c72747..cf7ed1b823 100644 --- a/src/pkg/packager/composer/list.go +++ b/src/pkg/packager/composer/list.go @@ -295,12 +295,6 @@ func (ic *ImportChain) Compose() (composed *types.ZarfComponent, err error) { // start overriding with the tail node node := ic.tail for node != nil { - // required can go from explicit X --> explicit Y - // but cannot go from explicit X --> implicit Y - if node.prev != nil && node.prev.Required == nil && node.Required != nil { - return nil, fmt.Errorf("component %q: required cannot be unset during composition from %q to %q", node.ImportName(), node.ImportLocation(), node.prev.ImportLocation()) - } - fixPaths(&node.ZarfComponent, node.relativeToHead) // perform overrides here