diff --git a/Makefile b/Makefile index 6730e5073f..5ab1738a69 100644 --- a/Makefile +++ b/Makefile @@ -8,8 +8,10 @@ KEY ?= "" # Figure out which Zarf binary we should use based on the operating system we are on ZARF_BIN := ./build/zarf +BUILD_CLI_FOR_SYSTEM := build-cli-linux-amd ifeq ($(OS),Windows_NT) ZARF_BIN := $(addsuffix .exe,$(ZARF_BIN)) + BUILD_CLI_FOR_SYSTEM := build-cli-windows-amd else UNAME_S := $(shell uname -s) UNAME_P := $(shell uname -p) @@ -19,13 +21,14 @@ else endif ifeq ($(UNAME_P),i386) ZARF_BIN := $(addsuffix -intel,$(ZARF_BIN)) + BUILD_CLI_FOR_SYSTEM = build-cli-mac-intel endif ifeq ($(UNAME_P),arm) ZARF_BIN := $(addsuffix -apple,$(ZARF_BIN)) + BUILD_CLI_FOR_SYSTEM = build-cli-mac-apple endif endif endif -.DEFAULT_GOAL := help CLI_VERSION ?= $(if $(shell git describe --tags),$(shell git describe --tags),"UnknownVersion") BUILD_ARGS := -s -w -X github.com/defenseunicorns/zarf/src/config.CLIVersion=$(CLI_VERSION) @@ -45,11 +48,13 @@ BUILD_DATE := $(shell date -u +'%Y-%m-%dT%H:%M:%SZ') BUILD_ARGS += -X k8s.io/component-base/version.gitCommit=$(GIT_SHA) BUILD_ARGS += -X k8s.io/component-base/version.buildDate=$(BUILD_DATE) +.DEFAULT_GOAL := build + .PHONY: help help: ## Display this help information @grep -E '^[a-zA-Z0-9_-]+:.*?## .*$$' $(MAKEFILE_LIST) \ - | sort | awk 'BEGIN {FS = ":.*?## "}; \ - {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' + | sort | awk 'BEGIN {FS = ":.*?## "}; \ + {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' clean: ## Clean the build directory rm -rf build @@ -62,6 +67,9 @@ delete-packages: ## Delete all Zarf package tarballs in the project recursively find . -type f -name 'zarf-package-*' -delete # Note: the path to the main.go file is not used due to https://github.com/golang/go/issues/51831#issuecomment-1074188363 +.PHONY: build +build: ## Build the Zarf CLI for the machines OS and architecture + $(MAKE) $(BUILD_CLI_FOR_SYSTEM) build-cli-linux-amd: ## Build the Zarf CLI for Linux on AMD64 CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags="$(BUILD_ARGS)" -o build/zarf . @@ -89,6 +97,9 @@ docs-and-schema: ## Generate the Zarf Documentation and Schema hack/gen-cli-docs.sh ZARF_CONFIG=hack/empty-config.toml hack/create-zarf-schema.sh +lint-packages-and-examples: build-cli-for-system ## Recursively lint all zarf.yaml files in the repo except for those dedicated to tests + hack/lint_all_zarf_packages.sh $(ZARF_BIN) + # INTERNAL: a shim used to build the agent image only if needed on Windows using the `test` command init-package-local-agent: @test "$(AGENT_IMAGE_TAG)" != "local" || $(MAKE) build-local-agent-image diff --git a/docs/2-the-zarf-cli/100-cli-commands/zarf_dev.md b/docs/2-the-zarf-cli/100-cli-commands/zarf_dev.md index ae76b14632..0e44236b00 100644 --- a/docs/2-the-zarf-cli/100-cli-commands/zarf_dev.md +++ b/docs/2-the-zarf-cli/100-cli-commands/zarf_dev.md @@ -28,7 +28,7 @@ Commands useful for developing packages * [zarf dev deploy](zarf_dev_deploy.md) - [beta] Creates and deploys a Zarf package from a given directory * [zarf dev find-images](zarf_dev_find-images.md) - Evaluates components in a Zarf file to identify images specified in their helm charts and manifests * [zarf dev generate-config](zarf_dev_generate-config.md) - Generates a config file for Zarf -* [zarf dev lint](zarf_dev_lint.md) - Verifies the package schema +* [zarf dev lint](zarf_dev_lint.md) - Lints the given package for valid schema and recommended practices * [zarf dev patch-git](zarf_dev_patch-git.md) - Converts all .git URLs to the specified Zarf HOST and with the Zarf URL pattern in a given FILE. NOTE: This should only be used for manifests that are not mutated by the Zarf Agent Mutating Webhook. * [zarf dev sha256sum](zarf_dev_sha256sum.md) - Generates a SHA256SUM for the given file diff --git a/docs/2-the-zarf-cli/100-cli-commands/zarf_dev_lint.md b/docs/2-the-zarf-cli/100-cli-commands/zarf_dev_lint.md index 209b7f1b4a..7ec1068b0b 100644 --- a/docs/2-the-zarf-cli/100-cli-commands/zarf_dev_lint.md +++ b/docs/2-the-zarf-cli/100-cli-commands/zarf_dev_lint.md @@ -1,11 +1,11 @@ # zarf dev lint -Verifies the package schema +Lints the given package for valid schema and recommended practices ## Synopsis -Verifies the package schema and warns the user if they have variables that won't be evaluated +Verifies the package schema, checks if any variables won't be evaluated, and checks for unpinned images/repos/files ``` zarf dev lint [ DIRECTORY ] [flags] @@ -14,7 +14,9 @@ zarf dev lint [ DIRECTORY ] [flags] ## Options ``` - -h, --help help for lint + -f, --flavor string The flavor of components to include in the resulting package (i.e. have a matching or empty "only.flavor" key) + -h, --help help for lint + --set stringToString Specify package variables to set on the command line (KEY=value) (default []) ``` ## Options inherited from parent commands diff --git a/hack/lint_all_zarf_packages.sh b/hack/lint_all_zarf_packages.sh new file mode 100755 index 0000000000..5f41d3d887 --- /dev/null +++ b/hack/lint_all_zarf_packages.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +ZARF_BIN=$1 +LINT_SRC_TEST=$2 +SCRIPT=$(realpath "$0") +SCRIPTPATH=$(dirname "$SCRIPT") +cd "$SCRIPTPATH" || exit +cd .. +find "." -type f -name 'zarf.yaml' | while read -r yaml_file; do + dir=$(dirname "$yaml_file") + if [[ "$dir" == *src/test/* ]] && [ "$LINT_SRC_TEST" != true ]; then + continue + fi + echo "Running 'zarf prepare lint' in directory: $dir" + $ZARF_BIN prepare lint "$dir" + echo "---" +done diff --git a/src/cmd/common/utils.go b/src/cmd/common/utils.go new file mode 100644 index 0000000000..01a6b104d6 --- /dev/null +++ b/src/cmd/common/utils.go @@ -0,0 +1,18 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package common handles command configuration across all commands +package common + +import ( + "github.com/defenseunicorns/zarf/src/types" +) + +// SetBaseDirectory sets base directory on package config when given in args +func SetBaseDirectory(args []string, pkgConfig *types.PackagerConfig) { + if len(args) > 0 { + pkgConfig.CreateOpts.BaseDir = args[0] + } else { + pkgConfig.CreateOpts.BaseDir = "." + } +} diff --git a/src/cmd/dev.go b/src/cmd/dev.go index 0b76ee55f8..63426692ea 100644 --- a/src/cmd/dev.go +++ b/src/cmd/dev.go @@ -38,17 +38,9 @@ var devDeployCmd = &cobra.Command{ Use: "deploy", Args: cobra.MaximumNArgs(1), Short: lang.CmdDevDeployShort, - Long: lang.CmdDevDeployLong, + Long: lang.CmdDevDeployLong, Run: func(cmd *cobra.Command, args []string) { - if len(args) > 0 { - pkgConfig.CreateOpts.BaseDir = args[0] - } else { - var err error - pkgConfig.CreateOpts.BaseDir, err = os.Getwd() - if err != nil { - message.Fatalf(err, lang.CmdPackageCreateErr, err.Error()) - } - } + common.SetBaseDirectory(args, &pkgConfig) v := common.GetViper() pkgConfig.CreateOpts.SetVariables = helpers.TransformAndMergeMap( @@ -71,7 +63,7 @@ var devDeployCmd = &cobra.Command{ var devTransformGitLinksCmd = &cobra.Command{ Use: "patch-git HOST FILE", Aliases: []string{"p"}, - Short: lang.CmdPreparePatchGitShort, + Short: lang.CmdDevPatchGitShort, Args: cobra.ExactArgs(2), Run: func(cmd *cobra.Command, args []string) { host, fileName := args[0], args[1] @@ -79,7 +71,7 @@ var devTransformGitLinksCmd = &cobra.Command{ // Read the contents of the given file content, err := os.ReadFile(fileName) if err != nil { - message.Fatalf(err, lang.CmdPreparePatchGitFileReadErr, fileName) + message.Fatalf(err, lang.CmdDevPatchGitFileReadErr, fileName) } pkgConfig.InitOpts.GitServer.Address = host @@ -94,17 +86,17 @@ var devTransformGitLinksCmd = &cobra.Command{ // Ask the user before this destructive action confirm := false prompt := &survey.Confirm{ - Message: fmt.Sprintf(lang.CmdPreparePatchGitOverwritePrompt, fileName), + Message: fmt.Sprintf(lang.CmdDevPatchGitOverwritePrompt, fileName), } if err := survey.AskOne(prompt, &confirm); err != nil { - message.Fatalf(nil, lang.CmdPreparePatchGitOverwriteErr, err.Error()) + message.Fatalf(nil, lang.CmdDevPatchGitOverwriteErr, err.Error()) } if confirm { // Overwrite the file err = os.WriteFile(fileName, []byte(processedText), 0640) if err != nil { - message.Fatal(err, lang.CmdPreparePatchGitFileWriteErr) + message.Fatal(err, lang.CmdDevPatchGitFileWriteErr) } } @@ -114,7 +106,7 @@ var devTransformGitLinksCmd = &cobra.Command{ var devSha256SumCmd = &cobra.Command{ Use: "sha256sum { FILE | URL }", Aliases: []string{"s"}, - Short: lang.CmdPrepareSha256sumShort, + Short: lang.CmdDevSha256sumShort, Args: cobra.ExactArgs(1), Run: func(cmd *cobra.Command, args []string) { fileName := args[0] @@ -124,11 +116,11 @@ var devSha256SumCmd = &cobra.Command{ var err error if helpers.IsURL(fileName) { - message.Warn(lang.CmdPrepareSha256sumRemoteWarning) + message.Warn(lang.CmdDevSha256sumRemoteWarning) fileBase, err := helpers.ExtractBasePathFromURL(fileName) if err != nil { - message.Fatalf(err, lang.CmdPrepareSha256sumHashErr, err.Error()) + message.Fatalf(err, lang.CmdDevSha256sumHashErr, err.Error()) } if fileBase == "" { @@ -137,13 +129,13 @@ var devSha256SumCmd = &cobra.Command{ tmp, err = utils.MakeTempDir(config.CommonOptions.TempDirectory) if err != nil { - message.Fatalf(err, lang.CmdPrepareSha256sumHashErr, err.Error()) + message.Fatalf(err, lang.CmdDevSha256sumHashErr, err.Error()) } downloadPath := filepath.Join(tmp, fileBase) err = utils.DownloadToFile(fileName, downloadPath, "") if err != nil { - message.Fatalf(err, lang.CmdPrepareSha256sumHashErr, err.Error()) + message.Fatalf(err, lang.CmdDevSha256sumHashErr, err.Error()) } fileName = downloadPath @@ -155,7 +147,7 @@ var devSha256SumCmd = &cobra.Command{ if tmp == "" { tmp, err = utils.MakeTempDir(config.CommonOptions.TempDirectory) if err != nil { - message.Fatalf(err, lang.CmdPrepareSha256sumHashErr, err.Error()) + message.Fatalf(err, lang.CmdDevSha256sumHashErr, err.Error()) } defer os.RemoveAll(tmp) } @@ -164,7 +156,7 @@ var devSha256SumCmd = &cobra.Command{ err = archiver.Extract(fileName, extractPath, tmp) if err != nil { - message.Fatalf(err, lang.CmdPrepareSha256sumHashErr, err.Error()) + message.Fatalf(err, lang.CmdDevSha256sumHashErr, err.Error()) } fileName = extractedFile @@ -172,14 +164,14 @@ var devSha256SumCmd = &cobra.Command{ data, err = os.Open(fileName) if err != nil { - message.Fatalf(err, lang.CmdPrepareSha256sumHashErr, err.Error()) + message.Fatalf(err, lang.CmdDevSha256sumHashErr, err.Error()) } defer data.Close() var hash string hash, err = helpers.GetSHA256Hash(data) if err != nil { - message.Fatalf(err, lang.CmdPrepareSha256sumHashErr, err.Error()) + message.Fatalf(err, lang.CmdDevSha256sumHashErr, err.Error()) } else { fmt.Println(hash) } @@ -190,19 +182,11 @@ var devFindImagesCmd = &cobra.Command{ Use: "find-images [ PACKAGE ]", Aliases: []string{"f"}, Args: cobra.MaximumNArgs(1), - Short: lang.CmdPrepareFindImagesShort, - Long: lang.CmdPrepareFindImagesLong, + Short: lang.CmdDevFindImagesShort, + Long: lang.CmdDevFindImagesLong, Run: func(cmd *cobra.Command, args []string) { // If a directory was provided, use that as the base directory - if len(args) > 0 { - pkgConfig.CreateOpts.BaseDir = args[0] - } else { - cwd, err := os.Getwd() - if err != nil { - message.Fatalf(err, lang.CmdPrepareFindImagesErr, err.Error()) - } - pkgConfig.CreateOpts.BaseDir = cwd - } + common.SetBaseDirectory(args, &pkgConfig) // Ensure uppercase keys from viper v := common.GetViper() @@ -215,7 +199,7 @@ var devFindImagesCmd = &cobra.Command{ // Find all the images the package might need if _, err := pkgClient.FindImages(); err != nil { - message.Fatalf(err, lang.CmdPrepareFindImagesErr, err.Error()) + message.Fatalf(err, lang.CmdDevFindImagesErr, err.Error()) } }, } @@ -224,8 +208,8 @@ var devGenConfigFileCmd = &cobra.Command{ Use: "generate-config [ FILENAME ]", Aliases: []string{"gc"}, Args: cobra.MaximumNArgs(1), - Short: lang.CmdPrepareGenerateConfigShort, - Long: lang.CmdPrepareGenerateConfigLong, + Short: lang.CmdDevGenerateConfigShort, + Long: lang.CmdDevGenerateConfigLong, Run: func(cmd *cobra.Command, args []string) { fileName := "zarf-config.toml" @@ -236,7 +220,7 @@ var devGenConfigFileCmd = &cobra.Command{ v := common.GetViper() if err := v.SafeWriteConfigAs(fileName); err != nil { - message.Fatalf(err, lang.CmdPrepareGenerateConfigErr, fileName) + message.Fatalf(err, lang.CmdDevGenerateConfigErr, fileName) } }, } @@ -245,20 +229,14 @@ var devLintCmd = &cobra.Command{ Use: "lint [ DIRECTORY ]", Args: cobra.MaximumNArgs(1), Aliases: []string{"l"}, - Short: lang.CmdPrepareLintShort, - Long: lang.CmdPrepareLintLong, + Short: lang.CmdDevLintShort, + Long: lang.CmdDevLintLong, Run: func(cmd *cobra.Command, args []string) { - baseDir := "" - if len(args) > 0 { - baseDir = args[0] - } else { - var err error - baseDir, err = os.Getwd() - if err != nil { - message.Fatalf(err, lang.CmdPrepareLintErr, err.Error()) - } - } - validator, err := lint.ValidateZarfSchema(baseDir) + common.SetBaseDirectory(args, &pkgConfig) + v := common.GetViper() + pkgConfig.CreateOpts.SetVariables = helpers.TransformAndMergeMap( + v.GetStringMapString(common.VPkgCreateSet), pkgConfig.CreateOpts.SetVariables, strings.ToUpper) + validator, err := lint.Validate(pkgConfig.CreateOpts) if err != nil { message.Fatal(err, err.Error()) } @@ -282,15 +260,17 @@ func init() { bindDevDeployFlags(v) - devSha256SumCmd.Flags().StringVarP(&extractPath, "extract-path", "e", "", lang.CmdPrepareFlagExtractPath) + devSha256SumCmd.Flags().StringVarP(&extractPath, "extract-path", "e", "", lang.CmdDevFlagExtractPath) - devFindImagesCmd.Flags().StringVarP(&pkgConfig.FindImagesOpts.RepoHelmChartPath, "repo-chart-path", "p", "", lang.CmdPrepareFlagRepoChartPath) + devFindImagesCmd.Flags().StringVarP(&pkgConfig.FindImagesOpts.RepoHelmChartPath, "repo-chart-path", "p", "", lang.CmdDevFlagRepoChartPath) // use the package create config for this and reset it here to avoid overwriting the config.CreateOptions.SetVariables - devFindImagesCmd.Flags().StringToStringVar(&pkgConfig.CreateOpts.SetVariables, "set", v.GetStringMapString(common.VPkgCreateSet), lang.CmdPrepareFlagSet) + devFindImagesCmd.Flags().StringToStringVar(&pkgConfig.CreateOpts.SetVariables, "set", v.GetStringMapString(common.VPkgCreateSet), lang.CmdDevFlagSet) // allow for the override of the default helm KubeVersion - devFindImagesCmd.Flags().StringVar(&pkgConfig.FindImagesOpts.KubeVersionOverride, "kube-version", "", lang.CmdPrepareFlagKubeVersion) + devFindImagesCmd.Flags().StringVar(&pkgConfig.FindImagesOpts.KubeVersionOverride, "kube-version", "", lang.CmdDevFlagKubeVersion) - devTransformGitLinksCmd.Flags().StringVar(&pkgConfig.InitOpts.GitServer.PushUsername, "git-account", config.ZarfGitPushUser, lang.CmdPrepareFlagGitAccount) + devLintCmd.Flags().StringToStringVar(&pkgConfig.CreateOpts.SetVariables, "set", v.GetStringMapString(common.VPkgCreateSet), lang.CmdPackageCreateFlagSet) + devLintCmd.Flags().StringVarP(&pkgConfig.CreateOpts.Flavor, "flavor", "f", v.GetString(common.VPkgCreateFlavor), lang.CmdPackageCreateFlagFlavor) + devTransformGitLinksCmd.Flags().StringVar(&pkgConfig.InitOpts.GitServer.PushUsername, "git-account", config.ZarfGitPushUser, lang.CmdDevFlagGitAccount) } func bindDevDeployFlags(v *viper.Viper) { diff --git a/src/cmd/package.go b/src/cmd/package.go index c626412beb..1ccd3e3061 100644 --- a/src/cmd/package.go +++ b/src/cmd/package.go @@ -6,7 +6,6 @@ package cmd import ( "fmt" - "os" "path/filepath" "regexp" "strings" @@ -41,17 +40,7 @@ var packageCreateCmd = &cobra.Command{ Short: lang.CmdPackageCreateShort, Long: lang.CmdPackageCreateLong, Run: func(cmd *cobra.Command, args []string) { - - // If a directory was provided, use that as the base directory - if len(args) > 0 { - pkgConfig.CreateOpts.BaseDir = args[0] - } else { - var err error - pkgConfig.CreateOpts.BaseDir, err = os.Getwd() - if err != nil { - message.Fatalf(err, lang.CmdPackageCreateErr, err.Error()) - } - } + common.SetBaseDirectory(args, &pkgConfig) var isCleanPathRegex = regexp.MustCompile(`^[a-zA-Z0-9\_\-\/\.\~\\:]+$`) if !isCleanPathRegex.MatchString(config.CommonOptions.CachePath) { diff --git a/src/cmd/tools/zarf.go b/src/cmd/tools/zarf.go index bbd8020e07..8f86593268 100644 --- a/src/cmd/tools/zarf.go +++ b/src/cmd/tools/zarf.go @@ -88,8 +88,10 @@ var updateCredsCmd = &cobra.Command{ // If no distro the zarf secret did not load properly message.Fatalf(nil, lang.ErrLoadState) } - - newState := c.MergeZarfState(oldState, updateCredsInitOpts, args) + var newState *types.ZarfState + if newState, err = c.MergeZarfState(oldState, updateCredsInitOpts, args); err != nil { + message.Fatal(err, lang.CmdToolsUpdateCredsUnableUpdateCreds) + } message.PrintCredentialUpdates(oldState, newState, args) diff --git a/src/config/lang/english.go b/src/config/lang/english.go index 47563cf656..60f4d7617a 100644 --- a/src/config/lang/english.go +++ b/src/config/lang/english.go @@ -15,20 +15,26 @@ import "errors" // Debug messages will not be a part of the language strings since they are not intended to be user facing // Include sprintf formatting directives in the string if needed. const ( - ErrLoadState = "Failed to load the Zarf State from the cluster." - ErrSaveState = "Failed to save the Zarf State to the cluster." - ErrLoadPackageSecret = "Failed to load %s's secret from the cluster" - ErrNoClusterConnection = "Failed to connect to the cluster." - ErrTunnelFailed = "Failed to create a tunnel to the cluster." - ErrUnmarshal = "failed to unmarshal file: %w" - ErrWritingFile = "failed to write file %s: %s" - ErrDownloading = "failed to download %s: %s" - ErrCreatingDir = "failed to create directory %s: %s" - ErrRemoveFile = "failed to remove file %s: %s" - ErrUnarchive = "failed to unarchive %s: %s" - ErrConfirmCancel = "confirm selection canceled: %s" - ErrFileExtract = "failed to extract filename %s from archive %s: %s" - ErrFileNameExtract = "failed to extract filename from URL %s: %s" + ErrLoadState = "Failed to load the Zarf State from the cluster." + ErrSaveState = "Failed to save the Zarf State to the cluster." + ErrLoadPackageSecret = "Failed to load %s's secret from the cluster" + ErrNoClusterConnection = "Failed to connect to the cluster." + ErrTunnelFailed = "Failed to create a tunnel to the cluster." + ErrUnmarshal = "failed to unmarshal file: %w" + ErrWritingFile = "failed to write file %s: %s" + ErrDownloading = "failed to download %s: %s" + ErrCreatingDir = "failed to create directory %s: %s" + ErrRemoveFile = "failed to remove file %s: %s" + ErrUnarchive = "failed to unarchive %s: %s" + ErrConfirmCancel = "confirm selection canceled: %s" + ErrFileExtract = "failed to extract filename %s from archive %s: %s" + ErrFileNameExtract = "failed to extract filename from URL %s: %s" + ErrUnableToGenerateRandomSecret = "unable to generate a random secret" +) + +// Lint messages +const ( + UnsetVarLintWarning = "There are templates that are not set and won't be evaluated during lint" ) // Zarf CLI commands. @@ -336,38 +342,38 @@ $ zarf package publish ./path/to/dir oci://my-registry.com/my-namespace CmdDevDeployFlagNoYolo = "Disable the YOLO mode default override and create / deploy the package as-defined" CmdDevDeployErr = "Failed to dev deploy: %s" - CmdPreparePatchGitShort = "Converts all .git URLs to the specified Zarf HOST and with the Zarf URL pattern in a given FILE. NOTE:\n" + + CmdDevPatchGitShort = "Converts all .git URLs to the specified Zarf HOST and with the Zarf URL pattern in a given FILE. NOTE:\n" + "This should only be used for manifests that are not mutated by the Zarf Agent Mutating Webhook." - CmdPreparePatchGitOverwritePrompt = "Overwrite the file %s with these changes?" - CmdPreparePatchGitOverwriteErr = "Confirm overwrite canceled: %s" - CmdPreparePatchGitFileReadErr = "Unable to read the file %s" - CmdPreparePatchGitFileWriteErr = "Unable to write the changes back to the file" + CmdDevPatchGitOverwritePrompt = "Overwrite the file %s with these changes?" + CmdDevPatchGitOverwriteErr = "Confirm overwrite canceled: %s" + CmdDevPatchGitFileReadErr = "Unable to read the file %s" + CmdDevPatchGitFileWriteErr = "Unable to write the changes back to the file" - CmdPrepareSha256sumShort = "Generates a SHA256SUM for the given file" - CmdPrepareSha256sumRemoteWarning = "This is a remote source. If a published checksum is available you should use that rather than calculating it directly from the remote link." - CmdPrepareSha256sumHashErr = "Unable to compute the SHA256SUM hash: %s" + CmdDevSha256sumShort = "Generates a SHA256SUM for the given file" + CmdDevSha256sumRemoteWarning = "This is a remote source. If a published checksum is available you should use that rather than calculating it directly from the remote link." + CmdDevSha256sumHashErr = "Unable to compute the SHA256SUM hash: %s" - CmdPrepareFindImagesShort = "Evaluates components in a Zarf file to identify images specified in their helm charts and manifests" - CmdPrepareFindImagesLong = "Evaluates components in a Zarf file to identify images specified in their helm charts and manifests.\n\n" + + CmdDevFindImagesShort = "Evaluates components in a Zarf file to identify images specified in their helm charts and manifests" + CmdDevFindImagesLong = "Evaluates components in a Zarf file to identify images specified in their helm charts and manifests.\n\n" + "Components that have repos that host helm charts can be processed by providing the --repo-chart-path." - CmdPrepareFindImagesErr = "Unable to find images: %s" + CmdDevFindImagesErr = "Unable to find images: %s" - CmdPrepareGenerateConfigShort = "Generates a config file for Zarf" - CmdPrepareGenerateConfigLong = "Generates a Zarf config file for controlling how the Zarf CLI operates. Optionally accepts a filename to write the config to.\n\n" + + CmdDevGenerateConfigShort = "Generates a config file for Zarf" + CmdDevGenerateConfigLong = "Generates a Zarf config file for controlling how the Zarf CLI operates. Optionally accepts a filename to write the config to.\n\n" + "The extension will determine the format of the config file, e.g. env-1.yaml, env-2.json, env-3.toml etc.\n" + "Accepted extensions are json, toml, yaml.\n\n" + "NOTE: This file must not already exist. If no filename is provided, the config will be written to the current working directory as zarf-config.toml." - CmdPrepareGenerateConfigErr = "Unable to write the config file %s, make sure the file doesn't already exist" + CmdDevGenerateConfigErr = "Unable to write the config file %s, make sure the file doesn't already exist" - CmdPrepareFlagExtractPath = `The path inside of an archive to use to calculate the sha256sum (i.e. for use with "files.extractPath")` - CmdPrepareFlagSet = "Specify package variables to set on the command line (KEY=value). Note, if using a config file, this will be set by [package.create.set]." - CmdPrepareFlagRepoChartPath = `If git repos hold helm charts, often found with gitops tools, specify the chart path, e.g. "/" or "/chart"` - CmdPrepareFlagGitAccount = "User or organization name for the git account that the repos are created under." - CmdPrepareFlagKubeVersion = "Override the default helm template KubeVersion when performing a package chart template" + CmdDevFlagExtractPath = `The path inside of an archive to use to calculate the sha256sum (i.e. for use with "files.extractPath")` + CmdDevFlagSet = "Specify package variables to set on the command line (KEY=value). Note, if using a config file, this will be set by [package.create.set]." + CmdDevFlagRepoChartPath = `If git repos hold helm charts, often found with gitops tools, specify the chart path, e.g. "/" or "/chart"` + CmdDevFlagGitAccount = "User or organization name for the git account that the repos are created under." + CmdDevFlagKubeVersion = "Override the default helm template KubeVersion when performing a package chart template" - CmdPrepareLintShort = "Verifies the package schema" - CmdPrepareLintLong = "Verifies the package schema and warns the user if they have variables that won't be evaluated" - CmdPrepareLintErr = "Unable to lint package: %s" + CmdDevLintShort = "Lints the given package for valid schema and recommended practices" + CmdDevLintLong = "Verifies the package schema, checks if any variables won't be evaluated, and checks for unpinned images/repos/files" + CmdDevLintErr = "Unable to lint package: %s" // zarf tools CmdToolsShort = "Collection of additional tools to make airgap easier" @@ -563,6 +569,7 @@ $ zarf tools update-creds artifact --artifact-push-username={USERNAME} --artifac CmdToolsUpdateCredsUnableUpdateRegistry = "Unable to update Zarf Registry values: %s" CmdToolsUpdateCredsUnableUpdateGit = "Unable to update Zarf Git Server values: %s" CmdToolsUpdateCredsUnableUpdateAgent = "Unable to update Zarf Agent TLS secrets: %s" + CmdToolsUpdateCredsUnableUpdateCreds = "Unable to update Zarf credentials" // zarf version CmdVersionShort = "Shows the version of the running Zarf binary" @@ -604,7 +611,7 @@ const ( // src/internal/packager/validate. const ( - PkgValidateTemplateDeprecation = "Package template %q is using the deprecated syntax ###ZARF_PKG_VAR_%s###. This will be removed in Zarf v1.0.0. Please update to ###ZARF_PKG_TMPL_%s###." + PkgValidateTemplateDeprecation = "Package template %q is using the deprecated syntax ###ZARF_PKG_VAR_%s###. This will be removed in Zarf v1.0.0. Please update to ###ZARF_PKG_TMPL_%s###." PkgValidateMustBeUppercase = "variable name %q must be all uppercase and contain no special characters except _" PkgValidateErrAction = "invalid action: %w" PkgValidateErrActionVariables = "component %q cannot contain setVariables outside of onDeploy in actions" diff --git a/src/pkg/cluster/state.go b/src/pkg/cluster/state.go index b9025b7b47..5fe6e4d9dd 100644 --- a/src/pkg/cluster/state.go +++ b/src/pkg/cluster/state.go @@ -12,13 +12,13 @@ import ( "slices" "github.com/defenseunicorns/zarf/src/config" + "github.com/defenseunicorns/zarf/src/config/lang" "github.com/defenseunicorns/zarf/src/types" "github.com/fatih/color" "github.com/defenseunicorns/zarf/src/pkg/k8s" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/pki" - "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -72,7 +72,9 @@ func (c *Cluster) InitZarfState(initOptions types.ZarfInitOptions) error { // Defaults state.Distro = distro - state.LoggingSecret = utils.RandomString(config.ZarfGeneratedPasswordLen) + if state.LoggingSecret, err = helpers.RandomString(config.ZarfGeneratedPasswordLen); err != nil { + return fmt.Errorf("%s: %w", lang.ErrUnableToGenerateRandomSecret, err) + } // Setup zarf agent PKI state.AgentTLS = pki.GeneratePKI(config.ZarfAgentHost) @@ -110,8 +112,12 @@ func (c *Cluster) InitZarfState(initOptions types.ZarfInitOptions) error { return fmt.Errorf("unable get default Zarf service account: %w", err) } - state.GitServer = c.fillInEmptyGitServerValues(initOptions.GitServer) - state.RegistryInfo = c.fillInEmptyContainerRegistryValues(initOptions.RegistryInfo) + if state.GitServer, err = c.fillInEmptyGitServerValues(initOptions.GitServer); err != nil { + return err + } + if state.RegistryInfo, err = c.fillInEmptyContainerRegistryValues(initOptions.RegistryInfo); err != nil { + return err + } state.ArtifactServer = c.fillInEmptyArtifactServerValues(initOptions.ArtifactServer) } else { if helpers.IsNotZeroAndNotEqual(initOptions.GitServer, state.GitServer) { @@ -158,7 +164,7 @@ func (c *Cluster) LoadZarfState() (state *types.ZarfState, err error) { // Set up the API connection secret, err := c.GetSecret(ZarfNamespaceName, ZarfStateSecretName) if err != nil { - return nil, fmt.Errorf("%w. %s", err, utils.ColorWrap("Did you remember to zarf init?", color.Bold)) + return nil, fmt.Errorf("%w. %s", err, message.ColorWrap("Did you remember to zarf init?", color.Bold)) } err = json.Unmarshal(secret.Data[ZarfStateDataKey], &state) @@ -245,9 +251,9 @@ func (c *Cluster) SaveZarfState(state *types.ZarfState) error { } // MergeZarfState merges init options for provided services into the provided state to create a new state struct -func (c *Cluster) MergeZarfState(oldState *types.ZarfState, initOptions types.ZarfInitOptions, services []string) *types.ZarfState { +func (c *Cluster) MergeZarfState(oldState *types.ZarfState, initOptions types.ZarfInitOptions, services []string) (*types.ZarfState, error) { newState := *oldState - + var err error if slices.Contains(services, message.RegistryKey) { newState.RegistryInfo = helpers.MergeNonZero(newState.RegistryInfo, initOptions.RegistryInfo) // Set the state of the internal registry if it has changed @@ -259,10 +265,14 @@ func (c *Cluster) MergeZarfState(oldState *types.ZarfState, initOptions types.Za // Set the new passwords if they should be autogenerated if newState.RegistryInfo.PushPassword == oldState.RegistryInfo.PushPassword && oldState.RegistryInfo.InternalRegistry { - newState.RegistryInfo.PushPassword = utils.RandomString(config.ZarfGeneratedPasswordLen) + if newState.RegistryInfo.PushPassword, err = helpers.RandomString(config.ZarfGeneratedPasswordLen); err != nil { + return nil, fmt.Errorf("%s: %w", lang.ErrUnableToGenerateRandomSecret, err) + } } if newState.RegistryInfo.PullPassword == oldState.RegistryInfo.PullPassword && oldState.RegistryInfo.InternalRegistry { - newState.RegistryInfo.PullPassword = utils.RandomString(config.ZarfGeneratedPasswordLen) + if newState.RegistryInfo.PullPassword, err = helpers.RandomString(config.ZarfGeneratedPasswordLen); err != nil { + return nil, fmt.Errorf("%s: %w", lang.ErrUnableToGenerateRandomSecret, err) + } } } if slices.Contains(services, message.GitKey) { @@ -277,10 +287,14 @@ func (c *Cluster) MergeZarfState(oldState *types.ZarfState, initOptions types.Za // Set the new passwords if they should be autogenerated if newState.GitServer.PushPassword == oldState.GitServer.PushPassword && oldState.GitServer.InternalServer { - newState.GitServer.PushPassword = utils.RandomString(config.ZarfGeneratedPasswordLen) + if newState.GitServer.PushPassword, err = helpers.RandomString(config.ZarfGeneratedPasswordLen); err != nil { + return nil, fmt.Errorf("%s: %w", lang.ErrUnableToGenerateRandomSecret, err) + } } if newState.GitServer.PullPassword == oldState.GitServer.PullPassword && oldState.GitServer.InternalServer { - newState.GitServer.PullPassword = utils.RandomString(config.ZarfGeneratedPasswordLen) + if newState.GitServer.PullPassword, err = helpers.RandomString(config.ZarfGeneratedPasswordLen); err != nil { + return nil, fmt.Errorf("%s: %w", lang.ErrUnableToGenerateRandomSecret, err) + } } } if slices.Contains(services, message.ArtifactKey) { @@ -302,10 +316,11 @@ func (c *Cluster) MergeZarfState(oldState *types.ZarfState, initOptions types.Za newState.AgentTLS = pki.GeneratePKI(config.ZarfAgentHost) } - return &newState + return &newState, nil } -func (c *Cluster) fillInEmptyContainerRegistryValues(containerRegistry types.RegistryInfo) types.RegistryInfo { +func (c *Cluster) fillInEmptyContainerRegistryValues(containerRegistry types.RegistryInfo) (types.RegistryInfo, error) { + var err error // Set default NodePort if none was provided if containerRegistry.NodePort == 0 { containerRegistry.NodePort = config.ZarfInClusterContainerRegistryNodePort @@ -319,7 +334,9 @@ func (c *Cluster) fillInEmptyContainerRegistryValues(containerRegistry types.Reg // Generate a push-user password if not provided by init flag if containerRegistry.PushPassword == "" { - containerRegistry.PushPassword = utils.RandomString(config.ZarfGeneratedPasswordLen) + if containerRegistry.PushPassword, err = helpers.RandomString(config.ZarfGeneratedPasswordLen); err != nil { + return containerRegistry, fmt.Errorf("%s: %w", lang.ErrUnableToGenerateRandomSecret, err) + } } // Set pull-username if not provided by init flag @@ -333,7 +350,9 @@ func (c *Cluster) fillInEmptyContainerRegistryValues(containerRegistry types.Reg } if containerRegistry.PullPassword == "" { if containerRegistry.InternalRegistry { - containerRegistry.PullPassword = utils.RandomString(config.ZarfGeneratedPasswordLen) + if containerRegistry.PullPassword, err = helpers.RandomString(config.ZarfGeneratedPasswordLen); err != nil { + return containerRegistry, fmt.Errorf("%s: %w", lang.ErrUnableToGenerateRandomSecret, err) + } } else { // If this is an external registry and a pull-user wasn't provided, use the same credentials as the push user containerRegistry.PullPassword = containerRegistry.PushPassword @@ -341,14 +360,17 @@ func (c *Cluster) fillInEmptyContainerRegistryValues(containerRegistry types.Reg } if containerRegistry.Secret == "" { - containerRegistry.Secret = utils.RandomString(config.ZarfGeneratedSecretLen) + if containerRegistry.Secret, err = helpers.RandomString(config.ZarfGeneratedPasswordLen); err != nil { + return containerRegistry, fmt.Errorf("%s: %w", lang.ErrUnableToGenerateRandomSecret, err) + } } - return containerRegistry + return containerRegistry, nil } // Fill in empty GitServerInfo values with the defaults. -func (c *Cluster) fillInEmptyGitServerValues(gitServer types.GitServerInfo) types.GitServerInfo { +func (c *Cluster) fillInEmptyGitServerValues(gitServer types.GitServerInfo) (types.GitServerInfo, error) { + var err error // Set default svc url if an external repository was not provided if gitServer.Address == "" { gitServer.Address = config.ZarfInClusterGitServiceURL @@ -357,7 +379,9 @@ func (c *Cluster) fillInEmptyGitServerValues(gitServer types.GitServerInfo) type // Generate a push-user password if not provided by init flag if gitServer.PushPassword == "" { - gitServer.PushPassword = utils.RandomString(config.ZarfGeneratedPasswordLen) + if gitServer.PushPassword, err = helpers.RandomString(config.ZarfGeneratedPasswordLen); err != nil { + return gitServer, fmt.Errorf("%s: %w", lang.ErrUnableToGenerateRandomSecret, err) + } } // Set read-user information if using an internal repository, otherwise copy from the push-user @@ -370,13 +394,15 @@ func (c *Cluster) fillInEmptyGitServerValues(gitServer types.GitServerInfo) type } if gitServer.PullPassword == "" { if gitServer.InternalServer { - gitServer.PullPassword = utils.RandomString(config.ZarfGeneratedPasswordLen) + if gitServer.PullPassword, err = helpers.RandomString(config.ZarfGeneratedPasswordLen); err != nil { + return gitServer, fmt.Errorf("%s: %w", lang.ErrUnableToGenerateRandomSecret, err) + } } else { gitServer.PullPassword = gitServer.PushPassword } } - return gitServer + return gitServer, nil } // Fill in empty ArtifactServerInfo values with the defaults. diff --git a/src/pkg/message/message.go b/src/pkg/message/message.go index ef1d2e235f..b4b6834a0f 100644 --- a/src/pkg/message/message.go +++ b/src/pkg/message/message.go @@ -14,6 +14,8 @@ import ( "strings" "time" + "github.com/defenseunicorns/zarf/src/config" + "github.com/fatih/color" "github.com/pterm/pterm" "github.com/sergi/go-diff/diffmatchpatch" ) @@ -322,15 +324,20 @@ func Truncate(text string, length int, invert bool) string { func Table(header []string, data [][]string) { pterm.Println() - if len(header) > 0 { - header[0] = fmt.Sprintf(" %s", header[0]) + // To avoid side effects make copies of the header and data before adding padding + headerCopy := make([]string, len(header)) + copy(headerCopy, header) + dataCopy := make([][]string, len(data)) + copy(dataCopy, data) + if len(headerCopy) > 0 { + headerCopy[0] = fmt.Sprintf(" %s", headerCopy[0]) } table := pterm.TableData{ - header, + headerCopy, } - for _, row := range data { + for _, row := range dataCopy { if len(row) > 0 { row[0] = fmt.Sprintf(" %s", row[0]) } @@ -340,6 +347,25 @@ func Table(header []string, data [][]string) { pterm.DefaultTable.WithHasHeader().WithData(table).Render() } +// ColorWrap changes a string to an ansi color code and appends the default color to the end +// preventing future characters from taking on the given color +// returns string as normal if color is disabled +func ColorWrap(str string, attr color.Attribute) string { + if config.NoColor { + return str + } + return fmt.Sprintf("\x1b[%dm%s\x1b[0m", attr, str) +} + +// First30last30 returns the source string that has been trimmed to 30 characters at the beginning and end. +func First30last30(s string) string { + if len(s) > 60 { + return s[0:27] + "..." + s[len(s)-26:] + } + + return s +} + func debugPrinter(offset int, a ...any) { printer := pterm.Debug.WithShowLineNumber(logLevel > 2).WithLineNumberOffset(offset) now := time.Now().Format(time.RFC3339) diff --git a/src/pkg/oci/push.go b/src/pkg/oci/push.go index 2d057003f1..7c678ab9b5 100644 --- a/src/pkg/oci/push.go +++ b/src/pkg/oci/push.go @@ -11,7 +11,6 @@ import ( "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/message" - "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/types" ocispec "github.com/opencontainers/image-spec/specs-go/v1" "oras.land/oras-go/v2" @@ -116,7 +115,7 @@ func (o *OrasRemote) PublishPackage(pkg *types.ZarfPackage, paths *layout.Packag // Get all of the layers in the package var descs []ocispec.Descriptor for name, path := range paths.Files() { - spinner.Updatef("Preparing layer %s", utils.First30last30(name)) + spinner.Updatef("Preparing layer %s", message.First30last30(name)) mediaType := ZarfLayerMediaTypeBlob diff --git a/src/pkg/oci/utils.go b/src/pkg/oci/utils.go index b15b14be19..1134e97132 100644 --- a/src/pkg/oci/utils.go +++ b/src/pkg/oci/utils.go @@ -11,7 +11,6 @@ import ( "strings" "github.com/defenseunicorns/zarf/src/pkg/message" - "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "github.com/defenseunicorns/zarf/src/types" ocispec "github.com/opencontainers/image-spec/specs-go/v1" @@ -63,7 +62,7 @@ func (o *OrasRemote) printLayer(desc ocispec.Descriptor, suffix string) error { title := desc.Annotations[ocispec.AnnotationTitle] var layerInfo string if title != "" { - layerInfo = fmt.Sprintf("%s %s", desc.Digest.Encoded()[:12], utils.First30last30(title)) + layerInfo = fmt.Sprintf("%s %s", desc.Digest.Encoded()[:12], message.First30last30(title)) } else { layerInfo = fmt.Sprintf("%s [%s]", desc.Digest.Encoded()[:12], desc.MediaType) } diff --git a/src/pkg/packager/compose.go b/src/pkg/packager/compose.go index 660659c5a2..6bbbf22fff 100644 --- a/src/pkg/packager/compose.go +++ b/src/pkg/packager/compose.go @@ -17,7 +17,7 @@ func (p *Packager) composeComponents() error { pkgVars := p.cfg.Pkg.Variables pkgConsts := p.cfg.Pkg.Constants - for _, component := range p.cfg.Pkg.Components { + for i, component := range p.cfg.Pkg.Components { arch := p.arch // filter by architecture if !composer.CompatibleComponent(component, arch, p.cfg.CreateOpts.Flavor) { @@ -29,7 +29,7 @@ func (p *Packager) composeComponents() error { component.Only.Flavor = "" // build the import chain - chain, err := composer.NewImportChain(component, arch, p.cfg.CreateOpts.Flavor) + chain, err := composer.NewImportChain(component, i, p.cfg.Pkg.Metadata.Name, arch, p.cfg.CreateOpts.Flavor) if err != nil { return err } diff --git a/src/pkg/packager/composer/list.go b/src/pkg/packager/composer/list.go index f38eeac49d..f46235f212 100644 --- a/src/pkg/packager/composer/list.go +++ b/src/pkg/packager/composer/list.go @@ -22,17 +22,49 @@ import ( type Node struct { types.ZarfComponent + index int + vars []types.ZarfPackageVariable consts []types.ZarfPackageConstant - relativeToHead string + relativeToHead string + originalPackageName string prev *Node next *Node } +// GetIndex returns the .components index location for this node's source `zarf.yaml` +func (n *Node) GetIndex() int { + return n.index +} + +// GetOriginalPackageName returns the .metadata.name of the zarf package the component originated from +func (n *Node) GetOriginalPackageName() string { + return n.originalPackageName +} + +// ImportLocation gets the path from the base zarf file to the imported zarf file +func (n *Node) ImportLocation() string { + if n.prev != nil { + if n.prev.ZarfComponent.Import.URL != "" { + return n.prev.ZarfComponent.Import.URL + } + } + return n.relativeToHead +} + +// Next returns next node in the chain +func (n *Node) Next() *Node { + return n.next +} + +// Prev returns previous node in the chain +func (n *Node) Prev() *Node { + return n.prev +} + // ImportName returns the name of the component to import -// // If the component import has a ComponentName defined, that will be used // otherwise the name of the component will be used func (n *Node) ImportName() string { @@ -51,14 +83,27 @@ type ImportChain struct { remote *oci.OrasRemote } -func (ic *ImportChain) append(c types.ZarfComponent, relativeToHead string, vars []types.ZarfPackageVariable, consts []types.ZarfPackageConstant) { +// Head returns the first node in the import chain +func (ic *ImportChain) Head() *Node { + return ic.head +} + +// Tail returns the last node in the import chain +func (ic *ImportChain) Tail() *Node { + return ic.tail +} + +func (ic *ImportChain) append(c types.ZarfComponent, index int, originalPackageName string, + relativeToHead string, vars []types.ZarfPackageVariable, consts []types.ZarfPackageConstant) { node := &Node{ - ZarfComponent: c, - relativeToHead: relativeToHead, - vars: vars, - consts: consts, - prev: nil, - next: nil, + ZarfComponent: c, + index: index, + originalPackageName: originalPackageName, + relativeToHead: relativeToHead, + vars: vars, + consts: consts, + prev: nil, + next: nil, } if ic.head == nil { ic.head = node @@ -72,14 +117,14 @@ func (ic *ImportChain) append(c types.ZarfComponent, relativeToHead string, vars } // NewImportChain creates a new import chain from a component -func NewImportChain(head types.ZarfComponent, arch, flavor string) (*ImportChain, error) { +// Returning the chain on error so we can have additional information to use during lint +func NewImportChain(head types.ZarfComponent, index int, originalPackageName, arch, flavor string) (*ImportChain, error) { + ic := &ImportChain{} if arch == "" { - return nil, fmt.Errorf("cannot build import chain: architecture must be provided") + return ic, fmt.Errorf("cannot build import chain: architecture must be provided") } - ic := &ImportChain{} - - ic.append(head, ".", nil, nil) + ic.append(head, index, originalPackageName, ".", nil, nil) history := []string{} @@ -110,9 +155,11 @@ func NewImportChain(head types.ZarfComponent, arch, flavor string) (*ImportChain var pkg types.ZarfPackage + var relativeToHead string + var importURL string if isLocal { history = append(history, node.Import.Path) - relativeToHead := filepath.Join(history...) + relativeToHead = filepath.Join(history...) // prevent circular imports (including self-imports) // this is O(n^2) but the import chain should be small @@ -129,6 +176,7 @@ func NewImportChain(head types.ZarfComponent, arch, flavor string) (*ImportChain return ic, err } } else if isRemote { + importURL = node.Import.URL remote, err := ic.getRemote(node.Import.URL) if err != nil { return ic, err @@ -141,26 +189,34 @@ func NewImportChain(head types.ZarfComponent, arch, flavor string) (*ImportChain name := node.ImportName() - found := helpers.Filter(pkg.Components, func(c types.ZarfComponent) bool { - matchesName := c.Name == name - return matchesName && CompatibleComponent(c, arch, flavor) - }) + // 'found' and 'index' are parallel slices. Each element in found[x] corresponds to pkg[index[x]] + // found[0] and pkg[index[0]] would be the same componenet for example + found := []types.ZarfComponent{} + index := []int{} + for i, component := range pkg.Components { + if component.Name == name && CompatibleComponent(component, arch, flavor) { + found = append(found, component) + index = append(index, i) + } + } if len(found) == 0 { + componentNotFound := "component %q not found in %q" if isLocal { - return ic, fmt.Errorf("component %q not found in %q", name, filepath.Join(history...)) + return ic, fmt.Errorf(componentNotFound, name, relativeToHead) } else if isRemote { - return ic, fmt.Errorf("component %q not found in %q", name, node.Import.URL) + return ic, fmt.Errorf(componentNotFound, name, importURL) } } else if len(found) > 1 { + multipleComponentsFound := "multiple components named %q found in %q satisfying %q" if isLocal { - return ic, fmt.Errorf("multiple components named %q found in %q satisfying %q", name, filepath.Join(history...), arch) + return ic, fmt.Errorf(multipleComponentsFound, name, relativeToHead, arch) } else if isRemote { - return ic, fmt.Errorf("multiple components named %q found in %q satisfying %q", name, node.Import.URL, arch) + return ic, fmt.Errorf(multipleComponentsFound, name, importURL, arch) } } - ic.append(found[0], filepath.Join(history...), pkg.Variables, pkg.Constants) + ic.append(found[0], index[0], pkg.Metadata.Name, relativeToHead, pkg.Variables, pkg.Constants) node = node.next } return ic, nil diff --git a/src/pkg/packager/composer/list_test.go b/src/pkg/packager/composer/list_test.go index f255803fde..d80f3633c3 100644 --- a/src/pkg/packager/composer/list_test.go +++ b/src/pkg/packager/composer/list_test.go @@ -43,14 +43,14 @@ func TestNewImportChain(t *testing.T) { expectedErrorMessage: "detected circular import chain", }, } - + testPackageName := "test-package" for _, testCase := range testCases { testCase := testCase t.Run(testCase.name, func(t *testing.T) { t.Parallel() - _, err := NewImportChain(testCase.head, testCase.arch, testCase.flavor) + _, err := NewImportChain(testCase.head, 0, testPackageName, testCase.arch, testCase.flavor) require.Contains(t, err.Error(), testCase.expectedErrorMessage) }) } @@ -441,17 +441,18 @@ func TestMerging(t *testing.T) { func createChainFromSlice(components []types.ZarfComponent) (ic *ImportChain) { ic = &ImportChain{} + testPackageName := "test-package" if len(components) == 0 { return ic } - ic.append(components[0], ".", nil, nil) + ic.append(components[0], 0, testPackageName, ".", nil, nil) history := []string{} for idx := 1; idx < len(components); idx++ { history = append(history, components[idx-1].Import.Path) - ic.append(components[idx], filepath.Join(history...), nil, nil) + ic.append(components[idx], idx, testPackageName, filepath.Join(history...), nil, nil) } return ic diff --git a/src/pkg/packager/lint/lint.go b/src/pkg/packager/lint/lint.go index b9142cc3fb..bc8b5df336 100644 --- a/src/pkg/packager/lint/lint.go +++ b/src/pkg/packager/lint/lint.go @@ -7,12 +7,19 @@ package lint import ( "embed" "fmt" + "os" "path/filepath" "regexp" "strings" + "github.com/defenseunicorns/zarf/src/config" + "github.com/defenseunicorns/zarf/src/config/lang" "github.com/defenseunicorns/zarf/src/pkg/layout" + "github.com/defenseunicorns/zarf/src/pkg/packager" + "github.com/defenseunicorns/zarf/src/pkg/packager/composer" + "github.com/defenseunicorns/zarf/src/pkg/transform" "github.com/defenseunicorns/zarf/src/pkg/utils" + "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "github.com/defenseunicorns/zarf/src/types" "github.com/xeipuuv/gojsonschema" ) @@ -24,22 +31,29 @@ func getSchemaFile() ([]byte, error) { return ZarfSchema.ReadFile("zarf.schema.json") } -// ValidateZarfSchema validates a zarf file against the zarf schema, returns *validator with warnings or errors if they exist +// Validate validates a zarf file against the zarf schema, returns *validator with warnings or errors if they exist // along with an error if the validation itself failed -func ValidateZarfSchema(path string) (*Validator, error) { +func Validate(createOpts types.ZarfCreateOptions) (*Validator, error) { validator := Validator{} var err error - if err := utils.ReadYaml(filepath.Join(path, layout.ZarfYAML), &validator.typedZarfPackage); err != nil { + + if err := utils.ReadYaml(filepath.Join(createOpts.BaseDir, layout.ZarfYAML), &validator.typedZarfPackage); err != nil { return nil, err } - checkForVarInComponentImport(&validator) - - if validator.jsonSchema, err = getSchemaFile(); err != nil { + if err := utils.ReadYaml(filepath.Join(createOpts.BaseDir, layout.ZarfYAML), &validator.untypedZarfPackage); err != nil { return nil, err } - if err := utils.ReadYaml(filepath.Join(path, layout.ZarfYAML), &validator.untypedZarfPackage); err != nil { + if err := os.Chdir(createOpts.BaseDir); err != nil { + return nil, fmt.Errorf("unable to access directory '%s': %w", createOpts.BaseDir, err) + } + + validator.baseDir = createOpts.BaseDir + + lintComponents(&validator, &createOpts) + + if validator.jsonSchema, err = getSchemaFile(); err != nil { return nil, err } @@ -50,16 +64,194 @@ func ValidateZarfSchema(path string) (*Validator, error) { return &validator, nil } -func checkForVarInComponentImport(validator *Validator) { +func lintComponents(validator *Validator, createOpts *types.ZarfCreateOptions) { for i, component := range validator.typedZarfPackage.Components { - if strings.Contains(component.Import.Path, types.ZarfPackageTemplatePrefix) { - validator.addWarning(fmt.Sprintf(".components.[%d].import.path: Will not resolve ZARF_PKG_TMPL_* variables", i)) + arch := config.GetArch(validator.typedZarfPackage.Metadata.Architecture) + + if !composer.CompatibleComponent(component, arch, createOpts.Flavor) { + continue } - if strings.Contains(component.Import.URL, types.ZarfPackageTemplatePrefix) { - validator.addWarning(fmt.Sprintf(".components.[%d].import.url: Will not resolve ZARF_PKG_TMPL_* variables", i)) + + chain, err := composer.NewImportChain(component, i, validator.typedZarfPackage.Metadata.Name, arch, createOpts.Flavor) + baseComponent := chain.Head() + + var badImportYqPath string + if baseComponent != nil { + if baseComponent.Import.URL != "" { + badImportYqPath = fmt.Sprintf(".components.[%d].import.url", i) + } + if baseComponent.Import.Path != "" { + badImportYqPath = fmt.Sprintf(".components.[%d].import.path", i) + } + } + if err != nil { + validator.addError(validatorMessage{ + description: err.Error(), + packageRelPath: ".", + packageName: validator.typedZarfPackage.Metadata.Name, + yqPath: badImportYqPath, + }) + } + + node := baseComponent + for node != nil { + checkForVarInComponentImport(validator, node) + fillComponentTemplate(validator, node, createOpts) + lintComponent(validator, node) + node = node.Next() } } +} + +func fillComponentTemplate(validator *Validator, node *composer.Node, createOpts *types.ZarfCreateOptions) { + + err := packager.ReloadComponentTemplate(&node.ZarfComponent) + if err != nil { + validator.addWarning(validatorMessage{ + description: err.Error(), + packageRelPath: node.ImportLocation(), + packageName: node.GetOriginalPackageName(), + }) + } + templateMap := map[string]string{} + + setVarsAndWarn := func(templatePrefix string, deprecated bool) { + yamlTemplates, err := utils.FindYamlTemplates(node, templatePrefix, "###") + if err != nil { + validator.addWarning(validatorMessage{ + description: err.Error(), + packageRelPath: node.ImportLocation(), + packageName: node.GetOriginalPackageName(), + }) + } + + for key := range yamlTemplates { + if deprecated { + validator.addWarning(validatorMessage{ + description: fmt.Sprintf(lang.PkgValidateTemplateDeprecation, key, key, key), + packageRelPath: node.ImportLocation(), + packageName: node.GetOriginalPackageName(), + }) + } + _, present := createOpts.SetVariables[key] + if !present { + validator.addWarning(validatorMessage{ + description: lang.UnsetVarLintWarning, + packageRelPath: node.ImportLocation(), + packageName: node.GetOriginalPackageName(), + }) + } + } + for key, value := range createOpts.SetVariables { + templateMap[fmt.Sprintf("%s%s###", templatePrefix, key)] = value + } + } + + setVarsAndWarn(types.ZarfPackageTemplatePrefix, false) + + // [DEPRECATION] Set the Package Variable syntax as well for backward compatibility + setVarsAndWarn(types.ZarfPackageVariablePrefix, true) + utils.ReloadYamlTemplate(node, templateMap) +} + +func isPinnedImage(image string) (bool, error) { + transformedImage, err := transform.ParseImageRef(image) + if err != nil { + if strings.Contains(image, types.ZarfPackageTemplatePrefix) || + strings.Contains(image, types.ZarfPackageVariablePrefix) { + return true, nil + } + return false, err + } + return (transformedImage.Digest != ""), err +} + +func isPinnedRepo(repo string) bool { + return (strings.Contains(repo, "@")) +} + +func lintComponent(validator *Validator, node *composer.Node) { + checkForUnpinnedRepos(validator, node) + checkForUnpinnedImages(validator, node) + checkForUnpinnedFiles(validator, node) +} + +func checkForUnpinnedRepos(validator *Validator, node *composer.Node) { + for j, repo := range node.Repos { + repoYqPath := fmt.Sprintf(".components.[%d].repos.[%d]", node.GetIndex(), j) + if !isPinnedRepo(repo) { + validator.addWarning(validatorMessage{ + yqPath: repoYqPath, + packageRelPath: node.ImportLocation(), + packageName: node.GetOriginalPackageName(), + description: "Unpinned repository", + item: repo, + }) + } + } +} + +func checkForUnpinnedImages(validator *Validator, node *composer.Node) { + for j, image := range node.Images { + imageYqPath := fmt.Sprintf(".components.[%d].images.[%d]", node.GetIndex(), j) + pinnedImage, err := isPinnedImage(image) + if err != nil { + validator.addError(validatorMessage{ + yqPath: imageYqPath, + packageRelPath: node.ImportLocation(), + packageName: node.GetOriginalPackageName(), + description: "Invalid image reference", + item: image, + }) + continue + } + if !pinnedImage { + validator.addWarning(validatorMessage{ + yqPath: imageYqPath, + packageRelPath: node.ImportLocation(), + packageName: node.GetOriginalPackageName(), + description: "Image not pinned with digest", + item: image, + }) + } + } +} + +func checkForUnpinnedFiles(validator *Validator, node *composer.Node) { + for j, file := range node.Files { + fileYqPath := fmt.Sprintf(".components.[%d].files.[%d]", node.GetIndex(), j) + if file.Shasum == "" && helpers.IsURL(file.Source) { + validator.addWarning(validatorMessage{ + yqPath: fileYqPath, + packageRelPath: node.ImportLocation(), + packageName: node.GetOriginalPackageName(), + description: "No shasum for remote file", + item: file.Source, + }) + } + } +} + +func checkForVarInComponentImport(validator *Validator, node *composer.Node) { + if strings.Contains(node.Import.Path, types.ZarfPackageTemplatePrefix) { + validator.addWarning(validatorMessage{ + yqPath: fmt.Sprintf(".components.[%d].import.path", node.GetIndex()), + packageRelPath: node.ImportLocation(), + packageName: node.GetOriginalPackageName(), + description: "Zarf does not evaluate variables at component.x.import.path", + item: node.Import.Path, + }) + } + if strings.Contains(node.Import.URL, types.ZarfPackageTemplatePrefix) { + validator.addWarning(validatorMessage{ + yqPath: fmt.Sprintf(".components.[%d].import.url", node.GetIndex()), + packageRelPath: node.ImportLocation(), + packageName: node.GetOriginalPackageName(), + description: "Zarf does not evaluate variables at component.x.import.url", + item: node.Import.URL, + }) + } } func makeFieldPathYqCompat(field string) string { @@ -86,9 +278,12 @@ func validateSchema(validator *Validator) error { if !result.Valid() { for _, desc := range result.Errors() { - err := fmt.Errorf( - "%s: %s", makeFieldPathYqCompat(desc.Field()), desc.Description()) - validator.addError(err) + validator.addError(validatorMessage{ + yqPath: makeFieldPathYqCompat(desc.Field()), + description: desc.Description(), + packageRelPath: ".", + packageName: validator.typedZarfPackage.Metadata.Name, + }) } } diff --git a/src/pkg/packager/lint/lint_test.go b/src/pkg/packager/lint/lint_test.go index ddc9907e0f..5f81660d32 100644 --- a/src/pkg/packager/lint/lint_test.go +++ b/src/pkg/packager/lint/lint_test.go @@ -5,9 +5,14 @@ package lint import ( + "errors" + "fmt" "os" + "path/filepath" "testing" + "github.com/defenseunicorns/zarf/src/config" + "github.com/defenseunicorns/zarf/src/pkg/packager/composer" "github.com/defenseunicorns/zarf/src/types" goyaml "github.com/goccy/go-yaml" "github.com/stretchr/testify/require" @@ -26,14 +31,6 @@ components: - name: import-test import: path: 123123 - -- name: import-test - import: - path: "###ZARF_PKG_TMPL_ZEBRA###" - -- name: import-url - import: - url: "oci://###ZARF_PKG_TMPL_ZEBRA###" ` const goodZarfPackage = ` @@ -71,7 +68,7 @@ func TestValidateSchema(t *testing.T) { validator := Validator{untypedZarfPackage: unmarshalledYaml, jsonSchema: getZarfSchema(t)} err := validateSchema(&validator) require.NoError(t, err) - require.Empty(t, validator.errors) + require.Empty(t, validator.findings) }) t.Run("validate schema fail", func(t *testing.T) { @@ -79,23 +76,82 @@ func TestValidateSchema(t *testing.T) { validator := Validator{untypedZarfPackage: unmarshalledYaml, jsonSchema: getZarfSchema(t)} err := validateSchema(&validator) require.NoError(t, err) - require.EqualError(t, validator.errors[0], ".components.[0].import: Additional property not-path is not allowed") - require.EqualError(t, validator.errors[1], ".components.[1].import.path: Invalid type. Expected: string, given: integer") + config.NoColor = true + require.Equal(t, "Additional property not-path is not allowed", validator.findings[0].String()) + require.Equal(t, "Invalid type. Expected: string, given: integer", validator.findings[1].String()) }) t.Run("Template in component import success", func(t *testing.T) { unmarshalledYaml := readAndUnmarshalYaml[types.ZarfPackage](t, goodZarfPackage) validator := Validator{typedZarfPackage: unmarshalledYaml} - checkForVarInComponentImport(&validator) - require.Empty(t, validator.warnings) + for _, component := range validator.typedZarfPackage.Components { + lintComponent(&validator, &composer.Node{ZarfComponent: component}) + } + require.Empty(t, validator.findings) }) - t.Run("Template in component import failure", func(t *testing.T) { - unmarshalledYaml := readAndUnmarshalYaml[types.ZarfPackage](t, badZarfPackage) - validator := Validator{typedZarfPackage: unmarshalledYaml} - checkForVarInComponentImport(&validator) - require.Equal(t, validator.warnings[0], ".components.[2].import.path: Will not resolve ZARF_PKG_TMPL_* variables") - require.Equal(t, validator.warnings[1], ".components.[3].import.url: Will not resolve ZARF_PKG_TMPL_* variables") + t.Run("Path template in component import failure", func(t *testing.T) { + pathVar := "###ZARF_PKG_TMPL_PATH###" + pathComponent := types.ZarfComponent{Import: types.ZarfComponentImport{Path: pathVar}} + validator := Validator{typedZarfPackage: types.ZarfPackage{Components: []types.ZarfComponent{pathComponent}}} + checkForVarInComponentImport(&validator, &composer.Node{ZarfComponent: pathComponent}) + require.Equal(t, pathVar, validator.findings[0].item) + }) + + t.Run("OCI template in component import failure", func(t *testing.T) { + ociPathVar := "oci://###ZARF_PKG_TMPL_PATH###" + URLComponent := types.ZarfComponent{Import: types.ZarfComponentImport{URL: ociPathVar}} + validator := Validator{typedZarfPackage: types.ZarfPackage{Components: []types.ZarfComponent{URLComponent}}} + checkForVarInComponentImport(&validator, &composer.Node{ZarfComponent: URLComponent}) + require.Equal(t, ociPathVar, validator.findings[0].item) + }) + + t.Run("Unpinnned repo warning", func(t *testing.T) { + validator := Validator{} + unpinnedRepo := "https://github.com/defenseunicorns/zarf-public-test.git" + component := types.ZarfComponent{Repos: []string{ + unpinnedRepo, + "https://dev.azure.com/defenseunicorns/zarf-public-test/_git/zarf-public-test@v0.0.1"}} + checkForUnpinnedRepos(&validator, &composer.Node{ZarfComponent: component}) + require.Equal(t, unpinnedRepo, validator.findings[0].item) + require.Equal(t, len(validator.findings), 1) + }) + + t.Run("Unpinnned image warning", func(t *testing.T) { + validator := Validator{} + unpinnedImage := "registry.com:9001/whatever/image:1.0.0" + badImage := "badimage:badimage@@sha256:3fbc632167424a6d997e74f5" + component := types.ZarfComponent{Images: []string{ + unpinnedImage, + "busybox:latest@sha256:3fbc632167424a6d997e74f52b878d7cc478225cffac6bc977eedfe51c7f4e79", + badImage}} + checkForUnpinnedImages(&validator, &composer.Node{ZarfComponent: component}) + require.Equal(t, unpinnedImage, validator.findings[0].item) + require.Equal(t, badImage, validator.findings[1].item) + require.Equal(t, 2, len(validator.findings)) + + }) + + t.Run("Unpinnned file warning", func(t *testing.T) { + validator := Validator{} + fileURL := "http://example.com/file.zip" + localFile := "local.txt" + zarfFiles := []types.ZarfFile{ + { + Source: fileURL, + }, + { + Source: localFile, + }, + { + Source: fileURL, + Shasum: "fake-shasum", + }, + } + component := types.ZarfComponent{Files: zarfFiles} + checkForUnpinnedFiles(&validator, &composer.Node{ZarfComponent: component}) + require.Equal(t, fileURL, validator.findings[0].item) + require.Equal(t, 1, len(validator.findings)) }) t.Run("Wrap standalone numbers in bracket", func(t *testing.T) { @@ -110,4 +166,68 @@ func TestValidateSchema(t *testing.T) { acutal := makeFieldPathYqCompat(input) require.Equal(t, input, acutal) }) + + t.Run("Test composable components", func(t *testing.T) { + pathVar := "fake-path" + unpinnedImage := "unpinned:latest" + pathComponent := types.ZarfComponent{ + Import: types.ZarfComponentImport{Path: pathVar}, + Images: []string{unpinnedImage}} + validator := Validator{ + typedZarfPackage: types.ZarfPackage{Components: []types.ZarfComponent{pathComponent}, + Metadata: types.ZarfMetadata{Name: "test-zarf-package"}}} + + createOpts := types.ZarfCreateOptions{Flavor: "", BaseDir: "."} + lintComponents(&validator, &createOpts) + // Require.contains rather than equals since the error message changes from linux to windows + require.Contains(t, validator.findings[0].description, fmt.Sprintf("open %s", filepath.Join("fake-path", "zarf.yaml"))) + require.Equal(t, ".components.[0].import.path", validator.findings[0].yqPath) + require.Equal(t, ".", validator.findings[0].packageRelPath) + require.Equal(t, unpinnedImage, validator.findings[1].item) + require.Equal(t, ".", validator.findings[1].packageRelPath) + }) + + t.Run("isImagePinned", func(t *testing.T) { + t.Parallel() + tests := []struct { + input string + expected bool + err error + }{ + { + input: "registry.com:8080/defenseunicorns/whatever", + expected: false, + err: nil, + }, + { + input: "ghcr.io/defenseunicorns/pepr/controller:v0.15.0", + expected: false, + err: nil, + }, + { + input: "busybox:latest@sha256:3fbc632167424a6d997e74f52b878d7cc478225cffac6bc977eedfe51c7f4e79", + expected: true, + err: nil, + }, + { + input: "busybox:bad/image", + expected: false, + err: errors.New("invalid reference format"), + }, + { + input: "busybox:###ZARF_PKG_TMPL_BUSYBOX_IMAGE###", + expected: true, + err: nil, + }, + } + for _, tc := range tests { + t.Run(tc.input, func(t *testing.T) { + acutal, err := isPinnedImage(tc.input) + if err != nil { + require.EqualError(t, err, tc.err.Error()) + } + require.Equal(t, tc.expected, acutal) + }) + } + }) } diff --git a/src/pkg/packager/lint/validator.go b/src/pkg/packager/lint/validator.go index f88c22de5f..830ac21ff2 100644 --- a/src/pkg/packager/lint/validator.go +++ b/src/pkg/packager/lint/validator.go @@ -6,63 +6,146 @@ package lint import ( "fmt" + "path/filepath" "github.com/defenseunicorns/zarf/src/pkg/message" - "github.com/defenseunicorns/zarf/src/pkg/utils" + "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "github.com/defenseunicorns/zarf/src/types" "github.com/fatih/color" ) +type category int + +const ( + categoryError category = 1 + categoryWarning category = 2 +) + +type validatorMessage struct { + yqPath string + description string + item string + packageRelPath string + packageName string + category category +} + +func (c category) String() string { + if c == categoryError { + return message.ColorWrap("Error", color.FgRed) + } else if c == categoryWarning { + return message.ColorWrap("Warning", color.FgYellow) + } + return "" +} + +func (vm validatorMessage) String() string { + if vm.item != "" { + vm.item = fmt.Sprintf(" - %s", vm.item) + } + return fmt.Sprintf("%s%s", vm.description, vm.item) +} + // Validator holds the warnings/errors and messaging that we get from validation type Validator struct { - warnings []string - errors []error + findings []validatorMessage jsonSchema []byte typedZarfPackage types.ZarfPackage untypedZarfPackage interface{} + baseDir string } // DisplayFormattedMessage message sent to user based on validator results func (v Validator) DisplayFormattedMessage() { - if !v.hasWarnings() && !v.hasErrors() { - message.Successf("Schema validation successful for %q", v.typedZarfPackage.Metadata.Name) + if !v.hasFindings() { + message.Successf("0 findings for %q", v.typedZarfPackage.Metadata.Name) } v.printValidationTable() } // IsSuccess returns true if there are not any errors func (v Validator) IsSuccess() bool { - return !v.hasErrors() + for _, finding := range v.findings { + if finding.category == categoryError { + return false + } + } + return true +} + +func (v Validator) packageRelPathToUser(vm validatorMessage) string { + if helpers.IsOCIURL(vm.packageRelPath) { + return vm.packageRelPath + } + return filepath.Join(v.baseDir, vm.packageRelPath) } func (v Validator) printValidationTable() { - if v.hasWarnings() || v.hasErrors() { - header := []string{"Type", "Message"} - connectData := [][]string{} - for _, warning := range v.warnings { - connectData = append(connectData, []string{utils.ColorWrap("Warning", color.FgYellow), warning}) + if !v.hasFindings() { + return + } + + mapOfFindingsByPath := make(map[string][]validatorMessage) + for _, finding := range v.findings { + mapOfFindingsByPath[finding.packageRelPath] = append(mapOfFindingsByPath[finding.packageRelPath], finding) + } + + header := []string{"Type", "Path", "Message"} + + for packageRelPath, findings := range mapOfFindingsByPath { + lintData := [][]string{} + for _, finding := range findings { + lintData = append(lintData, []string{finding.category.String(), finding.getPath(), finding.String()}) + } + message.Notef("Linting package %q at %s", findings[0].packageName, v.packageRelPathToUser(findings[0])) + message.Table(header, lintData) + message.Info(v.getFormattedFindingCount(packageRelPath, findings[0].packageName)) + } +} + +func (v Validator) getFormattedFindingCount(relPath string, packageName string) string { + warningCount := 0 + errorCount := 0 + for _, finding := range v.findings { + if finding.packageRelPath != relPath { + continue + } + if finding.category == categoryWarning { + warningCount++ } - for _, err := range v.errors { - connectData = append(connectData, []string{utils.ColorWrap("Error", color.FgRed), err.Error()}) + if finding.category == categoryError { + errorCount++ } - message.Table(header, connectData) - message.Info(fmt.Sprintf("%d warnings and %d errors in %q", - len(v.warnings), len(v.errors), v.typedZarfPackage.Metadata.Name)) } + wordWarning := "warnings" + if warningCount == 1 { + wordWarning = "warning" + } + wordError := "errors" + if errorCount == 1 { + wordError = "error" + } + return fmt.Sprintf("%d %s and %d %s in %q", + warningCount, wordWarning, errorCount, wordError, packageName) } -func (v Validator) hasWarnings() bool { - return len(v.warnings) > 0 +func (vm validatorMessage) getPath() string { + if vm.yqPath == "" { + return "" + } + return message.ColorWrap(vm.yqPath, color.FgCyan) } -func (v Validator) hasErrors() bool { - return len(v.errors) > 0 +func (v Validator) hasFindings() bool { + return len(v.findings) > 0 } -func (v *Validator) addWarning(message string) { - v.warnings = append(v.warnings, message) +func (v *Validator) addWarning(vmessage validatorMessage) { + vmessage.category = categoryWarning + v.findings = helpers.Unique(append(v.findings, vmessage)) } -func (v *Validator) addError(err error) { - v.errors = append(v.errors, err) +func (v *Validator) addError(vMessage validatorMessage) { + vMessage.category = categoryError + v.findings = helpers.Unique(append(v.findings, vMessage)) } diff --git a/src/pkg/packager/variables.go b/src/pkg/packager/variables.go index 2caa135907..fb3e20f6d3 100644 --- a/src/pkg/packager/variables.go +++ b/src/pkg/packager/variables.go @@ -15,6 +15,30 @@ import ( "github.com/defenseunicorns/zarf/src/types" ) +// ReloadComponentTemplate appends ###ZARF_COMPONENT_NAME### for the component, assigns value, and reloads +// Any instance of ###ZARF_COMPONENT_NAME### within a component will be replaced with that components name +func ReloadComponentTemplate(component *types.ZarfComponent) error { + mappings := map[string]string{} + mappings[types.ZarfComponentName] = component.Name + err := utils.ReloadYamlTemplate(component, mappings) + if err != nil { + return err + } + return nil +} + +// ReloadComponentTemplatesInPackage appends ###ZARF_COMPONENT_NAME### for each component, assigns value, and reloads +func ReloadComponentTemplatesInPackage(zarfPackage *types.ZarfPackage) error { + // iterate through components to and find all ###ZARF_COMPONENT_NAME, assign to component Name and value + for i := range zarfPackage.Components { + if err := ReloadComponentTemplate(&zarfPackage.Components[i]); err != nil { + return err + } + } + + return nil +} + // fillActiveTemplate handles setting the active variables and reloading the base template. func (p *Packager) fillActiveTemplate() error { templateMap := map[string]string{} @@ -54,7 +78,7 @@ func (p *Packager) fillActiveTemplate() error { } // update the component templates on the package - err := p.findComponentTemplatesAndReload() + err := ReloadComponentTemplatesInPackage(&p.cfg.Pkg) if err != nil { return err } @@ -126,21 +150,6 @@ func (p *Packager) setVariableInConfig(name, value string, sensitive bool, autoI } } -// findComponentTemplatesAndReload appends ###ZARF_COMPONENT_NAME### for each component, assigns value, and reloads -func (p *Packager) findComponentTemplatesAndReload() error { - // iterate through components to and find all ###ZARF_COMPONENT_NAME, assign to component Name and value - for i, component := range p.cfg.Pkg.Components { - mappings := map[string]string{} - mappings[types.ZarfComponentName] = component.Name - err := utils.ReloadYamlTemplate(&p.cfg.Pkg.Components[i], mappings) - if err != nil { - return err - } - } - - return nil -} - // checkVariablePattern checks to see if a current variable is set to a value that matches its pattern func (p *Packager) checkVariablePattern(name, pattern string) error { if regexp.MustCompile(pattern).MatchString(p.cfg.SetVariableMap[name].Value) { diff --git a/src/pkg/transform/image_test.go b/src/pkg/transform/image_test.go index 819f9f672e..2edc2e87ec 100644 --- a/src/pkg/transform/image_test.go +++ b/src/pkg/transform/image_test.go @@ -15,6 +15,7 @@ var imageRefs = []string{ "nginx:1.23.3", "defenseunicorns/zarf-agent:v0.22.1", "defenseunicorns/zarf-agent@sha256:84605f731c6a18194794c51e70021c671ab064654b751aa57e905bce55be13de", + "busybox:latest@sha256:3fbc632167424a6d997e74f52b878d7cc478225cffac6bc977eedfe51c7f4e79", "ghcr.io/stefanprodan/podinfo:6.3.3", "registry1.dso.mil/ironbank/opensource/defenseunicorns/zarf/zarf-agent:v0.25.0", "gitlab.com/project/gitea/gitea:1.19.3-rootless-zarf-3431384023", @@ -33,6 +34,7 @@ func TestImageTransformHost(t *testing.T) { "gitlab.com/project/library/nginx:1.23.3-zarf-3793515731", "gitlab.com/project/defenseunicorns/zarf-agent:v0.22.1-zarf-4283503412", "gitlab.com/project/defenseunicorns/zarf-agent@sha256:84605f731c6a18194794c51e70021c671ab064654b751aa57e905bce55be13de", + "gitlab.com/project/library/busybox@sha256:3fbc632167424a6d997e74f52b878d7cc478225cffac6bc977eedfe51c7f4e79", "gitlab.com/project/stefanprodan/podinfo:6.3.3-zarf-2985051089", "gitlab.com/project/ironbank/opensource/defenseunicorns/zarf/zarf-agent:v0.25.0-zarf-2003217571", "gitlab.com/project/gitea/gitea:1.19.3-rootless-zarf-3431384023", @@ -56,6 +58,7 @@ func TestImageTransformHostWithoutChecksum(t *testing.T) { "gitlab.com/project/library/nginx:1.23.3", "gitlab.com/project/defenseunicorns/zarf-agent:v0.22.1", "gitlab.com/project/defenseunicorns/zarf-agent@sha256:84605f731c6a18194794c51e70021c671ab064654b751aa57e905bce55be13de", + "gitlab.com/project/library/busybox@sha256:3fbc632167424a6d997e74f52b878d7cc478225cffac6bc977eedfe51c7f4e79", "gitlab.com/project/stefanprodan/podinfo:6.3.3", "gitlab.com/project/ironbank/opensource/defenseunicorns/zarf/zarf-agent:v0.25.0", "gitlab.com/project/gitea/gitea:1.19.3-rootless-zarf-3431384023", @@ -79,6 +82,7 @@ func TestParseImageRef(t *testing.T) { {"docker.io/", "library/nginx", "1.23.3", ""}, {"docker.io/", "defenseunicorns/zarf-agent", "v0.22.1", ""}, {"docker.io/", "defenseunicorns/zarf-agent", "", "sha256:84605f731c6a18194794c51e70021c671ab064654b751aa57e905bce55be13de"}, + {"docker.io/", "library/busybox", "latest", "sha256:3fbc632167424a6d997e74f52b878d7cc478225cffac6bc977eedfe51c7f4e79"}, {"ghcr.io/", "stefanprodan/podinfo", "6.3.3", ""}, {"registry1.dso.mil/", "ironbank/opensource/defenseunicorns/zarf/zarf-agent", "v0.25.0", ""}, {"gitlab.com/", "project/gitea/gitea", "1.19.3-rootless-zarf-3431384023", ""}, @@ -89,13 +93,19 @@ func TestParseImageRef(t *testing.T) { require.NoError(t, err) tag := expectedResult[idx][2] digest := expectedResult[idx][3] - tagOrDigest := ":" + tag - if tag == "" { + var tagOrDigest string + var tagAndDigest string + if tag != "" { + tagOrDigest = ":" + tag + tagAndDigest = ":" + tag + } + if digest != "" { tagOrDigest = "@" + digest + tagAndDigest += "@" + digest } path := expectedResult[idx][1] name := expectedResult[idx][0] + path - reference := name + tagOrDigest + reference := name + tagAndDigest require.Equal(t, reference, img.Reference) require.Equal(t, name, img.Name) diff --git a/src/pkg/utils/helpers/random.go b/src/pkg/utils/helpers/random.go new file mode 100644 index 0000000000..63e7bfa89c --- /dev/null +++ b/src/pkg/utils/helpers/random.go @@ -0,0 +1,29 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package helpers provides generic helper functions with no external imports +package helpers + +import ( + "crypto/rand" +) + +// Very limited special chars for git / basic auth +// https://owasp.org/www-community/password-special-characters has complete list of safe chars. +const randomStringChars = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ!~-" + +// RandomString generates a secure random string of the specified length. +func RandomString(length int) (string, error) { + bytes := make([]byte, length) + + if _, err := rand.Read(bytes); err != nil { + //message.Fatal(err, "unable to generate a random secret") + return "", err + } + + for i, b := range bytes { + bytes[i] = randomStringChars[b%byte(len(randomStringChars))] + } + + return string(bytes), nil +} diff --git a/src/pkg/utils/random.go b/src/pkg/utils/random.go deleted file mode 100644 index 7966cd765a..0000000000 --- a/src/pkg/utils/random.go +++ /dev/null @@ -1,47 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: 2021-Present The Zarf Authors - -// Package utils provides generic utility functions. -package utils - -import ( - "crypto/rand" - "fmt" - - "github.com/defenseunicorns/zarf/src/pkg/message" - "github.com/fatih/color" -) - -// Very limited special chars for git / basic auth -// https://owasp.org/www-community/password-special-characters has complete list of safe chars. -const randomStringChars = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ!~-" - -// RandomString generates a secure random string of the specified length. -func RandomString(length int) string { - bytes := make([]byte, length) - - if _, err := rand.Read(bytes); err != nil { - message.Fatal(err, "unable to generate a random secret") - } - - for i, b := range bytes { - bytes[i] = randomStringChars[b%byte(len(randomStringChars))] - } - - return string(bytes) -} - -// First30last30 returns the source string that has been trimmed to 30 characters at the beginning and end. -func First30last30(s string) string { - if len(s) > 60 { - return s[0:27] + "..." + s[len(s)-26:] - } - - return s -} - -// ColorWrap changes a string to an ansi color code and appends the default color to the end -// preventing future characters from taking on the given color -func ColorWrap(str string, attr color.Attribute) string { - return fmt.Sprintf("\x1b[%dm%s\x1b[0m", attr, str) -} diff --git a/src/test/e2e/12_lint_test.go b/src/test/e2e/12_lint_test.go index 98257fdc89..a2d6443ed8 100644 --- a/src/test/e2e/12_lint_test.go +++ b/src/test/e2e/12_lint_test.go @@ -1,31 +1,60 @@ package test import ( + "fmt" + "os" "path/filepath" "testing" + "github.com/defenseunicorns/zarf/src/config/lang" "github.com/stretchr/testify/require" ) func TestLint(t *testing.T) { t.Log("E2E: Lint") + t.Run("zarf test lint success", func(t *testing.T) { + t.Log("E2E: Test lint on schema success") + + // This runs lint on the zarf.yaml in the base directory of the repo + _, _, err := e2e.Zarf("dev", "lint") + require.NoError(t, err, "Expect no error here because the yaml file is following schema") + }) + t.Run("zarf test lint fail", func(t *testing.T) { t.Log("E2E: Test lint on schema fail") - path := filepath.Join("src", "test", "packages", "12-lint") - _, stderr, err := e2e.Zarf("prepare", "lint", path) + testPackagePath := filepath.Join("src", "test", "packages", "12-lint") + configPath := filepath.Join(testPackagePath, "zarf-config.toml") + os.Setenv("ZARF_CONFIG", configPath) + _, stderr, err := e2e.Zarf("dev", "lint", testPackagePath, "-f", "good-flavor") require.Error(t, err, "Require an exit code since there was warnings / errors") - require.Contains(t, stderr, ".components.[0].import: Additional property not-path is not allowed") - require.Contains(t, stderr, ".components.[2].import.path: Will not resolve ZARF_PKG_TMPL_* variables") - require.Contains(t, stderr, ".variables: Invalid type. Expected: array, given: null") - }) + strippedStderr := e2e.StripMessageFormatting(stderr) - t.Run("zarf test lint success", func(t *testing.T) { - t.Log("E2E: Test lint on schema success") + key := "WHATEVER_IMAGE" + require.Contains(t, strippedStderr, lang.UnsetVarLintWarning) + require.Contains(t, strippedStderr, fmt.Sprintf(lang.PkgValidateTemplateDeprecation, key, key, key)) + require.Contains(t, strippedStderr, ".components.[2].repos.[0] | Unpinned repository") + require.Contains(t, strippedStderr, ".metadata | Additional property description1 is not allowed") + require.Contains(t, strippedStderr, ".components.[0].import | Additional property not-path is not allowed") + // Testing the import / compose on lint is working + require.Contains(t, strippedStderr, ".components.[1].images.[0] | Image not pinned with digest - registry.com:9001/whatever/image:latest") + // Testing import / compose + variables are working + require.Contains(t, strippedStderr, ".components.[2].images.[3] | Image not pinned with digest - busybox:latest") + require.Contains(t, strippedStderr, ".components.[3].import.path | Zarf does not evaluate variables at component.x.import.path - ###ZARF_PKG_TMPL_PATH###") + // Testing OCI imports get linted + require.Contains(t, strippedStderr, ".components.[0].images.[0] | Image not pinned with digest - defenseunicorns/zarf-game:multi-tile-dark") + // Testing a bad path leads to a finding in lint + require.Contains(t, strippedStderr, fmt.Sprintf(".components.[3].import.path | open %s", filepath.Join("###ZARF_PKG_TMPL_PATH###", "zarf.yaml"))) + + // Check flavors + require.NotContains(t, strippedStderr, "image-in-bad-flavor-component:unpinned") + require.Contains(t, strippedStderr, "image-in-good-flavor-component:unpinned") + + // Check reported filepaths + require.Contains(t, strippedStderr, "Linting package \"dos-games\" at oci://🦄/dos-games:1.0.0-skeleton") + require.Contains(t, strippedStderr, fmt.Sprintf("Linting package \"lint\" at %s", testPackagePath)) - // This runs lint on the zarf.yaml in the base directory of the repo - _, _, err := e2e.Zarf("prepare", "lint") - require.NoError(t, err, "Expect no error here because the yaml file is following schema") }) + } diff --git a/src/test/packages/12-lint/linted-import/zarf.yaml b/src/test/packages/12-lint/linted-import/zarf.yaml new file mode 100644 index 0000000000..f5f21981f6 --- /dev/null +++ b/src/test/packages/12-lint/linted-import/zarf.yaml @@ -0,0 +1,23 @@ +kind: ZarfPackageConfig +metadata: + name: linted-import + description: Testing bad yaml imported + +variables: + - name: BUSYBOX_IMAGE + description: "whatever" + +components: + - name: dont-care + + - name: import-test + images: + - registry.com:9001/whatever/image:latest + - busybox@sha256:3fbc632167424a6d997e74f52b878d7cc478225cffac6bc977eedfe51c7f4e79 + - busybox:###ZARF_PKG_TMPL_BUSYBOX_IMAGE### + - busybox:###ZARF_PKG_TMPL_UNSET### + + - name: oci-games-url + import: + url: oci://🦄/dos-games:1.0.0-skeleton + name: baseline diff --git a/src/test/packages/12-lint/zarf-config.toml b/src/test/packages/12-lint/zarf-config.toml new file mode 100644 index 0000000000..1d1c57f33e --- /dev/null +++ b/src/test/packages/12-lint/zarf-config.toml @@ -0,0 +1,3 @@ +[package.create.set] +BUSYBOX_IMAGE = "latest" +PATH = "linted-import" diff --git a/src/test/packages/12-lint/zarf.yaml b/src/test/packages/12-lint/zarf.yaml index 7c026a15b7..efddf42eea 100644 --- a/src/test/packages/12-lint/zarf.yaml +++ b/src/test/packages/12-lint/zarf.yaml @@ -1,11 +1,8 @@ -kind: ZarfInitConfig +kind: ZarfPackageConfig metadata: - name: init + name: lint description1: Testing bad yaml - -variables: - components: - name: first-test-component import: @@ -13,12 +10,47 @@ components: - name: import-test import: - path: 123123 + path: linted-import - - name: import-test + - name: full-repo + repos: + - https://github.com/defenseunicorns/zarf-public-test.git + - https://dev.azure.com/defenseunicorns/zarf-public-test/_git/zarf-public-test@v0.0.1 + - https://gitlab.com/gitlab-org/build/omnibus-mirror/pcre2/-/tree/vreverse?ref_type=heads + images: + - registry.com:9001/whatever/image:1.0.0 + - busybox@sha256:3fbc632167424a6d997e74f52b878d7cc478225cffac6bc977eedfe51c7f4e79 + - busybox:###ZARF_PKG_VAR_WHATEVER_IMAGE### + - busybox:###ZARF_PKG_TMPL_BUSYBOX_IMAGE### + - ubuntu:###ZARF_PKG_TMPL_UBUNTU_IMAGE### + files: + - source: https://github.com/k3s-io/k3s/releases/download/v1.28.2+k3s1/k3s + shasum: 2f041d37a2c6d54d53e106e1c7713bc48f806f3919b0d9e092f5fcbdc55b41cf + target: src/ + - source: file-without-shasum.txt + target: src/ + + - name: import import: - path: "###ZARF_PKG_TMPL_ZEBRA###" + path: "###ZARF_PKG_TMPL_PATH###" - - name: import-url + - name: oci-games-url import: - url: "oci://###ZARF_PKG_TMPL_ZEBRA###" + url: oci://🦄/dos-games:1.0.0-skeleton + name: baseline + + - name: oci-games-url + import: + path: linted-import + + - name: import-bad-flavor + only: + flavor: bad-flavor + images: + - image-in-bad-flavor-component:unpinned + + - name: import-good-flavor + only: + flavor: good-flavor + images: + - image-in-good-flavor-component:unpinned