From 9e83f8f03a6833817dc77ea2686790a46220832a Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Mon, 16 Oct 2023 15:38:59 +0000 Subject: [PATCH 001/192] early WIP have an example of a yaml file being read and validated --- go.mod | 7 +- go.sum | 4 + src/cmd/prepare.go | 14 +++ src/pkg/utils/helpers/example-schema.json | 31 +++++++ src/pkg/utils/helpers/example.yaml | 6 ++ src/pkg/utils/helpers/validate_schema.go | 88 +++++++++++++++++++ src/pkg/utils/helpers/validate_schema_test.go | 28 ++++++ zarf.schema.json | 3 +- 8 files changed, 177 insertions(+), 4 deletions(-) create mode 100644 src/pkg/utils/helpers/example-schema.json create mode 100644 src/pkg/utils/helpers/example.yaml create mode 100644 src/pkg/utils/helpers/validate_schema.go create mode 100644 src/pkg/utils/helpers/validate_schema_test.go diff --git a/go.mod b/go.mod index fb60c15555..a47cd7d54e 100644 --- a/go.mod +++ b/go.mod @@ -42,6 +42,7 @@ require ( github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.16.0 github.com/stretchr/testify v1.8.4 + github.com/xeipuuv/gojsonschema v1.2.0 golang.org/x/crypto v0.13.0 golang.org/x/sync v0.3.0 golang.org/x/term v0.12.0 @@ -58,6 +59,8 @@ require ( sigs.k8s.io/yaml v1.3.0 ) +require github.com/santhosh-tekuri/jsonschema v1.2.4 + require ( atomicgo.dev/cursor v0.2.0 // indirect atomicgo.dev/keyboard v0.2.9 // indirect @@ -367,6 +370,7 @@ require ( github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/ryanuber/go-glob v1.0.0 // indirect github.com/sahilm/fuzzy v0.1.0 // indirect + github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 github.com/sassoftware/go-rpmutils v0.2.0 // indirect github.com/sassoftware/relic v7.2.1+incompatible // indirect github.com/scylladb/go-set v1.0.3-0.20200225121959-cc7b2070d91e // indirect @@ -411,7 +415,6 @@ require ( github.com/xanzy/ssh-agent v0.3.3 // indirect github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect - github.com/xeipuuv/gojsonschema v1.2.0 // indirect github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect github.com/xlab/treeprint v1.2.0 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect @@ -452,7 +455,7 @@ require ( gopkg.in/square/go-jose.v2 v2.6.0 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect + gopkg.in/yaml.v3 v3.0.1 inet.af/netaddr v0.0.0-20220811202034-502d2d690317 // indirect k8s.io/apiextensions-apiserver v0.28.2 // indirect k8s.io/apiserver v0.28.2 // indirect diff --git a/go.sum b/go.sum index 7032a61d88..03cea811c7 100644 --- a/go.sum +++ b/go.sum @@ -1412,6 +1412,10 @@ github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIH github.com/sagikazarmark/crypt v0.3.0/go.mod h1:uD/D+6UF4SrIR1uGEv7bBNkNqLGqUr43MRiaGWX1Nig= github.com/sahilm/fuzzy v0.1.0 h1:FzWGaw2Opqyu+794ZQ9SYifWv2EIXpwP4q8dY1kDAwI= github.com/sahilm/fuzzy v0.1.0/go.mod h1:VFvziUEIMCrT6A6tw2RFIXPXXmzXbOsSHF0DOI8ZK9Y= +github.com/santhosh-tekuri/jsonschema v1.2.4 h1:hNhW8e7t+H1vgY+1QeEQpveR6D4+OwKPXCfD2aieJis= +github.com/santhosh-tekuri/jsonschema v1.2.4/go.mod h1:TEAUOeZSmIxTTuHatJzrvARHiuO9LYd+cIxzgEHCQI4= +github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 h1:lZUw3E0/J3roVtGQ+SCrUrg3ON6NgVqpn3+iol9aGu4= +github.com/santhosh-tekuri/jsonschema/v5 v5.3.1/go.mod h1:uToXkOrWAZ6/Oc07xWQrPOhJotwFIyu2bBVN41fcDUY= github.com/sassoftware/go-rpmutils v0.2.0 h1:pKW0HDYMFWQ5b4JQPiI3WI12hGsVoW0V8+GMoZiI/JE= github.com/sassoftware/go-rpmutils v0.2.0/go.mod h1:TJJQYtLe/BeEmEjelI3b7xNZjzAukEkeWKmoakvaOoI= github.com/sassoftware/relic v7.2.1+incompatible h1:Pwyh1F3I0r4clFJXkSI8bOyJINGqpgjJU3DYAZeI05A= diff --git a/src/cmd/prepare.go b/src/cmd/prepare.go index f53dcdcae6..a4613ba8e4 100644 --- a/src/cmd/prepare.go +++ b/src/cmd/prepare.go @@ -7,6 +7,7 @@ package cmd import ( "fmt" "io" + "log" "os" "strings" @@ -20,6 +21,8 @@ import ( "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "github.com/spf13/cobra" + "github.com/xeipuuv/gojsonschema" + "sigs.k8s.io/yaml" ) var prepareCmd = &cobra.Command{ @@ -157,6 +160,16 @@ var prepareGenerateConfigFile = &cobra.Command{ }, } +var prepareValidateSchema = &cobra.Command{ + Use: "lint [ FILENAME ] [ FILENAME ]", + Args: cobra.MaximumNArgs(2), + Short: lang.CmdPrepareGenerateConfigShort, + Long: lang.CmdPrepareGenerateConfigLong, + Run: func(cmd *cobra.Command, args []string) { + + }, +} + func init() { v := common.InitViper() @@ -165,6 +178,7 @@ func init() { prepareCmd.AddCommand(prepareComputeFileSha256sum) prepareCmd.AddCommand(prepareFindImages) prepareCmd.AddCommand(prepareGenerateConfigFile) + prepareCmd.AddCommand(prepareValidateSchema) prepareFindImages.Flags().StringVarP(&pkgConfig.FindImagesOpts.RepoHelmChartPath, "repo-chart-path", "p", "", lang.CmdPrepareFlagRepoChartPath) // use the package create config for this and reset it here to avoid overwriting the config.CreateOptions.SetVariables diff --git a/src/pkg/utils/helpers/example-schema.json b/src/pkg/utils/helpers/example-schema.json new file mode 100644 index 0000000000..513869f972 --- /dev/null +++ b/src/pkg/utils/helpers/example-schema.json @@ -0,0 +1,31 @@ +{ + "$id": "https://example.com/product.schema.json", + "title": "Product", + "description": "A product from Acme's catalog", + "type": "object", + "properties": { + "productId": { + "description": "The unique identifier for a product", + "type": "integer" + }, + "productName": { + "description": "Name of the product", + "type": "string" + }, + "price": { + "description": "The price of the product", + "type": "number", + "exclusiveMinimum": 0 + }, + "tags": { + "description": "Tags for the product", + "type": "array", + "items": { + "type": "string" + }, + "minItems": 1, + "uniqueItems": true + } + }, + "required": [ "productId", "productName", "price" ] +} diff --git a/src/pkg/utils/helpers/example.yaml b/src/pkg/utils/helpers/example.yaml new file mode 100644 index 0000000000..5d19224c8c --- /dev/null +++ b/src/pkg/utils/helpers/example.yaml @@ -0,0 +1,6 @@ +productId: 1 +productName: A green door +price: 12.50 +tags: +- home +- green diff --git a/src/pkg/utils/helpers/validate_schema.go b/src/pkg/utils/helpers/validate_schema.go new file mode 100644 index 0000000000..a59364df8f --- /dev/null +++ b/src/pkg/utils/helpers/validate_schema.go @@ -0,0 +1,88 @@ +package helpers + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "log" + "os" + + "github.com/santhosh-tekuri/jsonschema" + "gopkg.in/yaml.v3" +) + +func ReadSchema(yamlFile, jsonFile string) bool { + + yamlBytes, err := os.ReadFile(yamlFile) + if err != nil { + log.Fatalf("Error reading YAML file: %s", yamlFile) + } + + jsonBytes, err := os.ReadFile(jsonFile) + if err != nil { + log.Fatalf("Error reading YAML file: %s", jsonFile) + } + + var m interface{} + err = yaml.Unmarshal(yamlBytes, &m) + if err != nil { + panic(err) + } + m, err = toStringKeys(m) + if err != nil { + panic(err) + } + + mJSON, err := json.Marshal(m) + if err != nil { + panic(err) + } + yamlJsonReader := bytes.NewReader(mJSON) + schemaReader := bytes.NewReader(jsonBytes) + + compiler := jsonschema.NewCompiler() + + if err := compiler.AddResource("schema.json", schemaReader); err != nil { + panic(err) + } + schema, err := compiler.Compile("schema.json") + if err != nil { + panic(err) + } + if err := schema.Validate(yamlJsonReader); err != nil { + panic(err) + } + fmt.Println("validation successfull") + return true +} + +func toStringKeys(val interface{}) (interface{}, error) { + var err error + switch val := val.(type) { + case map[interface{}]interface{}: + m := make(map[string]interface{}) + for k, v := range val { + k, ok := k.(string) + if !ok { + return nil, errors.New("found non-string key") + } + m[k], err = toStringKeys(v) + if err != nil { + return nil, err + } + } + return m, nil + case []interface{}: + var l = make([]interface{}, len(val)) + for i, v := range val { + l[i], err = toStringKeys(v) + if err != nil { + return nil, err + } + } + return l, nil + default: + return val, nil + } +} diff --git a/src/pkg/utils/helpers/validate_schema_test.go b/src/pkg/utils/helpers/validate_schema_test.go new file mode 100644 index 0000000000..cb243189aa --- /dev/null +++ b/src/pkg/utils/helpers/validate_schema_test.go @@ -0,0 +1,28 @@ +package helpers + +import "testing" + +func TestReadSchema(t *testing.T) { + t.Run("basic read schema", func(t *testing.T) { + want := true + + if got := ReadSchema("example.yaml","example-schema.json"); got != want { + t.Errorf("AppleSauce() = %v, want %v", got, want) + } + }) +} + + +// func TestIsZarfValid(t *testing.T) { +// t.Run("first test", func(t *testing.T) { +// want := false + +// if got := ZarfYamlIsValid("/home/austin/code/zarf/zarf1.yaml"); got != want { +// t.Errorf("ZarfYamlIsValid = %v, want %v", got, want) +// } +// // if got := ZarfYamlIsValid("/home/austin/code/zarf/zarf.yaml"); got != want { +// // t.Errorf("AppleSauce() = %v, want %v", got, want) +// // } +// }) +// } + diff --git a/zarf.schema.json b/zarf.schema.json index 483c37ea02..65592ec16f 100644 --- a/zarf.schema.json +++ b/zarf.schema.json @@ -658,12 +658,11 @@ "description": "The name of the component to import from the referenced zarf.yaml" }, "path": { - "pattern": "^(?!.*###ZARF_PKG_TMPL_).*$", "type": "string", "description": "The relative path to a directory containing a zarf.yaml to import from" }, "url": { - "pattern": "^oci://(?!.*###ZARF_PKG_TMPL_).*$", + "pattern": "^oci://.*$", "type": "string", "description": "[beta] The URL to a Zarf package to import via OCI" } From 1a77654d88aecd3c67ab90ad380cde22cc1c7d98 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Mon, 16 Oct 2023 15:40:24 +0000 Subject: [PATCH 002/192] refactor --- src/pkg/utils/helpers/validate_schema.go | 35 ------------------------ 1 file changed, 35 deletions(-) diff --git a/src/pkg/utils/helpers/validate_schema.go b/src/pkg/utils/helpers/validate_schema.go index a59364df8f..18d41c5921 100644 --- a/src/pkg/utils/helpers/validate_schema.go +++ b/src/pkg/utils/helpers/validate_schema.go @@ -3,7 +3,6 @@ package helpers import ( "bytes" "encoding/json" - "errors" "fmt" "log" "os" @@ -29,10 +28,6 @@ func ReadSchema(yamlFile, jsonFile string) bool { if err != nil { panic(err) } - m, err = toStringKeys(m) - if err != nil { - panic(err) - } mJSON, err := json.Marshal(m) if err != nil { @@ -56,33 +51,3 @@ func ReadSchema(yamlFile, jsonFile string) bool { fmt.Println("validation successfull") return true } - -func toStringKeys(val interface{}) (interface{}, error) { - var err error - switch val := val.(type) { - case map[interface{}]interface{}: - m := make(map[string]interface{}) - for k, v := range val { - k, ok := k.(string) - if !ok { - return nil, errors.New("found non-string key") - } - m[k], err = toStringKeys(v) - if err != nil { - return nil, err - } - } - return m, nil - case []interface{}: - var l = make([]interface{}, len(val)) - for i, v := range val { - l[i], err = toStringKeys(v) - if err != nil { - return nil, err - } - } - return l, nil - default: - return val, nil - } -} From fb3a56e1d5f3a663d4877d56696048c2ef40db03 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Mon, 16 Oct 2023 15:47:34 +0000 Subject: [PATCH 003/192] WIP refactor --- src/pkg/utils/helpers/validate_schema.go | 25 ++++++++----------- src/pkg/utils/helpers/validate_schema_test.go | 17 +------------ 2 files changed, 12 insertions(+), 30 deletions(-) diff --git a/src/pkg/utils/helpers/validate_schema.go b/src/pkg/utils/helpers/validate_schema.go index 18d41c5921..c98b3706dc 100644 --- a/src/pkg/utils/helpers/validate_schema.go +++ b/src/pkg/utils/helpers/validate_schema.go @@ -3,7 +3,6 @@ package helpers import ( "bytes" "encoding/json" - "fmt" "log" "os" @@ -11,43 +10,41 @@ import ( "gopkg.in/yaml.v3" ) -func ReadSchema(yamlFile, jsonFile string) bool { +func ValidateZarfSchema(yamlFile, jsonFile string) bool { yamlBytes, err := os.ReadFile(yamlFile) if err != nil { - log.Fatalf("Error reading YAML file: %s", yamlFile) + log.Fatalf("Error reading file: %s", err) } - jsonBytes, err := os.ReadFile(jsonFile) + zarfSchema, err := os.ReadFile(jsonFile) if err != nil { - log.Fatalf("Error reading YAML file: %s", jsonFile) + log.Fatalf("Error reading file: %s", err) } - var m interface{} - err = yaml.Unmarshal(yamlBytes, &m) + var unmarshalledYaml interface{} + err = yaml.Unmarshal(yamlBytes, &unmarshalledYaml) if err != nil { panic(err) } - mJSON, err := json.Marshal(m) + zarfYamlAsJsonBytes, err := json.Marshal(unmarshalledYaml) if err != nil { panic(err) } - yamlJsonReader := bytes.NewReader(mJSON) - schemaReader := bytes.NewReader(jsonBytes) compiler := jsonschema.NewCompiler() + inMemoryZarfSchema := "zarf.schema.json" - if err := compiler.AddResource("schema.json", schemaReader); err != nil { + if err := compiler.AddResource(inMemoryZarfSchema, bytes.NewReader(zarfSchema)); err != nil { panic(err) } - schema, err := compiler.Compile("schema.json") + schema, err := compiler.Compile(inMemoryZarfSchema) if err != nil { panic(err) } - if err := schema.Validate(yamlJsonReader); err != nil { + if err := schema.Validate(bytes.NewReader(zarfYamlAsJsonBytes)); err != nil { panic(err) } - fmt.Println("validation successfull") return true } diff --git a/src/pkg/utils/helpers/validate_schema_test.go b/src/pkg/utils/helpers/validate_schema_test.go index cb243189aa..d97ad3c87e 100644 --- a/src/pkg/utils/helpers/validate_schema_test.go +++ b/src/pkg/utils/helpers/validate_schema_test.go @@ -6,23 +6,8 @@ func TestReadSchema(t *testing.T) { t.Run("basic read schema", func(t *testing.T) { want := true - if got := ReadSchema("example.yaml","example-schema.json"); got != want { + if got := ValidateZarfSchema("/home/austin/code/zarf/zarf.yaml", "../../../../zarf.schema.json"); got != want { t.Errorf("AppleSauce() = %v, want %v", got, want) } }) } - - -// func TestIsZarfValid(t *testing.T) { -// t.Run("first test", func(t *testing.T) { -// want := false - -// if got := ZarfYamlIsValid("/home/austin/code/zarf/zarf1.yaml"); got != want { -// t.Errorf("ZarfYamlIsValid = %v, want %v", got, want) -// } -// // if got := ZarfYamlIsValid("/home/austin/code/zarf/zarf.yaml"); got != want { -// // t.Errorf("AppleSauce() = %v, want %v", got, want) -// // } -// }) -// } - From 353ed3f8803c2272583f86bbb9ba4ba5822e61c7 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Mon, 16 Oct 2023 17:58:52 +0000 Subject: [PATCH 004/192] refactor validate schema to work with unmarshalled yaml --- src/cmd/prepare.go | 17 +++++++----- src/pkg/utils/helpers/validate_schema.go | 16 ++---------- src/pkg/utils/helpers/validate_schema_test.go | 26 +++++++++++++++---- 3 files changed, 33 insertions(+), 26 deletions(-) diff --git a/src/cmd/prepare.go b/src/cmd/prepare.go index a4613ba8e4..48f5544186 100644 --- a/src/cmd/prepare.go +++ b/src/cmd/prepare.go @@ -7,22 +7,21 @@ package cmd import ( "fmt" "io" - "log" "os" + "path/filepath" "strings" "github.com/AlecAivazis/survey/v2" "github.com/defenseunicorns/zarf/src/cmd/common" "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/config/lang" + "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/packager" "github.com/defenseunicorns/zarf/src/pkg/transform" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "github.com/spf13/cobra" - "github.com/xeipuuv/gojsonschema" - "sigs.k8s.io/yaml" ) var prepareCmd = &cobra.Command{ @@ -161,12 +160,16 @@ var prepareGenerateConfigFile = &cobra.Command{ } var prepareValidateSchema = &cobra.Command{ - Use: "lint [ FILENAME ] [ FILENAME ]", - Args: cobra.MaximumNArgs(2), - Short: lang.CmdPrepareGenerateConfigShort, - Long: lang.CmdPrepareGenerateConfigLong, + Use: "lint [ FILENAME ] [ FILENAME ]", + Args: cobra.MaximumNArgs(2), + Short: lang.CmdPrepareGenerateConfigShort, + Long: lang.CmdPrepareGenerateConfigLong, Run: func(cmd *cobra.Command, args []string) { + pkgClient := packager.NewOrDie(&pkgConfig) + defer pkgClient.ClearTempPaths() + // Need to find where we load in the file + helpers.ValidateZarfSchema() }, } diff --git a/src/pkg/utils/helpers/validate_schema.go b/src/pkg/utils/helpers/validate_schema.go index c98b3706dc..14ba0ef9ef 100644 --- a/src/pkg/utils/helpers/validate_schema.go +++ b/src/pkg/utils/helpers/validate_schema.go @@ -7,28 +7,16 @@ import ( "os" "github.com/santhosh-tekuri/jsonschema" - "gopkg.in/yaml.v3" ) -func ValidateZarfSchema(yamlFile, jsonFile string) bool { - - yamlBytes, err := os.ReadFile(yamlFile) - if err != nil { - log.Fatalf("Error reading file: %s", err) - } +func ValidateZarfSchema(zarfUnmarshaledYaml interface{}, jsonFile string) bool { zarfSchema, err := os.ReadFile(jsonFile) if err != nil { log.Fatalf("Error reading file: %s", err) } - var unmarshalledYaml interface{} - err = yaml.Unmarshal(yamlBytes, &unmarshalledYaml) - if err != nil { - panic(err) - } - - zarfYamlAsJsonBytes, err := json.Marshal(unmarshalledYaml) + zarfYamlAsJsonBytes, err := json.Marshal(zarfUnmarshaledYaml) if err != nil { panic(err) } diff --git a/src/pkg/utils/helpers/validate_schema_test.go b/src/pkg/utils/helpers/validate_schema_test.go index d97ad3c87e..e2b62bf77f 100644 --- a/src/pkg/utils/helpers/validate_schema_test.go +++ b/src/pkg/utils/helpers/validate_schema_test.go @@ -1,13 +1,29 @@ package helpers -import "testing" +import ( + "os" + "testing" -func TestReadSchema(t *testing.T) { + goyaml "github.com/goccy/go-yaml" +) + +func TestValidateZarfSchema(t *testing.T) { t.Run("basic read schema", func(t *testing.T) { want := true - - if got := ValidateZarfSchema("/home/austin/code/zarf/zarf.yaml", "../../../../zarf.schema.json"); got != want { - t.Errorf("AppleSauce() = %v, want %v", got, want) + var unmarshalledYaml interface{} + readYaml("../../../../zarf.yaml", &unmarshalledYaml) + if got := ValidateZarfSchema(unmarshalledYaml, "../../../../zarf.schema.json"); got != want { + t.Errorf("ValidateZarfSchema = %v, want %v", got, want) } }) } + +func readYaml(path string, destConfig any) error { + file, err := os.ReadFile(path) + + if err != nil { + return err + } + + return goyaml.Unmarshal(file, destConfig) +} From 719bf9a6e671ef47d99c16a21fefcd77eb111214 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Mon, 16 Oct 2023 23:13:29 +0000 Subject: [PATCH 005/192] wip --- main.go | 4 +++ src/cmd/prepare.go | 4 +-- src/config/config.go | 5 ++++ src/pkg/packager/lint.go | 26 +++++++++++++++++++ src/pkg/utils/helpers/validate_schema.go | 11 ++------ src/pkg/utils/helpers/validate_schema_test.go | 10 ++++++- 6 files changed, 47 insertions(+), 13 deletions(-) create mode 100644 src/pkg/packager/lint.go diff --git a/main.go b/main.go index b3c621ef93..397b240fc6 100644 --- a/main.go +++ b/main.go @@ -17,8 +17,12 @@ var assets embed.FS //go:embed cosign.pub var cosignPublicKey string +//go:embed zarf.schema.json +var zarfSchema embed.FS + func main() { config.UIAssets = assets config.CosignPublicKey = cosignPublicKey + config.ZarfSchema = zarfSchema cmd.Execute() } diff --git a/src/cmd/prepare.go b/src/cmd/prepare.go index 48f5544186..e028af5f0d 100644 --- a/src/cmd/prepare.go +++ b/src/cmd/prepare.go @@ -8,14 +8,12 @@ import ( "fmt" "io" "os" - "path/filepath" "strings" "github.com/AlecAivazis/survey/v2" "github.com/defenseunicorns/zarf/src/cmd/common" "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/config/lang" - "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/packager" "github.com/defenseunicorns/zarf/src/pkg/transform" @@ -169,7 +167,7 @@ var prepareValidateSchema = &cobra.Command{ pkgClient := packager.NewOrDie(&pkgConfig) defer pkgClient.ClearTempPaths() // Need to find where we load in the file - helpers.ValidateZarfSchema() + //helpers.ValidateZarfSchema() }, } diff --git a/src/config/config.go b/src/config/config.go index 669efd4fb4..a0eca247ac 100644 --- a/src/config/config.go +++ b/src/config/config.go @@ -94,6 +94,7 @@ var ( CosignPublicKey string UIAssets embed.FS + ZarfSchema embed.FS // Timestamp of when the CLI was started operationStartTime = time.Now().Unix() @@ -102,6 +103,10 @@ var ( ZarfDefaultCachePath = filepath.Join("~", ".zarf-cache") ) +func GetSchemaFile() ([]byte, error) { + return ZarfSchema.ReadFile("zarf.schema.json") +} + // GetArch returns the arch based on a priority list with options for overriding. func GetArch(archs ...string) string { // List of architecture overrides. diff --git a/src/pkg/packager/lint.go b/src/pkg/packager/lint.go new file mode 100644 index 0000000000..5e3b708d4c --- /dev/null +++ b/src/pkg/packager/lint.go @@ -0,0 +1,26 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package packager contains functions for interacting with, managing and deploying Zarf packages. +package packager + +import ( + "fmt" + "path/filepath" + + "github.com/defenseunicorns/zarf/src/config" + "github.com/defenseunicorns/zarf/src/pkg/layout" + "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" +) + +// Create generates a Zarf package tarball for a given PackageConfig and optional base directory. +func (p *Packager) Lint() (err error) { + + if err = p.readZarfYAML(filepath.Join(p.cfg.CreateOpts.BaseDir, layout.ZarfYAML)); err != nil { + return fmt.Errorf("unable to read the zarf.yaml file: %s", err.Error()) + } + data, _ := config.ZarfSchema.ReadFile("zarf.schema.json") + helpers.ValidateZarfSchema(p.cfg.Pkg, data) + + return nil +} diff --git a/src/pkg/utils/helpers/validate_schema.go b/src/pkg/utils/helpers/validate_schema.go index 14ba0ef9ef..03e1a34d15 100644 --- a/src/pkg/utils/helpers/validate_schema.go +++ b/src/pkg/utils/helpers/validate_schema.go @@ -3,18 +3,11 @@ package helpers import ( "bytes" "encoding/json" - "log" - "os" "github.com/santhosh-tekuri/jsonschema" ) -func ValidateZarfSchema(zarfUnmarshaledYaml interface{}, jsonFile string) bool { - - zarfSchema, err := os.ReadFile(jsonFile) - if err != nil { - log.Fatalf("Error reading file: %s", err) - } +func ValidateZarfSchema(zarfUnmarshaledYaml interface{}, zarfJsonSchema []byte) bool { zarfYamlAsJsonBytes, err := json.Marshal(zarfUnmarshaledYaml) if err != nil { @@ -24,7 +17,7 @@ func ValidateZarfSchema(zarfUnmarshaledYaml interface{}, jsonFile string) bool { compiler := jsonschema.NewCompiler() inMemoryZarfSchema := "zarf.schema.json" - if err := compiler.AddResource(inMemoryZarfSchema, bytes.NewReader(zarfSchema)); err != nil { + if err := compiler.AddResource(inMemoryZarfSchema, bytes.NewReader(zarfJsonSchema)); err != nil { panic(err) } schema, err := compiler.Compile(inMemoryZarfSchema) diff --git a/src/pkg/utils/helpers/validate_schema_test.go b/src/pkg/utils/helpers/validate_schema_test.go index e2b62bf77f..cbfbe0d8ea 100644 --- a/src/pkg/utils/helpers/validate_schema_test.go +++ b/src/pkg/utils/helpers/validate_schema_test.go @@ -1,6 +1,7 @@ package helpers import ( + "log" "os" "testing" @@ -12,7 +13,14 @@ func TestValidateZarfSchema(t *testing.T) { want := true var unmarshalledYaml interface{} readYaml("../../../../zarf.yaml", &unmarshalledYaml) - if got := ValidateZarfSchema(unmarshalledYaml, "../../../../zarf.schema.json"); got != want { + zarfSchema, err := os.ReadFile("../../../../zarf.schema.json") + if err != nil { + log.Fatalf("Error reading file: %s", err) + } + if err != nil { + panic(err) + } + if got := ValidateZarfSchema(unmarshalledYaml, zarfSchema); got != want { t.Errorf("ValidateZarfSchema = %v, want %v", got, want) } }) From ccf26adeff4d96894262682f72c0cb04d7898c95 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Tue, 17 Oct 2023 13:12:51 +0000 Subject: [PATCH 006/192] wip --- src/cmd/prepare.go | 13 ++++++---- src/pkg/packager/lint.go | 2 +- src/pkg/utils/helpers/example-schema.json | 31 ----------------------- src/pkg/utils/helpers/example.yaml | 6 ----- src/pkg/utils/helpers/validate_schema.go | 3 +++ 5 files changed, 12 insertions(+), 43 deletions(-) delete mode 100644 src/pkg/utils/helpers/example-schema.json delete mode 100644 src/pkg/utils/helpers/example.yaml diff --git a/src/cmd/prepare.go b/src/cmd/prepare.go index e028af5f0d..4db581a4ac 100644 --- a/src/cmd/prepare.go +++ b/src/cmd/prepare.go @@ -158,16 +158,19 @@ var prepareGenerateConfigFile = &cobra.Command{ } var prepareValidateSchema = &cobra.Command{ - Use: "lint [ FILENAME ] [ FILENAME ]", - Args: cobra.MaximumNArgs(2), + Use: "lint ", + Args: cobra.MaximumNArgs(0), Short: lang.CmdPrepareGenerateConfigShort, Long: lang.CmdPrepareGenerateConfigLong, Run: func(cmd *cobra.Command, args []string) { - + cwd, err := os.Getwd() + if err != nil { + message.Fatalf(err, lang.CmdPrepareFindImagesErr, err.Error()) + } + pkgConfig.CreateOpts.BaseDir = cwd pkgClient := packager.NewOrDie(&pkgConfig) defer pkgClient.ClearTempPaths() - // Need to find where we load in the file - //helpers.ValidateZarfSchema() + pkgClient.Lint() }, } diff --git a/src/pkg/packager/lint.go b/src/pkg/packager/lint.go index 5e3b708d4c..f613e6f5f6 100644 --- a/src/pkg/packager/lint.go +++ b/src/pkg/packager/lint.go @@ -19,7 +19,7 @@ func (p *Packager) Lint() (err error) { if err = p.readZarfYAML(filepath.Join(p.cfg.CreateOpts.BaseDir, layout.ZarfYAML)); err != nil { return fmt.Errorf("unable to read the zarf.yaml file: %s", err.Error()) } - data, _ := config.ZarfSchema.ReadFile("zarf.schema.json") + data, _ := config.GetSchemaFile() helpers.ValidateZarfSchema(p.cfg.Pkg, data) return nil diff --git a/src/pkg/utils/helpers/example-schema.json b/src/pkg/utils/helpers/example-schema.json deleted file mode 100644 index 513869f972..0000000000 --- a/src/pkg/utils/helpers/example-schema.json +++ /dev/null @@ -1,31 +0,0 @@ -{ - "$id": "https://example.com/product.schema.json", - "title": "Product", - "description": "A product from Acme's catalog", - "type": "object", - "properties": { - "productId": { - "description": "The unique identifier for a product", - "type": "integer" - }, - "productName": { - "description": "Name of the product", - "type": "string" - }, - "price": { - "description": "The price of the product", - "type": "number", - "exclusiveMinimum": 0 - }, - "tags": { - "description": "Tags for the product", - "type": "array", - "items": { - "type": "string" - }, - "minItems": 1, - "uniqueItems": true - } - }, - "required": [ "productId", "productName", "price" ] -} diff --git a/src/pkg/utils/helpers/example.yaml b/src/pkg/utils/helpers/example.yaml deleted file mode 100644 index 5d19224c8c..0000000000 --- a/src/pkg/utils/helpers/example.yaml +++ /dev/null @@ -1,6 +0,0 @@ -productId: 1 -productName: A green door -price: 12.50 -tags: -- home -- green diff --git a/src/pkg/utils/helpers/validate_schema.go b/src/pkg/utils/helpers/validate_schema.go index 03e1a34d15..4e884b2759 100644 --- a/src/pkg/utils/helpers/validate_schema.go +++ b/src/pkg/utils/helpers/validate_schema.go @@ -3,6 +3,7 @@ package helpers import ( "bytes" "encoding/json" + "fmt" "github.com/santhosh-tekuri/jsonschema" ) @@ -16,6 +17,7 @@ func ValidateZarfSchema(zarfUnmarshaledYaml interface{}, zarfJsonSchema []byte) compiler := jsonschema.NewCompiler() inMemoryZarfSchema := "zarf.schema.json" + fmt.Println(string(zarfYamlAsJsonBytes)) if err := compiler.AddResource(inMemoryZarfSchema, bytes.NewReader(zarfJsonSchema)); err != nil { panic(err) @@ -27,5 +29,6 @@ func ValidateZarfSchema(zarfUnmarshaledYaml interface{}, zarfJsonSchema []byte) if err := schema.Validate(bytes.NewReader(zarfYamlAsJsonBytes)); err != nil { panic(err) } + fmt.Println("validation succesful") return true } From 610c088da2e29bcad7458fc322497a0ab0e249b9 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Thu, 19 Oct 2023 15:00:46 +0000 Subject: [PATCH 007/192] WIP have test working with sucessful and unsuccsful example --- src/cmd/prepare.go | 16 +- src/pkg/packager/lint.go | 2 + .../helpers/successful_validation/zarf.yaml | 43 ++++ src/pkg/utils/helpers/validate_schema.go | 16 +- src/pkg/utils/helpers/validate_schema_test.go | 21 +- src/test/e2e/09_lint_test.go | 17 ++ src/test/packages/09-test-lint/README.md | 1 + src/test/packages/09-test-lint/zarf.json | 216 ++++++++++++++++++ src/test/packages/09-test-lint/zarf.yaml | 9 + 9 files changed, 320 insertions(+), 21 deletions(-) create mode 100644 src/pkg/utils/helpers/successful_validation/zarf.yaml create mode 100644 src/test/e2e/09_lint_test.go create mode 100644 src/test/packages/09-test-lint/README.md create mode 100644 src/test/packages/09-test-lint/zarf.json create mode 100644 src/test/packages/09-test-lint/zarf.yaml diff --git a/src/cmd/prepare.go b/src/cmd/prepare.go index 4db581a4ac..f46b3fcab4 100644 --- a/src/cmd/prepare.go +++ b/src/cmd/prepare.go @@ -158,16 +158,20 @@ var prepareGenerateConfigFile = &cobra.Command{ } var prepareValidateSchema = &cobra.Command{ - Use: "lint ", - Args: cobra.MaximumNArgs(0), + Use: "lint [ DIRECTORY ]", + Args: cobra.MaximumNArgs(1), Short: lang.CmdPrepareGenerateConfigShort, Long: lang.CmdPrepareGenerateConfigLong, Run: func(cmd *cobra.Command, args []string) { - cwd, err := os.Getwd() - if err != nil { - message.Fatalf(err, lang.CmdPrepareFindImagesErr, err.Error()) + if len(args) > 0 { + pkgConfig.CreateOpts.BaseDir = args[0] + } else { + var err error + pkgConfig.CreateOpts.BaseDir, err = os.Getwd() + if err != nil { + message.Fatalf(err, lang.CmdPackageCreateErr, err.Error()) + } } - pkgConfig.CreateOpts.BaseDir = cwd pkgClient := packager.NewOrDie(&pkgConfig) defer pkgClient.ClearTempPaths() pkgClient.Lint() diff --git a/src/pkg/packager/lint.go b/src/pkg/packager/lint.go index f613e6f5f6..79b0656145 100644 --- a/src/pkg/packager/lint.go +++ b/src/pkg/packager/lint.go @@ -19,7 +19,9 @@ func (p *Packager) Lint() (err error) { if err = p.readZarfYAML(filepath.Join(p.cfg.CreateOpts.BaseDir, layout.ZarfYAML)); err != nil { return fmt.Errorf("unable to read the zarf.yaml file: %s", err.Error()) } + fmt.Printf("This is the struct %+v\n", p.cfg.Pkg) data, _ := config.GetSchemaFile() + helpers.ValidateZarfSchema(p.cfg.Pkg, data) return nil diff --git a/src/pkg/utils/helpers/successful_validation/zarf.yaml b/src/pkg/utils/helpers/successful_validation/zarf.yaml new file mode 100644 index 0000000000..859813fde5 --- /dev/null +++ b/src/pkg/utils/helpers/successful_validation/zarf.yaml @@ -0,0 +1,43 @@ +kind: ZarfInitConfig +metadata: + name: init + description: Used to establish a new Zarf cluster + +components: + - name: k3s + import: + path: packages/distros/k3s + + # This package moves the injector & registries binaries + - name: zarf-injector + required: true + import: + path: packages/zarf-registry + + # Creates the temporary seed-registry + - name: zarf-seed-registry + required: true + import: + path: packages/zarf-registry + + # Creates the permanent registry + - name: zarf-registry + required: true + import: + path: packages/zarf-registry + + # Creates the pod+git mutating webhook + - name: zarf-agent + required: true + import: + path: packages/zarf-agent + + # (Optional) Adds logging to the cluster + - name: logging + import: + path: packages/logging-pgl + + # (Optional) Adds a git server to the cluster + - name: git-server + import: + path: packages/gitea diff --git a/src/pkg/utils/helpers/validate_schema.go b/src/pkg/utils/helpers/validate_schema.go index 4e884b2759..355c06642c 100644 --- a/src/pkg/utils/helpers/validate_schema.go +++ b/src/pkg/utils/helpers/validate_schema.go @@ -3,32 +3,30 @@ package helpers import ( "bytes" "encoding/json" - "fmt" "github.com/santhosh-tekuri/jsonschema" ) -func ValidateZarfSchema(zarfUnmarshaledYaml interface{}, zarfJsonSchema []byte) bool { +func ValidateZarfSchema(zarfUnmarshaledYaml interface{}, zarfJsonSchema []byte) error { zarfYamlAsJsonBytes, err := json.Marshal(zarfUnmarshaledYaml) if err != nil { - panic(err) + return err } compiler := jsonschema.NewCompiler() inMemoryZarfSchema := "zarf.schema.json" - fmt.Println(string(zarfYamlAsJsonBytes)) if err := compiler.AddResource(inMemoryZarfSchema, bytes.NewReader(zarfJsonSchema)); err != nil { - panic(err) + return err } schema, err := compiler.Compile(inMemoryZarfSchema) if err != nil { - panic(err) + return err } if err := schema.Validate(bytes.NewReader(zarfYamlAsJsonBytes)); err != nil { - panic(err) + return err } - fmt.Println("validation succesful") - return true + + return nil } diff --git a/src/pkg/utils/helpers/validate_schema_test.go b/src/pkg/utils/helpers/validate_schema_test.go index cbfbe0d8ea..8d7d1736d8 100644 --- a/src/pkg/utils/helpers/validate_schema_test.go +++ b/src/pkg/utils/helpers/validate_schema_test.go @@ -9,19 +9,28 @@ import ( ) func TestValidateZarfSchema(t *testing.T) { - t.Run("basic read schema", func(t *testing.T) { - want := true + t.Run("Read schema success", func(t *testing.T) { var unmarshalledYaml interface{} - readYaml("../../../../zarf.yaml", &unmarshalledYaml) + readYaml("successful_validation/zarf.yaml", &unmarshalledYaml) zarfSchema, err := os.ReadFile("../../../../zarf.schema.json") if err != nil { log.Fatalf("Error reading file: %s", err) } + if got := ValidateZarfSchema(unmarshalledYaml, zarfSchema); got != nil { + t.Errorf("ValidateZarfSchema = %v, want %v", got, nil) + } + }) + + t.Run("Read schema fail", func(t *testing.T) { + var unmarshalledYaml interface{} + readYaml("unsuccessful_validation/bad_zarf.yaml", &unmarshalledYaml) + zarfSchema, err := os.ReadFile("../../../../zarf.schema.json") if err != nil { - panic(err) + log.Fatalf("Error reading file: %s", err) } - if got := ValidateZarfSchema(unmarshalledYaml, zarfSchema); got != want { - t.Errorf("ValidateZarfSchema = %v, want %v", got, want) + err = ValidateZarfSchema(unmarshalledYaml, zarfSchema) + if err == nil { + t.Errorf("ValidateZarfSchema worked on a bad file") } }) } diff --git a/src/test/e2e/09_lint_test.go b/src/test/e2e/09_lint_test.go new file mode 100644 index 0000000000..e16ba396c5 --- /dev/null +++ b/src/test/e2e/09_lint_test.go @@ -0,0 +1,17 @@ +package test + +import ( + "path/filepath" + "testing" +) + +func TestLint(t *testing.T) { + t.Log("E2E: Lint") + + t.Run("zarf test lint", func(t *testing.T) { + t.Log("E2E: Test Differential Package Behavior") + + zarfYaml := filepath.Join("src", "test", "packages", "09-lint") + e2e.Zarf("prepare", "lint", zarfYaml) + }) +} diff --git a/src/test/packages/09-test-lint/README.md b/src/test/packages/09-test-lint/README.md new file mode 100644 index 0000000000..e93d9a79da --- /dev/null +++ b/src/test/packages/09-test-lint/README.md @@ -0,0 +1 @@ +# Lint test Package diff --git a/src/test/packages/09-test-lint/zarf.json b/src/test/packages/09-test-lint/zarf.json new file mode 100644 index 0000000000..fcb0df33db --- /dev/null +++ b/src/test/packages/09-test-lint/zarf.json @@ -0,0 +1,216 @@ +{ + "kind": "ZarfInitConfig", + "metadata": { + "name": "init", + "description": "Used to establish a new Zarf cluster" + }, + "build": { + "terminal": "", + "user": "", + "architecture": "", + "timestamp": "", + "version": "" + }, + "components": [ + { + "name": "k3s", + "only": { + "cluster": {} + }, + "import": { + "path": "packages/distros/k3s" + }, + "scripts": {}, + "extensions": {}, + "actions": { + "onCreate": { + "defaults": { + "shell": {} + } + }, + "onDeploy": { + "defaults": { + "shell": {} + } + }, + "onRemove": { + "defaults": { + "shell": {} + } + } + } + }, + { + "name": "zarf-injector", + "required": true, + "only": { + "cluster": {} + }, + "import": { + "path": "packages/zarf-registry" + }, + "scripts": {}, + "extensions": {}, + "actions": { + "onCreate": { + "defaults": { + "shell": {} + } + }, + "onDeploy": { + "defaults": { + "shell": {} + } + }, + "onRemove": { + "defaults": { + "shell": {} + } + } + } + }, + { + "name": "zarf-seed-registry", + "required": true, + "only": { + "cluster": {} + }, + "import": { + "path": "packages/zarf-registry" + }, + "scripts": {}, + "extensions": {}, + "actions": { + "onCreate": { + "defaults": { + "shell": {} + } + }, + "onDeploy": { + "defaults": { + "shell": {} + } + }, + "onRemove": { + "defaults": { + "shell": {} + } + } + } + }, + { + "name": "zarf-registry", + "required": true, + "only": { + "cluster": {} + }, + "import": { + "path": "packages/zarf-registry" + }, + "scripts": {}, + "extensions": {}, + "actions": { + "onCreate": { + "defaults": { + "shell": {} + } + }, + "onDeploy": { + "defaults": { + "shell": {} + } + }, + "onRemove": { + "defaults": { + "shell": {} + } + } + } + }, + { + "name": "zarf-agent", + "required": true, + "only": { + "cluster": {} + }, + "import": { + "path": "packages/zarf-agent" + }, + "scripts": {}, + "extensions": {}, + "actions": { + "onCreate": { + "defaults": { + "shell": {} + } + }, + "onDeploy": { + "defaults": { + "shell": {} + } + }, + "onRemove": { + "defaults": { + "shell": {} + } + } + } + }, + { + "name": "logging", + "only": { + "cluster": {} + }, + "import": { + "path": "packages/logging-pgl" + }, + "scripts": {}, + "extensions": {}, + "actions": { + "onCreate": { + "defaults": { + "shell": {} + } + }, + "onDeploy": { + "defaults": { + "shell": {} + } + }, + "onRemove": { + "defaults": { + "shell": {} + } + } + } + }, + { + "name": "git-server", + "only": { + "cluster": {} + }, + "import": { + "path": "packages/gitea" + }, + "scripts": {}, + "extensions": {}, + "actions": { + "onCreate": { + "defaults": { + "shell": {} + } + }, + "onDeploy": { + "defaults": { + "shell": {} + } + }, + "onRemove": { + "defaults": { + "shell": {} + } + } + } + } + ] +} diff --git a/src/test/packages/09-test-lint/zarf.yaml b/src/test/packages/09-test-lint/zarf.yaml new file mode 100644 index 0000000000..53bef86117 --- /dev/null +++ b/src/test/packages/09-test-lint/zarf.yaml @@ -0,0 +1,9 @@ +kind: ZarfInitConfig +metadata: + name: init + description: Used to establish a new Zarf cluster + +components: + - name: k3s + import: + path: packages/distros/k3s From db1c19ab467cf4b27125dc4910e47b1322e35b4e Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Thu, 19 Oct 2023 15:07:18 +0000 Subject: [PATCH 008/192] adding bad zarf file for testing --- .pre-commit-config.yaml | 2 + .../helpers/unsuccessful_validation/zarf.yaml | 43 +++++++++++++++++++ 2 files changed, 45 insertions(+) create mode 100644 src/pkg/utils/helpers/unsuccessful_validation/zarf.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7028b9df9c..81e7aafc71 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -47,3 +47,5 @@ repos: files: "zarf.yaml" types: [yaml] args: ["--schemafile", "zarf.schema.json"] + exclude: | + (?x)^(src/pkg/utils/helpers/unsuccessful_validation/zarf.yaml) diff --git a/src/pkg/utils/helpers/unsuccessful_validation/zarf.yaml b/src/pkg/utils/helpers/unsuccessful_validation/zarf.yaml new file mode 100644 index 0000000000..7efc69a9a8 --- /dev/null +++ b/src/pkg/utils/helpers/unsuccessful_validation/zarf.yaml @@ -0,0 +1,43 @@ +kind: ZarfInitConfig +metadata: + name: init + description: Used to establish a new Zarf cluster + +components: + - name: k3s + import: + pa324234th: packages/distros/k3s + + # This package moves the injector & registries binaries + - name: zarf-injector + required: true + import: + path: packages/zarf-registry + + # Creates the temporary seed-registry + - name: zarf-seed-registry + required: true + import: + path: packages/zarf-registry + + # Creates the permanent registry + - name: zarf-registry + required: true + import: + path: packages/zarf-registry + + # Creates the pod+git mutating webhook + - name: zarf-agent + required: true + import: + path: packages/zarf-agent + + # (Optional) Adds logging to the cluster + - name: logging + import: + path: packages/logging-pgl + + # (Optional) Adds a git server to the cluster + - name: git-server + import: + path: packages/gitea From faebd0a03d1eff09b7fb975254e078c2c752446c Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Thu, 19 Oct 2023 17:46:47 +0000 Subject: [PATCH 009/192] WIP --- .pre-commit-config.yaml | 5 +- src/pkg/packager/lint.go | 10 +- src/pkg/utils/helpers/validate_schema.go | 3 + src/test/e2e/09_lint_test.go | 2 +- src/test/packages/09-test-lint/zarf.json | 216 ----------------------- src/test/packages/09-test-lint/zarf.yaml | 2 +- 6 files changed, 17 insertions(+), 221 deletions(-) delete mode 100644 src/test/packages/09-test-lint/zarf.json diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 81e7aafc71..230312d079 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -48,4 +48,7 @@ repos: types: [yaml] args: ["--schemafile", "zarf.schema.json"] exclude: | - (?x)^(src/pkg/utils/helpers/unsuccessful_validation/zarf.yaml) + (?x)^( + src/pkg/utils/helpers/unsuccessful_validation/zarf.yaml| + src/test/packages/09-test-lint/ + ) diff --git a/src/pkg/packager/lint.go b/src/pkg/packager/lint.go index 79b0656145..3d0ed0fa21 100644 --- a/src/pkg/packager/lint.go +++ b/src/pkg/packager/lint.go @@ -10,6 +10,7 @@ import ( "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/pkg/layout" + "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" ) @@ -20,9 +21,14 @@ func (p *Packager) Lint() (err error) { return fmt.Errorf("unable to read the zarf.yaml file: %s", err.Error()) } fmt.Printf("This is the struct %+v\n", p.cfg.Pkg) - data, _ := config.GetSchemaFile() + zarfSchema, _ := config.GetSchemaFile() + var zarfData interface{} - helpers.ValidateZarfSchema(p.cfg.Pkg, data) + if err := utils.ReadYaml(filepath.Join(p.cfg.CreateOpts.BaseDir, layout.ZarfYAML), &zarfData); err != nil { + return err + } + + helpers.ValidateZarfSchema(zarfData, zarfSchema) return nil } diff --git a/src/pkg/utils/helpers/validate_schema.go b/src/pkg/utils/helpers/validate_schema.go index 355c06642c..939aec1a87 100644 --- a/src/pkg/utils/helpers/validate_schema.go +++ b/src/pkg/utils/helpers/validate_schema.go @@ -3,6 +3,7 @@ package helpers import ( "bytes" "encoding/json" + "fmt" "github.com/santhosh-tekuri/jsonschema" ) @@ -16,6 +17,8 @@ func ValidateZarfSchema(zarfUnmarshaledYaml interface{}, zarfJsonSchema []byte) compiler := jsonschema.NewCompiler() inMemoryZarfSchema := "zarf.schema.json" + fmt.Println("we are here") + fmt.Println(zarfYamlAsJsonBytes) if err := compiler.AddResource(inMemoryZarfSchema, bytes.NewReader(zarfJsonSchema)); err != nil { return err diff --git a/src/test/e2e/09_lint_test.go b/src/test/e2e/09_lint_test.go index e16ba396c5..418de72ee6 100644 --- a/src/test/e2e/09_lint_test.go +++ b/src/test/e2e/09_lint_test.go @@ -9,7 +9,7 @@ func TestLint(t *testing.T) { t.Log("E2E: Lint") t.Run("zarf test lint", func(t *testing.T) { - t.Log("E2E: Test Differential Package Behavior") + t.Log("E2E: Test lint") zarfYaml := filepath.Join("src", "test", "packages", "09-lint") e2e.Zarf("prepare", "lint", zarfYaml) diff --git a/src/test/packages/09-test-lint/zarf.json b/src/test/packages/09-test-lint/zarf.json deleted file mode 100644 index fcb0df33db..0000000000 --- a/src/test/packages/09-test-lint/zarf.json +++ /dev/null @@ -1,216 +0,0 @@ -{ - "kind": "ZarfInitConfig", - "metadata": { - "name": "init", - "description": "Used to establish a new Zarf cluster" - }, - "build": { - "terminal": "", - "user": "", - "architecture": "", - "timestamp": "", - "version": "" - }, - "components": [ - { - "name": "k3s", - "only": { - "cluster": {} - }, - "import": { - "path": "packages/distros/k3s" - }, - "scripts": {}, - "extensions": {}, - "actions": { - "onCreate": { - "defaults": { - "shell": {} - } - }, - "onDeploy": { - "defaults": { - "shell": {} - } - }, - "onRemove": { - "defaults": { - "shell": {} - } - } - } - }, - { - "name": "zarf-injector", - "required": true, - "only": { - "cluster": {} - }, - "import": { - "path": "packages/zarf-registry" - }, - "scripts": {}, - "extensions": {}, - "actions": { - "onCreate": { - "defaults": { - "shell": {} - } - }, - "onDeploy": { - "defaults": { - "shell": {} - } - }, - "onRemove": { - "defaults": { - "shell": {} - } - } - } - }, - { - "name": "zarf-seed-registry", - "required": true, - "only": { - "cluster": {} - }, - "import": { - "path": "packages/zarf-registry" - }, - "scripts": {}, - "extensions": {}, - "actions": { - "onCreate": { - "defaults": { - "shell": {} - } - }, - "onDeploy": { - "defaults": { - "shell": {} - } - }, - "onRemove": { - "defaults": { - "shell": {} - } - } - } - }, - { - "name": "zarf-registry", - "required": true, - "only": { - "cluster": {} - }, - "import": { - "path": "packages/zarf-registry" - }, - "scripts": {}, - "extensions": {}, - "actions": { - "onCreate": { - "defaults": { - "shell": {} - } - }, - "onDeploy": { - "defaults": { - "shell": {} - } - }, - "onRemove": { - "defaults": { - "shell": {} - } - } - } - }, - { - "name": "zarf-agent", - "required": true, - "only": { - "cluster": {} - }, - "import": { - "path": "packages/zarf-agent" - }, - "scripts": {}, - "extensions": {}, - "actions": { - "onCreate": { - "defaults": { - "shell": {} - } - }, - "onDeploy": { - "defaults": { - "shell": {} - } - }, - "onRemove": { - "defaults": { - "shell": {} - } - } - } - }, - { - "name": "logging", - "only": { - "cluster": {} - }, - "import": { - "path": "packages/logging-pgl" - }, - "scripts": {}, - "extensions": {}, - "actions": { - "onCreate": { - "defaults": { - "shell": {} - } - }, - "onDeploy": { - "defaults": { - "shell": {} - } - }, - "onRemove": { - "defaults": { - "shell": {} - } - } - } - }, - { - "name": "git-server", - "only": { - "cluster": {} - }, - "import": { - "path": "packages/gitea" - }, - "scripts": {}, - "extensions": {}, - "actions": { - "onCreate": { - "defaults": { - "shell": {} - } - }, - "onDeploy": { - "defaults": { - "shell": {} - } - }, - "onRemove": { - "defaults": { - "shell": {} - } - } - } - } - ] -} diff --git a/src/test/packages/09-test-lint/zarf.yaml b/src/test/packages/09-test-lint/zarf.yaml index 53bef86117..f30a6b8f9a 100644 --- a/src/test/packages/09-test-lint/zarf.yaml +++ b/src/test/packages/09-test-lint/zarf.yaml @@ -6,4 +6,4 @@ metadata: components: - name: k3s import: - path: packages/distros/k3s + pat12312h: packages/distros/k3s From 063c4383f5c546e54a311eda13a8ced9314b04a3 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Thu, 19 Oct 2023 20:03:18 +0000 Subject: [PATCH 010/192] refactor schema unit tests --- src/cmd/prepare.go | 1 + src/pkg/packager/lint.go | 10 ++-- src/pkg/utils/helpers/validate_schema.go | 3 -- src/pkg/utils/helpers/validate_schema_test.go | 50 +++++++++---------- src/test/e2e/09_lint_test.go | 10 ++-- src/test/packages/09-test-lint/README.md | 1 - src/test/packages/09-test-lint/zarf.yaml | 9 ---- 7 files changed, 36 insertions(+), 48 deletions(-) delete mode 100644 src/test/packages/09-test-lint/README.md delete mode 100644 src/test/packages/09-test-lint/zarf.yaml diff --git a/src/cmd/prepare.go b/src/cmd/prepare.go index f46b3fcab4..5cdda72c06 100644 --- a/src/cmd/prepare.go +++ b/src/cmd/prepare.go @@ -175,6 +175,7 @@ var prepareValidateSchema = &cobra.Command{ pkgClient := packager.NewOrDie(&pkgConfig) defer pkgClient.ClearTempPaths() pkgClient.Lint() + }, } diff --git a/src/pkg/packager/lint.go b/src/pkg/packager/lint.go index 3d0ed0fa21..0785c38047 100644 --- a/src/pkg/packager/lint.go +++ b/src/pkg/packager/lint.go @@ -16,19 +16,15 @@ import ( // Create generates a Zarf package tarball for a given PackageConfig and optional base directory. func (p *Packager) Lint() (err error) { - - if err = p.readZarfYAML(filepath.Join(p.cfg.CreateOpts.BaseDir, layout.ZarfYAML)); err != nil { - return fmt.Errorf("unable to read the zarf.yaml file: %s", err.Error()) - } - fmt.Printf("This is the struct %+v\n", p.cfg.Pkg) zarfSchema, _ := config.GetSchemaFile() var zarfData interface{} - if err := utils.ReadYaml(filepath.Join(p.cfg.CreateOpts.BaseDir, layout.ZarfYAML), &zarfData); err != nil { return err } - helpers.ValidateZarfSchema(zarfData, zarfSchema) + if err = helpers.ValidateZarfSchema(zarfData, zarfSchema); err != nil { + return err + } return nil } diff --git a/src/pkg/utils/helpers/validate_schema.go b/src/pkg/utils/helpers/validate_schema.go index 939aec1a87..355c06642c 100644 --- a/src/pkg/utils/helpers/validate_schema.go +++ b/src/pkg/utils/helpers/validate_schema.go @@ -3,7 +3,6 @@ package helpers import ( "bytes" "encoding/json" - "fmt" "github.com/santhosh-tekuri/jsonschema" ) @@ -17,8 +16,6 @@ func ValidateZarfSchema(zarfUnmarshaledYaml interface{}, zarfJsonSchema []byte) compiler := jsonschema.NewCompiler() inMemoryZarfSchema := "zarf.schema.json" - fmt.Println("we are here") - fmt.Println(zarfYamlAsJsonBytes) if err := compiler.AddResource(inMemoryZarfSchema, bytes.NewReader(zarfJsonSchema)); err != nil { return err diff --git a/src/pkg/utils/helpers/validate_schema_test.go b/src/pkg/utils/helpers/validate_schema_test.go index 8d7d1736d8..3000885a6f 100644 --- a/src/pkg/utils/helpers/validate_schema_test.go +++ b/src/pkg/utils/helpers/validate_schema_test.go @@ -1,7 +1,6 @@ package helpers import ( - "log" "os" "testing" @@ -9,38 +8,39 @@ import ( ) func TestValidateZarfSchema(t *testing.T) { - t.Run("Read schema success", func(t *testing.T) { - var unmarshalledYaml interface{} - readYaml("successful_validation/zarf.yaml", &unmarshalledYaml) + readSchema := func(t *testing.T) []byte { + t.Helper() zarfSchema, err := os.ReadFile("../../../../zarf.schema.json") if err != nil { - log.Fatalf("Error reading file: %s", err) + t.Fatalf("Error reading schema file: %s", err) } - if got := ValidateZarfSchema(unmarshalledYaml, zarfSchema); got != nil { - t.Errorf("ValidateZarfSchema = %v, want %v", got, nil) - } - }) + return zarfSchema + } - t.Run("Read schema fail", func(t *testing.T) { + readAndUnmarshalYaml := func(t *testing.T, path string) interface{} { + t.Helper() var unmarshalledYaml interface{} - readYaml("unsuccessful_validation/bad_zarf.yaml", &unmarshalledYaml) - zarfSchema, err := os.ReadFile("../../../../zarf.schema.json") + file, err := os.ReadFile(path) + goyaml.Unmarshal(file, &unmarshalledYaml) if err != nil { - log.Fatalf("Error reading file: %s", err) + return err } - err = ValidateZarfSchema(unmarshalledYaml, zarfSchema) - if err == nil { - t.Errorf("ValidateZarfSchema worked on a bad file") + return unmarshalledYaml + } + + t.Run("Read schema success", func(t *testing.T) { + unmarshalledYaml := readAndUnmarshalYaml(t, "successful_validation/zarf.yaml") + zarfSchema := readSchema(t) + if got := ValidateZarfSchema(unmarshalledYaml, zarfSchema); got != nil { + t.Errorf("Expected successful validation, got error: %v", got) } }) -} -func readYaml(path string, destConfig any) error { - file, err := os.ReadFile(path) - - if err != nil { - return err - } - - return goyaml.Unmarshal(file, destConfig) + t.Run("Read schema fail", func(t *testing.T) { + unmarshalledYaml := readAndUnmarshalYaml(t, "unsuccessful_validation/bad_zarf.yaml") + zarfSchema := readSchema(t) + if err := ValidateZarfSchema(unmarshalledYaml, zarfSchema); err == nil { + t.Errorf("Expected validation to fail, but it succeeded.") + } + }) } diff --git a/src/test/e2e/09_lint_test.go b/src/test/e2e/09_lint_test.go index 418de72ee6..d2666be4ba 100644 --- a/src/test/e2e/09_lint_test.go +++ b/src/test/e2e/09_lint_test.go @@ -1,7 +1,7 @@ package test import ( - "path/filepath" + "fmt" "testing" ) @@ -11,7 +11,11 @@ func TestLint(t *testing.T) { t.Run("zarf test lint", func(t *testing.T) { t.Log("E2E: Test lint") - zarfYaml := filepath.Join("src", "test", "packages", "09-lint") - e2e.Zarf("prepare", "lint", zarfYaml) + //zarfYaml := filepath.Join("src", "test", "packages", "09-lint") + path := "src/test/packages/09-lint" + stdOut, stdErr, err := e2e.Zarf("prepare", "lint", path) + fmt.Println("printing stdout", stdOut) + fmt.Println(stdErr) + fmt.Println(err) }) } diff --git a/src/test/packages/09-test-lint/README.md b/src/test/packages/09-test-lint/README.md deleted file mode 100644 index e93d9a79da..0000000000 --- a/src/test/packages/09-test-lint/README.md +++ /dev/null @@ -1 +0,0 @@ -# Lint test Package diff --git a/src/test/packages/09-test-lint/zarf.yaml b/src/test/packages/09-test-lint/zarf.yaml deleted file mode 100644 index f30a6b8f9a..0000000000 --- a/src/test/packages/09-test-lint/zarf.yaml +++ /dev/null @@ -1,9 +0,0 @@ -kind: ZarfInitConfig -metadata: - name: init - description: Used to establish a new Zarf cluster - -components: - - name: k3s - import: - pat12312h: packages/distros/k3s From 96efdb81e9b652a226eff937b1530b2233808405 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Thu, 19 Oct 2023 20:31:32 +0000 Subject: [PATCH 011/192] moving lint --- .pre-commit-config.yaml | 6 +++--- src/cmd/prepare.go | 4 +--- src/pkg/{packager => lint}/lint.go | 12 +++++------- .../helpers => lint}/successful_validation/zarf.yaml | 0 .../unsuccessful_validation/zarf.yaml | 0 src/pkg/{utils/helpers => lint}/validate_schema.go | 10 +++++----- .../{utils/helpers => lint}/validate_schema_test.go | 8 ++++---- src/test/packages/09-lint/README.md | 1 + src/test/packages/09-lint/zarf.yaml | 9 +++++++++ 9 files changed, 28 insertions(+), 22 deletions(-) rename src/pkg/{packager => lint}/lint.go (61%) rename src/pkg/{utils/helpers => lint}/successful_validation/zarf.yaml (100%) rename src/pkg/{utils/helpers => lint}/unsuccessful_validation/zarf.yaml (100%) rename src/pkg/{utils/helpers => lint}/validate_schema.go (63%) rename src/pkg/{utils/helpers => lint}/validate_schema_test.go (81%) create mode 100644 src/test/packages/09-lint/README.md create mode 100644 src/test/packages/09-lint/zarf.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 230312d079..2fb8ee0763 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -49,6 +49,6 @@ repos: args: ["--schemafile", "zarf.schema.json"] exclude: | (?x)^( - src/pkg/utils/helpers/unsuccessful_validation/zarf.yaml| - src/test/packages/09-test-lint/ - ) + src/pkg/lint/unsuccessful_validation/zarf.yaml| + src/test/packages/09-lint/zarf.yaml + )$ diff --git a/src/cmd/prepare.go b/src/cmd/prepare.go index 5cdda72c06..83fe2bab78 100644 --- a/src/cmd/prepare.go +++ b/src/cmd/prepare.go @@ -172,9 +172,7 @@ var prepareValidateSchema = &cobra.Command{ message.Fatalf(err, lang.CmdPackageCreateErr, err.Error()) } } - pkgClient := packager.NewOrDie(&pkgConfig) - defer pkgClient.ClearTempPaths() - pkgClient.Lint() + Lint.ValidateSchema() }, } diff --git a/src/pkg/packager/lint.go b/src/pkg/lint/lint.go similarity index 61% rename from src/pkg/packager/lint.go rename to src/pkg/lint/lint.go index 0785c38047..fa3a20bbcd 100644 --- a/src/pkg/packager/lint.go +++ b/src/pkg/lint/lint.go @@ -1,28 +1,26 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: 2021-Present The Zarf Authors -// Package packager contains functions for interacting with, managing and deploying Zarf packages. -package packager +// Package packager contains functions for interacting with, managing and deploying Zarf packages +package lint import ( - "fmt" "path/filepath" "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/utils" - "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" ) // Create generates a Zarf package tarball for a given PackageConfig and optional base directory. -func (p *Packager) Lint() (err error) { +func Lint(baseDir string) (err error) { zarfSchema, _ := config.GetSchemaFile() var zarfData interface{} - if err := utils.ReadYaml(filepath.Join(p.cfg.CreateOpts.BaseDir, layout.ZarfYAML), &zarfData); err != nil { + if err := utils.ReadYaml(filepath.Join(baseDir, layout.ZarfYAML), &zarfData); err != nil { return err } - if err = helpers.ValidateZarfSchema(zarfData, zarfSchema); err != nil { + if err = ValidateSchema(zarfData, zarfSchema); err != nil { return err } diff --git a/src/pkg/utils/helpers/successful_validation/zarf.yaml b/src/pkg/lint/successful_validation/zarf.yaml similarity index 100% rename from src/pkg/utils/helpers/successful_validation/zarf.yaml rename to src/pkg/lint/successful_validation/zarf.yaml diff --git a/src/pkg/utils/helpers/unsuccessful_validation/zarf.yaml b/src/pkg/lint/unsuccessful_validation/zarf.yaml similarity index 100% rename from src/pkg/utils/helpers/unsuccessful_validation/zarf.yaml rename to src/pkg/lint/unsuccessful_validation/zarf.yaml diff --git a/src/pkg/utils/helpers/validate_schema.go b/src/pkg/lint/validate_schema.go similarity index 63% rename from src/pkg/utils/helpers/validate_schema.go rename to src/pkg/lint/validate_schema.go index 355c06642c..24e4545e3b 100644 --- a/src/pkg/utils/helpers/validate_schema.go +++ b/src/pkg/lint/validate_schema.go @@ -1,4 +1,4 @@ -package helpers +package lint import ( "bytes" @@ -7,17 +7,17 @@ import ( "github.com/santhosh-tekuri/jsonschema" ) -func ValidateZarfSchema(zarfUnmarshaledYaml interface{}, zarfJsonSchema []byte) error { +func ValidateSchema(unmarshalledYaml interface{}, jsonSchema []byte) error { - zarfYamlAsJsonBytes, err := json.Marshal(zarfUnmarshaledYaml) + zarfYamlAsJsonBytes, err := json.Marshal(unmarshalledYaml) if err != nil { return err } compiler := jsonschema.NewCompiler() - inMemoryZarfSchema := "zarf.schema.json" + inMemoryZarfSchema := "schema.json" - if err := compiler.AddResource(inMemoryZarfSchema, bytes.NewReader(zarfJsonSchema)); err != nil { + if err := compiler.AddResource(inMemoryZarfSchema, bytes.NewReader(jsonSchema)); err != nil { return err } schema, err := compiler.Compile(inMemoryZarfSchema) diff --git a/src/pkg/utils/helpers/validate_schema_test.go b/src/pkg/lint/validate_schema_test.go similarity index 81% rename from src/pkg/utils/helpers/validate_schema_test.go rename to src/pkg/lint/validate_schema_test.go index 3000885a6f..a8b4c729a6 100644 --- a/src/pkg/utils/helpers/validate_schema_test.go +++ b/src/pkg/lint/validate_schema_test.go @@ -1,4 +1,4 @@ -package helpers +package lint import ( "os" @@ -10,7 +10,7 @@ import ( func TestValidateZarfSchema(t *testing.T) { readSchema := func(t *testing.T) []byte { t.Helper() - zarfSchema, err := os.ReadFile("../../../../zarf.schema.json") + zarfSchema, err := os.ReadFile("../../../zarf.schema.json") if err != nil { t.Fatalf("Error reading schema file: %s", err) } @@ -31,7 +31,7 @@ func TestValidateZarfSchema(t *testing.T) { t.Run("Read schema success", func(t *testing.T) { unmarshalledYaml := readAndUnmarshalYaml(t, "successful_validation/zarf.yaml") zarfSchema := readSchema(t) - if got := ValidateZarfSchema(unmarshalledYaml, zarfSchema); got != nil { + if got := ValidateSchema(unmarshalledYaml, zarfSchema); got != nil { t.Errorf("Expected successful validation, got error: %v", got) } }) @@ -39,7 +39,7 @@ func TestValidateZarfSchema(t *testing.T) { t.Run("Read schema fail", func(t *testing.T) { unmarshalledYaml := readAndUnmarshalYaml(t, "unsuccessful_validation/bad_zarf.yaml") zarfSchema := readSchema(t) - if err := ValidateZarfSchema(unmarshalledYaml, zarfSchema); err == nil { + if err := ValidateSchema(unmarshalledYaml, zarfSchema); err == nil { t.Errorf("Expected validation to fail, but it succeeded.") } }) diff --git a/src/test/packages/09-lint/README.md b/src/test/packages/09-lint/README.md new file mode 100644 index 0000000000..e93d9a79da --- /dev/null +++ b/src/test/packages/09-lint/README.md @@ -0,0 +1 @@ +# Lint test Package diff --git a/src/test/packages/09-lint/zarf.yaml b/src/test/packages/09-lint/zarf.yaml new file mode 100644 index 0000000000..f30a6b8f9a --- /dev/null +++ b/src/test/packages/09-lint/zarf.yaml @@ -0,0 +1,9 @@ +kind: ZarfInitConfig +metadata: + name: init + description: Used to establish a new Zarf cluster + +components: + - name: k3s + import: + pat12312h: packages/distros/k3s From 6485174e38319c32e876cfa4cd2d9b55c40d0e5b Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Thu, 19 Oct 2023 20:40:14 +0000 Subject: [PATCH 012/192] WIP --- src/cmd/prepare.go | 8 +++++--- src/pkg/lint/lint.go | 2 +- src/pkg/lint/validate_schema_test.go | 2 +- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/cmd/prepare.go b/src/cmd/prepare.go index 83fe2bab78..1894403ca7 100644 --- a/src/cmd/prepare.go +++ b/src/cmd/prepare.go @@ -14,6 +14,7 @@ import ( "github.com/defenseunicorns/zarf/src/cmd/common" "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/config/lang" + "github.com/defenseunicorns/zarf/src/pkg/lint" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/packager" "github.com/defenseunicorns/zarf/src/pkg/transform" @@ -163,16 +164,17 @@ var prepareValidateSchema = &cobra.Command{ Short: lang.CmdPrepareGenerateConfigShort, Long: lang.CmdPrepareGenerateConfigLong, Run: func(cmd *cobra.Command, args []string) { + baseDir := "" if len(args) > 0 { - pkgConfig.CreateOpts.BaseDir = args[0] + baseDir = args[0] } else { var err error - pkgConfig.CreateOpts.BaseDir, err = os.Getwd() + baseDir, err = os.Getwd() if err != nil { message.Fatalf(err, lang.CmdPackageCreateErr, err.Error()) } } - Lint.ValidateSchema() + lint.ValidateZarfSchema(baseDir) }, } diff --git a/src/pkg/lint/lint.go b/src/pkg/lint/lint.go index fa3a20bbcd..7c1cabf4db 100644 --- a/src/pkg/lint/lint.go +++ b/src/pkg/lint/lint.go @@ -13,7 +13,7 @@ import ( ) // Create generates a Zarf package tarball for a given PackageConfig and optional base directory. -func Lint(baseDir string) (err error) { +func ValidateZarfSchema(baseDir string) (err error) { zarfSchema, _ := config.GetSchemaFile() var zarfData interface{} if err := utils.ReadYaml(filepath.Join(baseDir, layout.ZarfYAML), &zarfData); err != nil { diff --git a/src/pkg/lint/validate_schema_test.go b/src/pkg/lint/validate_schema_test.go index a8b4c729a6..e457532315 100644 --- a/src/pkg/lint/validate_schema_test.go +++ b/src/pkg/lint/validate_schema_test.go @@ -7,7 +7,7 @@ import ( goyaml "github.com/goccy/go-yaml" ) -func TestValidateZarfSchema(t *testing.T) { +func TestValidateSchema(t *testing.T) { readSchema := func(t *testing.T) []byte { t.Helper() zarfSchema, err := os.ReadFile("../../../zarf.schema.json") From 93238ed21b269d6cede4215141ae8d56bdeae115 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Thu, 19 Oct 2023 20:49:27 +0000 Subject: [PATCH 013/192] WIP e2e tests functional --- src/cmd/prepare.go | 4 ++-- src/test/e2e/09_lint_test.go | 7 +++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/cmd/prepare.go b/src/cmd/prepare.go index 1894403ca7..6582b784cd 100644 --- a/src/cmd/prepare.go +++ b/src/cmd/prepare.go @@ -174,8 +174,8 @@ var prepareValidateSchema = &cobra.Command{ message.Fatalf(err, lang.CmdPackageCreateErr, err.Error()) } } - lint.ValidateZarfSchema(baseDir) - + err := lint.ValidateZarfSchema(baseDir) + message.Fatal(err, err.Error()) }, } diff --git a/src/test/e2e/09_lint_test.go b/src/test/e2e/09_lint_test.go index d2666be4ba..7f527f9181 100644 --- a/src/test/e2e/09_lint_test.go +++ b/src/test/e2e/09_lint_test.go @@ -2,7 +2,10 @@ package test import ( "fmt" + "path/filepath" "testing" + + "github.com/stretchr/testify/require" ) func TestLint(t *testing.T) { @@ -11,9 +14,9 @@ func TestLint(t *testing.T) { t.Run("zarf test lint", func(t *testing.T) { t.Log("E2E: Test lint") - //zarfYaml := filepath.Join("src", "test", "packages", "09-lint") - path := "src/test/packages/09-lint" + path := filepath.Join("src", "test", "packages", "09-lint") stdOut, stdErr, err := e2e.Zarf("prepare", "lint", path) + require.NoError(t, err, "We don't get an error") fmt.Println("printing stdout", stdOut) fmt.Println(stdErr) fmt.Println(err) From ad05c6b37184b9c49b182b8806c1e287556c80da Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Fri, 20 Oct 2023 14:19:17 +0000 Subject: [PATCH 014/192] fix test lint --- src/pkg/lint/validate_schema.go | 3 ++- src/test/e2e/09_lint_test.go | 10 +++------- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/src/pkg/lint/validate_schema.go b/src/pkg/lint/validate_schema.go index 24e4545e3b..f7fd8c5153 100644 --- a/src/pkg/lint/validate_schema.go +++ b/src/pkg/lint/validate_schema.go @@ -3,6 +3,7 @@ package lint import ( "bytes" "encoding/json" + "fmt" "github.com/santhosh-tekuri/jsonschema" ) @@ -25,7 +26,7 @@ func ValidateSchema(unmarshalledYaml interface{}, jsonSchema []byte) error { return err } if err := schema.Validate(bytes.NewReader(zarfYamlAsJsonBytes)); err != nil { - return err + return fmt.Errorf("this is an error I'm messing with %s", err) } return nil diff --git a/src/test/e2e/09_lint_test.go b/src/test/e2e/09_lint_test.go index 7f527f9181..bfb02a2c76 100644 --- a/src/test/e2e/09_lint_test.go +++ b/src/test/e2e/09_lint_test.go @@ -1,7 +1,6 @@ package test import ( - "fmt" "path/filepath" "testing" @@ -12,13 +11,10 @@ func TestLint(t *testing.T) { t.Log("E2E: Lint") t.Run("zarf test lint", func(t *testing.T) { - t.Log("E2E: Test lint") + t.Log("E2E: Test lint on schema") path := filepath.Join("src", "test", "packages", "09-lint") - stdOut, stdErr, err := e2e.Zarf("prepare", "lint", path) - require.NoError(t, err, "We don't get an error") - fmt.Println("printing stdout", stdOut) - fmt.Println(stdErr) - fmt.Println(err) + _, _, err := e2e.Zarf("prepare", "lint", path) + require.Error(t, err, "We should get an error here because the yaml file is bad") }) } From 5b285773ecbad86d7d353fcc0250d3fd9c9b994b Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Fri, 20 Oct 2023 14:33:02 +0000 Subject: [PATCH 015/192] lint tests working --- src/cmd/lint.go | 39 ++++++++++++++++++++++++++++++++++++ src/cmd/prepare.go | 23 --------------------- src/config/lang/english.go | 3 +++ src/test/e2e/09_lint_test.go | 2 +- 4 files changed, 43 insertions(+), 24 deletions(-) create mode 100644 src/cmd/lint.go diff --git a/src/cmd/lint.go b/src/cmd/lint.go new file mode 100644 index 0000000000..d1504ace23 --- /dev/null +++ b/src/cmd/lint.go @@ -0,0 +1,39 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package cmd contains the CLI commands for Zarf. +package cmd + +import ( + "os" + + "github.com/defenseunicorns/zarf/src/config/lang" + "github.com/defenseunicorns/zarf/src/pkg/lint" + "github.com/defenseunicorns/zarf/src/pkg/message" + "github.com/spf13/cobra" +) + +var lintCmd = &cobra.Command{ + Use: "lint [ DIRECTORY ]", + Args: cobra.MaximumNArgs(1), + Aliases: []string{"l"}, + Short: lang.CmdLintShort, + Run: func(cmd *cobra.Command, args []string) { + baseDir := "" + if len(args) > 0 { + baseDir = args[0] + } else { + var err error + baseDir, err = os.Getwd() + if err != nil { + message.Fatalf(err, lang.CmdPackageCreateErr, err.Error()) + } + } + err := lint.ValidateZarfSchema(baseDir) + message.Fatal(err, err.Error()) + }, +} + +func init() { + rootCmd.AddCommand(lintCmd) +} diff --git a/src/cmd/prepare.go b/src/cmd/prepare.go index 6582b784cd..f53dcdcae6 100644 --- a/src/cmd/prepare.go +++ b/src/cmd/prepare.go @@ -14,7 +14,6 @@ import ( "github.com/defenseunicorns/zarf/src/cmd/common" "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/config/lang" - "github.com/defenseunicorns/zarf/src/pkg/lint" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/packager" "github.com/defenseunicorns/zarf/src/pkg/transform" @@ -158,27 +157,6 @@ var prepareGenerateConfigFile = &cobra.Command{ }, } -var prepareValidateSchema = &cobra.Command{ - Use: "lint [ DIRECTORY ]", - Args: cobra.MaximumNArgs(1), - Short: lang.CmdPrepareGenerateConfigShort, - Long: lang.CmdPrepareGenerateConfigLong, - Run: func(cmd *cobra.Command, args []string) { - baseDir := "" - if len(args) > 0 { - baseDir = args[0] - } else { - var err error - baseDir, err = os.Getwd() - if err != nil { - message.Fatalf(err, lang.CmdPackageCreateErr, err.Error()) - } - } - err := lint.ValidateZarfSchema(baseDir) - message.Fatal(err, err.Error()) - }, -} - func init() { v := common.InitViper() @@ -187,7 +165,6 @@ func init() { prepareCmd.AddCommand(prepareComputeFileSha256sum) prepareCmd.AddCommand(prepareFindImages) prepareCmd.AddCommand(prepareGenerateConfigFile) - prepareCmd.AddCommand(prepareValidateSchema) prepareFindImages.Flags().StringVarP(&pkgConfig.FindImagesOpts.RepoHelmChartPath, "repo-chart-path", "p", "", lang.CmdPrepareFlagRepoChartPath) // use the package create config for this and reset it here to avoid overwriting the config.CreateOptions.SetVariables diff --git a/src/config/lang/english.go b/src/config/lang/english.go index 1642e605d1..9d30058643 100644 --- a/src/config/lang/english.go +++ b/src/config/lang/english.go @@ -302,6 +302,9 @@ const ( CmdPackageClusterSourceFallback = "%q does not satisfy any current sources, assuming it is a package deployed to a cluster" CmdPackageInvalidSource = "Unable to identify source from %q: %s" + //zarf lint + CmdLintShort = "Tools for validating schema / ensuring best practices" + // zarf prepare CmdPrepareShort = "Tools to help prepare assets for packaging" diff --git a/src/test/e2e/09_lint_test.go b/src/test/e2e/09_lint_test.go index bfb02a2c76..9d206e47a5 100644 --- a/src/test/e2e/09_lint_test.go +++ b/src/test/e2e/09_lint_test.go @@ -14,7 +14,7 @@ func TestLint(t *testing.T) { t.Log("E2E: Test lint on schema") path := filepath.Join("src", "test", "packages", "09-lint") - _, _, err := e2e.Zarf("prepare", "lint", path) + _, _, err := e2e.Zarf("lint", path) require.Error(t, err, "We should get an error here because the yaml file is bad") }) } From bb8eaf6670763ef280e71d9fd1d1f696a6f0244d Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Tue, 24 Oct 2023 18:40:15 +0000 Subject: [PATCH 016/192] adding tests, adding success mesage, taking away negative lookahead regex --- src/cmd/lint.go | 4 +++- src/pkg/lint/lint.go | 2 ++ src/pkg/lint/validate_schema.go | 2 +- src/test/e2e/09_lint_test.go | 14 +++++++++++--- src/test/packages/09-lint/README.md | 1 - src/test/packages/09-lint/zarf.yaml | 9 --------- src/types/component.go | 4 ++-- 7 files changed, 19 insertions(+), 17 deletions(-) delete mode 100644 src/test/packages/09-lint/README.md delete mode 100644 src/test/packages/09-lint/zarf.yaml diff --git a/src/cmd/lint.go b/src/cmd/lint.go index d1504ace23..13ee8385af 100644 --- a/src/cmd/lint.go +++ b/src/cmd/lint.go @@ -30,7 +30,9 @@ var lintCmd = &cobra.Command{ } } err := lint.ValidateZarfSchema(baseDir) - message.Fatal(err, err.Error()) + if err != nil { + message.Fatal(err, err.Error()) + } }, } diff --git a/src/pkg/lint/lint.go b/src/pkg/lint/lint.go index 7c1cabf4db..d0e2db596d 100644 --- a/src/pkg/lint/lint.go +++ b/src/pkg/lint/lint.go @@ -9,6 +9,7 @@ import ( "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/pkg/layout" + "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/utils" ) @@ -24,5 +25,6 @@ func ValidateZarfSchema(baseDir string) (err error) { return err } + message.Success("Validation successful") return nil } diff --git a/src/pkg/lint/validate_schema.go b/src/pkg/lint/validate_schema.go index f7fd8c5153..d8f869255c 100644 --- a/src/pkg/lint/validate_schema.go +++ b/src/pkg/lint/validate_schema.go @@ -26,7 +26,7 @@ func ValidateSchema(unmarshalledYaml interface{}, jsonSchema []byte) error { return err } if err := schema.Validate(bytes.NewReader(zarfYamlAsJsonBytes)); err != nil { - return fmt.Errorf("this is an error I'm messing with %s", err) + return fmt.Errorf("schema validation error: %s", err) } return nil diff --git a/src/test/e2e/09_lint_test.go b/src/test/e2e/09_lint_test.go index 9d206e47a5..1fdfade530 100644 --- a/src/test/e2e/09_lint_test.go +++ b/src/test/e2e/09_lint_test.go @@ -10,11 +10,19 @@ import ( func TestLint(t *testing.T) { t.Log("E2E: Lint") - t.Run("zarf test lint", func(t *testing.T) { + t.Run("zarf test lint fail", func(t *testing.T) { t.Log("E2E: Test lint on schema") - path := filepath.Join("src", "test", "packages", "09-lint") + path := filepath.Join("src", "test", "packages", "09-lint", "invalid_yaml") _, _, err := e2e.Zarf("lint", path) - require.Error(t, err, "We should get an error here because the yaml file is bad") + require.Error(t, err, "Expect error here because the yaml file is not following schema") + }) + + t.Run("zarf test lint success", func(t *testing.T) { + t.Log("E2E: Test lint on schema") + + path := filepath.Join("src", "test", "packages", "09-lint", "valid_yaml") + _, _, err := e2e.Zarf("lint", path) + require.NoError(t, err, "Expect no error here because the yaml file is following schema") }) } diff --git a/src/test/packages/09-lint/README.md b/src/test/packages/09-lint/README.md deleted file mode 100644 index e93d9a79da..0000000000 --- a/src/test/packages/09-lint/README.md +++ /dev/null @@ -1 +0,0 @@ -# Lint test Package diff --git a/src/test/packages/09-lint/zarf.yaml b/src/test/packages/09-lint/zarf.yaml deleted file mode 100644 index f30a6b8f9a..0000000000 --- a/src/test/packages/09-lint/zarf.yaml +++ /dev/null @@ -1,9 +0,0 @@ -kind: ZarfInitConfig -metadata: - name: init - description: Used to establish a new Zarf cluster - -components: - - name: k3s - import: - pat12312h: packages/distros/k3s diff --git a/src/types/component.go b/src/types/component.go index 89b8e00111..883a852637 100644 --- a/src/types/component.go +++ b/src/types/component.go @@ -217,9 +217,9 @@ type ZarfDataInjection struct { type ZarfComponentImport struct { ComponentName string `json:"name,omitempty" jsonschema:"description=The name of the component to import from the referenced zarf.yaml"` // For further explanation see https://regex101.com/r/nxX8vx/1 - Path string `json:"path,omitempty" jsonschema:"description=The relative path to a directory containing a zarf.yaml to import from,pattern=^(?!.*###ZARF_PKG_TMPL_).*$"` + Path string `json:"path,omitempty" jsonschema:"description=The relative path to a directory containing a zarf.yaml to import from"` // For further explanation see https://regex101.com/r/nxX8vx/1 - URL string `json:"url,omitempty" jsonschema:"description=[beta] The URL to a Zarf package to import via OCI,pattern=^oci://(?!.*###ZARF_PKG_TMPL_).*$"` + URL string `json:"url,omitempty" jsonschema:"description=[beta] The URL to a Zarf package to import via OCI,pattern=^oci://.*$"` } // IsEmpty returns if the components fields (other than the fields we were told to ignore) are empty or set to the types zero-value From e28be8e87131b2a3e0f20414ac90674dfee4db67 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Tue, 24 Oct 2023 18:55:02 +0000 Subject: [PATCH 017/192] add yamls for tests --- .pre-commit-config.yaml | 2 +- .../packages/09-lint/invalid_yaml/zarf.yaml | 9 ++++ .../packages/09-lint/valid_yaml/zarf.yaml | 43 +++++++++++++++++++ 3 files changed, 53 insertions(+), 1 deletion(-) create mode 100644 src/test/packages/09-lint/invalid_yaml/zarf.yaml create mode 100644 src/test/packages/09-lint/valid_yaml/zarf.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2fb8ee0763..9784951ef6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -50,5 +50,5 @@ repos: exclude: | (?x)^( src/pkg/lint/unsuccessful_validation/zarf.yaml| - src/test/packages/09-lint/zarf.yaml + src/test/packages/09-lint/.* )$ diff --git a/src/test/packages/09-lint/invalid_yaml/zarf.yaml b/src/test/packages/09-lint/invalid_yaml/zarf.yaml new file mode 100644 index 0000000000..f30a6b8f9a --- /dev/null +++ b/src/test/packages/09-lint/invalid_yaml/zarf.yaml @@ -0,0 +1,9 @@ +kind: ZarfInitConfig +metadata: + name: init + description: Used to establish a new Zarf cluster + +components: + - name: k3s + import: + pat12312h: packages/distros/k3s diff --git a/src/test/packages/09-lint/valid_yaml/zarf.yaml b/src/test/packages/09-lint/valid_yaml/zarf.yaml new file mode 100644 index 0000000000..859813fde5 --- /dev/null +++ b/src/test/packages/09-lint/valid_yaml/zarf.yaml @@ -0,0 +1,43 @@ +kind: ZarfInitConfig +metadata: + name: init + description: Used to establish a new Zarf cluster + +components: + - name: k3s + import: + path: packages/distros/k3s + + # This package moves the injector & registries binaries + - name: zarf-injector + required: true + import: + path: packages/zarf-registry + + # Creates the temporary seed-registry + - name: zarf-seed-registry + required: true + import: + path: packages/zarf-registry + + # Creates the permanent registry + - name: zarf-registry + required: true + import: + path: packages/zarf-registry + + # Creates the pod+git mutating webhook + - name: zarf-agent + required: true + import: + path: packages/zarf-agent + + # (Optional) Adds logging to the cluster + - name: logging + import: + path: packages/logging-pgl + + # (Optional) Adds a git server to the cluster + - name: git-server + import: + path: packages/gitea From e2a632da57a5839a41ee980187bce891077b9e84 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Wed, 1 Nov 2023 01:14:04 +0000 Subject: [PATCH 018/192] yaml extension partially tested --- .pre-commit-config.yaml | 1 + src/pkg/lint/validate_schema_test.go | 10 +++++++++- src/pkg/lint/yaml-extension/zarf.yaml | 24 ++++++++++++++++++++++++ src/test/e2e/09_lint_test.go | 1 + 4 files changed, 35 insertions(+), 1 deletion(-) create mode 100644 src/pkg/lint/yaml-extension/zarf.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9784951ef6..cf0e67c06b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -50,5 +50,6 @@ repos: exclude: | (?x)^( src/pkg/lint/unsuccessful_validation/zarf.yaml| + src/pkg/lint/yaml-extension/zarf.yaml| src/test/packages/09-lint/.* )$ diff --git a/src/pkg/lint/validate_schema_test.go b/src/pkg/lint/validate_schema_test.go index e457532315..e098d71896 100644 --- a/src/pkg/lint/validate_schema_test.go +++ b/src/pkg/lint/validate_schema_test.go @@ -37,10 +37,18 @@ func TestValidateSchema(t *testing.T) { }) t.Run("Read schema fail", func(t *testing.T) { - unmarshalledYaml := readAndUnmarshalYaml(t, "unsuccessful_validation/bad_zarf.yaml") + unmarshalledYaml := readAndUnmarshalYaml(t, "unsuccessful_validation/zarf.yaml") zarfSchema := readSchema(t) if err := ValidateSchema(unmarshalledYaml, zarfSchema); err == nil { t.Errorf("Expected validation to fail, but it succeeded.") } }) + + t.Run("Read schema yaml extension", func(t *testing.T) { + unmarshalledYaml := readAndUnmarshalYaml(t, "yaml-extension/zarf.yaml") + zarfSchema := readSchema(t) + if got := ValidateSchema(unmarshalledYaml, zarfSchema); got != nil { + t.Errorf("Expected successful validation, got error: %v", got) + } + }) } diff --git a/src/pkg/lint/yaml-extension/zarf.yaml b/src/pkg/lint/yaml-extension/zarf.yaml new file mode 100644 index 0000000000..c55c123e75 --- /dev/null +++ b/src/pkg/lint/yaml-extension/zarf.yaml @@ -0,0 +1,24 @@ +# Plan to have the schema or the lint code allow these soon +# x-env: &env +# environment: +# - CONFIG_KEY +# - EXAMPLE_KEY + +# x-version: &version +# version: 1.0.0 + +kind: ZarfPackageConfig +metadata: + name: hello + <<: &version 1.0.0 + +components: + - name: c1 + required: true + charts: + - name: redis + namespace: redis-system + version: *version + localPath: ./redis + images: + - &hello_image docker.io/library/redis:latest diff --git a/src/test/e2e/09_lint_test.go b/src/test/e2e/09_lint_test.go index 1fdfade530..49641c3750 100644 --- a/src/test/e2e/09_lint_test.go +++ b/src/test/e2e/09_lint_test.go @@ -16,6 +16,7 @@ func TestLint(t *testing.T) { path := filepath.Join("src", "test", "packages", "09-lint", "invalid_yaml") _, _, err := e2e.Zarf("lint", path) require.Error(t, err, "Expect error here because the yaml file is not following schema") + //Require contains to make sure the messaging is roughly how we want }) t.Run("zarf test lint success", func(t *testing.T) { From 09d1c07f3471096ac85cdd777ab5f8c4df792c3c Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Wed, 1 Nov 2023 01:28:10 +0000 Subject: [PATCH 019/192] moving lint back to prepare --- src/cmd/lint.go | 41 ------------------------------------ src/cmd/prepare.go | 25 ++++++++++++++++++++++ src/test/e2e/09_lint_test.go | 4 ++-- 3 files changed, 27 insertions(+), 43 deletions(-) delete mode 100644 src/cmd/lint.go diff --git a/src/cmd/lint.go b/src/cmd/lint.go deleted file mode 100644 index 13ee8385af..0000000000 --- a/src/cmd/lint.go +++ /dev/null @@ -1,41 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: 2021-Present The Zarf Authors - -// Package cmd contains the CLI commands for Zarf. -package cmd - -import ( - "os" - - "github.com/defenseunicorns/zarf/src/config/lang" - "github.com/defenseunicorns/zarf/src/pkg/lint" - "github.com/defenseunicorns/zarf/src/pkg/message" - "github.com/spf13/cobra" -) - -var lintCmd = &cobra.Command{ - Use: "lint [ DIRECTORY ]", - Args: cobra.MaximumNArgs(1), - Aliases: []string{"l"}, - Short: lang.CmdLintShort, - Run: func(cmd *cobra.Command, args []string) { - baseDir := "" - if len(args) > 0 { - baseDir = args[0] - } else { - var err error - baseDir, err = os.Getwd() - if err != nil { - message.Fatalf(err, lang.CmdPackageCreateErr, err.Error()) - } - } - err := lint.ValidateZarfSchema(baseDir) - if err != nil { - message.Fatal(err, err.Error()) - } - }, -} - -func init() { - rootCmd.AddCommand(lintCmd) -} diff --git a/src/cmd/prepare.go b/src/cmd/prepare.go index f53dcdcae6..c71f056938 100644 --- a/src/cmd/prepare.go +++ b/src/cmd/prepare.go @@ -14,6 +14,7 @@ import ( "github.com/defenseunicorns/zarf/src/cmd/common" "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/config/lang" + "github.com/defenseunicorns/zarf/src/pkg/lint" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/packager" "github.com/defenseunicorns/zarf/src/pkg/transform" @@ -157,6 +158,29 @@ var prepareGenerateConfigFile = &cobra.Command{ }, } +var lintCmd = &cobra.Command{ + Use: "lint [ DIRECTORY ]", + Args: cobra.MaximumNArgs(1), + Aliases: []string{"l"}, + Short: lang.CmdLintShort, + Run: func(cmd *cobra.Command, args []string) { + baseDir := "" + if len(args) > 0 { + baseDir = args[0] + } else { + var err error + baseDir, err = os.Getwd() + if err != nil { + message.Fatalf(err, lang.CmdPackageCreateErr, err.Error()) + } + } + err := lint.ValidateZarfSchema(baseDir) + if err != nil { + message.Fatal(err, err.Error()) + } + }, +} + func init() { v := common.InitViper() @@ -165,6 +189,7 @@ func init() { prepareCmd.AddCommand(prepareComputeFileSha256sum) prepareCmd.AddCommand(prepareFindImages) prepareCmd.AddCommand(prepareGenerateConfigFile) + prepareCmd.AddCommand(lintCmd) prepareFindImages.Flags().StringVarP(&pkgConfig.FindImagesOpts.RepoHelmChartPath, "repo-chart-path", "p", "", lang.CmdPrepareFlagRepoChartPath) // use the package create config for this and reset it here to avoid overwriting the config.CreateOptions.SetVariables diff --git a/src/test/e2e/09_lint_test.go b/src/test/e2e/09_lint_test.go index 49641c3750..4ac54f88ae 100644 --- a/src/test/e2e/09_lint_test.go +++ b/src/test/e2e/09_lint_test.go @@ -14,7 +14,7 @@ func TestLint(t *testing.T) { t.Log("E2E: Test lint on schema") path := filepath.Join("src", "test", "packages", "09-lint", "invalid_yaml") - _, _, err := e2e.Zarf("lint", path) + _, _, err := e2e.Zarf("prepare", "lint", path) require.Error(t, err, "Expect error here because the yaml file is not following schema") //Require contains to make sure the messaging is roughly how we want }) @@ -23,7 +23,7 @@ func TestLint(t *testing.T) { t.Log("E2E: Test lint on schema") path := filepath.Join("src", "test", "packages", "09-lint", "valid_yaml") - _, _, err := e2e.Zarf("lint", path) + _, _, err := e2e.Zarf("prepare", "lint", path) require.NoError(t, err, "Expect no error here because the yaml file is following schema") }) } From 61e8f60ad2d2ea2ad4790b7a225693d2b491c940 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Mon, 6 Nov 2023 17:10:24 +0000 Subject: [PATCH 020/192] WIP test working --- src/pkg/lint/lint.go | 16 ++++++--- .../lint/unsuccessful_validation/zarf.yaml | 34 ------------------- src/pkg/lint/validate_schema.go | 30 +++++++++++++++- src/pkg/lint/validate_schema_test.go | 26 ++++++++++++-- 4 files changed, 64 insertions(+), 42 deletions(-) diff --git a/src/pkg/lint/lint.go b/src/pkg/lint/lint.go index d0e2db596d..f545f92d28 100644 --- a/src/pkg/lint/lint.go +++ b/src/pkg/lint/lint.go @@ -7,7 +7,6 @@ package lint import ( "path/filepath" - "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/utils" @@ -15,15 +14,22 @@ import ( // Create generates a Zarf package tarball for a given PackageConfig and optional base directory. func ValidateZarfSchema(baseDir string) (err error) { - zarfSchema, _ := config.GetSchemaFile() + // zarfSchema, _ := config.GetSchemaFile() var zarfData interface{} if err := utils.ReadYaml(filepath.Join(baseDir, layout.ZarfYAML), &zarfData); err != nil { return err } - if err = ValidateSchema(zarfData, zarfSchema); err != nil { - return err - } + // result := ValidateSchema(zarfData, zarfSchema) + + // if result.Valid() { + // message.Success("The document is valid\n") + // } else { + // fmt.Printf("The document is not valid. see errors :\n") + // for _, desc := range result.Errors() { + // fmt.Printf("- %s\n", desc) + // } + // } message.Success("Validation successful") return nil diff --git a/src/pkg/lint/unsuccessful_validation/zarf.yaml b/src/pkg/lint/unsuccessful_validation/zarf.yaml index 7efc69a9a8..b46d66c4e0 100644 --- a/src/pkg/lint/unsuccessful_validation/zarf.yaml +++ b/src/pkg/lint/unsuccessful_validation/zarf.yaml @@ -7,37 +7,3 @@ components: - name: k3s import: pa324234th: packages/distros/k3s - - # This package moves the injector & registries binaries - - name: zarf-injector - required: true - import: - path: packages/zarf-registry - - # Creates the temporary seed-registry - - name: zarf-seed-registry - required: true - import: - path: packages/zarf-registry - - # Creates the permanent registry - - name: zarf-registry - required: true - import: - path: packages/zarf-registry - - # Creates the pod+git mutating webhook - - name: zarf-agent - required: true - import: - path: packages/zarf-agent - - # (Optional) Adds logging to the cluster - - name: logging - import: - path: packages/logging-pgl - - # (Optional) Adds a git server to the cluster - - name: git-server - import: - path: packages/gitea diff --git a/src/pkg/lint/validate_schema.go b/src/pkg/lint/validate_schema.go index d8f869255c..3bccfa3ffd 100644 --- a/src/pkg/lint/validate_schema.go +++ b/src/pkg/lint/validate_schema.go @@ -3,12 +3,15 @@ package lint import ( "bytes" "encoding/json" + "errors" "fmt" + "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/santhosh-tekuri/jsonschema" + "github.com/xeipuuv/gojsonschema" ) -func ValidateSchema(unmarshalledYaml interface{}, jsonSchema []byte) error { +func ValidateSchema2(unmarshalledYaml interface{}, jsonSchema []byte) error { zarfYamlAsJsonBytes, err := json.Marshal(unmarshalledYaml) if err != nil { @@ -31,3 +34,28 @@ func ValidateSchema(unmarshalledYaml interface{}, jsonSchema []byte) error { return nil } + +func ValidateSchema(unmarshalledYaml interface{}, jsonSchema []byte) error { + schemaLoader := gojsonschema.NewBytesLoader(jsonSchema) + documentLoader := gojsonschema.NewGoLoader(unmarshalledYaml) + + result, err := gojsonschema.Validate(schemaLoader, documentLoader) + if err != nil { + return err + } + + if result.Valid() { + message.Success("The document is valid\n") + } else { + errorMessage := "The document is not valid: " + for i, desc := range result.Errors() { + if i != 0 { + errorMessage = errorMessage + ", " + } + errorMessage = errorMessage + desc.String() + } + err = errors.New(errorMessage) + } + + return err +} diff --git a/src/pkg/lint/validate_schema_test.go b/src/pkg/lint/validate_schema_test.go index e098d71896..8c6bff2486 100644 --- a/src/pkg/lint/validate_schema_test.go +++ b/src/pkg/lint/validate_schema_test.go @@ -5,6 +5,7 @@ import ( "testing" goyaml "github.com/goccy/go-yaml" + "github.com/stretchr/testify/require" ) func TestValidateSchema(t *testing.T) { @@ -28,6 +29,16 @@ func TestValidateSchema(t *testing.T) { return unmarshalledYaml } + unmarshallStringYaml := func(t *testing.T, str string) interface{} { + t.Helper() + var unmarshalledYaml interface{} + err := goyaml.Unmarshal([]byte(str), &unmarshalledYaml) + if err != nil { + return err + } + return unmarshalledYaml + } + t.Run("Read schema success", func(t *testing.T) { unmarshalledYaml := readAndUnmarshalYaml(t, "successful_validation/zarf.yaml") zarfSchema := readSchema(t) @@ -37,9 +48,20 @@ func TestValidateSchema(t *testing.T) { }) t.Run("Read schema fail", func(t *testing.T) { - unmarshalledYaml := readAndUnmarshalYaml(t, "unsuccessful_validation/zarf.yaml") + var yamlContents = ` +kind: ZarfInitConfig +metadata: + name: init +components: + - name: k3s + import: + pa324234th: test +` + unmarshalledYaml := unmarshallStringYaml(t, yamlContents) zarfSchema := readSchema(t) - if err := ValidateSchema(unmarshalledYaml, zarfSchema); err == nil { + err := ValidateSchema(unmarshalledYaml, zarfSchema) + require.Contains(t, err.Error(), "components.0.import: Additional property pa324234th is not allowed") + if err == nil { t.Errorf("Expected validation to fail, but it succeeded.") } }) From 35c8e09f5464f6c989343ed71dbc930ace342e21 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Tue, 7 Nov 2023 13:47:49 +0000 Subject: [PATCH 021/192] lint feature is working --- src/pkg/lint/lint.go | 16 ++--- .../lint/unsuccessful_validation/zarf.yaml | 10 ++- src/pkg/lint/validate_schema.go | 14 ++--- src/pkg/lint/validate_schema_test.go | 62 ++++++++++++------- src/test/e2e/09_lint_test.go | 8 +-- 5 files changed, 60 insertions(+), 50 deletions(-) diff --git a/src/pkg/lint/lint.go b/src/pkg/lint/lint.go index f545f92d28..46a6d1b3eb 100644 --- a/src/pkg/lint/lint.go +++ b/src/pkg/lint/lint.go @@ -7,6 +7,7 @@ package lint import ( "path/filepath" + "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/utils" @@ -14,22 +15,17 @@ import ( // Create generates a Zarf package tarball for a given PackageConfig and optional base directory. func ValidateZarfSchema(baseDir string) (err error) { - // zarfSchema, _ := config.GetSchemaFile() + zarfSchema, _ := config.GetSchemaFile() var zarfData interface{} if err := utils.ReadYaml(filepath.Join(baseDir, layout.ZarfYAML), &zarfData); err != nil { return err } - // result := ValidateSchema(zarfData, zarfSchema) + err = ValidateSchema(zarfData, zarfSchema) - // if result.Valid() { - // message.Success("The document is valid\n") - // } else { - // fmt.Printf("The document is not valid. see errors :\n") - // for _, desc := range result.Errors() { - // fmt.Printf("- %s\n", desc) - // } - // } + if err != nil { + return err + } message.Success("Validation successful") return nil diff --git a/src/pkg/lint/unsuccessful_validation/zarf.yaml b/src/pkg/lint/unsuccessful_validation/zarf.yaml index b46d66c4e0..6ebb779c23 100644 --- a/src/pkg/lint/unsuccessful_validation/zarf.yaml +++ b/src/pkg/lint/unsuccessful_validation/zarf.yaml @@ -1,9 +1,13 @@ kind: ZarfInitConfig metadata: name: init - description: Used to establish a new Zarf cluster + description: Testing bad yaml components: - - name: k3s + - name: first-test-component import: - pa324234th: packages/distros/k3s + not-path: packages/distros/k3s + + - name: second-test-component + not-import: + path: packages/distros/k3s diff --git a/src/pkg/lint/validate_schema.go b/src/pkg/lint/validate_schema.go index 3bccfa3ffd..348007871d 100644 --- a/src/pkg/lint/validate_schema.go +++ b/src/pkg/lint/validate_schema.go @@ -6,7 +6,6 @@ import ( "errors" "fmt" - "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/santhosh-tekuri/jsonschema" "github.com/xeipuuv/gojsonschema" ) @@ -44,15 +43,10 @@ func ValidateSchema(unmarshalledYaml interface{}, jsonSchema []byte) error { return err } - if result.Valid() { - message.Success("The document is valid\n") - } else { - errorMessage := "The document is not valid: " - for i, desc := range result.Errors() { - if i != 0 { - errorMessage = errorMessage + ", " - } - errorMessage = errorMessage + desc.String() + if !result.Valid() { + errorMessage := "The document is not valid:" + for _, desc := range result.Errors() { + errorMessage = errorMessage + "\n - " + desc.String() } err = errors.New(errorMessage) } diff --git a/src/pkg/lint/validate_schema_test.go b/src/pkg/lint/validate_schema_test.go index 8c6bff2486..41ece0f9d0 100644 --- a/src/pkg/lint/validate_schema_test.go +++ b/src/pkg/lint/validate_schema_test.go @@ -29,46 +29,62 @@ func TestValidateSchema(t *testing.T) { return unmarshalledYaml } - unmarshallStringYaml := func(t *testing.T, str string) interface{} { - t.Helper() - var unmarshalledYaml interface{} - err := goyaml.Unmarshal([]byte(str), &unmarshalledYaml) - if err != nil { - return err - } - return unmarshalledYaml - } + // unmarshallStringYaml := func(t *testing.T, str string) (interface{}, error) { + // t.Helper() + // var unmarshalledYaml interface{} + // err := goyaml.Unmarshal([]byte(str), &unmarshalledYaml) + // return unmarshalledYaml, err + // } t.Run("Read schema success", func(t *testing.T) { unmarshalledYaml := readAndUnmarshalYaml(t, "successful_validation/zarf.yaml") zarfSchema := readSchema(t) + err := ValidateSchema(unmarshalledYaml, zarfSchema) + require.NoError(t, err) if got := ValidateSchema(unmarshalledYaml, zarfSchema); got != nil { t.Errorf("Expected successful validation, got error: %v", got) } }) t.Run("Read schema fail", func(t *testing.T) { - var yamlContents = ` -kind: ZarfInitConfig -metadata: - name: init -components: - - name: k3s - import: - pa324234th: test -` - unmarshalledYaml := unmarshallStringYaml(t, yamlContents) + // var yamlContents = ` + // kind: ZarfInitConfig + // metadata: + // name: init + // components: + // - name: k3s + // import: + // pa324234th: test + // - name: test + // import: + // pa3th: test + // ` + // unmarshalledYaml,_ := unmarshallStringYaml(t, yamlContents) zarfSchema := readSchema(t) + unmarshalledYaml := readAndUnmarshalYaml(t, "unsuccessful_validation/zarf.yaml") err := ValidateSchema(unmarshalledYaml, zarfSchema) - require.Contains(t, err.Error(), "components.0.import: Additional property pa324234th is not allowed") - if err == nil { - t.Errorf("Expected validation to fail, but it succeeded.") - } + errorMessage := `The document is not valid: + - components.0.import: Additional property not-path is not allowed + - components.1: Additional property not-import is not allowed` + require.EqualError(t, err, errorMessage) }) + // t.Run("bad yaml", func(t *testing.T) { + // var yamlContents = "unquoted_string_with_colons: key: value" + // unmarshalledYaml,err := unmarshallStringYaml(t, yamlContents) + // zarfSchema := readSchema(t) + // err = ValidateSchema(unmarshalledYaml, zarfSchema) + // require.Contains(t, err.Error(), "components.0.import: Additional property pa324234th is not allowed") + // if err == nil { + // t.Errorf("Expected validation to fail, but it succeeded.") + // } + // }) + t.Run("Read schema yaml extension", func(t *testing.T) { unmarshalledYaml := readAndUnmarshalYaml(t, "yaml-extension/zarf.yaml") zarfSchema := readSchema(t) + err := ValidateSchema(unmarshalledYaml, zarfSchema) + require.NoError(t, err) if got := ValidateSchema(unmarshalledYaml, zarfSchema); got != nil { t.Errorf("Expected successful validation, got error: %v", got) } diff --git a/src/test/e2e/09_lint_test.go b/src/test/e2e/09_lint_test.go index 4ac54f88ae..f7fa1da8c7 100644 --- a/src/test/e2e/09_lint_test.go +++ b/src/test/e2e/09_lint_test.go @@ -11,16 +11,16 @@ func TestLint(t *testing.T) { t.Log("E2E: Lint") t.Run("zarf test lint fail", func(t *testing.T) { - t.Log("E2E: Test lint on schema") + t.Log("E2E: Test lint on schema fail") path := filepath.Join("src", "test", "packages", "09-lint", "invalid_yaml") - _, _, err := e2e.Zarf("prepare", "lint", path) + _, stderr, err := e2e.Zarf("prepare", "lint", path) require.Error(t, err, "Expect error here because the yaml file is not following schema") - //Require contains to make sure the messaging is roughly how we want + require.Contains(t, stderr, "- components.0.import: Additional property pat12312h is not allowed") }) t.Run("zarf test lint success", func(t *testing.T) { - t.Log("E2E: Test lint on schema") + t.Log("E2E: Test lint on schema success") path := filepath.Join("src", "test", "packages", "09-lint", "valid_yaml") _, _, err := e2e.Zarf("prepare", "lint", path) From fa03f3529c6ca6faad8f3b35bf3ff8549c11999e Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Tue, 7 Nov 2023 13:48:13 +0000 Subject: [PATCH 022/192] refactoring --- src/pkg/lint/validate_schema.go | 28 ---------------------------- 1 file changed, 28 deletions(-) diff --git a/src/pkg/lint/validate_schema.go b/src/pkg/lint/validate_schema.go index 348007871d..a614400b2d 100644 --- a/src/pkg/lint/validate_schema.go +++ b/src/pkg/lint/validate_schema.go @@ -1,39 +1,11 @@ package lint import ( - "bytes" - "encoding/json" "errors" - "fmt" - "github.com/santhosh-tekuri/jsonschema" "github.com/xeipuuv/gojsonschema" ) -func ValidateSchema2(unmarshalledYaml interface{}, jsonSchema []byte) error { - - zarfYamlAsJsonBytes, err := json.Marshal(unmarshalledYaml) - if err != nil { - return err - } - - compiler := jsonschema.NewCompiler() - inMemoryZarfSchema := "schema.json" - - if err := compiler.AddResource(inMemoryZarfSchema, bytes.NewReader(jsonSchema)); err != nil { - return err - } - schema, err := compiler.Compile(inMemoryZarfSchema) - if err != nil { - return err - } - if err := schema.Validate(bytes.NewReader(zarfYamlAsJsonBytes)); err != nil { - return fmt.Errorf("schema validation error: %s", err) - } - - return nil -} - func ValidateSchema(unmarshalledYaml interface{}, jsonSchema []byte) error { schemaLoader := gojsonschema.NewBytesLoader(jsonSchema) documentLoader := gojsonschema.NewGoLoader(unmarshalledYaml) From a0c307dd09fbfa64581386589e66b681ecf0a0ce Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Tue, 7 Nov 2023 13:54:01 +0000 Subject: [PATCH 023/192] adding comment to exported function --- src/config/config.go | 1 + src/pkg/lint/lint.go | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/src/config/config.go b/src/config/config.go index b71950386a..f5d24b2e05 100644 --- a/src/config/config.go +++ b/src/config/config.go @@ -102,6 +102,7 @@ var ( ZarfDefaultCachePath = filepath.Join("~", ".zarf-cache") ) +// GetSchemaFile returns the zarf schema file func GetSchemaFile() ([]byte, error) { return ZarfSchema.ReadFile("zarf.schema.json") } diff --git a/src/pkg/lint/lint.go b/src/pkg/lint/lint.go index 46a6d1b3eb..00029836e6 100644 --- a/src/pkg/lint/lint.go +++ b/src/pkg/lint/lint.go @@ -7,7 +7,6 @@ package lint import ( "path/filepath" - "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/utils" From 3376a3eed0b9aed8d4dac1ad45ca0d0ee2226a12 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Tue, 7 Nov 2023 14:03:43 +0000 Subject: [PATCH 024/192] unexporting validate schema --- src/pkg/lint/lint.go | 5 +++-- src/pkg/lint/validate_schema.go | 2 +- src/pkg/lint/validate_schema_test.go | 10 +++++----- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/src/pkg/lint/lint.go b/src/pkg/lint/lint.go index 00029836e6..5f66e05a71 100644 --- a/src/pkg/lint/lint.go +++ b/src/pkg/lint/lint.go @@ -7,12 +7,13 @@ package lint import ( "path/filepath" + "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/utils" ) -// Create generates a Zarf package tarball for a given PackageConfig and optional base directory. +// Validates a zarf file against the zarf schema, returns an error if the file is invalid func ValidateZarfSchema(baseDir string) (err error) { zarfSchema, _ := config.GetSchemaFile() var zarfData interface{} @@ -20,7 +21,7 @@ func ValidateZarfSchema(baseDir string) (err error) { return err } - err = ValidateSchema(zarfData, zarfSchema) + err = validateSchema(zarfData, zarfSchema) if err != nil { return err diff --git a/src/pkg/lint/validate_schema.go b/src/pkg/lint/validate_schema.go index a614400b2d..9b2e99a60d 100644 --- a/src/pkg/lint/validate_schema.go +++ b/src/pkg/lint/validate_schema.go @@ -6,7 +6,7 @@ import ( "github.com/xeipuuv/gojsonschema" ) -func ValidateSchema(unmarshalledYaml interface{}, jsonSchema []byte) error { +func validateSchema(unmarshalledYaml interface{}, jsonSchema []byte) error { schemaLoader := gojsonschema.NewBytesLoader(jsonSchema) documentLoader := gojsonschema.NewGoLoader(unmarshalledYaml) diff --git a/src/pkg/lint/validate_schema_test.go b/src/pkg/lint/validate_schema_test.go index 41ece0f9d0..aeed602a0a 100644 --- a/src/pkg/lint/validate_schema_test.go +++ b/src/pkg/lint/validate_schema_test.go @@ -39,9 +39,9 @@ func TestValidateSchema(t *testing.T) { t.Run("Read schema success", func(t *testing.T) { unmarshalledYaml := readAndUnmarshalYaml(t, "successful_validation/zarf.yaml") zarfSchema := readSchema(t) - err := ValidateSchema(unmarshalledYaml, zarfSchema) + err := validateSchema(unmarshalledYaml, zarfSchema) require.NoError(t, err) - if got := ValidateSchema(unmarshalledYaml, zarfSchema); got != nil { + if got := validateSchema(unmarshalledYaml, zarfSchema); got != nil { t.Errorf("Expected successful validation, got error: %v", got) } }) @@ -62,7 +62,7 @@ func TestValidateSchema(t *testing.T) { // unmarshalledYaml,_ := unmarshallStringYaml(t, yamlContents) zarfSchema := readSchema(t) unmarshalledYaml := readAndUnmarshalYaml(t, "unsuccessful_validation/zarf.yaml") - err := ValidateSchema(unmarshalledYaml, zarfSchema) + err := validateSchema(unmarshalledYaml, zarfSchema) errorMessage := `The document is not valid: - components.0.import: Additional property not-path is not allowed - components.1: Additional property not-import is not allowed` @@ -83,9 +83,9 @@ func TestValidateSchema(t *testing.T) { t.Run("Read schema yaml extension", func(t *testing.T) { unmarshalledYaml := readAndUnmarshalYaml(t, "yaml-extension/zarf.yaml") zarfSchema := readSchema(t) - err := ValidateSchema(unmarshalledYaml, zarfSchema) + err := validateSchema(unmarshalledYaml, zarfSchema) require.NoError(t, err) - if got := ValidateSchema(unmarshalledYaml, zarfSchema); got != nil { + if got := validateSchema(unmarshalledYaml, zarfSchema); got != nil { t.Errorf("Expected successful validation, got error: %v", got) } }) From 73d4de80aa941fd4d3341f4cb278b798be9085cd Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Tue, 7 Nov 2023 14:11:12 +0000 Subject: [PATCH 025/192] moving lint to test 11 --- .pre-commit-config.yaml | 2 +- src/test/e2e/{09_lint_test.go => 11_lint_test.go} | 4 ++-- src/test/packages/{09-lint => 11-lint}/invalid_yaml/zarf.yaml | 0 src/test/packages/{09-lint => 11-lint}/valid_yaml/zarf.yaml | 0 4 files changed, 3 insertions(+), 3 deletions(-) rename src/test/e2e/{09_lint_test.go => 11_lint_test.go} (85%) rename src/test/packages/{09-lint => 11-lint}/invalid_yaml/zarf.yaml (100%) rename src/test/packages/{09-lint => 11-lint}/valid_yaml/zarf.yaml (100%) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index cf0e67c06b..ec3a9f6a58 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -51,5 +51,5 @@ repos: (?x)^( src/pkg/lint/unsuccessful_validation/zarf.yaml| src/pkg/lint/yaml-extension/zarf.yaml| - src/test/packages/09-lint/.* + src/test/packages/11-lint/.* )$ diff --git a/src/test/e2e/09_lint_test.go b/src/test/e2e/11_lint_test.go similarity index 85% rename from src/test/e2e/09_lint_test.go rename to src/test/e2e/11_lint_test.go index f7fa1da8c7..c4578a2442 100644 --- a/src/test/e2e/09_lint_test.go +++ b/src/test/e2e/11_lint_test.go @@ -13,7 +13,7 @@ func TestLint(t *testing.T) { t.Run("zarf test lint fail", func(t *testing.T) { t.Log("E2E: Test lint on schema fail") - path := filepath.Join("src", "test", "packages", "09-lint", "invalid_yaml") + path := filepath.Join("src", "test", "packages", "11-lint", "invalid_yaml") _, stderr, err := e2e.Zarf("prepare", "lint", path) require.Error(t, err, "Expect error here because the yaml file is not following schema") require.Contains(t, stderr, "- components.0.import: Additional property pat12312h is not allowed") @@ -22,7 +22,7 @@ func TestLint(t *testing.T) { t.Run("zarf test lint success", func(t *testing.T) { t.Log("E2E: Test lint on schema success") - path := filepath.Join("src", "test", "packages", "09-lint", "valid_yaml") + path := filepath.Join("src", "test", "packages", "11-lint", "valid_yaml") _, _, err := e2e.Zarf("prepare", "lint", path) require.NoError(t, err, "Expect no error here because the yaml file is following schema") }) diff --git a/src/test/packages/09-lint/invalid_yaml/zarf.yaml b/src/test/packages/11-lint/invalid_yaml/zarf.yaml similarity index 100% rename from src/test/packages/09-lint/invalid_yaml/zarf.yaml rename to src/test/packages/11-lint/invalid_yaml/zarf.yaml diff --git a/src/test/packages/09-lint/valid_yaml/zarf.yaml b/src/test/packages/11-lint/valid_yaml/zarf.yaml similarity index 100% rename from src/test/packages/09-lint/valid_yaml/zarf.yaml rename to src/test/packages/11-lint/valid_yaml/zarf.yaml From d6dc0b5abf51b3ad23f1208b3fb4996950a39a8f Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Tue, 7 Nov 2023 14:39:44 +0000 Subject: [PATCH 026/192] tests working --- src/pkg/lint/lint.go | 2 +- src/pkg/lint/validate_schema.go | 2 +- src/pkg/lint/validate_schema_test.go | 43 ++++------------------------ 3 files changed, 8 insertions(+), 39 deletions(-) diff --git a/src/pkg/lint/lint.go b/src/pkg/lint/lint.go index 5f66e05a71..83db6cba36 100644 --- a/src/pkg/lint/lint.go +++ b/src/pkg/lint/lint.go @@ -21,7 +21,7 @@ func ValidateZarfSchema(baseDir string) (err error) { return err } - err = validateSchema(zarfData, zarfSchema) + err = ValidateSchema(zarfData, zarfSchema) if err != nil { return err diff --git a/src/pkg/lint/validate_schema.go b/src/pkg/lint/validate_schema.go index 9b2e99a60d..a614400b2d 100644 --- a/src/pkg/lint/validate_schema.go +++ b/src/pkg/lint/validate_schema.go @@ -6,7 +6,7 @@ import ( "github.com/xeipuuv/gojsonschema" ) -func validateSchema(unmarshalledYaml interface{}, jsonSchema []byte) error { +func ValidateSchema(unmarshalledYaml interface{}, jsonSchema []byte) error { schemaLoader := gojsonschema.NewBytesLoader(jsonSchema) documentLoader := gojsonschema.NewGoLoader(unmarshalledYaml) diff --git a/src/pkg/lint/validate_schema_test.go b/src/pkg/lint/validate_schema_test.go index aeed602a0a..0c1cd93aa0 100644 --- a/src/pkg/lint/validate_schema_test.go +++ b/src/pkg/lint/validate_schema_test.go @@ -29,63 +29,32 @@ func TestValidateSchema(t *testing.T) { return unmarshalledYaml } - // unmarshallStringYaml := func(t *testing.T, str string) (interface{}, error) { - // t.Helper() - // var unmarshalledYaml interface{} - // err := goyaml.Unmarshal([]byte(str), &unmarshalledYaml) - // return unmarshalledYaml, err - // } - t.Run("Read schema success", func(t *testing.T) { unmarshalledYaml := readAndUnmarshalYaml(t, "successful_validation/zarf.yaml") zarfSchema := readSchema(t) - err := validateSchema(unmarshalledYaml, zarfSchema) + err := ValidateSchema(unmarshalledYaml, zarfSchema) require.NoError(t, err) - if got := validateSchema(unmarshalledYaml, zarfSchema); got != nil { + if got := ValidateSchema(unmarshalledYaml, zarfSchema); got != nil { t.Errorf("Expected successful validation, got error: %v", got) } }) t.Run("Read schema fail", func(t *testing.T) { - // var yamlContents = ` - // kind: ZarfInitConfig - // metadata: - // name: init - // components: - // - name: k3s - // import: - // pa324234th: test - // - name: test - // import: - // pa3th: test - // ` - // unmarshalledYaml,_ := unmarshallStringYaml(t, yamlContents) - zarfSchema := readSchema(t) unmarshalledYaml := readAndUnmarshalYaml(t, "unsuccessful_validation/zarf.yaml") - err := validateSchema(unmarshalledYaml, zarfSchema) + zarfSchema := readSchema(t) + err := ValidateSchema(unmarshalledYaml, zarfSchema) errorMessage := `The document is not valid: - components.0.import: Additional property not-path is not allowed - components.1: Additional property not-import is not allowed` require.EqualError(t, err, errorMessage) }) - // t.Run("bad yaml", func(t *testing.T) { - // var yamlContents = "unquoted_string_with_colons: key: value" - // unmarshalledYaml,err := unmarshallStringYaml(t, yamlContents) - // zarfSchema := readSchema(t) - // err = ValidateSchema(unmarshalledYaml, zarfSchema) - // require.Contains(t, err.Error(), "components.0.import: Additional property pa324234th is not allowed") - // if err == nil { - // t.Errorf("Expected validation to fail, but it succeeded.") - // } - // }) - t.Run("Read schema yaml extension", func(t *testing.T) { unmarshalledYaml := readAndUnmarshalYaml(t, "yaml-extension/zarf.yaml") zarfSchema := readSchema(t) - err := validateSchema(unmarshalledYaml, zarfSchema) + err := ValidateSchema(unmarshalledYaml, zarfSchema) require.NoError(t, err) - if got := validateSchema(unmarshalledYaml, zarfSchema); got != nil { + if got := ValidateSchema(unmarshalledYaml, zarfSchema); got != nil { t.Errorf("Expected successful validation, got error: %v", got) } }) From 0d414eeb8726f81661fae841f835cea945c05b68 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Tue, 7 Nov 2023 18:01:56 +0000 Subject: [PATCH 027/192] warning if someone has zarf_pkg_tmpl somewhere --- src/pkg/lint/lint.go | 11 +++++++ .../lint/unsuccessful_validation/zarf.yaml | 8 +++++ src/pkg/lint/validate_schema.go | 32 ++++++++++++++++++- src/pkg/lint/validate_schema_test.go | 22 ++++++++++++- .../packages/11-lint/invalid_yaml/zarf.yaml | 7 ++++ 5 files changed, 78 insertions(+), 2 deletions(-) diff --git a/src/pkg/lint/lint.go b/src/pkg/lint/lint.go index 83db6cba36..1fc550b68d 100644 --- a/src/pkg/lint/lint.go +++ b/src/pkg/lint/lint.go @@ -11,10 +11,21 @@ import ( "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/utils" + "github.com/defenseunicorns/zarf/src/types" ) // Validates a zarf file against the zarf schema, returns an error if the file is invalid func ValidateZarfSchema(baseDir string) (err error) { + var typedZarfData types.ZarfPackage + if err := utils.ReadYaml(filepath.Join(baseDir, layout.ZarfYAML), &typedZarfData); err != nil { + return err + } + + err = checkForVarInComponentImport(typedZarfData) + if err != nil { + message.Warn(err.Error()) + } + zarfSchema, _ := config.GetSchemaFile() var zarfData interface{} if err := utils.ReadYaml(filepath.Join(baseDir, layout.ZarfYAML), &zarfData); err != nil { diff --git a/src/pkg/lint/unsuccessful_validation/zarf.yaml b/src/pkg/lint/unsuccessful_validation/zarf.yaml index 6ebb779c23..639f1cf462 100644 --- a/src/pkg/lint/unsuccessful_validation/zarf.yaml +++ b/src/pkg/lint/unsuccessful_validation/zarf.yaml @@ -11,3 +11,11 @@ components: - name: second-test-component not-import: path: packages/distros/k3s + + - name: import-test + import: + path: "###ZARF_PKG_TMPL_ZEBRA###" + + - name: import-url + import: + url: "oci://###ZARF_PKG_TMPL_ZEBRA###" diff --git a/src/pkg/lint/validate_schema.go b/src/pkg/lint/validate_schema.go index a614400b2d..950a426439 100644 --- a/src/pkg/lint/validate_schema.go +++ b/src/pkg/lint/validate_schema.go @@ -2,10 +2,18 @@ package lint import ( "errors" + "strconv" + "strings" + "github.com/defenseunicorns/zarf/src/types" "github.com/xeipuuv/gojsonschema" ) +const ( + zarfInvalidPrefix = "zarf.yaml is not valid:" + zarfWarningPrefix = "zarf schema warning:" +) + func ValidateSchema(unmarshalledYaml interface{}, jsonSchema []byte) error { schemaLoader := gojsonschema.NewBytesLoader(jsonSchema) documentLoader := gojsonschema.NewGoLoader(unmarshalledYaml) @@ -16,7 +24,7 @@ func ValidateSchema(unmarshalledYaml interface{}, jsonSchema []byte) error { } if !result.Valid() { - errorMessage := "The document is not valid:" + errorMessage := zarfInvalidPrefix for _, desc := range result.Errors() { errorMessage = errorMessage + "\n - " + desc.String() } @@ -25,3 +33,25 @@ func ValidateSchema(unmarshalledYaml interface{}, jsonSchema []byte) error { return err } + +func checkForVarInComponentImport(unmarshalledYaml types.ZarfPackage) error { + valid := true + errorMessage := zarfWarningPrefix + componentWarningStart := "\n - component." + for i, component := range unmarshalledYaml.Components { + if strings.Contains(component.Import.Path, "###ZARF_PKG_TMPL_") { + errorMessage = errorMessage + componentWarningStart + strconv.Itoa(i) + + ".import.path will not resolve ZARF_PKG_TMPL_* variables" + valid = false + } + if strings.Contains(component.Import.URL, "###ZARF_PKG_TMPL_") { + errorMessage = errorMessage + componentWarningStart + strconv.Itoa(i) + + ".import.url will not resolve ZARF_PKG_TMPL_* variables" + valid = false + } + } + if valid { + return nil + } + return errors.New(errorMessage) +} diff --git a/src/pkg/lint/validate_schema_test.go b/src/pkg/lint/validate_schema_test.go index 0c1cd93aa0..02a1062921 100644 --- a/src/pkg/lint/validate_schema_test.go +++ b/src/pkg/lint/validate_schema_test.go @@ -4,6 +4,7 @@ import ( "os" "testing" + "github.com/defenseunicorns/zarf/src/types" goyaml "github.com/goccy/go-yaml" "github.com/stretchr/testify/require" ) @@ -43,7 +44,7 @@ func TestValidateSchema(t *testing.T) { unmarshalledYaml := readAndUnmarshalYaml(t, "unsuccessful_validation/zarf.yaml") zarfSchema := readSchema(t) err := ValidateSchema(unmarshalledYaml, zarfSchema) - errorMessage := `The document is not valid: + errorMessage := zarfInvalidPrefix + ` - components.0.import: Additional property not-path is not allowed - components.1: Additional property not-import is not allowed` require.EqualError(t, err, errorMessage) @@ -58,4 +59,23 @@ func TestValidateSchema(t *testing.T) { t.Errorf("Expected successful validation, got error: %v", got) } }) + + t.Run("Read schema yaml extension", func(t *testing.T) { + var unmarshalledYaml types.ZarfPackage + file, _ := os.ReadFile("yaml-extension/zarf.yaml") + goyaml.Unmarshal(file, &unmarshalledYaml) + err := checkForVarInComponentImport(unmarshalledYaml) + require.NoError(t, err) + }) + + t.Run("Read schema import-error", func(t *testing.T) { + var unmarshalledYaml types.ZarfPackage + file, _ := os.ReadFile("unsuccessful_validation/zarf.yaml") + goyaml.Unmarshal(file, &unmarshalledYaml) + err := checkForVarInComponentImport(unmarshalledYaml) + errorMessage := zarfWarningPrefix + ` + - component.2.import.path will not resolve ZARF_PKG_TMPL_* variables + - component.3.import.url will not resolve ZARF_PKG_TMPL_* variables` + require.EqualError(t, err, errorMessage) + }) } diff --git a/src/test/packages/11-lint/invalid_yaml/zarf.yaml b/src/test/packages/11-lint/invalid_yaml/zarf.yaml index f30a6b8f9a..39f1b899da 100644 --- a/src/test/packages/11-lint/invalid_yaml/zarf.yaml +++ b/src/test/packages/11-lint/invalid_yaml/zarf.yaml @@ -7,3 +7,10 @@ components: - name: k3s import: pat12312h: packages/distros/k3s + - name: import-test + import: + path: "###ZARF_PKG_TMPL_ZEBRA###" + + - name: import-url + import: + url: "oci://###ZARF_PKG_TMPL_ZEBRA###" From 14e9804bd1b6416e571939946c4b7639cb5b23a7 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Tue, 7 Nov 2023 18:47:34 +0000 Subject: [PATCH 028/192] refactor --- src/pkg/lint/validate_schema.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/pkg/lint/validate_schema.go b/src/pkg/lint/validate_schema.go index 950a426439..6ccc0ea5e2 100644 --- a/src/pkg/lint/validate_schema.go +++ b/src/pkg/lint/validate_schema.go @@ -12,6 +12,7 @@ import ( const ( zarfInvalidPrefix = "zarf.yaml is not valid:" zarfWarningPrefix = "zarf schema warning:" + zarfTemplateVar = "###ZARF_PKG_TMPL_" ) func ValidateSchema(unmarshalledYaml interface{}, jsonSchema []byte) error { @@ -39,12 +40,12 @@ func checkForVarInComponentImport(unmarshalledYaml types.ZarfPackage) error { errorMessage := zarfWarningPrefix componentWarningStart := "\n - component." for i, component := range unmarshalledYaml.Components { - if strings.Contains(component.Import.Path, "###ZARF_PKG_TMPL_") { + if strings.Contains(component.Import.Path, zarfTemplateVar) { errorMessage = errorMessage + componentWarningStart + strconv.Itoa(i) + ".import.path will not resolve ZARF_PKG_TMPL_* variables" valid = false } - if strings.Contains(component.Import.URL, "###ZARF_PKG_TMPL_") { + if strings.Contains(component.Import.URL, zarfTemplateVar) { errorMessage = errorMessage + componentWarningStart + strconv.Itoa(i) + ".import.url will not resolve ZARF_PKG_TMPL_* variables" valid = false From 78b76f042a790b333f4ef87f303919796fe1460c Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Tue, 7 Nov 2023 19:26:11 +0000 Subject: [PATCH 029/192] changing format of message --- src/pkg/lint/lint.go | 2 +- src/pkg/lint/validate_schema.go | 12 ++++++------ src/pkg/lint/validate_schema_test.go | 10 +++++----- src/test/e2e/11_lint_test.go | 1 + 4 files changed, 13 insertions(+), 12 deletions(-) diff --git a/src/pkg/lint/lint.go b/src/pkg/lint/lint.go index 1fc550b68d..092d1b04f9 100644 --- a/src/pkg/lint/lint.go +++ b/src/pkg/lint/lint.go @@ -32,7 +32,7 @@ func ValidateZarfSchema(baseDir string) (err error) { return err } - err = ValidateSchema(zarfData, zarfSchema) + err = validateSchema(zarfData, zarfSchema) if err != nil { return err diff --git a/src/pkg/lint/validate_schema.go b/src/pkg/lint/validate_schema.go index 6ccc0ea5e2..a728f00e34 100644 --- a/src/pkg/lint/validate_schema.go +++ b/src/pkg/lint/validate_schema.go @@ -15,7 +15,7 @@ const ( zarfTemplateVar = "###ZARF_PKG_TMPL_" ) -func ValidateSchema(unmarshalledYaml interface{}, jsonSchema []byte) error { +func validateSchema(unmarshalledYaml interface{}, jsonSchema []byte) error { schemaLoader := gojsonschema.NewBytesLoader(jsonSchema) documentLoader := gojsonschema.NewGoLoader(unmarshalledYaml) @@ -35,19 +35,19 @@ func ValidateSchema(unmarshalledYaml interface{}, jsonSchema []byte) error { return err } -func checkForVarInComponentImport(unmarshalledYaml types.ZarfPackage) error { +func checkForVarInComponentImport(zarfYaml types.ZarfPackage) error { valid := true errorMessage := zarfWarningPrefix - componentWarningStart := "\n - component." - for i, component := range unmarshalledYaml.Components { + componentWarningStart := " component." + for i, component := range zarfYaml.Components { if strings.Contains(component.Import.Path, zarfTemplateVar) { errorMessage = errorMessage + componentWarningStart + strconv.Itoa(i) + - ".import.path will not resolve ZARF_PKG_TMPL_* variables" + ".import.path will not resolve ZARF_PKG_TMPL_* variables." valid = false } if strings.Contains(component.Import.URL, zarfTemplateVar) { errorMessage = errorMessage + componentWarningStart + strconv.Itoa(i) + - ".import.url will not resolve ZARF_PKG_TMPL_* variables" + ".import.url will not resolve ZARF_PKG_TMPL_* variables." valid = false } } diff --git a/src/pkg/lint/validate_schema_test.go b/src/pkg/lint/validate_schema_test.go index 02a1062921..2b71b2fae4 100644 --- a/src/pkg/lint/validate_schema_test.go +++ b/src/pkg/lint/validate_schema_test.go @@ -33,9 +33,9 @@ func TestValidateSchema(t *testing.T) { t.Run("Read schema success", func(t *testing.T) { unmarshalledYaml := readAndUnmarshalYaml(t, "successful_validation/zarf.yaml") zarfSchema := readSchema(t) - err := ValidateSchema(unmarshalledYaml, zarfSchema) + err := validateSchema(unmarshalledYaml, zarfSchema) require.NoError(t, err) - if got := ValidateSchema(unmarshalledYaml, zarfSchema); got != nil { + if got := validateSchema(unmarshalledYaml, zarfSchema); got != nil { t.Errorf("Expected successful validation, got error: %v", got) } }) @@ -43,7 +43,7 @@ func TestValidateSchema(t *testing.T) { t.Run("Read schema fail", func(t *testing.T) { unmarshalledYaml := readAndUnmarshalYaml(t, "unsuccessful_validation/zarf.yaml") zarfSchema := readSchema(t) - err := ValidateSchema(unmarshalledYaml, zarfSchema) + err := validateSchema(unmarshalledYaml, zarfSchema) errorMessage := zarfInvalidPrefix + ` - components.0.import: Additional property not-path is not allowed - components.1: Additional property not-import is not allowed` @@ -53,9 +53,9 @@ func TestValidateSchema(t *testing.T) { t.Run("Read schema yaml extension", func(t *testing.T) { unmarshalledYaml := readAndUnmarshalYaml(t, "yaml-extension/zarf.yaml") zarfSchema := readSchema(t) - err := ValidateSchema(unmarshalledYaml, zarfSchema) + err := validateSchema(unmarshalledYaml, zarfSchema) require.NoError(t, err) - if got := ValidateSchema(unmarshalledYaml, zarfSchema); got != nil { + if got := validateSchema(unmarshalledYaml, zarfSchema); got != nil { t.Errorf("Expected successful validation, got error: %v", got) } }) diff --git a/src/test/e2e/11_lint_test.go b/src/test/e2e/11_lint_test.go index c4578a2442..14b0865040 100644 --- a/src/test/e2e/11_lint_test.go +++ b/src/test/e2e/11_lint_test.go @@ -17,6 +17,7 @@ func TestLint(t *testing.T) { _, stderr, err := e2e.Zarf("prepare", "lint", path) require.Error(t, err, "Expect error here because the yaml file is not following schema") require.Contains(t, stderr, "- components.0.import: Additional property pat12312h is not allowed") + require.Contains(t, stderr, "component.1.import.path will not resolve ZARF_PKG_TMPL_* variables") }) t.Run("zarf test lint success", func(t *testing.T) { From 03ea4459b030cfc127718ca31a2e1f69247de159 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Tue, 7 Nov 2023 19:28:09 +0000 Subject: [PATCH 030/192] fixing comment --- src/pkg/lint/lint.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pkg/lint/lint.go b/src/pkg/lint/lint.go index 092d1b04f9..febc4f902f 100644 --- a/src/pkg/lint/lint.go +++ b/src/pkg/lint/lint.go @@ -14,7 +14,7 @@ import ( "github.com/defenseunicorns/zarf/src/types" ) -// Validates a zarf file against the zarf schema, returns an error if the file is invalid +// ValidatesZarfSchema a zarf file against the zarf schema, returns an error if the file is invalid func ValidateZarfSchema(baseDir string) (err error) { var typedZarfData types.ZarfPackage if err := utils.ReadYaml(filepath.Join(baseDir, layout.ZarfYAML), &typedZarfData); err != nil { From ec1eb04a4a6e7efc031e79045c0d31b570623fde Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Tue, 7 Nov 2023 19:35:40 +0000 Subject: [PATCH 031/192] fix comment --- src/pkg/lint/lint.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pkg/lint/lint.go b/src/pkg/lint/lint.go index febc4f902f..277020fcd8 100644 --- a/src/pkg/lint/lint.go +++ b/src/pkg/lint/lint.go @@ -14,7 +14,7 @@ import ( "github.com/defenseunicorns/zarf/src/types" ) -// ValidatesZarfSchema a zarf file against the zarf schema, returns an error if the file is invalid +// ValidateZarfSchema a zarf file against the zarf schema, returns an error if the file is invalid func ValidateZarfSchema(baseDir string) (err error) { var typedZarfData types.ZarfPackage if err := utils.ReadYaml(filepath.Join(baseDir, layout.ZarfYAML), &typedZarfData); err != nil { From f5f484f8088179b6b05b8ab7814cc8e441b538b3 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Tue, 7 Nov 2023 19:36:16 +0000 Subject: [PATCH 032/192] comment --- src/pkg/lint/lint.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pkg/lint/lint.go b/src/pkg/lint/lint.go index 277020fcd8..ee119ac663 100644 --- a/src/pkg/lint/lint.go +++ b/src/pkg/lint/lint.go @@ -1,7 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: 2021-Present The Zarf Authors -// Package packager contains functions for interacting with, managing and deploying Zarf packages +// Package lint contains functions for linting the zarf.yaml package lint import ( From b82f857b110437c7b00a4411931cd6c75f3c485e Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Tue, 7 Nov 2023 20:10:10 +0000 Subject: [PATCH 033/192] refactor tests --- .pre-commit-config.yaml | 1 - src/pkg/lint/successful_validation/zarf.yaml | 59 +++++++------------- src/pkg/lint/validate_schema_test.go | 44 ++++++--------- src/pkg/lint/yaml-extension/zarf.yaml | 24 -------- 4 files changed, 37 insertions(+), 91 deletions(-) delete mode 100644 src/pkg/lint/yaml-extension/zarf.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ec3a9f6a58..564e34c92a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -50,6 +50,5 @@ repos: exclude: | (?x)^( src/pkg/lint/unsuccessful_validation/zarf.yaml| - src/pkg/lint/yaml-extension/zarf.yaml| src/test/packages/11-lint/.* )$ diff --git a/src/pkg/lint/successful_validation/zarf.yaml b/src/pkg/lint/successful_validation/zarf.yaml index 859813fde5..a8640d1b4c 100644 --- a/src/pkg/lint/successful_validation/zarf.yaml +++ b/src/pkg/lint/successful_validation/zarf.yaml @@ -1,43 +1,24 @@ -kind: ZarfInitConfig -metadata: - name: init - description: Used to establish a new Zarf cluster - -components: - - name: k3s - import: - path: packages/distros/k3s +# Plan to have the schema or the lint code allow these soon +# x-env: &env +# environment: +# - CONFIG_KEY +# - EXAMPLE_KEY - # This package moves the injector & registries binaries - - name: zarf-injector - required: true - import: - path: packages/zarf-registry - - # Creates the temporary seed-registry - - name: zarf-seed-registry - required: true - import: - path: packages/zarf-registry +# x-version: &version +# version: 1.0.0 - # Creates the permanent registry - - name: zarf-registry - required: true - import: - path: packages/zarf-registry +kind: ZarfPackageConfig +metadata: + name: hello + version: &version 1.0.0 - # Creates the pod+git mutating webhook - - name: zarf-agent +components: + - name: c1 required: true - import: - path: packages/zarf-agent - - # (Optional) Adds logging to the cluster - - name: logging - import: - path: packages/logging-pgl - - # (Optional) Adds a git server to the cluster - - name: git-server - import: - path: packages/gitea + charts: + - name: redis + namespace: redis-system + version: *version + localPath: ./redis + images: + - &hello_image docker.io/library/redis:latest diff --git a/src/pkg/lint/validate_schema_test.go b/src/pkg/lint/validate_schema_test.go index 2b71b2fae4..ee4641fa79 100644 --- a/src/pkg/lint/validate_schema_test.go +++ b/src/pkg/lint/validate_schema_test.go @@ -22,15 +22,20 @@ func TestValidateSchema(t *testing.T) { readAndUnmarshalYaml := func(t *testing.T, path string) interface{} { t.Helper() var unmarshalledYaml interface{} - file, err := os.ReadFile(path) + file, _ := os.ReadFile(path) + goyaml.Unmarshal(file, &unmarshalledYaml) + return unmarshalledYaml + } + + readAndUnmarshallZarf := func(t *testing.T, path string) types.ZarfPackage { + t.Helper() + var unmarshalledYaml types.ZarfPackage + file, _ := os.ReadFile(path) goyaml.Unmarshal(file, &unmarshalledYaml) - if err != nil { - return err - } return unmarshalledYaml } - t.Run("Read schema success", func(t *testing.T) { + t.Run("validate schema success", func(t *testing.T) { unmarshalledYaml := readAndUnmarshalYaml(t, "successful_validation/zarf.yaml") zarfSchema := readSchema(t) err := validateSchema(unmarshalledYaml, zarfSchema) @@ -40,7 +45,7 @@ func TestValidateSchema(t *testing.T) { } }) - t.Run("Read schema fail", func(t *testing.T) { + t.Run("validate schema fail", func(t *testing.T) { unmarshalledYaml := readAndUnmarshalYaml(t, "unsuccessful_validation/zarf.yaml") zarfSchema := readSchema(t) err := validateSchema(unmarshalledYaml, zarfSchema) @@ -50,32 +55,17 @@ func TestValidateSchema(t *testing.T) { require.EqualError(t, err, errorMessage) }) - t.Run("Read schema yaml extension", func(t *testing.T) { - unmarshalledYaml := readAndUnmarshalYaml(t, "yaml-extension/zarf.yaml") - zarfSchema := readSchema(t) - err := validateSchema(unmarshalledYaml, zarfSchema) - require.NoError(t, err) - if got := validateSchema(unmarshalledYaml, zarfSchema); got != nil { - t.Errorf("Expected successful validation, got error: %v", got) - } - }) - - t.Run("Read schema yaml extension", func(t *testing.T) { - var unmarshalledYaml types.ZarfPackage - file, _ := os.ReadFile("yaml-extension/zarf.yaml") - goyaml.Unmarshal(file, &unmarshalledYaml) + t.Run("Template in component import success", func(t *testing.T) { + unmarshalledYaml := readAndUnmarshallZarf(t, "successful_validation/zarf.yaml") err := checkForVarInComponentImport(unmarshalledYaml) require.NoError(t, err) }) - t.Run("Read schema import-error", func(t *testing.T) { - var unmarshalledYaml types.ZarfPackage - file, _ := os.ReadFile("unsuccessful_validation/zarf.yaml") - goyaml.Unmarshal(file, &unmarshalledYaml) + t.Run("Template in component import failure", func(t *testing.T) { + unmarshalledYaml := readAndUnmarshallZarf(t, "unsuccessful_validation/zarf.yaml") err := checkForVarInComponentImport(unmarshalledYaml) - errorMessage := zarfWarningPrefix + ` - - component.2.import.path will not resolve ZARF_PKG_TMPL_* variables - - component.3.import.url will not resolve ZARF_PKG_TMPL_* variables` + errorMessage := zarfWarningPrefix + " component.2.import.path will not resolve ZARF_PKG_TMPL_* variables. " + + "component.3.import.url will not resolve ZARF_PKG_TMPL_* variables." require.EqualError(t, err, errorMessage) }) } diff --git a/src/pkg/lint/yaml-extension/zarf.yaml b/src/pkg/lint/yaml-extension/zarf.yaml deleted file mode 100644 index c55c123e75..0000000000 --- a/src/pkg/lint/yaml-extension/zarf.yaml +++ /dev/null @@ -1,24 +0,0 @@ -# Plan to have the schema or the lint code allow these soon -# x-env: &env -# environment: -# - CONFIG_KEY -# - EXAMPLE_KEY - -# x-version: &version -# version: 1.0.0 - -kind: ZarfPackageConfig -metadata: - name: hello - <<: &version 1.0.0 - -components: - - name: c1 - required: true - charts: - - name: redis - namespace: redis-system - version: *version - localPath: ./redis - images: - - &hello_image docker.io/library/redis:latest From 347a8d1435e7cfeb5ac77cb96d45e48fe2c36cfb Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Wed, 8 Nov 2023 14:09:58 +0000 Subject: [PATCH 034/192] refactor tests --- go.mod | 5 +--- go.sum | 4 ---- main.go | 1 - src/pkg/lint/successful_validation/zarf.yaml | 24 -------------------- src/pkg/lint/validate_schema_test.go | 11 ++++----- 5 files changed, 5 insertions(+), 40 deletions(-) delete mode 100644 src/pkg/lint/successful_validation/zarf.yaml diff --git a/go.mod b/go.mod index 229e3b8b51..958a9962e2 100644 --- a/go.mod +++ b/go.mod @@ -59,8 +59,6 @@ require ( sigs.k8s.io/yaml v1.3.0 ) -require github.com/santhosh-tekuri/jsonschema v1.2.4 - require ( atomicgo.dev/cursor v0.2.0 // indirect atomicgo.dev/keyboard v0.2.9 // indirect @@ -373,7 +371,6 @@ require ( github.com/sagikazarmark/locafero v0.3.0 // indirect github.com/sagikazarmark/slog-shim v0.1.0 // indirect github.com/sahilm/fuzzy v0.1.0 // indirect - github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 github.com/sassoftware/go-rpmutils v0.2.0 // indirect github.com/sassoftware/relic v7.2.1+incompatible // indirect github.com/scylladb/go-set v1.0.3-0.20200225121959-cc7b2070d91e // indirect @@ -458,7 +455,7 @@ require ( gopkg.in/square/go-jose.v2 v2.6.0 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect - gopkg.in/yaml.v3 v3.0.1 + gopkg.in/yaml.v3 v3.0.1 // indirect inet.af/netaddr v0.0.0-20220811202034-502d2d690317 // indirect k8s.io/apiextensions-apiserver v0.28.2 // indirect k8s.io/apiserver v0.28.2 // indirect diff --git a/go.sum b/go.sum index efb574297d..848584069a 100644 --- a/go.sum +++ b/go.sum @@ -1425,10 +1425,6 @@ github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6g github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= github.com/sahilm/fuzzy v0.1.0 h1:FzWGaw2Opqyu+794ZQ9SYifWv2EIXpwP4q8dY1kDAwI= github.com/sahilm/fuzzy v0.1.0/go.mod h1:VFvziUEIMCrT6A6tw2RFIXPXXmzXbOsSHF0DOI8ZK9Y= -github.com/santhosh-tekuri/jsonschema v1.2.4 h1:hNhW8e7t+H1vgY+1QeEQpveR6D4+OwKPXCfD2aieJis= -github.com/santhosh-tekuri/jsonschema v1.2.4/go.mod h1:TEAUOeZSmIxTTuHatJzrvARHiuO9LYd+cIxzgEHCQI4= -github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 h1:lZUw3E0/J3roVtGQ+SCrUrg3ON6NgVqpn3+iol9aGu4= -github.com/santhosh-tekuri/jsonschema/v5 v5.3.1/go.mod h1:uToXkOrWAZ6/Oc07xWQrPOhJotwFIyu2bBVN41fcDUY= github.com/sassoftware/go-rpmutils v0.2.0 h1:pKW0HDYMFWQ5b4JQPiI3WI12hGsVoW0V8+GMoZiI/JE= github.com/sassoftware/go-rpmutils v0.2.0/go.mod h1:TJJQYtLe/BeEmEjelI3b7xNZjzAukEkeWKmoakvaOoI= github.com/sassoftware/relic v7.2.1+incompatible h1:Pwyh1F3I0r4clFJXkSI8bOyJINGqpgjJU3DYAZeI05A= diff --git a/main.go b/main.go index e71065a19b..651ba68a26 100644 --- a/main.go +++ b/main.go @@ -6,7 +6,6 @@ package main import ( "embed" - _ "embed" "github.com/defenseunicorns/zarf/src/cmd" "github.com/defenseunicorns/zarf/src/config" diff --git a/src/pkg/lint/successful_validation/zarf.yaml b/src/pkg/lint/successful_validation/zarf.yaml deleted file mode 100644 index a8640d1b4c..0000000000 --- a/src/pkg/lint/successful_validation/zarf.yaml +++ /dev/null @@ -1,24 +0,0 @@ -# Plan to have the schema or the lint code allow these soon -# x-env: &env -# environment: -# - CONFIG_KEY -# - EXAMPLE_KEY - -# x-version: &version -# version: 1.0.0 - -kind: ZarfPackageConfig -metadata: - name: hello - version: &version 1.0.0 - -components: - - name: c1 - required: true - charts: - - name: redis - namespace: redis-system - version: *version - localPath: ./redis - images: - - &hello_image docker.io/library/redis:latest diff --git a/src/pkg/lint/validate_schema_test.go b/src/pkg/lint/validate_schema_test.go index ee4641fa79..8c914557d1 100644 --- a/src/pkg/lint/validate_schema_test.go +++ b/src/pkg/lint/validate_schema_test.go @@ -27,7 +27,7 @@ func TestValidateSchema(t *testing.T) { return unmarshalledYaml } - readAndUnmarshallZarf := func(t *testing.T, path string) types.ZarfPackage { + readAndUnmarshallZarfPackage := func(t *testing.T, path string) types.ZarfPackage { t.Helper() var unmarshalledYaml types.ZarfPackage file, _ := os.ReadFile(path) @@ -36,13 +36,10 @@ func TestValidateSchema(t *testing.T) { } t.Run("validate schema success", func(t *testing.T) { - unmarshalledYaml := readAndUnmarshalYaml(t, "successful_validation/zarf.yaml") + unmarshalledYaml := readAndUnmarshalYaml(t, "../../../zarf.yaml") zarfSchema := readSchema(t) err := validateSchema(unmarshalledYaml, zarfSchema) require.NoError(t, err) - if got := validateSchema(unmarshalledYaml, zarfSchema); got != nil { - t.Errorf("Expected successful validation, got error: %v", got) - } }) t.Run("validate schema fail", func(t *testing.T) { @@ -56,13 +53,13 @@ func TestValidateSchema(t *testing.T) { }) t.Run("Template in component import success", func(t *testing.T) { - unmarshalledYaml := readAndUnmarshallZarf(t, "successful_validation/zarf.yaml") + unmarshalledYaml := readAndUnmarshallZarfPackage(t, "successful_validation/zarf.yaml") err := checkForVarInComponentImport(unmarshalledYaml) require.NoError(t, err) }) t.Run("Template in component import failure", func(t *testing.T) { - unmarshalledYaml := readAndUnmarshallZarf(t, "unsuccessful_validation/zarf.yaml") + unmarshalledYaml := readAndUnmarshallZarfPackage(t, "unsuccessful_validation/zarf.yaml") err := checkForVarInComponentImport(unmarshalledYaml) errorMessage := zarfWarningPrefix + " component.2.import.path will not resolve ZARF_PKG_TMPL_* variables. " + "component.3.import.url will not resolve ZARF_PKG_TMPL_* variables." From fba2afed609c97c355ecc8a722b7dded4bb76e42 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Wed, 8 Nov 2023 14:31:24 +0000 Subject: [PATCH 035/192] refactor e2e, make docs and schema --- .../100-cli-commands/zarf_prepare.md | 1 + .../100-cli-commands/zarf_prepare_lint.md | 31 +++++++++++++ .../100-cli-commands/zarf_tools_helm.md | 1 + .../zarf_tools_helm_cm-push.md | 42 ++++++++++++++++++ docs/3-create-a-zarf-package/4-zarf-schema.md | 10 ++--- src/test/e2e/11_lint_test.go | 4 +- .../packages/11-lint/invalid_yaml/zarf.yaml | 1 + .../packages/11-lint/valid_yaml/zarf.yaml | 43 ------------------- 8 files changed, 81 insertions(+), 52 deletions(-) create mode 100644 docs/2-the-zarf-cli/100-cli-commands/zarf_prepare_lint.md create mode 100644 docs/2-the-zarf-cli/100-cli-commands/zarf_tools_helm_cm-push.md delete mode 100644 src/test/packages/11-lint/valid_yaml/zarf.yaml diff --git a/docs/2-the-zarf-cli/100-cli-commands/zarf_prepare.md b/docs/2-the-zarf-cli/100-cli-commands/zarf_prepare.md index d0c4598e7c..c8b2ecd222 100644 --- a/docs/2-the-zarf-cli/100-cli-commands/zarf_prepare.md +++ b/docs/2-the-zarf-cli/100-cli-commands/zarf_prepare.md @@ -27,6 +27,7 @@ Tools to help prepare assets for packaging * [zarf](zarf.md) - DevSecOps for Airgap * [zarf prepare find-images](zarf_prepare_find-images.md) - Evaluates components in a zarf file to identify images specified in their helm charts and manifests * [zarf prepare generate-config](zarf_prepare_generate-config.md) - Generates a config file for Zarf +* [zarf prepare lint](zarf_prepare_lint.md) - Tools for validating schema / ensuring best practices * [zarf prepare patch-git](zarf_prepare_patch-git.md) - Converts all .git URLs to the specified Zarf HOST and with the Zarf URL pattern in a given FILE. NOTE: This should only be used for manifests that are not mutated by the Zarf Agent Mutating Webhook. * [zarf prepare sha256sum](zarf_prepare_sha256sum.md) - Generates a SHA256SUM for the given file diff --git a/docs/2-the-zarf-cli/100-cli-commands/zarf_prepare_lint.md b/docs/2-the-zarf-cli/100-cli-commands/zarf_prepare_lint.md new file mode 100644 index 0000000000..a114cba47f --- /dev/null +++ b/docs/2-the-zarf-cli/100-cli-commands/zarf_prepare_lint.md @@ -0,0 +1,31 @@ +# zarf prepare lint + + +Tools for validating schema / ensuring best practices + +``` +zarf prepare lint [ DIRECTORY ] [flags] +``` + +## Options + +``` + -h, --help help for lint +``` + +## Options inherited from parent commands + +``` + -a, --architecture string Architecture for OCI images and Zarf packages + --insecure Allow access to insecure registries and disable other recommended security enforcements such as package checksum and signature validation. This flag should only be used if you have a specific reason and accept the reduced security posture. + -l, --log-level string Log level when running Zarf. Valid options are: warn, info, debug, trace (default "info") + --no-color Disable colors in output + --no-log-file Disable log file creation + --no-progress Disable fancy UI progress bars, spinners, logos, etc + --tmpdir string Specify the temporary directory to use for intermediate files + --zarf-cache string Specify the location of the Zarf cache directory (default "~/.zarf-cache") +``` + +## SEE ALSO + +* [zarf prepare](zarf_prepare.md) - Tools to help prepare assets for packaging diff --git a/docs/2-the-zarf-cli/100-cli-commands/zarf_tools_helm.md b/docs/2-the-zarf-cli/100-cli-commands/zarf_tools_helm.md index a0c5884216..cea6f81d37 100644 --- a/docs/2-the-zarf-cli/100-cli-commands/zarf_tools_helm.md +++ b/docs/2-the-zarf-cli/100-cli-commands/zarf_tools_helm.md @@ -31,5 +31,6 @@ Subset of the Helm CLI that includes the repo and dependency commands for managi ## SEE ALSO * [zarf tools](zarf_tools.md) - Collection of additional tools to make airgap easier +* [zarf tools helm cm-push](zarf_tools_helm_cm-push.md) - Please see https://github.com/chartmuseum/helm-push for usage * [zarf tools helm dependency](zarf_tools_helm_dependency.md) - manage a chart's dependencies * [zarf tools helm repo](zarf_tools_helm_repo.md) - add, list, remove, update, and index chart repositories diff --git a/docs/2-the-zarf-cli/100-cli-commands/zarf_tools_helm_cm-push.md b/docs/2-the-zarf-cli/100-cli-commands/zarf_tools_helm_cm-push.md new file mode 100644 index 0000000000..c82bd946ab --- /dev/null +++ b/docs/2-the-zarf-cli/100-cli-commands/zarf_tools_helm_cm-push.md @@ -0,0 +1,42 @@ +# zarf tools helm cm-push + + +Please see https://github.com/chartmuseum/helm-push for usage + +## Synopsis + +Push chart package to ChartMuseum + +``` +zarf tools helm cm-push [flags] +``` + +## Options + +``` + -h, --help help for cm-push +``` + +## Options inherited from parent commands + +``` + --burst-limit int client-side default throttling limit (default 100) + --debug enable verbose output + --kube-apiserver string the address and the port for the Kubernetes API server + --kube-as-group stringArray group to impersonate for the operation, this flag can be repeated to specify multiple groups. + --kube-as-user string username to impersonate for the operation + --kube-ca-file string the certificate authority file for the Kubernetes API server connection + --kube-context string name of the kubeconfig context to use + --kube-insecure-skip-tls-verify if true, the Kubernetes API server's certificate will not be checked for validity. This will make your HTTPS connections insecure + --kube-tls-server-name string server name to use for Kubernetes API server certificate validation. If it is not provided, the hostname used to contact the server is used + --kube-token string bearer token used for authentication + --kubeconfig string path to the kubeconfig file + -n, --namespace string namespace scope for this request + --registry-config string path to the registry config file + --repository-cache string path to the file containing cached repository indexes + --repository-config string path to the file containing repository names and URLs +``` + +## SEE ALSO + +* [zarf tools helm](zarf_tools_helm.md) - Subset of the Helm CLI included with Zarf to help manage helm charts. diff --git a/docs/3-create-a-zarf-package/4-zarf-schema.md b/docs/3-create-a-zarf-package/4-zarf-schema.md index 241382d6f6..3156977bce 100644 --- a/docs/3-create-a-zarf-package/4-zarf-schema.md +++ b/docs/3-create-a-zarf-package/4-zarf-schema.md @@ -783,10 +783,6 @@ Must be one of: | -------- | -------- | | **Type** | `string` | -| Restrictions | | -| --------------------------------- | ----------------------------------------------------------------------------------------------------------------------- | -| **Must match regular expression** | ```^(?!.*###ZARF_PKG_TMPL_).*$``` [Test](https://regex101.com/?regex=%5E%28%3F%21.%2A%23%23%23ZARF_PKG_TMPL_%29.%2A%24) | - @@ -803,9 +799,9 @@ Must be one of: | -------- | -------- | | **Type** | `string` | -| Restrictions | | -| --------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------- | -| **Must match regular expression** | ```^oci://(?!.*###ZARF_PKG_TMPL_).*$``` [Test](https://regex101.com/?regex=%5Eoci%3A%2F%2F%28%3F%21.%2A%23%23%23ZARF_PKG_TMPL_%29.%2A%24) | +| Restrictions | | +| --------------------------------- | --------------------------------------------------------------------------- | +| **Must match regular expression** | ```^oci://.*$``` [Test](https://regex101.com/?regex=%5Eoci%3A%2F%2F.%2A%24) | diff --git a/src/test/e2e/11_lint_test.go b/src/test/e2e/11_lint_test.go index 14b0865040..96e9cba750 100644 --- a/src/test/e2e/11_lint_test.go +++ b/src/test/e2e/11_lint_test.go @@ -23,8 +23,8 @@ func TestLint(t *testing.T) { t.Run("zarf test lint success", func(t *testing.T) { t.Log("E2E: Test lint on schema success") - path := filepath.Join("src", "test", "packages", "11-lint", "valid_yaml") - _, _, err := e2e.Zarf("prepare", "lint", path) + // This runs lint on the zarf.yaml in the base directory of the repo + _, _, err := e2e.Zarf("prepare", "lint") require.NoError(t, err, "Expect no error here because the yaml file is following schema") }) } diff --git a/src/test/packages/11-lint/invalid_yaml/zarf.yaml b/src/test/packages/11-lint/invalid_yaml/zarf.yaml index 39f1b899da..af38a32cb6 100644 --- a/src/test/packages/11-lint/invalid_yaml/zarf.yaml +++ b/src/test/packages/11-lint/invalid_yaml/zarf.yaml @@ -7,6 +7,7 @@ components: - name: k3s import: pat12312h: packages/distros/k3s + - name: import-test import: path: "###ZARF_PKG_TMPL_ZEBRA###" diff --git a/src/test/packages/11-lint/valid_yaml/zarf.yaml b/src/test/packages/11-lint/valid_yaml/zarf.yaml deleted file mode 100644 index 859813fde5..0000000000 --- a/src/test/packages/11-lint/valid_yaml/zarf.yaml +++ /dev/null @@ -1,43 +0,0 @@ -kind: ZarfInitConfig -metadata: - name: init - description: Used to establish a new Zarf cluster - -components: - - name: k3s - import: - path: packages/distros/k3s - - # This package moves the injector & registries binaries - - name: zarf-injector - required: true - import: - path: packages/zarf-registry - - # Creates the temporary seed-registry - - name: zarf-seed-registry - required: true - import: - path: packages/zarf-registry - - # Creates the permanent registry - - name: zarf-registry - required: true - import: - path: packages/zarf-registry - - # Creates the pod+git mutating webhook - - name: zarf-agent - required: true - import: - path: packages/zarf-agent - - # (Optional) Adds logging to the cluster - - name: logging - import: - path: packages/logging-pgl - - # (Optional) Adds a git server to the cluster - - name: git-server - import: - path: packages/gitea From 84e82c3bffd5cee077460eb15f5aef3d5b967216 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Wed, 8 Nov 2023 14:32:45 +0000 Subject: [PATCH 036/192] refactor e2e --- src/test/e2e/11_lint_test.go | 2 +- src/test/packages/11-lint/{invalid_yaml => }/zarf.yaml | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename src/test/packages/11-lint/{invalid_yaml => }/zarf.yaml (100%) diff --git a/src/test/e2e/11_lint_test.go b/src/test/e2e/11_lint_test.go index 96e9cba750..6232a06315 100644 --- a/src/test/e2e/11_lint_test.go +++ b/src/test/e2e/11_lint_test.go @@ -13,7 +13,7 @@ func TestLint(t *testing.T) { t.Run("zarf test lint fail", func(t *testing.T) { t.Log("E2E: Test lint on schema fail") - path := filepath.Join("src", "test", "packages", "11-lint", "invalid_yaml") + path := filepath.Join("src", "test", "packages", "11-lint") _, stderr, err := e2e.Zarf("prepare", "lint", path) require.Error(t, err, "Expect error here because the yaml file is not following schema") require.Contains(t, stderr, "- components.0.import: Additional property pat12312h is not allowed") diff --git a/src/test/packages/11-lint/invalid_yaml/zarf.yaml b/src/test/packages/11-lint/zarf.yaml similarity index 100% rename from src/test/packages/11-lint/invalid_yaml/zarf.yaml rename to src/test/packages/11-lint/zarf.yaml From 094a192d7dc66998575a1c9a775577f2f650cfd4 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Wed, 8 Nov 2023 14:55:02 +0000 Subject: [PATCH 037/192] refactor to use sprintf --- src/pkg/lint/validate_schema.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/pkg/lint/validate_schema.go b/src/pkg/lint/validate_schema.go index a728f00e34..24016553cf 100644 --- a/src/pkg/lint/validate_schema.go +++ b/src/pkg/lint/validate_schema.go @@ -2,7 +2,7 @@ package lint import ( "errors" - "strconv" + "fmt" "strings" "github.com/defenseunicorns/zarf/src/types" @@ -27,7 +27,7 @@ func validateSchema(unmarshalledYaml interface{}, jsonSchema []byte) error { if !result.Valid() { errorMessage := zarfInvalidPrefix for _, desc := range result.Errors() { - errorMessage = errorMessage + "\n - " + desc.String() + errorMessage = fmt.Sprintf("%s\n - %s", errorMessage, desc.String()) } err = errors.New(errorMessage) } @@ -38,16 +38,16 @@ func validateSchema(unmarshalledYaml interface{}, jsonSchema []byte) error { func checkForVarInComponentImport(zarfYaml types.ZarfPackage) error { valid := true errorMessage := zarfWarningPrefix - componentWarningStart := " component." + componentWarningStart := "component." for i, component := range zarfYaml.Components { if strings.Contains(component.Import.Path, zarfTemplateVar) { - errorMessage = errorMessage + componentWarningStart + strconv.Itoa(i) + - ".import.path will not resolve ZARF_PKG_TMPL_* variables." + errorMessage = fmt.Sprintf("%s %s%d.import.path will not resolve ZARF_PKG_TMPL_* variables.", + errorMessage, componentWarningStart, i) valid = false } if strings.Contains(component.Import.URL, zarfTemplateVar) { - errorMessage = errorMessage + componentWarningStart + strconv.Itoa(i) + - ".import.url will not resolve ZARF_PKG_TMPL_* variables." + errorMessage = fmt.Sprintf("%s %s%d.import.url will not resolve ZARF_PKG_TMPL_* variables.", + errorMessage, componentWarningStart, i) valid = false } } From 3b3aa75eae7ed126ab15e1b4589dd28d4037353b Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Wed, 8 Nov 2023 15:08:24 +0000 Subject: [PATCH 038/192] exporting template var string --- src/pkg/lint/validate_schema.go | 6 +++--- src/pkg/packager/variables.go | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/pkg/lint/validate_schema.go b/src/pkg/lint/validate_schema.go index 24016553cf..0b1d92179e 100644 --- a/src/pkg/lint/validate_schema.go +++ b/src/pkg/lint/validate_schema.go @@ -12,7 +12,7 @@ import ( const ( zarfInvalidPrefix = "zarf.yaml is not valid:" zarfWarningPrefix = "zarf schema warning:" - zarfTemplateVar = "###ZARF_PKG_TMPL_" + ZarfTemplateVar = "###ZARF_PKG_TMPL_" ) func validateSchema(unmarshalledYaml interface{}, jsonSchema []byte) error { @@ -40,12 +40,12 @@ func checkForVarInComponentImport(zarfYaml types.ZarfPackage) error { errorMessage := zarfWarningPrefix componentWarningStart := "component." for i, component := range zarfYaml.Components { - if strings.Contains(component.Import.Path, zarfTemplateVar) { + if strings.Contains(component.Import.Path, ZarfTemplateVar) { errorMessage = fmt.Sprintf("%s %s%d.import.path will not resolve ZARF_PKG_TMPL_* variables.", errorMessage, componentWarningStart, i) valid = false } - if strings.Contains(component.Import.URL, zarfTemplateVar) { + if strings.Contains(component.Import.URL, ZarfTemplateVar) { errorMessage = fmt.Sprintf("%s %s%d.import.url will not resolve ZARF_PKG_TMPL_* variables.", errorMessage, componentWarningStart, i) valid = false diff --git a/src/pkg/packager/variables.go b/src/pkg/packager/variables.go index f8f9dd20b4..b9e417ed6a 100644 --- a/src/pkg/packager/variables.go +++ b/src/pkg/packager/variables.go @@ -11,6 +11,7 @@ import ( "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/config/lang" "github.com/defenseunicorns/zarf/src/pkg/interactive" + "github.com/defenseunicorns/zarf/src/pkg/lint" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/types" ) @@ -59,7 +60,7 @@ func (p *Packager) fillActiveTemplate() error { return err } - if err := promptAndSetTemplate("###ZARF_PKG_TMPL_", false); err != nil { + if err := promptAndSetTemplate(lint.ZarfTemplateVar, false); err != nil { return err } // [DEPRECATION] Set the Package Variable syntax as well for backward compatibility From fa6ff162adbdfa2cb430735b74aa49c50ae6757c Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Wed, 8 Nov 2023 17:28:46 +0000 Subject: [PATCH 039/192] moved lint into packager --- .pre-commit-config.yaml | 2 +- src/cmd/prepare.go | 14 +++--- src/pkg/lint/lint.go | 43 ------------------- .../validate_schema.go => packager/lint.go} | 41 ++++++++++++++++-- .../lint_test.go} | 6 ++- .../unsuccessful_validation/zarf.yaml | 0 src/pkg/packager/variables.go | 3 +- 7 files changed, 51 insertions(+), 58 deletions(-) delete mode 100644 src/pkg/lint/lint.go rename src/pkg/{lint/validate_schema.go => packager/lint.go} (50%) rename src/pkg/{lint/validate_schema_test.go => packager/lint_test.go} (91%) rename src/pkg/{lint => packager}/unsuccessful_validation/zarf.yaml (100%) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 564e34c92a..a17e0ad46a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -49,6 +49,6 @@ repos: args: ["--schemafile", "zarf.schema.json"] exclude: | (?x)^( - src/pkg/lint/unsuccessful_validation/zarf.yaml| + src/pkg/packager/unsuccessful_validation/zarf.yaml| src/test/packages/11-lint/.* )$ diff --git a/src/cmd/prepare.go b/src/cmd/prepare.go index 2a9aab6358..1c5e658eb6 100644 --- a/src/cmd/prepare.go +++ b/src/cmd/prepare.go @@ -15,7 +15,6 @@ import ( "github.com/defenseunicorns/zarf/src/cmd/common" "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/config/lang" - "github.com/defenseunicorns/zarf/src/pkg/lint" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/packager" "github.com/defenseunicorns/zarf/src/pkg/transform" @@ -212,17 +211,18 @@ var lintCmd = &cobra.Command{ Aliases: []string{"l"}, Short: lang.CmdLintShort, Run: func(cmd *cobra.Command, args []string) { - baseDir := "" if len(args) > 0 { - baseDir = args[0] + pkgConfig.CreateOpts.BaseDir = args[0] } else { - var err error - baseDir, err = os.Getwd() + cwd, err := os.Getwd() if err != nil { - message.Fatalf(err, lang.CmdPackageCreateErr, err.Error()) + message.Fatalf(err, lang.CmdPrepareFindImagesErr, err.Error()) } + pkgConfig.CreateOpts.BaseDir = cwd } - err := lint.ValidateZarfSchema(baseDir) + pkgClient := packager.NewOrDie(&pkgConfig) + defer pkgClient.ClearTempPaths() + err := pkgClient.ValidateZarfSchema() if err != nil { message.Fatal(err, err.Error()) } diff --git a/src/pkg/lint/lint.go b/src/pkg/lint/lint.go deleted file mode 100644 index ee119ac663..0000000000 --- a/src/pkg/lint/lint.go +++ /dev/null @@ -1,43 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: 2021-Present The Zarf Authors - -// Package lint contains functions for linting the zarf.yaml -package lint - -import ( - "path/filepath" - - "github.com/defenseunicorns/zarf/src/config" - "github.com/defenseunicorns/zarf/src/pkg/layout" - "github.com/defenseunicorns/zarf/src/pkg/message" - "github.com/defenseunicorns/zarf/src/pkg/utils" - "github.com/defenseunicorns/zarf/src/types" -) - -// ValidateZarfSchema a zarf file against the zarf schema, returns an error if the file is invalid -func ValidateZarfSchema(baseDir string) (err error) { - var typedZarfData types.ZarfPackage - if err := utils.ReadYaml(filepath.Join(baseDir, layout.ZarfYAML), &typedZarfData); err != nil { - return err - } - - err = checkForVarInComponentImport(typedZarfData) - if err != nil { - message.Warn(err.Error()) - } - - zarfSchema, _ := config.GetSchemaFile() - var zarfData interface{} - if err := utils.ReadYaml(filepath.Join(baseDir, layout.ZarfYAML), &zarfData); err != nil { - return err - } - - err = validateSchema(zarfData, zarfSchema) - - if err != nil { - return err - } - - message.Success("Validation successful") - return nil -} diff --git a/src/pkg/lint/validate_schema.go b/src/pkg/packager/lint.go similarity index 50% rename from src/pkg/lint/validate_schema.go rename to src/pkg/packager/lint.go index 0b1d92179e..939b3b1193 100644 --- a/src/pkg/lint/validate_schema.go +++ b/src/pkg/packager/lint.go @@ -1,10 +1,19 @@ -package lint +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package packager contains functions for linting the zarf.yaml +package packager import ( "errors" "fmt" + "path/filepath" "strings" + "github.com/defenseunicorns/zarf/src/config" + "github.com/defenseunicorns/zarf/src/pkg/layout" + "github.com/defenseunicorns/zarf/src/pkg/message" + "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/types" "github.com/xeipuuv/gojsonschema" ) @@ -12,9 +21,33 @@ import ( const ( zarfInvalidPrefix = "zarf.yaml is not valid:" zarfWarningPrefix = "zarf schema warning:" - ZarfTemplateVar = "###ZARF_PKG_TMPL_" + zarfTemplateVar = "###ZARF_PKG_TMPL_" ) +// ValidateZarfSchema a zarf file against the zarf schema, returns an error if the file is invalid +func (p *Packager) ValidateZarfSchema() (err error) { + if err = p.readZarfYAML(filepath.Join(p.cfg.CreateOpts.BaseDir, layout.ZarfYAML)); err != nil { + return fmt.Errorf("unable to read the zarf.yaml file: %s", err.Error()) + } + + if err := checkForVarInComponentImport(p.cfg.Pkg); err != nil { + message.Warn(err.Error()) + } + + zarfSchema, _ := config.GetSchemaFile() + var zarfData interface{} + if err := utils.ReadYaml(filepath.Join(p.cfg.CreateOpts.BaseDir, layout.ZarfYAML), &zarfData); err != nil { + return err + } + + if err = validateSchema(zarfData, zarfSchema); err != nil { + return err + } + + message.Success("Validation successful") + return nil +} + func validateSchema(unmarshalledYaml interface{}, jsonSchema []byte) error { schemaLoader := gojsonschema.NewBytesLoader(jsonSchema) documentLoader := gojsonschema.NewGoLoader(unmarshalledYaml) @@ -40,12 +73,12 @@ func checkForVarInComponentImport(zarfYaml types.ZarfPackage) error { errorMessage := zarfWarningPrefix componentWarningStart := "component." for i, component := range zarfYaml.Components { - if strings.Contains(component.Import.Path, ZarfTemplateVar) { + if strings.Contains(component.Import.Path, zarfTemplateVar) { errorMessage = fmt.Sprintf("%s %s%d.import.path will not resolve ZARF_PKG_TMPL_* variables.", errorMessage, componentWarningStart, i) valid = false } - if strings.Contains(component.Import.URL, ZarfTemplateVar) { + if strings.Contains(component.Import.URL, zarfTemplateVar) { errorMessage = fmt.Sprintf("%s %s%d.import.url will not resolve ZARF_PKG_TMPL_* variables.", errorMessage, componentWarningStart, i) valid = false diff --git a/src/pkg/lint/validate_schema_test.go b/src/pkg/packager/lint_test.go similarity index 91% rename from src/pkg/lint/validate_schema_test.go rename to src/pkg/packager/lint_test.go index 8c914557d1..1b3320b3c9 100644 --- a/src/pkg/lint/validate_schema_test.go +++ b/src/pkg/packager/lint_test.go @@ -1,4 +1,8 @@ -package lint +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package packager contains functions for interacting with, managing and deploying Zarf packages. +package packager import ( "os" diff --git a/src/pkg/lint/unsuccessful_validation/zarf.yaml b/src/pkg/packager/unsuccessful_validation/zarf.yaml similarity index 100% rename from src/pkg/lint/unsuccessful_validation/zarf.yaml rename to src/pkg/packager/unsuccessful_validation/zarf.yaml diff --git a/src/pkg/packager/variables.go b/src/pkg/packager/variables.go index b9e417ed6a..2a07944c1c 100644 --- a/src/pkg/packager/variables.go +++ b/src/pkg/packager/variables.go @@ -11,7 +11,6 @@ import ( "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/config/lang" "github.com/defenseunicorns/zarf/src/pkg/interactive" - "github.com/defenseunicorns/zarf/src/pkg/lint" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/types" ) @@ -60,7 +59,7 @@ func (p *Packager) fillActiveTemplate() error { return err } - if err := promptAndSetTemplate(lint.ZarfTemplateVar, false); err != nil { + if err := promptAndSetTemplate(zarfTemplateVar, false); err != nil { return err } // [DEPRECATION] Set the Package Variable syntax as well for backward compatibility From 02a39b737d77b8dbe0af9dde9d4834b7b70ca1f9 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Wed, 8 Nov 2023 17:36:35 +0000 Subject: [PATCH 040/192] move 11 to 12 --- .pre-commit-config.yaml | 2 +- src/test/e2e/11_lint_test.go | 30 ----------------------------- src/test/packages/11-lint/zarf.yaml | 17 ---------------- 3 files changed, 1 insertion(+), 48 deletions(-) delete mode 100644 src/test/e2e/11_lint_test.go delete mode 100644 src/test/packages/11-lint/zarf.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a17e0ad46a..3217685488 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -50,5 +50,5 @@ repos: exclude: | (?x)^( src/pkg/packager/unsuccessful_validation/zarf.yaml| - src/test/packages/11-lint/.* + src/test/packages/12-lint/.* )$ diff --git a/src/test/e2e/11_lint_test.go b/src/test/e2e/11_lint_test.go deleted file mode 100644 index 6232a06315..0000000000 --- a/src/test/e2e/11_lint_test.go +++ /dev/null @@ -1,30 +0,0 @@ -package test - -import ( - "path/filepath" - "testing" - - "github.com/stretchr/testify/require" -) - -func TestLint(t *testing.T) { - t.Log("E2E: Lint") - - t.Run("zarf test lint fail", func(t *testing.T) { - t.Log("E2E: Test lint on schema fail") - - path := filepath.Join("src", "test", "packages", "11-lint") - _, stderr, err := e2e.Zarf("prepare", "lint", path) - require.Error(t, err, "Expect error here because the yaml file is not following schema") - require.Contains(t, stderr, "- components.0.import: Additional property pat12312h is not allowed") - require.Contains(t, stderr, "component.1.import.path will not resolve ZARF_PKG_TMPL_* variables") - }) - - t.Run("zarf test lint success", func(t *testing.T) { - t.Log("E2E: Test lint on schema success") - - // This runs lint on the zarf.yaml in the base directory of the repo - _, _, err := e2e.Zarf("prepare", "lint") - require.NoError(t, err, "Expect no error here because the yaml file is following schema") - }) -} diff --git a/src/test/packages/11-lint/zarf.yaml b/src/test/packages/11-lint/zarf.yaml deleted file mode 100644 index af38a32cb6..0000000000 --- a/src/test/packages/11-lint/zarf.yaml +++ /dev/null @@ -1,17 +0,0 @@ -kind: ZarfInitConfig -metadata: - name: init - description: Used to establish a new Zarf cluster - -components: - - name: k3s - import: - pat12312h: packages/distros/k3s - - - name: import-test - import: - path: "###ZARF_PKG_TMPL_ZEBRA###" - - - name: import-url - import: - url: "oci://###ZARF_PKG_TMPL_ZEBRA###" From 3bd8a7d7810889afe320ac78a3f16b956082db65 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Wed, 8 Nov 2023 17:38:13 +0000 Subject: [PATCH 041/192] with files this time --- src/test/e2e/12_lint_test.go | 30 +++++++++++++++++++++++++++++ src/test/packages/12-lint/zarf.yaml | 17 ++++++++++++++++ 2 files changed, 47 insertions(+) create mode 100644 src/test/e2e/12_lint_test.go create mode 100644 src/test/packages/12-lint/zarf.yaml diff --git a/src/test/e2e/12_lint_test.go b/src/test/e2e/12_lint_test.go new file mode 100644 index 0000000000..d0252a82f6 --- /dev/null +++ b/src/test/e2e/12_lint_test.go @@ -0,0 +1,30 @@ +package test + +import ( + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestLint(t *testing.T) { + t.Log("E2E: Lint") + + t.Run("zarf test lint fail", func(t *testing.T) { + t.Log("E2E: Test lint on schema fail") + + path := filepath.Join("src", "test", "packages", "12-lint") + _, stderr, err := e2e.Zarf("prepare", "lint", path) + require.Error(t, err, "Expect error here because the yaml file is not following schema") + require.Contains(t, stderr, "- components.0.import: Additional property pat12312h is not allowed") + require.Contains(t, stderr, "component.1.import.path will not resolve ZARF_PKG_TMPL_* variables") + }) + + t.Run("zarf test lint success", func(t *testing.T) { + t.Log("E2E: Test lint on schema success") + + // This runs lint on the zarf.yaml in the base directory of the repo + _, _, err := e2e.Zarf("prepare", "lint") + require.NoError(t, err, "Expect no error here because the yaml file is following schema") + }) +} diff --git a/src/test/packages/12-lint/zarf.yaml b/src/test/packages/12-lint/zarf.yaml new file mode 100644 index 0000000000..af38a32cb6 --- /dev/null +++ b/src/test/packages/12-lint/zarf.yaml @@ -0,0 +1,17 @@ +kind: ZarfInitConfig +metadata: + name: init + description: Used to establish a new Zarf cluster + +components: + - name: k3s + import: + pat12312h: packages/distros/k3s + + - name: import-test + import: + path: "###ZARF_PKG_TMPL_ZEBRA###" + + - name: import-url + import: + url: "oci://###ZARF_PKG_TMPL_ZEBRA###" From 5013c9f4b06ff789700806f2de7e290e333ad21e Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Thu, 9 Nov 2023 15:17:52 +0000 Subject: [PATCH 042/192] changing to use more maintained json package --- go.mod | 1 + go.sum | 2 + src/pkg/packager/lint.go | 57 +++++++++++++++++-- src/pkg/packager/lint_test.go | 8 +-- .../unsuccessful_validation/zarf.yaml | 6 +- src/test/e2e/12_lint_test.go | 4 +- 6 files changed, 64 insertions(+), 14 deletions(-) diff --git a/go.mod b/go.mod index 958a9962e2..e56edc2bd9 100644 --- a/go.mod +++ b/go.mod @@ -371,6 +371,7 @@ require ( github.com/sagikazarmark/locafero v0.3.0 // indirect github.com/sagikazarmark/slog-shim v0.1.0 // indirect github.com/sahilm/fuzzy v0.1.0 // indirect + github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 // indirect github.com/sassoftware/go-rpmutils v0.2.0 // indirect github.com/sassoftware/relic v7.2.1+incompatible // indirect github.com/scylladb/go-set v1.0.3-0.20200225121959-cc7b2070d91e // indirect diff --git a/go.sum b/go.sum index 848584069a..59465d5175 100644 --- a/go.sum +++ b/go.sum @@ -1425,6 +1425,8 @@ github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6g github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= github.com/sahilm/fuzzy v0.1.0 h1:FzWGaw2Opqyu+794ZQ9SYifWv2EIXpwP4q8dY1kDAwI= github.com/sahilm/fuzzy v0.1.0/go.mod h1:VFvziUEIMCrT6A6tw2RFIXPXXmzXbOsSHF0DOI8ZK9Y= +github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 h1:lZUw3E0/J3roVtGQ+SCrUrg3ON6NgVqpn3+iol9aGu4= +github.com/santhosh-tekuri/jsonschema/v5 v5.3.1/go.mod h1:uToXkOrWAZ6/Oc07xWQrPOhJotwFIyu2bBVN41fcDUY= github.com/sassoftware/go-rpmutils v0.2.0 h1:pKW0HDYMFWQ5b4JQPiI3WI12hGsVoW0V8+GMoZiI/JE= github.com/sassoftware/go-rpmutils v0.2.0/go.mod h1:TJJQYtLe/BeEmEjelI3b7xNZjzAukEkeWKmoakvaOoI= github.com/sassoftware/relic v7.2.1+incompatible h1:Pwyh1F3I0r4clFJXkSI8bOyJINGqpgjJU3DYAZeI05A= diff --git a/src/pkg/packager/lint.go b/src/pkg/packager/lint.go index 939b3b1193..811e43ad07 100644 --- a/src/pkg/packager/lint.go +++ b/src/pkg/packager/lint.go @@ -5,6 +5,7 @@ package packager import ( + "bytes" "errors" "fmt" "path/filepath" @@ -15,11 +16,12 @@ import ( "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/types" + "github.com/santhosh-tekuri/jsonschema/v5" "github.com/xeipuuv/gojsonschema" ) const ( - zarfInvalidPrefix = "zarf.yaml is not valid:" + zarfInvalidPrefix = "schema is invalid:" zarfWarningPrefix = "zarf schema warning:" zarfTemplateVar = "###ZARF_PKG_TMPL_" ) @@ -48,7 +50,7 @@ func (p *Packager) ValidateZarfSchema() (err error) { return nil } -func validateSchema(unmarshalledYaml interface{}, jsonSchema []byte) error { +func validateSchema2(unmarshalledYaml interface{}, jsonSchema []byte) error { schemaLoader := gojsonschema.NewBytesLoader(jsonSchema) documentLoader := gojsonschema.NewGoLoader(unmarshalledYaml) @@ -71,15 +73,15 @@ func validateSchema(unmarshalledYaml interface{}, jsonSchema []byte) error { func checkForVarInComponentImport(zarfYaml types.ZarfPackage) error { valid := true errorMessage := zarfWarningPrefix - componentWarningStart := "component." + componentWarningStart := "component/" for i, component := range zarfYaml.Components { if strings.Contains(component.Import.Path, zarfTemplateVar) { - errorMessage = fmt.Sprintf("%s %s%d.import.path will not resolve ZARF_PKG_TMPL_* variables.", + errorMessage = fmt.Sprintf("%s %s%d/import/path will not resolve ZARF_PKG_TMPL_* variables.", errorMessage, componentWarningStart, i) valid = false } if strings.Contains(component.Import.URL, zarfTemplateVar) { - errorMessage = fmt.Sprintf("%s %s%d.import.url will not resolve ZARF_PKG_TMPL_* variables.", + errorMessage = fmt.Sprintf("%s %s%d/import/url will not resolve ZARF_PKG_TMPL_* variables.", errorMessage, componentWarningStart, i) valid = false } @@ -89,3 +91,48 @@ func checkForVarInComponentImport(zarfYaml types.ZarfPackage) error { } return errors.New(errorMessage) } + +func validateSchema(unmarshalledYaml interface{}, jsonSchema []byte) error { + compiler := jsonschema.NewCompiler() + inMemoryZarfSchema := "schema.json" + + if err := compiler.AddResource(inMemoryZarfSchema, bytes.NewReader(jsonSchema)); err != nil { + return err + } + + schema, err := compiler.Compile(inMemoryZarfSchema) + if err != nil { + return err + } + + if err := schema.Validate(unmarshalledYaml); err != nil { + if validationError, ok := err.(*jsonschema.ValidationError); ok { + allSchemaErrors := printAllCauses(validationError, []error{}) + var errMessage strings.Builder + errMessage.WriteString(zarfInvalidPrefix) + for _, err := range allSchemaErrors { + errMessage.WriteString("\n") + errMessage.WriteString(err.Error()) + } + return errors.New(errMessage.String()) + } + return err + } + + return nil +} + +func printAllCauses(validationErr *jsonschema.ValidationError, errToUser []error) []error { + if validationErr == nil { + return errToUser + } + if validationErr.Causes == nil { + errMessage := fmt.Sprintf(" - %s: %s", validationErr.InstanceLocation, validationErr.Message) + return append(errToUser, fmt.Errorf("%s", errMessage)) + } + + for _, subCause := range validationErr.Causes { + errToUser = printAllCauses(subCause, errToUser) + } + return errToUser +} diff --git a/src/pkg/packager/lint_test.go b/src/pkg/packager/lint_test.go index 1b3320b3c9..0fec409b31 100644 --- a/src/pkg/packager/lint_test.go +++ b/src/pkg/packager/lint_test.go @@ -51,8 +51,8 @@ func TestValidateSchema(t *testing.T) { zarfSchema := readSchema(t) err := validateSchema(unmarshalledYaml, zarfSchema) errorMessage := zarfInvalidPrefix + ` - - components.0.import: Additional property not-path is not allowed - - components.1: Additional property not-import is not allowed` + - /components/0/import: additionalProperties 'not-path' not allowed + - /components/1/import/path: expected string, but got number` require.EqualError(t, err, errorMessage) }) @@ -65,8 +65,8 @@ func TestValidateSchema(t *testing.T) { t.Run("Template in component import failure", func(t *testing.T) { unmarshalledYaml := readAndUnmarshallZarfPackage(t, "unsuccessful_validation/zarf.yaml") err := checkForVarInComponentImport(unmarshalledYaml) - errorMessage := zarfWarningPrefix + " component.2.import.path will not resolve ZARF_PKG_TMPL_* variables. " + - "component.3.import.url will not resolve ZARF_PKG_TMPL_* variables." + errorMessage := zarfWarningPrefix + " component/2/import/path will not resolve ZARF_PKG_TMPL_* variables. " + + "component/3/import/url will not resolve ZARF_PKG_TMPL_* variables." require.EqualError(t, err, errorMessage) }) } diff --git a/src/pkg/packager/unsuccessful_validation/zarf.yaml b/src/pkg/packager/unsuccessful_validation/zarf.yaml index 639f1cf462..3c2d75c816 100644 --- a/src/pkg/packager/unsuccessful_validation/zarf.yaml +++ b/src/pkg/packager/unsuccessful_validation/zarf.yaml @@ -8,9 +8,9 @@ components: import: not-path: packages/distros/k3s - - name: second-test-component - not-import: - path: packages/distros/k3s + - name: import-test + import: + path: 123123 - name: import-test import: diff --git a/src/test/e2e/12_lint_test.go b/src/test/e2e/12_lint_test.go index d0252a82f6..598480a62c 100644 --- a/src/test/e2e/12_lint_test.go +++ b/src/test/e2e/12_lint_test.go @@ -16,8 +16,8 @@ func TestLint(t *testing.T) { path := filepath.Join("src", "test", "packages", "12-lint") _, stderr, err := e2e.Zarf("prepare", "lint", path) require.Error(t, err, "Expect error here because the yaml file is not following schema") - require.Contains(t, stderr, "- components.0.import: Additional property pat12312h is not allowed") - require.Contains(t, stderr, "component.1.import.path will not resolve ZARF_PKG_TMPL_* variables") + require.Contains(t, stderr, "- /components/0/import: additionalProperties 'pat12312h' not allowed") + require.Contains(t, stderr, "component/1/import/path will not resolve ZARF_PKG_TMPL_* variables") }) t.Run("zarf test lint success", func(t *testing.T) { From 5b67deabf35f9a488d0f2c2d04b8462b73123b1e Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Thu, 9 Nov 2023 15:19:07 +0000 Subject: [PATCH 043/192] removing old jsonschema package --- go.mod | 4 ++-- src/pkg/packager/lint.go | 21 --------------------- 2 files changed, 2 insertions(+), 23 deletions(-) diff --git a/go.mod b/go.mod index e56edc2bd9..e0eb0e7732 100644 --- a/go.mod +++ b/go.mod @@ -31,6 +31,7 @@ require ( github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.17.0 github.com/pterm/pterm v0.12.69 + github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 github.com/sergi/go-diff v1.3.1 github.com/sigstore/cosign/v2 v2.2.0 github.com/sigstore/sigstore/pkg/signature/kms/aws v1.7.4 @@ -41,7 +42,6 @@ require ( github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.17.0 github.com/stretchr/testify v1.8.4 - github.com/xeipuuv/gojsonschema v1.2.0 golang.org/x/crypto v0.14.0 golang.org/x/sync v0.4.0 golang.org/x/term v0.13.0 @@ -371,7 +371,6 @@ require ( github.com/sagikazarmark/locafero v0.3.0 // indirect github.com/sagikazarmark/slog-shim v0.1.0 // indirect github.com/sahilm/fuzzy v0.1.0 // indirect - github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 // indirect github.com/sassoftware/go-rpmutils v0.2.0 // indirect github.com/sassoftware/relic v7.2.1+incompatible // indirect github.com/scylladb/go-set v1.0.3-0.20200225121959-cc7b2070d91e // indirect @@ -416,6 +415,7 @@ require ( github.com/xanzy/ssh-agent v0.3.3 // indirect github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect + github.com/xeipuuv/gojsonschema v1.2.0 // indirect github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect github.com/xlab/treeprint v1.2.0 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect diff --git a/src/pkg/packager/lint.go b/src/pkg/packager/lint.go index 811e43ad07..87a80ba05a 100644 --- a/src/pkg/packager/lint.go +++ b/src/pkg/packager/lint.go @@ -17,7 +17,6 @@ import ( "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/types" "github.com/santhosh-tekuri/jsonschema/v5" - "github.com/xeipuuv/gojsonschema" ) const ( @@ -50,26 +49,6 @@ func (p *Packager) ValidateZarfSchema() (err error) { return nil } -func validateSchema2(unmarshalledYaml interface{}, jsonSchema []byte) error { - schemaLoader := gojsonschema.NewBytesLoader(jsonSchema) - documentLoader := gojsonschema.NewGoLoader(unmarshalledYaml) - - result, err := gojsonschema.Validate(schemaLoader, documentLoader) - if err != nil { - return err - } - - if !result.Valid() { - errorMessage := zarfInvalidPrefix - for _, desc := range result.Errors() { - errorMessage = fmt.Sprintf("%s\n - %s", errorMessage, desc.String()) - } - err = errors.New(errorMessage) - } - - return err -} - func checkForVarInComponentImport(zarfYaml types.ZarfPackage) error { valid := true errorMessage := zarfWarningPrefix From 1e998459c97b1657a457db6f0f4b55c9ef10265e Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Thu, 9 Nov 2023 15:31:26 +0000 Subject: [PATCH 044/192] changed language --- src/cmd/prepare.go | 2 +- src/config/lang/english.go | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/cmd/prepare.go b/src/cmd/prepare.go index 1c5e658eb6..6154249036 100644 --- a/src/cmd/prepare.go +++ b/src/cmd/prepare.go @@ -209,7 +209,7 @@ var lintCmd = &cobra.Command{ Use: "lint [ DIRECTORY ]", Args: cobra.MaximumNArgs(1), Aliases: []string{"l"}, - Short: lang.CmdLintShort, + Short: lang.CmdPrepareLintShort, Run: func(cmd *cobra.Command, args []string) { if len(args) > 0 { pkgConfig.CreateOpts.BaseDir = args[0] diff --git a/src/config/lang/english.go b/src/config/lang/english.go index 91edf55868..296b9ad419 100644 --- a/src/config/lang/english.go +++ b/src/config/lang/english.go @@ -305,9 +305,6 @@ const ( CmdPackageClusterSourceFallback = "%q does not satisfy any current sources, assuming it is a package deployed to a cluster" CmdPackageInvalidSource = "Unable to identify source from %q: %s" - //zarf lint - CmdLintShort = "Tools for validating schema / ensuring best practices" - // zarf prepare CmdPrepareShort = "Tools to help prepare assets for packaging" @@ -340,6 +337,8 @@ const ( CmdPrepareFlagGitAccount = "User or organization name for the git account that the repos are created under." CmdPrepareFlagKubeVersion = "Override the default helm template KubeVersion when performing a package chart template" + CmdPrepareLintShort = "Verifies the package schema" + // zarf tools CmdToolsShort = "Collection of additional tools to make airgap easier" From 94fad1cf693c3ed7fefe79f41d0224508a1dbfa9 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Thu, 9 Nov 2023 15:32:08 +0000 Subject: [PATCH 045/192] make docs --- docs/2-the-zarf-cli/100-cli-commands/zarf_prepare.md | 2 +- docs/2-the-zarf-cli/100-cli-commands/zarf_prepare_lint.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/2-the-zarf-cli/100-cli-commands/zarf_prepare.md b/docs/2-the-zarf-cli/100-cli-commands/zarf_prepare.md index c8b2ecd222..837e4c03ba 100644 --- a/docs/2-the-zarf-cli/100-cli-commands/zarf_prepare.md +++ b/docs/2-the-zarf-cli/100-cli-commands/zarf_prepare.md @@ -27,7 +27,7 @@ Tools to help prepare assets for packaging * [zarf](zarf.md) - DevSecOps for Airgap * [zarf prepare find-images](zarf_prepare_find-images.md) - Evaluates components in a zarf file to identify images specified in their helm charts and manifests * [zarf prepare generate-config](zarf_prepare_generate-config.md) - Generates a config file for Zarf -* [zarf prepare lint](zarf_prepare_lint.md) - Tools for validating schema / ensuring best practices +* [zarf prepare lint](zarf_prepare_lint.md) - Verifies the package schema * [zarf prepare patch-git](zarf_prepare_patch-git.md) - Converts all .git URLs to the specified Zarf HOST and with the Zarf URL pattern in a given FILE. NOTE: This should only be used for manifests that are not mutated by the Zarf Agent Mutating Webhook. * [zarf prepare sha256sum](zarf_prepare_sha256sum.md) - Generates a SHA256SUM for the given file diff --git a/docs/2-the-zarf-cli/100-cli-commands/zarf_prepare_lint.md b/docs/2-the-zarf-cli/100-cli-commands/zarf_prepare_lint.md index a114cba47f..4de015b12d 100644 --- a/docs/2-the-zarf-cli/100-cli-commands/zarf_prepare_lint.md +++ b/docs/2-the-zarf-cli/100-cli-commands/zarf_prepare_lint.md @@ -1,7 +1,7 @@ # zarf prepare lint -Tools for validating schema / ensuring best practices +Verifies the package schema ``` zarf prepare lint [ DIRECTORY ] [flags] From 4750a1acb73f4adfe20fe78eaafa58b330ae5780 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Thu, 9 Nov 2023 16:59:09 +0000 Subject: [PATCH 046/192] refactor tests --- src/pkg/packager/lint_test.go | 34 ++++++++++++------- .../unsuccessful_validation/zarf.yaml | 21 ------------ src/test/e2e/12_lint_test.go | 4 +-- src/test/packages/12-lint/zarf.yaml | 10 ++++-- 4 files changed, 31 insertions(+), 38 deletions(-) delete mode 100644 src/pkg/packager/unsuccessful_validation/zarf.yaml diff --git a/src/pkg/packager/lint_test.go b/src/pkg/packager/lint_test.go index 0fec409b31..cb1012e44f 100644 --- a/src/pkg/packager/lint_test.go +++ b/src/pkg/packager/lint_test.go @@ -14,28 +14,38 @@ import ( ) func TestValidateSchema(t *testing.T) { - readSchema := func(t *testing.T) []byte { - t.Helper() - zarfSchema, err := os.ReadFile("../../../zarf.schema.json") + readFileFailFatally := func(t *testing.T, path string) []byte { + file, err := os.ReadFile(path) if err != nil { - t.Fatalf("Error reading schema file: %s", err) + t.Errorf("error reading file: %s", err) } - return zarfSchema + return file + } + + readSchema := func(t *testing.T) []byte { + t.Helper() + return readFileFailFatally(t, "../../../zarf.schema.json") } readAndUnmarshalYaml := func(t *testing.T, path string) interface{} { t.Helper() var unmarshalledYaml interface{} - file, _ := os.ReadFile(path) - goyaml.Unmarshal(file, &unmarshalledYaml) + file := readFileFailFatally(t, path) + err := goyaml.Unmarshal(file, &unmarshalledYaml) + if err != nil { + t.Errorf("error unmarshalling yaml %s", err) + } return unmarshalledYaml } readAndUnmarshallZarfPackage := func(t *testing.T, path string) types.ZarfPackage { t.Helper() var unmarshalledYaml types.ZarfPackage - file, _ := os.ReadFile(path) - goyaml.Unmarshal(file, &unmarshalledYaml) + file := readFileFailFatally(t, path) + err := goyaml.Unmarshal(file, &unmarshalledYaml) + if err != nil { + t.Errorf("error unmarshalling yaml %s", err) + } return unmarshalledYaml } @@ -47,7 +57,7 @@ func TestValidateSchema(t *testing.T) { }) t.Run("validate schema fail", func(t *testing.T) { - unmarshalledYaml := readAndUnmarshalYaml(t, "unsuccessful_validation/zarf.yaml") + unmarshalledYaml := readAndUnmarshalYaml(t, "../../test/packages/12-lint/zarf.yaml") zarfSchema := readSchema(t) err := validateSchema(unmarshalledYaml, zarfSchema) errorMessage := zarfInvalidPrefix + ` @@ -57,13 +67,13 @@ func TestValidateSchema(t *testing.T) { }) t.Run("Template in component import success", func(t *testing.T) { - unmarshalledYaml := readAndUnmarshallZarfPackage(t, "successful_validation/zarf.yaml") + unmarshalledYaml := readAndUnmarshallZarfPackage(t, "../../../zarf.yaml") err := checkForVarInComponentImport(unmarshalledYaml) require.NoError(t, err) }) t.Run("Template in component import failure", func(t *testing.T) { - unmarshalledYaml := readAndUnmarshallZarfPackage(t, "unsuccessful_validation/zarf.yaml") + unmarshalledYaml := readAndUnmarshallZarfPackage(t, "../../test/packages/12-lint/zarf.yaml") err := checkForVarInComponentImport(unmarshalledYaml) errorMessage := zarfWarningPrefix + " component/2/import/path will not resolve ZARF_PKG_TMPL_* variables. " + "component/3/import/url will not resolve ZARF_PKG_TMPL_* variables." diff --git a/src/pkg/packager/unsuccessful_validation/zarf.yaml b/src/pkg/packager/unsuccessful_validation/zarf.yaml deleted file mode 100644 index 3c2d75c816..0000000000 --- a/src/pkg/packager/unsuccessful_validation/zarf.yaml +++ /dev/null @@ -1,21 +0,0 @@ -kind: ZarfInitConfig -metadata: - name: init - description: Testing bad yaml - -components: - - name: first-test-component - import: - not-path: packages/distros/k3s - - - name: import-test - import: - path: 123123 - - - name: import-test - import: - path: "###ZARF_PKG_TMPL_ZEBRA###" - - - name: import-url - import: - url: "oci://###ZARF_PKG_TMPL_ZEBRA###" diff --git a/src/test/e2e/12_lint_test.go b/src/test/e2e/12_lint_test.go index 598480a62c..c88f10b664 100644 --- a/src/test/e2e/12_lint_test.go +++ b/src/test/e2e/12_lint_test.go @@ -16,8 +16,8 @@ func TestLint(t *testing.T) { path := filepath.Join("src", "test", "packages", "12-lint") _, stderr, err := e2e.Zarf("prepare", "lint", path) require.Error(t, err, "Expect error here because the yaml file is not following schema") - require.Contains(t, stderr, "- /components/0/import: additionalProperties 'pat12312h' not allowed") - require.Contains(t, stderr, "component/1/import/path will not resolve ZARF_PKG_TMPL_* variables") + require.Contains(t, stderr, "- /components/0/import: additionalProperties 'not-path' not allowed") + require.Contains(t, stderr, "component/2/import/path will not resolve ZARF_PKG_TMPL_* variables") }) t.Run("zarf test lint success", func(t *testing.T) { diff --git a/src/test/packages/12-lint/zarf.yaml b/src/test/packages/12-lint/zarf.yaml index af38a32cb6..3c2d75c816 100644 --- a/src/test/packages/12-lint/zarf.yaml +++ b/src/test/packages/12-lint/zarf.yaml @@ -1,12 +1,16 @@ kind: ZarfInitConfig metadata: name: init - description: Used to establish a new Zarf cluster + description: Testing bad yaml components: - - name: k3s + - name: first-test-component import: - pat12312h: packages/distros/k3s + not-path: packages/distros/k3s + + - name: import-test + import: + path: 123123 - name: import-test import: From 167a78dada35c83db7d6a52e6d3c4a3dcbee20c2 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Thu, 9 Nov 2023 17:37:42 +0000 Subject: [PATCH 047/192] refactor func name --- src/pkg/packager/lint.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/pkg/packager/lint.go b/src/pkg/packager/lint.go index 87a80ba05a..edd513762f 100644 --- a/src/pkg/packager/lint.go +++ b/src/pkg/packager/lint.go @@ -86,7 +86,7 @@ func validateSchema(unmarshalledYaml interface{}, jsonSchema []byte) error { if err := schema.Validate(unmarshalledYaml); err != nil { if validationError, ok := err.(*jsonschema.ValidationError); ok { - allSchemaErrors := printAllCauses(validationError, []error{}) + allSchemaErrors := getChildCauses(validationError, []error{}) var errMessage strings.Builder errMessage.WriteString(zarfInvalidPrefix) for _, err := range allSchemaErrors { @@ -101,7 +101,7 @@ func validateSchema(unmarshalledYaml interface{}, jsonSchema []byte) error { return nil } -func printAllCauses(validationErr *jsonschema.ValidationError, errToUser []error) []error { +func getChildCauses(validationErr *jsonschema.ValidationError, errToUser []error) []error { if validationErr == nil { return errToUser } @@ -111,7 +111,7 @@ func printAllCauses(validationErr *jsonschema.ValidationError, errToUser []error } for _, subCause := range validationErr.Causes { - errToUser = printAllCauses(subCause, errToUser) + errToUser = getChildCauses(subCause, errToUser) } return errToUser } From 762c34e37a7a54326004af121ee2c42369dbbd7b Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Thu, 9 Nov 2023 17:48:58 +0000 Subject: [PATCH 048/192] refactor lint --- src/pkg/packager/lint.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/pkg/packager/lint.go b/src/pkg/packager/lint.go index edd513762f..b0853f4d28 100644 --- a/src/pkg/packager/lint.go +++ b/src/pkg/packager/lint.go @@ -101,17 +101,17 @@ func validateSchema(unmarshalledYaml interface{}, jsonSchema []byte) error { return nil } -func getChildCauses(validationErr *jsonschema.ValidationError, errToUser []error) []error { +func getChildCauses(validationErr *jsonschema.ValidationError, childCauses []error) []error { if validationErr == nil { - return errToUser + return childCauses } if validationErr.Causes == nil { - errMessage := fmt.Sprintf(" - %s: %s", validationErr.InstanceLocation, validationErr.Message) - return append(errToUser, fmt.Errorf("%s", errMessage)) + errMessage := fmt.Errorf(" - %s: %s", validationErr.InstanceLocation, validationErr.Message) + return append(childCauses, errMessage) } for _, subCause := range validationErr.Causes { - errToUser = getChildCauses(subCause, errToUser) + childCauses = getChildCauses(subCause, childCauses) } - return errToUser + return childCauses } From c8a66e281394c601f8da7a4b40db197b63178cf4 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Tue, 14 Nov 2023 17:48:29 +0000 Subject: [PATCH 049/192] matching style guide better --- src/pkg/packager/lint.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/pkg/packager/lint.go b/src/pkg/packager/lint.go index b0853f4d28..6377c4ac1a 100644 --- a/src/pkg/packager/lint.go +++ b/src/pkg/packager/lint.go @@ -85,7 +85,8 @@ func validateSchema(unmarshalledYaml interface{}, jsonSchema []byte) error { } if err := schema.Validate(unmarshalledYaml); err != nil { - if validationError, ok := err.(*jsonschema.ValidationError); ok { + var validationError *jsonschema.ValidationError + if errors.As(err, &validationError) { allSchemaErrors := getChildCauses(validationError, []error{}) var errMessage strings.Builder errMessage.WriteString(zarfInvalidPrefix) From 527c26b13ed371e155e4bfebad54a4fcc0322142 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Tue, 14 Nov 2023 19:51:08 +0000 Subject: [PATCH 050/192] removing local helm plugin so docs are correct --- .../100-cli-commands/zarf_tools_helm.md | 1 - .../zarf_tools_helm_cm-push.md | 42 ------------------- src/pkg/packager/lint.go | 4 +- src/pkg/packager/lint_test.go | 4 +- 4 files changed, 5 insertions(+), 46 deletions(-) delete mode 100644 docs/2-the-zarf-cli/100-cli-commands/zarf_tools_helm_cm-push.md diff --git a/docs/2-the-zarf-cli/100-cli-commands/zarf_tools_helm.md b/docs/2-the-zarf-cli/100-cli-commands/zarf_tools_helm.md index cea6f81d37..a0c5884216 100644 --- a/docs/2-the-zarf-cli/100-cli-commands/zarf_tools_helm.md +++ b/docs/2-the-zarf-cli/100-cli-commands/zarf_tools_helm.md @@ -31,6 +31,5 @@ Subset of the Helm CLI that includes the repo and dependency commands for managi ## SEE ALSO * [zarf tools](zarf_tools.md) - Collection of additional tools to make airgap easier -* [zarf tools helm cm-push](zarf_tools_helm_cm-push.md) - Please see https://github.com/chartmuseum/helm-push for usage * [zarf tools helm dependency](zarf_tools_helm_dependency.md) - manage a chart's dependencies * [zarf tools helm repo](zarf_tools_helm_repo.md) - add, list, remove, update, and index chart repositories diff --git a/docs/2-the-zarf-cli/100-cli-commands/zarf_tools_helm_cm-push.md b/docs/2-the-zarf-cli/100-cli-commands/zarf_tools_helm_cm-push.md deleted file mode 100644 index c82bd946ab..0000000000 --- a/docs/2-the-zarf-cli/100-cli-commands/zarf_tools_helm_cm-push.md +++ /dev/null @@ -1,42 +0,0 @@ -# zarf tools helm cm-push - - -Please see https://github.com/chartmuseum/helm-push for usage - -## Synopsis - -Push chart package to ChartMuseum - -``` -zarf tools helm cm-push [flags] -``` - -## Options - -``` - -h, --help help for cm-push -``` - -## Options inherited from parent commands - -``` - --burst-limit int client-side default throttling limit (default 100) - --debug enable verbose output - --kube-apiserver string the address and the port for the Kubernetes API server - --kube-as-group stringArray group to impersonate for the operation, this flag can be repeated to specify multiple groups. - --kube-as-user string username to impersonate for the operation - --kube-ca-file string the certificate authority file for the Kubernetes API server connection - --kube-context string name of the kubeconfig context to use - --kube-insecure-skip-tls-verify if true, the Kubernetes API server's certificate will not be checked for validity. This will make your HTTPS connections insecure - --kube-tls-server-name string server name to use for Kubernetes API server certificate validation. If it is not provided, the hostname used to contact the server is used - --kube-token string bearer token used for authentication - --kubeconfig string path to the kubeconfig file - -n, --namespace string namespace scope for this request - --registry-config string path to the registry config file - --repository-cache string path to the file containing cached repository indexes - --repository-config string path to the file containing repository names and URLs -``` - -## SEE ALSO - -* [zarf tools helm](zarf_tools_helm.md) - Subset of the Helm CLI included with Zarf to help manage helm charts. diff --git a/src/pkg/packager/lint.go b/src/pkg/packager/lint.go index 6377c4ac1a..0d5aa7e1ac 100644 --- a/src/pkg/packager/lint.go +++ b/src/pkg/packager/lint.go @@ -26,6 +26,8 @@ const ( ) // ValidateZarfSchema a zarf file against the zarf schema, returns an error if the file is invalid + +//Move this out to a validater package func (p *Packager) ValidateZarfSchema() (err error) { if err = p.readZarfYAML(filepath.Join(p.cfg.CreateOpts.BaseDir, layout.ZarfYAML)); err != nil { return fmt.Errorf("unable to read the zarf.yaml file: %s", err.Error()) @@ -52,7 +54,7 @@ func (p *Packager) ValidateZarfSchema() (err error) { func checkForVarInComponentImport(zarfYaml types.ZarfPackage) error { valid := true errorMessage := zarfWarningPrefix - componentWarningStart := "component/" + componentWarningStart := "/component/" for i, component := range zarfYaml.Components { if strings.Contains(component.Import.Path, zarfTemplateVar) { errorMessage = fmt.Sprintf("%s %s%d/import/path will not resolve ZARF_PKG_TMPL_* variables.", diff --git a/src/pkg/packager/lint_test.go b/src/pkg/packager/lint_test.go index cb1012e44f..252ca1caff 100644 --- a/src/pkg/packager/lint_test.go +++ b/src/pkg/packager/lint_test.go @@ -75,8 +75,8 @@ func TestValidateSchema(t *testing.T) { t.Run("Template in component import failure", func(t *testing.T) { unmarshalledYaml := readAndUnmarshallZarfPackage(t, "../../test/packages/12-lint/zarf.yaml") err := checkForVarInComponentImport(unmarshalledYaml) - errorMessage := zarfWarningPrefix + " component/2/import/path will not resolve ZARF_PKG_TMPL_* variables. " + - "component/3/import/url will not resolve ZARF_PKG_TMPL_* variables." + errorMessage := zarfWarningPrefix + " /component/2/import/path will not resolve ZARF_PKG_TMPL_* variables. " + + "/component/3/import/url will not resolve ZARF_PKG_TMPL_* variables." require.EqualError(t, err, errorMessage) }) } From 251b5e8a95595fda14ad75540e23ebc0ade500ca Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Wed, 15 Nov 2023 16:24:46 +0000 Subject: [PATCH 051/192] changing back to xeipuuv/gojsonschema --- go.mod | 3 +- go.sum | 2 -- src/pkg/packager/lint.go | 55 +++++++++-------------------------- src/pkg/packager/lint_test.go | 8 ++--- src/test/e2e/12_lint_test.go | 4 +-- 5 files changed, 21 insertions(+), 51 deletions(-) diff --git a/go.mod b/go.mod index a157de6255..2709b50399 100644 --- a/go.mod +++ b/go.mod @@ -31,7 +31,6 @@ require ( github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.17.0 github.com/pterm/pterm v0.12.69 - github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 github.com/sergi/go-diff v1.3.1 github.com/sigstore/cosign/v2 v2.2.1 github.com/sigstore/sigstore/pkg/signature/kms/aws v1.7.5 @@ -42,6 +41,7 @@ require ( github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.17.0 github.com/stretchr/testify v1.8.4 + github.com/xeipuuv/gojsonschema v1.2.0 golang.org/x/crypto v0.14.0 golang.org/x/sync v0.5.0 golang.org/x/term v0.13.0 @@ -414,7 +414,6 @@ require ( github.com/xanzy/ssh-agent v0.3.3 // indirect github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect - github.com/xeipuuv/gojsonschema v1.2.0 // indirect github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect github.com/xlab/treeprint v1.2.0 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect diff --git a/go.sum b/go.sum index d48f22130d..f0d06523ef 100644 --- a/go.sum +++ b/go.sum @@ -1352,8 +1352,6 @@ github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6g github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= github.com/sahilm/fuzzy v0.1.0 h1:FzWGaw2Opqyu+794ZQ9SYifWv2EIXpwP4q8dY1kDAwI= github.com/sahilm/fuzzy v0.1.0/go.mod h1:VFvziUEIMCrT6A6tw2RFIXPXXmzXbOsSHF0DOI8ZK9Y= -github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 h1:lZUw3E0/J3roVtGQ+SCrUrg3ON6NgVqpn3+iol9aGu4= -github.com/santhosh-tekuri/jsonschema/v5 v5.3.1/go.mod h1:uToXkOrWAZ6/Oc07xWQrPOhJotwFIyu2bBVN41fcDUY= github.com/sassoftware/go-rpmutils v0.2.0 h1:pKW0HDYMFWQ5b4JQPiI3WI12hGsVoW0V8+GMoZiI/JE= github.com/sassoftware/go-rpmutils v0.2.0/go.mod h1:TJJQYtLe/BeEmEjelI3b7xNZjzAukEkeWKmoakvaOoI= github.com/sassoftware/relic v7.2.1+incompatible h1:Pwyh1F3I0r4clFJXkSI8bOyJINGqpgjJU3DYAZeI05A= diff --git a/src/pkg/packager/lint.go b/src/pkg/packager/lint.go index 0d5aa7e1ac..15891922aa 100644 --- a/src/pkg/packager/lint.go +++ b/src/pkg/packager/lint.go @@ -5,7 +5,6 @@ package packager import ( - "bytes" "errors" "fmt" "path/filepath" @@ -16,7 +15,7 @@ import ( "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/types" - "github.com/santhosh-tekuri/jsonschema/v5" + "github.com/xeipuuv/gojsonschema" ) const ( @@ -27,7 +26,7 @@ const ( // ValidateZarfSchema a zarf file against the zarf schema, returns an error if the file is invalid -//Move this out to a validater package +// Move this out to a validater package func (p *Packager) ValidateZarfSchema() (err error) { if err = p.readZarfYAML(filepath.Join(p.cfg.CreateOpts.BaseDir, layout.ZarfYAML)); err != nil { return fmt.Errorf("unable to read the zarf.yaml file: %s", err.Error()) @@ -54,15 +53,15 @@ func (p *Packager) ValidateZarfSchema() (err error) { func checkForVarInComponentImport(zarfYaml types.ZarfPackage) error { valid := true errorMessage := zarfWarningPrefix - componentWarningStart := "/component/" + componentWarningStart := "component." for i, component := range zarfYaml.Components { if strings.Contains(component.Import.Path, zarfTemplateVar) { - errorMessage = fmt.Sprintf("%s %s%d/import/path will not resolve ZARF_PKG_TMPL_* variables.", + errorMessage = fmt.Sprintf("%s %s%d.import.path will not resolve ZARF_PKG_TMPL_* variables.", errorMessage, componentWarningStart, i) valid = false } if strings.Contains(component.Import.URL, zarfTemplateVar) { - errorMessage = fmt.Sprintf("%s %s%d/import/url will not resolve ZARF_PKG_TMPL_* variables.", + errorMessage = fmt.Sprintf("%s %s%d.import.url will not resolve ZARF_PKG_TMPL_* variables.", errorMessage, componentWarningStart, i) valid = false } @@ -74,47 +73,21 @@ func checkForVarInComponentImport(zarfYaml types.ZarfPackage) error { } func validateSchema(unmarshalledYaml interface{}, jsonSchema []byte) error { - compiler := jsonschema.NewCompiler() - inMemoryZarfSchema := "schema.json" + schemaLoader := gojsonschema.NewBytesLoader(jsonSchema) + documentLoader := gojsonschema.NewGoLoader(unmarshalledYaml) - if err := compiler.AddResource(inMemoryZarfSchema, bytes.NewReader(jsonSchema)); err != nil { - return err - } - - schema, err := compiler.Compile(inMemoryZarfSchema) + result, err := gojsonschema.Validate(schemaLoader, documentLoader) if err != nil { return err } - if err := schema.Validate(unmarshalledYaml); err != nil { - var validationError *jsonschema.ValidationError - if errors.As(err, &validationError) { - allSchemaErrors := getChildCauses(validationError, []error{}) - var errMessage strings.Builder - errMessage.WriteString(zarfInvalidPrefix) - for _, err := range allSchemaErrors { - errMessage.WriteString("\n") - errMessage.WriteString(err.Error()) - } - return errors.New(errMessage.String()) + if !result.Valid() { + errorMessage := zarfInvalidPrefix + for _, desc := range result.Errors() { + errorMessage = fmt.Sprintf("%s\n - %s", errorMessage, desc.String()) } - return err + err = errors.New(errorMessage) } - return nil -} - -func getChildCauses(validationErr *jsonschema.ValidationError, childCauses []error) []error { - if validationErr == nil { - return childCauses - } - if validationErr.Causes == nil { - errMessage := fmt.Errorf(" - %s: %s", validationErr.InstanceLocation, validationErr.Message) - return append(childCauses, errMessage) - } - - for _, subCause := range validationErr.Causes { - childCauses = getChildCauses(subCause, childCauses) - } - return childCauses + return err } diff --git a/src/pkg/packager/lint_test.go b/src/pkg/packager/lint_test.go index 252ca1caff..21ab56c059 100644 --- a/src/pkg/packager/lint_test.go +++ b/src/pkg/packager/lint_test.go @@ -61,8 +61,8 @@ func TestValidateSchema(t *testing.T) { zarfSchema := readSchema(t) err := validateSchema(unmarshalledYaml, zarfSchema) errorMessage := zarfInvalidPrefix + ` - - /components/0/import: additionalProperties 'not-path' not allowed - - /components/1/import/path: expected string, but got number` + - components.0.import: Additional property not-path is not allowed + - components.1.import.path: Invalid type. Expected: string, given: integer` require.EqualError(t, err, errorMessage) }) @@ -75,8 +75,8 @@ func TestValidateSchema(t *testing.T) { t.Run("Template in component import failure", func(t *testing.T) { unmarshalledYaml := readAndUnmarshallZarfPackage(t, "../../test/packages/12-lint/zarf.yaml") err := checkForVarInComponentImport(unmarshalledYaml) - errorMessage := zarfWarningPrefix + " /component/2/import/path will not resolve ZARF_PKG_TMPL_* variables. " + - "/component/3/import/url will not resolve ZARF_PKG_TMPL_* variables." + errorMessage := zarfWarningPrefix + " component.2.import.path will not resolve ZARF_PKG_TMPL_* variables. " + + "component.3.import.url will not resolve ZARF_PKG_TMPL_* variables." require.EqualError(t, err, errorMessage) }) } diff --git a/src/test/e2e/12_lint_test.go b/src/test/e2e/12_lint_test.go index c88f10b664..0a3c8dc60d 100644 --- a/src/test/e2e/12_lint_test.go +++ b/src/test/e2e/12_lint_test.go @@ -16,8 +16,8 @@ func TestLint(t *testing.T) { path := filepath.Join("src", "test", "packages", "12-lint") _, stderr, err := e2e.Zarf("prepare", "lint", path) require.Error(t, err, "Expect error here because the yaml file is not following schema") - require.Contains(t, stderr, "- /components/0/import: additionalProperties 'not-path' not allowed") - require.Contains(t, stderr, "component/2/import/path will not resolve ZARF_PKG_TMPL_* variables") + require.Contains(t, stderr, "- components.0.import: Additional property not-path is not allowed") + require.Contains(t, stderr, "component.2.import.path will not resolve ZARF_PKG_TMPL_* variables") }) t.Run("zarf test lint success", func(t *testing.T) { From 509e71902ec2757ce0477bef8208a3bc034cb72f Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Wed, 15 Nov 2023 19:37:48 +0000 Subject: [PATCH 052/192] changing lint tests to be from yaml string rather than file --- go.mod | 8 +++--- go.sum | 2 ++ src/pkg/packager/lint.go | 2 +- src/pkg/packager/lint_test.go | 49 ++++++++++++++++++++++++++++++++--- 4 files changed, 54 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index 2709b50399..78f2456d85 100644 --- a/go.mod +++ b/go.mod @@ -1,8 +1,8 @@ module github.com/defenseunicorns/zarf -go 1.21.0 +go 1.21.1 -toolchain go1.21.3 +toolchain go1.21.4 require ( cuelang.org/go v0.6.0 @@ -11,6 +11,7 @@ require ( github.com/alecthomas/jsonschema v0.0.0-20220216202328-9eeeec9d044b github.com/anchore/stereoscope v0.0.0-20231027135531-5909e353ee88 github.com/anchore/syft v0.84.1 + github.com/defenseunicorns/gojsonschema v0.0.0-20231115180840-cc7edfd762d7 github.com/derailed/k9s v0.27.4 github.com/distribution/reference v0.5.0 github.com/docker/cli v24.0.7+incompatible @@ -41,7 +42,6 @@ require ( github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.17.0 github.com/stretchr/testify v1.8.4 - github.com/xeipuuv/gojsonschema v1.2.0 golang.org/x/crypto v0.14.0 golang.org/x/sync v0.5.0 golang.org/x/term v0.13.0 @@ -57,6 +57,7 @@ require ( sigs.k8s.io/kustomize/api v0.14.0 sigs.k8s.io/kustomize/kyaml v0.14.3 sigs.k8s.io/yaml v1.4.0 + ) require ( @@ -414,6 +415,7 @@ require ( github.com/xanzy/ssh-agent v0.3.3 // indirect github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect + github.com/xeipuuv/gojsonschema v1.2.0 // indirect github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect github.com/xlab/treeprint v1.2.0 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect diff --git a/go.sum b/go.sum index f0d06523ef..2fb41f61e1 100644 --- a/go.sum +++ b/go.sum @@ -445,6 +445,8 @@ github.com/daviddengcn/go-colortext v1.0.0/go.mod h1:zDqEI5NVUop5QPpVJUxE9UO10hR github.com/decred/dcrd/crypto/blake256 v1.0.1/go.mod h1:2OfgNZ5wDpcsFmHmCK5gZTPcCXqlm2ArzUIkw9czNJo= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 h1:8UrgZ3GkP4i/CLijOJx79Yu+etlyjdBU4sfcs2WYQMs= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0/go.mod h1:v57UDF4pDQJcEfFUCRop3lJL149eHGSe9Jvczhzjo/0= +github.com/defenseunicorns/gojsonschema v0.0.0-20231115180840-cc7edfd762d7 h1:PrAGq11REmGDdhHwERlPu7HmQP5rPu8iHiIevlUgiUs= +github.com/defenseunicorns/gojsonschema v0.0.0-20231115180840-cc7edfd762d7/go.mod h1:QJUPfuJ1T3ag4AYZr3Cq1VXct3rv94PZOKH+y/zkw80= github.com/deitch/magic v0.0.0-20230404182410-1ff89d7342da h1:ZOjWpVsFZ06eIhnh4mkaceTiVoktdU67+M7KDHJ268M= github.com/deitch/magic v0.0.0-20230404182410-1ff89d7342da/go.mod h1:B3tI9iGHi4imdLi4Asdha1Sc6feLMTfPLXh9IUYmysk= github.com/depcheck-test/depcheck-test v0.0.0-20220607135614-199033aaa936 h1:foGzavPWwtoyBvjWyKJYDYsyzy+23iBV7NKTwdk+LRY= diff --git a/src/pkg/packager/lint.go b/src/pkg/packager/lint.go index 15891922aa..9a23fda652 100644 --- a/src/pkg/packager/lint.go +++ b/src/pkg/packager/lint.go @@ -10,12 +10,12 @@ import ( "path/filepath" "strings" + "github.com/defenseunicorns/gojsonschema" "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/types" - "github.com/xeipuuv/gojsonschema" ) const ( diff --git a/src/pkg/packager/lint_test.go b/src/pkg/packager/lint_test.go index 21ab56c059..62130f5654 100644 --- a/src/pkg/packager/lint_test.go +++ b/src/pkg/packager/lint_test.go @@ -13,6 +13,29 @@ import ( "github.com/stretchr/testify/require" ) +const brokenSchemaZarfPackage = ` +kind: ZarfInitConfig +metadata: + name: init + description: Testing bad yaml + +components: +- name: first-test-component + import: + not-path: packages/distros/k3s +- name: import-test + import: + path: 123123 + +- name: import-test + import: + path: "###ZARF_PKG_TMPL_ZEBRA###" + +- name: import-url + import: + url: "oci://###ZARF_PKG_TMPL_ZEBRA###" +` + func TestValidateSchema(t *testing.T) { readFileFailFatally := func(t *testing.T, path string) []byte { file, err := os.ReadFile(path) @@ -33,7 +56,17 @@ func TestValidateSchema(t *testing.T) { file := readFileFailFatally(t, path) err := goyaml.Unmarshal(file, &unmarshalledYaml) if err != nil { - t.Errorf("error unmarshalling yaml %s", err) + t.Errorf("error unmarshalling yaml %v", err) + } + return unmarshalledYaml + } + + readAndUnmarshallYamlString := func(t *testing.T, yamlString string) interface{} { + t.Helper() + var unmarshalledYaml interface{} + err := goyaml.Unmarshal([]byte(yamlString), &unmarshalledYaml) + if err != nil { + t.Errorf("error unmarshalling yaml string %v", err) } return unmarshalledYaml } @@ -49,6 +82,16 @@ func TestValidateSchema(t *testing.T) { return unmarshalledYaml } + readAndUnmarshallZarfPackageString := func(t *testing.T, yamlString string) types.ZarfPackage { + t.Helper() + var unmarshalledYaml types.ZarfPackage + err := goyaml.Unmarshal([]byte(yamlString), &unmarshalledYaml) + if err != nil { + t.Errorf("error unmarshalling yaml %v", err) + } + return unmarshalledYaml + } + t.Run("validate schema success", func(t *testing.T) { unmarshalledYaml := readAndUnmarshalYaml(t, "../../../zarf.yaml") zarfSchema := readSchema(t) @@ -57,7 +100,7 @@ func TestValidateSchema(t *testing.T) { }) t.Run("validate schema fail", func(t *testing.T) { - unmarshalledYaml := readAndUnmarshalYaml(t, "../../test/packages/12-lint/zarf.yaml") + unmarshalledYaml := readAndUnmarshallYamlString(t, brokenSchemaZarfPackage) zarfSchema := readSchema(t) err := validateSchema(unmarshalledYaml, zarfSchema) errorMessage := zarfInvalidPrefix + ` @@ -73,7 +116,7 @@ func TestValidateSchema(t *testing.T) { }) t.Run("Template in component import failure", func(t *testing.T) { - unmarshalledYaml := readAndUnmarshallZarfPackage(t, "../../test/packages/12-lint/zarf.yaml") + unmarshalledYaml := readAndUnmarshallZarfPackageString(t, brokenSchemaZarfPackage) err := checkForVarInComponentImport(unmarshalledYaml) errorMessage := zarfWarningPrefix + " component.2.import.path will not resolve ZARF_PKG_TMPL_* variables. " + "component.3.import.url will not resolve ZARF_PKG_TMPL_* variables." From 5a42ffa9971ffba2cabe74a2b6c12553940038f9 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Thu, 16 Nov 2023 16:41:37 +0000 Subject: [PATCH 053/192] adding validator, moving to defenseunicorns/jsonschema --- go.mod | 6 ++--- go.sum | 7 ++---- src/cmd/prepare.go | 12 +++++----- .../validater_test.go} | 4 ++-- .../{lint.go => validator/validator.go} | 24 ++++++++++--------- src/pkg/packager/variables.go | 3 ++- 6 files changed, 28 insertions(+), 28 deletions(-) rename src/pkg/packager/{lint_test.go => validator/validater_test.go} (96%) rename src/pkg/packager/{lint.go => validator/validator.go} (74%) diff --git a/go.mod b/go.mod index 78f2456d85..3931c1ca09 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,8 @@ go 1.21.1 toolchain go1.21.4 +replace github.com/xeipuuv/gojsonschema => github.com/defenseunicorns/gojsonschema v0.0.0-20231116163348-e00f069122d6 + require ( cuelang.org/go v0.6.0 github.com/AlecAivazis/survey/v2 v2.3.7 @@ -11,7 +13,6 @@ require ( github.com/alecthomas/jsonschema v0.0.0-20220216202328-9eeeec9d044b github.com/anchore/stereoscope v0.0.0-20231027135531-5909e353ee88 github.com/anchore/syft v0.84.1 - github.com/defenseunicorns/gojsonschema v0.0.0-20231115180840-cc7edfd762d7 github.com/derailed/k9s v0.27.4 github.com/distribution/reference v0.5.0 github.com/docker/cli v24.0.7+incompatible @@ -42,6 +43,7 @@ require ( github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.17.0 github.com/stretchr/testify v1.8.4 + github.com/xeipuuv/gojsonschema v1.2.0 golang.org/x/crypto v0.14.0 golang.org/x/sync v0.5.0 golang.org/x/term v0.13.0 @@ -57,7 +59,6 @@ require ( sigs.k8s.io/kustomize/api v0.14.0 sigs.k8s.io/kustomize/kyaml v0.14.3 sigs.k8s.io/yaml v1.4.0 - ) require ( @@ -415,7 +416,6 @@ require ( github.com/xanzy/ssh-agent v0.3.3 // indirect github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect - github.com/xeipuuv/gojsonschema v1.2.0 // indirect github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect github.com/xlab/treeprint v1.2.0 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect diff --git a/go.sum b/go.sum index 2fb41f61e1..279b0e1f91 100644 --- a/go.sum +++ b/go.sum @@ -445,8 +445,8 @@ github.com/daviddengcn/go-colortext v1.0.0/go.mod h1:zDqEI5NVUop5QPpVJUxE9UO10hR github.com/decred/dcrd/crypto/blake256 v1.0.1/go.mod h1:2OfgNZ5wDpcsFmHmCK5gZTPcCXqlm2ArzUIkw9czNJo= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 h1:8UrgZ3GkP4i/CLijOJx79Yu+etlyjdBU4sfcs2WYQMs= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0/go.mod h1:v57UDF4pDQJcEfFUCRop3lJL149eHGSe9Jvczhzjo/0= -github.com/defenseunicorns/gojsonschema v0.0.0-20231115180840-cc7edfd762d7 h1:PrAGq11REmGDdhHwERlPu7HmQP5rPu8iHiIevlUgiUs= -github.com/defenseunicorns/gojsonschema v0.0.0-20231115180840-cc7edfd762d7/go.mod h1:QJUPfuJ1T3ag4AYZr3Cq1VXct3rv94PZOKH+y/zkw80= +github.com/defenseunicorns/gojsonschema v0.0.0-20231116163348-e00f069122d6 h1:gwevOZ0fxT2nzM9hrtdPbsiOHjFqDRIYMzJHba3/G6Q= +github.com/defenseunicorns/gojsonschema v0.0.0-20231116163348-e00f069122d6/go.mod h1:StKLYMmPj1R5yIs6CK49EkcW1TvUYuw5Vri+LRk7Dy8= github.com/deitch/magic v0.0.0-20230404182410-1ff89d7342da h1:ZOjWpVsFZ06eIhnh4mkaceTiVoktdU67+M7KDHJ268M= github.com/deitch/magic v0.0.0-20230404182410-1ff89d7342da/go.mod h1:B3tI9iGHi4imdLi4Asdha1Sc6feLMTfPLXh9IUYmysk= github.com/depcheck-test/depcheck-test v0.0.0-20220607135614-199033aaa936 h1:foGzavPWwtoyBvjWyKJYDYsyzy+23iBV7NKTwdk+LRY= @@ -1528,13 +1528,10 @@ github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3k github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM= github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8= github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM= -github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo= github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= -github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74= -github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 h1:nIPpBwaJSVYIxUFsDv3M8ofmx9yWTog9BfvIu0q41lo= github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos= github.com/xlab/treeprint v1.2.0 h1:HzHnuAF1plUN2zGlAFHbSQP2qJ0ZAD3XF5XD7OesXRQ= diff --git a/src/cmd/prepare.go b/src/cmd/prepare.go index 6154249036..9403e820c3 100644 --- a/src/cmd/prepare.go +++ b/src/cmd/prepare.go @@ -17,6 +17,7 @@ import ( "github.com/defenseunicorns/zarf/src/config/lang" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/packager" + validator "github.com/defenseunicorns/zarf/src/pkg/packager/validator" "github.com/defenseunicorns/zarf/src/pkg/transform" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" @@ -211,18 +212,17 @@ var lintCmd = &cobra.Command{ Aliases: []string{"l"}, Short: lang.CmdPrepareLintShort, Run: func(cmd *cobra.Command, args []string) { + baseDir := "" if len(args) > 0 { - pkgConfig.CreateOpts.BaseDir = args[0] + baseDir = args[0] } else { - cwd, err := os.Getwd() + var err error + baseDir, err = os.Getwd() if err != nil { message.Fatalf(err, lang.CmdPrepareFindImagesErr, err.Error()) } - pkgConfig.CreateOpts.BaseDir = cwd } - pkgClient := packager.NewOrDie(&pkgConfig) - defer pkgClient.ClearTempPaths() - err := pkgClient.ValidateZarfSchema() + err := validator.ValidateZarfSchema(baseDir) if err != nil { message.Fatal(err, err.Error()) } diff --git a/src/pkg/packager/lint_test.go b/src/pkg/packager/validator/validater_test.go similarity index 96% rename from src/pkg/packager/lint_test.go rename to src/pkg/packager/validator/validater_test.go index 62130f5654..1a82ef7bf8 100644 --- a/src/pkg/packager/lint_test.go +++ b/src/pkg/packager/validator/validater_test.go @@ -1,8 +1,8 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: 2021-Present The Zarf Authors -// Package packager contains functions for interacting with, managing and deploying Zarf packages. -package packager +// Package validator contains functions for interacting with, managing and deploying Zarf packages. +package validator import ( "os" diff --git a/src/pkg/packager/lint.go b/src/pkg/packager/validator/validator.go similarity index 74% rename from src/pkg/packager/lint.go rename to src/pkg/packager/validator/validator.go index 9a23fda652..c3dc525770 100644 --- a/src/pkg/packager/lint.go +++ b/src/pkg/packager/validator/validator.go @@ -1,8 +1,8 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: 2021-Present The Zarf Authors -// Package packager contains functions for linting the zarf.yaml -package packager +// Package validator contains functions for linting the zarf.yaml +package validator import ( "errors" @@ -10,35 +10,37 @@ import ( "path/filepath" "strings" - "github.com/defenseunicorns/gojsonschema" "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/types" + "github.com/xeipuuv/gojsonschema" ) const ( zarfInvalidPrefix = "schema is invalid:" zarfWarningPrefix = "zarf schema warning:" - zarfTemplateVar = "###ZARF_PKG_TMPL_" + ZarfTemplateVar = "###ZARF_PKG_TMPL_" ) // ValidateZarfSchema a zarf file against the zarf schema, returns an error if the file is invalid // Move this out to a validater package -func (p *Packager) ValidateZarfSchema() (err error) { - if err = p.readZarfYAML(filepath.Join(p.cfg.CreateOpts.BaseDir, layout.ZarfYAML)); err != nil { - return fmt.Errorf("unable to read the zarf.yaml file: %s", err.Error()) +func ValidateZarfSchema(path string) (err error) { + var zarfTypedData types.ZarfPackage + if err := utils.ReadYaml(filepath.Join(path, layout.ZarfYAML), &zarfTypedData); err != nil { + return err } - if err := checkForVarInComponentImport(p.cfg.Pkg); err != nil { + if err := checkForVarInComponentImport(zarfTypedData); err != nil { message.Warn(err.Error()) } zarfSchema, _ := config.GetSchemaFile() + var zarfData interface{} - if err := utils.ReadYaml(filepath.Join(p.cfg.CreateOpts.BaseDir, layout.ZarfYAML), &zarfData); err != nil { + if err := utils.ReadYaml(filepath.Join(path, layout.ZarfYAML), &zarfData); err != nil { return err } @@ -55,12 +57,12 @@ func checkForVarInComponentImport(zarfYaml types.ZarfPackage) error { errorMessage := zarfWarningPrefix componentWarningStart := "component." for i, component := range zarfYaml.Components { - if strings.Contains(component.Import.Path, zarfTemplateVar) { + if strings.Contains(component.Import.Path, ZarfTemplateVar) { errorMessage = fmt.Sprintf("%s %s%d.import.path will not resolve ZARF_PKG_TMPL_* variables.", errorMessage, componentWarningStart, i) valid = false } - if strings.Contains(component.Import.URL, zarfTemplateVar) { + if strings.Contains(component.Import.URL, ZarfTemplateVar) { errorMessage = fmt.Sprintf("%s %s%d.import.url will not resolve ZARF_PKG_TMPL_* variables.", errorMessage, componentWarningStart, i) valid = false diff --git a/src/pkg/packager/variables.go b/src/pkg/packager/variables.go index 2a07944c1c..0a7c411fbf 100644 --- a/src/pkg/packager/variables.go +++ b/src/pkg/packager/variables.go @@ -11,6 +11,7 @@ import ( "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/config/lang" "github.com/defenseunicorns/zarf/src/pkg/interactive" + validator "github.com/defenseunicorns/zarf/src/pkg/packager/validator" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/types" ) @@ -59,7 +60,7 @@ func (p *Packager) fillActiveTemplate() error { return err } - if err := promptAndSetTemplate(zarfTemplateVar, false); err != nil { + if err := promptAndSetTemplate(validator.ZarfTemplateVar, false); err != nil { return err } // [DEPRECATION] Set the Package Variable syntax as well for backward compatibility From 48acf97c4403ec06a0e4d2166f9f469bc3e94f9f Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Thu, 16 Nov 2023 16:50:00 +0000 Subject: [PATCH 054/192] fix file path unit tests --- src/pkg/packager/validator/validater_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/pkg/packager/validator/validater_test.go b/src/pkg/packager/validator/validater_test.go index 1a82ef7bf8..c2f7dda613 100644 --- a/src/pkg/packager/validator/validater_test.go +++ b/src/pkg/packager/validator/validater_test.go @@ -47,7 +47,7 @@ func TestValidateSchema(t *testing.T) { readSchema := func(t *testing.T) []byte { t.Helper() - return readFileFailFatally(t, "../../../zarf.schema.json") + return readFileFailFatally(t, "../../../../zarf.schema.json") } readAndUnmarshalYaml := func(t *testing.T, path string) interface{} { @@ -93,7 +93,7 @@ func TestValidateSchema(t *testing.T) { } t.Run("validate schema success", func(t *testing.T) { - unmarshalledYaml := readAndUnmarshalYaml(t, "../../../zarf.yaml") + unmarshalledYaml := readAndUnmarshalYaml(t, "../../../../zarf.yaml") zarfSchema := readSchema(t) err := validateSchema(unmarshalledYaml, zarfSchema) require.NoError(t, err) @@ -110,7 +110,7 @@ func TestValidateSchema(t *testing.T) { }) t.Run("Template in component import success", func(t *testing.T) { - unmarshalledYaml := readAndUnmarshallZarfPackage(t, "../../../zarf.yaml") + unmarshalledYaml := readAndUnmarshallZarfPackage(t, "../../../../zarf.yaml") err := checkForVarInComponentImport(unmarshalledYaml) require.NoError(t, err) }) From 7d695f90c5aef19a06a07c5e379c2ff04efbec69 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Thu, 16 Nov 2023 16:59:28 +0000 Subject: [PATCH 055/192] moving some magic strings into types --- src/pkg/packager/validator/validator.go | 5 ++--- src/pkg/packager/variables.go | 9 ++++----- src/types/runtime.go | 5 +++++ 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/src/pkg/packager/validator/validator.go b/src/pkg/packager/validator/validator.go index c3dc525770..e290c1cf34 100644 --- a/src/pkg/packager/validator/validator.go +++ b/src/pkg/packager/validator/validator.go @@ -21,7 +21,6 @@ import ( const ( zarfInvalidPrefix = "schema is invalid:" zarfWarningPrefix = "zarf schema warning:" - ZarfTemplateVar = "###ZARF_PKG_TMPL_" ) // ValidateZarfSchema a zarf file against the zarf schema, returns an error if the file is invalid @@ -57,12 +56,12 @@ func checkForVarInComponentImport(zarfYaml types.ZarfPackage) error { errorMessage := zarfWarningPrefix componentWarningStart := "component." for i, component := range zarfYaml.Components { - if strings.Contains(component.Import.Path, ZarfTemplateVar) { + if strings.Contains(component.Import.Path, types.ZarfPackageTemplatePrefix) { errorMessage = fmt.Sprintf("%s %s%d.import.path will not resolve ZARF_PKG_TMPL_* variables.", errorMessage, componentWarningStart, i) valid = false } - if strings.Contains(component.Import.URL, ZarfTemplateVar) { + if strings.Contains(component.Import.URL, types.ZarfPackageTemplatePrefix) { errorMessage = fmt.Sprintf("%s %s%d.import.url will not resolve ZARF_PKG_TMPL_* variables.", errorMessage, componentWarningStart, i) valid = false diff --git a/src/pkg/packager/variables.go b/src/pkg/packager/variables.go index 0a7c411fbf..2caa135907 100644 --- a/src/pkg/packager/variables.go +++ b/src/pkg/packager/variables.go @@ -11,7 +11,6 @@ import ( "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/config/lang" "github.com/defenseunicorns/zarf/src/pkg/interactive" - validator "github.com/defenseunicorns/zarf/src/pkg/packager/validator" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/types" ) @@ -60,16 +59,16 @@ func (p *Packager) fillActiveTemplate() error { return err } - if err := promptAndSetTemplate(validator.ZarfTemplateVar, false); err != nil { + if err := promptAndSetTemplate(types.ZarfPackageTemplatePrefix, false); err != nil { return err } // [DEPRECATION] Set the Package Variable syntax as well for backward compatibility - if err := promptAndSetTemplate("###ZARF_PKG_VAR_", true); err != nil { + if err := promptAndSetTemplate(types.ZarfPackageVariablePrefix, true); err != nil { return err } // Add special variable for the current package architecture - templateMap["###ZARF_PKG_ARCH###"] = p.arch + templateMap[types.ZarfPackageArch] = p.arch return utils.ReloadYamlTemplate(&p.cfg.Pkg, templateMap) } @@ -132,7 +131,7 @@ func (p *Packager) findComponentTemplatesAndReload() error { // iterate through components to and find all ###ZARF_COMPONENT_NAME, assign to component Name and value for i, component := range p.cfg.Pkg.Components { mappings := map[string]string{} - mappings["###ZARF_COMPONENT_NAME###"] = component.Name + mappings[types.ZarfComponentName] = component.Name err := utils.ReloadYamlTemplate(&p.cfg.Pkg.Components[i], mappings) if err != nil { return err diff --git a/src/types/runtime.go b/src/types/runtime.go index b9546c1658..b4a0653033 100644 --- a/src/types/runtime.go +++ b/src/types/runtime.go @@ -9,6 +9,11 @@ const ( RawVariableType VariableType = "raw" // FileVariableType is a type for a Zarf package variable that loads its contents from a file FileVariableType VariableType = "file" + + ZarfPackageTemplatePrefix = "###ZARF_PKG_TMPL_" + ZarfPackageVariablePrefix = "###ZARF_PKG_VAR_" + ZarfPackageArch = "###ZARF_PKG_ARCH###" + ZarfComponentName = "###ZARF_COMPONENT_NAME###" ) // VariableType represents a type of a Zarf package variable From 18069573bb50255d85b1f5dd106b09fa2f64f667 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Thu, 16 Nov 2023 17:00:45 +0000 Subject: [PATCH 056/192] fix comment --- src/pkg/packager/validator/validator.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/pkg/packager/validator/validator.go b/src/pkg/packager/validator/validator.go index e290c1cf34..929e1980cf 100644 --- a/src/pkg/packager/validator/validator.go +++ b/src/pkg/packager/validator/validator.go @@ -24,8 +24,6 @@ const ( ) // ValidateZarfSchema a zarf file against the zarf schema, returns an error if the file is invalid - -// Move this out to a validater package func ValidateZarfSchema(path string) (err error) { var zarfTypedData types.ZarfPackage if err := utils.ReadYaml(filepath.Join(path, layout.ZarfYAML), &zarfTypedData); err != nil { From 269c118fccb8c54a74a1d6700604c7ec405726ef Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Thu, 16 Nov 2023 21:25:44 +0000 Subject: [PATCH 057/192] refactoring --- .pre-commit-config.yaml | 1 - src/cmd/prepare.go | 2 +- src/pkg/packager/validator/validater_test.go | 84 +++++++------------- src/pkg/packager/validator/validator.go | 4 +- src/types/runtime.go | 3 + 5 files changed, 35 insertions(+), 59 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3217685488..9b3caa47f8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -49,6 +49,5 @@ repos: args: ["--schemafile", "zarf.schema.json"] exclude: | (?x)^( - src/pkg/packager/unsuccessful_validation/zarf.yaml| src/test/packages/12-lint/.* )$ diff --git a/src/cmd/prepare.go b/src/cmd/prepare.go index 9403e820c3..03e9cabfd0 100644 --- a/src/cmd/prepare.go +++ b/src/cmd/prepare.go @@ -17,7 +17,7 @@ import ( "github.com/defenseunicorns/zarf/src/config/lang" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/packager" - validator "github.com/defenseunicorns/zarf/src/pkg/packager/validator" + "github.com/defenseunicorns/zarf/src/pkg/packager/validator" "github.com/defenseunicorns/zarf/src/pkg/transform" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" diff --git a/src/pkg/packager/validator/validater_test.go b/src/pkg/packager/validator/validater_test.go index c2f7dda613..021ce772c8 100644 --- a/src/pkg/packager/validator/validater_test.go +++ b/src/pkg/packager/validator/validater_test.go @@ -1,7 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: 2021-Present The Zarf Authors -// Package validator contains functions for interacting with, managing and deploying Zarf packages. +// Package validator contains functions for verifying zarf yaml files are valid package validator import ( @@ -13,7 +13,7 @@ import ( "github.com/stretchr/testify/require" ) -const brokenSchemaZarfPackage = ` +const badZarfPackage = ` kind: ZarfInitConfig metadata: name: init @@ -36,72 +36,46 @@ components: url: "oci://###ZARF_PKG_TMPL_ZEBRA###" ` -func TestValidateSchema(t *testing.T) { - readFileFailFatally := func(t *testing.T, path string) []byte { - file, err := os.ReadFile(path) - if err != nil { - t.Errorf("error reading file: %s", err) - } - return file - } - - readSchema := func(t *testing.T) []byte { - t.Helper() - return readFileFailFatally(t, "../../../../zarf.schema.json") - } - - readAndUnmarshalYaml := func(t *testing.T, path string) interface{} { - t.Helper() - var unmarshalledYaml interface{} - file := readFileFailFatally(t, path) - err := goyaml.Unmarshal(file, &unmarshalledYaml) - if err != nil { - t.Errorf("error unmarshalling yaml %v", err) - } - return unmarshalledYaml - } +const goodZarfPackage = ` +kind: ZarfPackageConfig +metadata: + name: good-zarf-package - readAndUnmarshallYamlString := func(t *testing.T, yamlString string) interface{} { - t.Helper() - var unmarshalledYaml interface{} - err := goyaml.Unmarshal([]byte(yamlString), &unmarshalledYaml) - if err != nil { - t.Errorf("error unmarshalling yaml string %v", err) - } - return unmarshalledYaml - } +components: + - name: baseline + required: true +` - readAndUnmarshallZarfPackage := func(t *testing.T, path string) types.ZarfPackage { - t.Helper() - var unmarshalledYaml types.ZarfPackage - file := readFileFailFatally(t, path) - err := goyaml.Unmarshal(file, &unmarshalledYaml) - if err != nil { - t.Errorf("error unmarshalling yaml %s", err) - } - return unmarshalledYaml +func readAndUnmarshalYaml[T interface{}](t *testing.T, yamlString string) T { + t.Helper() + var unmarshalledYaml T + err := goyaml.Unmarshal([]byte(yamlString), &unmarshalledYaml) + if err != nil { + t.Errorf("error unmarshalling yaml %v", err) } + return unmarshalledYaml +} - readAndUnmarshallZarfPackageString := func(t *testing.T, yamlString string) types.ZarfPackage { +func TestValidateSchema(t *testing.T) { + getZarfSchema := func(t *testing.T) []byte { t.Helper() - var unmarshalledYaml types.ZarfPackage - err := goyaml.Unmarshal([]byte(yamlString), &unmarshalledYaml) + file, err := os.ReadFile("../../../../zarf.schema.json") if err != nil { - t.Errorf("error unmarshalling yaml %v", err) + t.Errorf("error reading file: %s", err) } - return unmarshalledYaml + return file } t.Run("validate schema success", func(t *testing.T) { - unmarshalledYaml := readAndUnmarshalYaml(t, "../../../../zarf.yaml") - zarfSchema := readSchema(t) + unmarshalledYaml := readAndUnmarshalYaml[interface{}](t, goodZarfPackage) + zarfSchema := getZarfSchema(t) err := validateSchema(unmarshalledYaml, zarfSchema) require.NoError(t, err) }) t.Run("validate schema fail", func(t *testing.T) { - unmarshalledYaml := readAndUnmarshallYamlString(t, brokenSchemaZarfPackage) - zarfSchema := readSchema(t) + unmarshalledYaml := readAndUnmarshalYaml[interface{}](t, badZarfPackage) + zarfSchema := getZarfSchema(t) err := validateSchema(unmarshalledYaml, zarfSchema) errorMessage := zarfInvalidPrefix + ` - components.0.import: Additional property not-path is not allowed @@ -110,13 +84,13 @@ func TestValidateSchema(t *testing.T) { }) t.Run("Template in component import success", func(t *testing.T) { - unmarshalledYaml := readAndUnmarshallZarfPackage(t, "../../../../zarf.yaml") + unmarshalledYaml := readAndUnmarshalYaml[types.ZarfPackage](t, goodZarfPackage) err := checkForVarInComponentImport(unmarshalledYaml) require.NoError(t, err) }) t.Run("Template in component import failure", func(t *testing.T) { - unmarshalledYaml := readAndUnmarshallZarfPackageString(t, brokenSchemaZarfPackage) + unmarshalledYaml := readAndUnmarshalYaml[types.ZarfPackage](t, badZarfPackage) err := checkForVarInComponentImport(unmarshalledYaml) errorMessage := zarfWarningPrefix + " component.2.import.path will not resolve ZARF_PKG_TMPL_* variables. " + "component.3.import.url will not resolve ZARF_PKG_TMPL_* variables." diff --git a/src/pkg/packager/validator/validator.go b/src/pkg/packager/validator/validator.go index 929e1980cf..d4e1eacaa2 100644 --- a/src/pkg/packager/validator/validator.go +++ b/src/pkg/packager/validator/validator.go @@ -1,7 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: 2021-Present The Zarf Authors -// Package validator contains functions for linting the zarf.yaml +// Package validator contains functions for verifying zarf yaml files are valid package validator import ( @@ -45,7 +45,7 @@ func ValidateZarfSchema(path string) (err error) { return err } - message.Success("Validation successful") + message.Success(fmt.Sprintf("Schema validation successful for %q", zarfTypedData.Metadata.Name)) return nil } diff --git a/src/types/runtime.go b/src/types/runtime.go index b4a0653033..0cd53e18a0 100644 --- a/src/types/runtime.go +++ b/src/types/runtime.go @@ -9,7 +9,10 @@ const ( RawVariableType VariableType = "raw" // FileVariableType is a type for a Zarf package variable that loads its contents from a file FileVariableType VariableType = "file" +) +// Zarf looks for these strings in zarf.yaml to make dynamic changes +const ( ZarfPackageTemplatePrefix = "###ZARF_PKG_TMPL_" ZarfPackageVariablePrefix = "###ZARF_PKG_VAR_" ZarfPackageArch = "###ZARF_PKG_ARCH###" From 2e785cf42bdcdfc1ba45035803c49f350fafc2c9 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Thu, 16 Nov 2023 21:33:18 +0000 Subject: [PATCH 058/192] refactor validator --- src/pkg/packager/validator/validater_test.go | 4 ++-- src/pkg/packager/validator/validator.go | 20 ++++++++------------ 2 files changed, 10 insertions(+), 14 deletions(-) diff --git a/src/pkg/packager/validator/validater_test.go b/src/pkg/packager/validator/validater_test.go index 021ce772c8..6c07a8df93 100644 --- a/src/pkg/packager/validator/validater_test.go +++ b/src/pkg/packager/validator/validater_test.go @@ -92,8 +92,8 @@ func TestValidateSchema(t *testing.T) { t.Run("Template in component import failure", func(t *testing.T) { unmarshalledYaml := readAndUnmarshalYaml[types.ZarfPackage](t, badZarfPackage) err := checkForVarInComponentImport(unmarshalledYaml) - errorMessage := zarfWarningPrefix + " component.2.import.path will not resolve ZARF_PKG_TMPL_* variables. " + - "component.3.import.url will not resolve ZARF_PKG_TMPL_* variables." + errorMessage := zarfWarningPrefix + " component.2.import.path will not resolve ZARF_PKG_TMPL_* variables, " + + "component.3.import.url will not resolve ZARF_PKG_TMPL_* variables" require.EqualError(t, err, errorMessage) }) } diff --git a/src/pkg/packager/validator/validator.go b/src/pkg/packager/validator/validator.go index d4e1eacaa2..0dd998b107 100644 --- a/src/pkg/packager/validator/validator.go +++ b/src/pkg/packager/validator/validator.go @@ -50,25 +50,21 @@ func ValidateZarfSchema(path string) (err error) { } func checkForVarInComponentImport(zarfYaml types.ZarfPackage) error { - valid := true - errorMessage := zarfWarningPrefix - componentWarningStart := "component." + var errorMessages []string for i, component := range zarfYaml.Components { if strings.Contains(component.Import.Path, types.ZarfPackageTemplatePrefix) { - errorMessage = fmt.Sprintf("%s %s%d.import.path will not resolve ZARF_PKG_TMPL_* variables.", - errorMessage, componentWarningStart, i) - valid = false + errorMessages = append(errorMessages, fmt.Sprintf("component.%d.import.path will not resolve ZARF_PKG_TMPL_* variables", i)) } if strings.Contains(component.Import.URL, types.ZarfPackageTemplatePrefix) { - errorMessage = fmt.Sprintf("%s %s%d.import.url will not resolve ZARF_PKG_TMPL_* variables.", - errorMessage, componentWarningStart, i) - valid = false + errorMessages = append(errorMessages, fmt.Sprintf("component.%d.import.url will not resolve ZARF_PKG_TMPL_* variables", i)) } } - if valid { - return nil + + if len(errorMessages) > 0 { + return fmt.Errorf("%s %s", zarfWarningPrefix, strings.Join(errorMessages, ", ")) } - return errors.New(errorMessage) + + return nil } func validateSchema(unmarshalledYaml interface{}, jsonSchema []byte) error { From 743ecd715327a642940587ecfbca898977671be2 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Fri, 17 Nov 2023 14:26:01 +0000 Subject: [PATCH 059/192] WIP introducing validator struct --- src/pkg/packager/validator/validater_test.go | 49 ++++++++--- src/pkg/packager/validator/validator.go | 91 +++++++++++--------- 2 files changed, 88 insertions(+), 52 deletions(-) diff --git a/src/pkg/packager/validator/validater_test.go b/src/pkg/packager/validator/validater_test.go index 6c07a8df93..3d21b7f0f2 100644 --- a/src/pkg/packager/validator/validater_test.go +++ b/src/pkg/packager/validator/validater_test.go @@ -5,6 +5,7 @@ package validator import ( + "fmt" "os" "testing" @@ -69,31 +70,55 @@ func TestValidateSchema(t *testing.T) { t.Run("validate schema success", func(t *testing.T) { unmarshalledYaml := readAndUnmarshalYaml[interface{}](t, goodZarfPackage) zarfSchema := getZarfSchema(t) - err := validateSchema(unmarshalledYaml, zarfSchema) + validator := Validator{} + validator, err := validateSchema(validator, unmarshalledYaml, zarfSchema) require.NoError(t, err) + require.Empty(t, validator.errors) }) t.Run("validate schema fail", func(t *testing.T) { unmarshalledYaml := readAndUnmarshalYaml[interface{}](t, badZarfPackage) zarfSchema := getZarfSchema(t) - err := validateSchema(unmarshalledYaml, zarfSchema) - errorMessage := zarfInvalidPrefix + ` - - components.0.import: Additional property not-path is not allowed - - components.1.import.path: Invalid type. Expected: string, given: integer` - require.EqualError(t, err, errorMessage) + validator := Validator{} + validator, err := validateSchema(validator, unmarshalledYaml, zarfSchema) + // errorMessage := zarfInvalidPrefix + ` + // - components.0.import: Additional property not-path is not allowed + // - components.1.import.path: Invalid type. Expected: string, given: integer` + // require.EqualError(t, err, errorMessage) + require.NoError(t, err) + require.Equal(t, validator.errors[0], "components.0.import: Additional property not-path is not allowed") + require.Equal(t, validator.errors[1], "components.1.import.path: Invalid type. Expected: string, given: integer") }) t.Run("Template in component import success", func(t *testing.T) { unmarshalledYaml := readAndUnmarshalYaml[types.ZarfPackage](t, goodZarfPackage) - err := checkForVarInComponentImport(unmarshalledYaml) - require.NoError(t, err) + validator := checkForVarInComponentImport(Validator{}, unmarshalledYaml) + require.Empty(t, validator) }) t.Run("Template in component import failure", func(t *testing.T) { unmarshalledYaml := readAndUnmarshalYaml[types.ZarfPackage](t, badZarfPackage) - err := checkForVarInComponentImport(unmarshalledYaml) - errorMessage := zarfWarningPrefix + " component.2.import.path will not resolve ZARF_PKG_TMPL_* variables, " + - "component.3.import.url will not resolve ZARF_PKG_TMPL_* variables" - require.EqualError(t, err, errorMessage) + validator := checkForVarInComponentImport(Validator{}, unmarshalledYaml) + // errorMessage := zarfWarningPrefix + " component.2.import.path will not resolve ZARF_PKG_TMPL_* variables, " + + // "component.3.import.url will not resolve ZARF_PKG_TMPL_* variables" + // require.EqualError(t, err, errorMessage) + require.Equal(t, validator.warnings[0], "component.2.import.path will not resolve ZARF_PKG_TMPL_* variables") + require.Equal(t, validator.warnings[1], "component.3.import.url will not resolve ZARF_PKG_TMPL_* variables") + }) + + t.Run("Validator Error formatting", func(t *testing.T) { + error1 := "components.0.import: Additional property not-path is not allowed" + error2 := "components.1.import.path: Invalid type. Expected: string, given: integer" + validator := Validator{errors: []string{error1, error2}} + errorMessage := fmt.Sprintf("%s\n - %s\n - %s", validatorInvalidPrefix, error1, error2) + require.EqualError(t, validator.getFormmatedError(), errorMessage) + }) + + t.Run("Validator Warning formatting", func(t *testing.T) { + warning1 := "components.0.import: Additional property not-path is not allowed" + warning2 := "components.1.import.path: Invalid type. Expected: string, given: integer" + validator := Validator{warnings: []string{warning1, warning2}} + message := fmt.Sprintf("%s %s, %s", validatorWarningPrefix, warning1, warning2) + require.Equal(t, validator.getFormmatedWarning(), message) }) } diff --git a/src/pkg/packager/validator/validator.go b/src/pkg/packager/validator/validator.go index 0dd998b107..8848d3e67c 100644 --- a/src/pkg/packager/validator/validator.go +++ b/src/pkg/packager/validator/validator.go @@ -7,82 +7,93 @@ package validator import ( "errors" "fmt" - "path/filepath" "strings" - "github.com/defenseunicorns/zarf/src/config" - "github.com/defenseunicorns/zarf/src/pkg/layout" - "github.com/defenseunicorns/zarf/src/pkg/message" - "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/types" "github.com/xeipuuv/gojsonschema" ) const ( - zarfInvalidPrefix = "schema is invalid:" - zarfWarningPrefix = "zarf schema warning:" + validatorInvalidPrefix = "schema is invalid:" + validatorWarningPrefix = "zarf schema warning:" ) -// ValidateZarfSchema a zarf file against the zarf schema, returns an error if the file is invalid -func ValidateZarfSchema(path string) (err error) { - var zarfTypedData types.ZarfPackage - if err := utils.ReadYaml(filepath.Join(path, layout.ZarfYAML), &zarfTypedData); err != nil { - return err - } +type Validator struct { + warnings []string + errors []string +} - if err := checkForVarInComponentImport(zarfTypedData); err != nil { - message.Warn(err.Error()) +func (v Validator) getFormmatedError() error { + if len(v.errors) == 0 { + return nil } - - zarfSchema, _ := config.GetSchemaFile() - - var zarfData interface{} - if err := utils.ReadYaml(filepath.Join(path, layout.ZarfYAML), &zarfData); err != nil { - return err + errorMessage := validatorInvalidPrefix + for _, errorStr := range v.errors { + errorMessage = fmt.Sprintf("%s\n - %s", errorMessage, errorStr) } + return errors.New(errorMessage) +} - if err = validateSchema(zarfData, zarfSchema); err != nil { - return err +func (v Validator) getFormmatedWarning() string { + if len(v.warnings) == 0 { + return "" } + return fmt.Sprintf("%s %s", validatorWarningPrefix, strings.Join(v.warnings, ", ")) +} - message.Success(fmt.Sprintf("Schema validation successful for %q", zarfTypedData.Metadata.Name)) +// ValidateZarfSchema a zarf file against the zarf schema, returns an error if the file is invalid +func ValidateZarfSchema(path string) (err error) { + // var zarfTypedData types.ZarfPackage + // if err := utils.ReadYaml(filepath.Join(path, layout.ZarfYAML), &zarfTypedData); err != nil { + // return err + // } + + // if err := checkForVarInComponentImport(zarfTypedData); err != nil { + // message.Warn("") + // } + + // zarfSchema, _ := config.GetSchemaFile() + + // var zarfData interface{} + // if err := utils.ReadYaml(filepath.Join(path, layout.ZarfYAML), &zarfData); err != nil { + // return err + // } + // var validator Validator + // if validator, err = validateSchema(nil,zarfData, zarfSchema); err != nil { + // return err + // } + + // message.Success(fmt.Sprintf("Schema validation successful for %q", zarfTypedData.Metadata.Name)) return nil } -func checkForVarInComponentImport(zarfYaml types.ZarfPackage) error { - var errorMessages []string - for i, component := range zarfYaml.Components { +func checkForVarInComponentImport(validator Validator, zarfPackage types.ZarfPackage) Validator { + for i, component := range zarfPackage.Components { if strings.Contains(component.Import.Path, types.ZarfPackageTemplatePrefix) { - errorMessages = append(errorMessages, fmt.Sprintf("component.%d.import.path will not resolve ZARF_PKG_TMPL_* variables", i)) + validator.warnings = append(validator.warnings, fmt.Sprintf("component.%d.import.path will not resolve ZARF_PKG_TMPL_* variables", i)) } if strings.Contains(component.Import.URL, types.ZarfPackageTemplatePrefix) { - errorMessages = append(errorMessages, fmt.Sprintf("component.%d.import.url will not resolve ZARF_PKG_TMPL_* variables", i)) + validator.warnings = append(validator.warnings, fmt.Sprintf("component.%d.import.url will not resolve ZARF_PKG_TMPL_* variables", i)) } } - if len(errorMessages) > 0 { - return fmt.Errorf("%s %s", zarfWarningPrefix, strings.Join(errorMessages, ", ")) - } - - return nil + return validator } -func validateSchema(unmarshalledYaml interface{}, jsonSchema []byte) error { +func validateSchema(validator Validator, unmarshalledYaml interface{}, jsonSchema []byte) (Validator, error) { schemaLoader := gojsonschema.NewBytesLoader(jsonSchema) documentLoader := gojsonschema.NewGoLoader(unmarshalledYaml) result, err := gojsonschema.Validate(schemaLoader, documentLoader) if err != nil { - return err + return validator, err } if !result.Valid() { - errorMessage := zarfInvalidPrefix for _, desc := range result.Errors() { - errorMessage = fmt.Sprintf("%s\n - %s", errorMessage, desc.String()) + validator.errors = append(validator.errors, desc.String()) } - err = errors.New(errorMessage) } - return err + return validator, err } From d240e86a7a9d89e702b9bb5f603ec539f3078ea0 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Fri, 17 Nov 2023 15:54:59 +0000 Subject: [PATCH 060/192] tests working with refactored validator, still more refactoring todo --- src/cmd/prepare.go | 12 +++- src/pkg/packager/validator/validater_test.go | 11 +-- src/pkg/packager/validator/validator.go | 70 ++++++++++++-------- 3 files changed, 56 insertions(+), 37 deletions(-) diff --git a/src/cmd/prepare.go b/src/cmd/prepare.go index 03e9cabfd0..60db3cf4f4 100644 --- a/src/cmd/prepare.go +++ b/src/cmd/prepare.go @@ -222,10 +222,20 @@ var lintCmd = &cobra.Command{ message.Fatalf(err, lang.CmdPrepareFindImagesErr, err.Error()) } } - err := validator.ValidateZarfSchema(baseDir) + validator, err := validator.ValidateZarfSchema(baseDir) if err != nil { message.Fatal(err, err.Error()) } + if validator.HasWarnings() { + message.Warn(validator.GetFormmatedWarning()) + } + if validator.HasErrors() { + message.Fatal(validator.GetFormmatedError(), validator.GetFormmatedError().Error()) + } + if !validator.HasWarnings() && !validator.HasErrors() { + message.Success(validator.GetFormmatedSuccess()) + } + }, } diff --git a/src/pkg/packager/validator/validater_test.go b/src/pkg/packager/validator/validater_test.go index 3d21b7f0f2..297b68ea26 100644 --- a/src/pkg/packager/validator/validater_test.go +++ b/src/pkg/packager/validator/validater_test.go @@ -81,10 +81,6 @@ func TestValidateSchema(t *testing.T) { zarfSchema := getZarfSchema(t) validator := Validator{} validator, err := validateSchema(validator, unmarshalledYaml, zarfSchema) - // errorMessage := zarfInvalidPrefix + ` - // - components.0.import: Additional property not-path is not allowed - // - components.1.import.path: Invalid type. Expected: string, given: integer` - // require.EqualError(t, err, errorMessage) require.NoError(t, err) require.Equal(t, validator.errors[0], "components.0.import: Additional property not-path is not allowed") require.Equal(t, validator.errors[1], "components.1.import.path: Invalid type. Expected: string, given: integer") @@ -99,9 +95,6 @@ func TestValidateSchema(t *testing.T) { t.Run("Template in component import failure", func(t *testing.T) { unmarshalledYaml := readAndUnmarshalYaml[types.ZarfPackage](t, badZarfPackage) validator := checkForVarInComponentImport(Validator{}, unmarshalledYaml) - // errorMessage := zarfWarningPrefix + " component.2.import.path will not resolve ZARF_PKG_TMPL_* variables, " + - // "component.3.import.url will not resolve ZARF_PKG_TMPL_* variables" - // require.EqualError(t, err, errorMessage) require.Equal(t, validator.warnings[0], "component.2.import.path will not resolve ZARF_PKG_TMPL_* variables") require.Equal(t, validator.warnings[1], "component.3.import.url will not resolve ZARF_PKG_TMPL_* variables") }) @@ -111,7 +104,7 @@ func TestValidateSchema(t *testing.T) { error2 := "components.1.import.path: Invalid type. Expected: string, given: integer" validator := Validator{errors: []string{error1, error2}} errorMessage := fmt.Sprintf("%s\n - %s\n - %s", validatorInvalidPrefix, error1, error2) - require.EqualError(t, validator.getFormmatedError(), errorMessage) + require.EqualError(t, validator.GetFormmatedError(), errorMessage) }) t.Run("Validator Warning formatting", func(t *testing.T) { @@ -119,6 +112,6 @@ func TestValidateSchema(t *testing.T) { warning2 := "components.1.import.path: Invalid type. Expected: string, given: integer" validator := Validator{warnings: []string{warning1, warning2}} message := fmt.Sprintf("%s %s, %s", validatorWarningPrefix, warning1, warning2) - require.Equal(t, validator.getFormmatedWarning(), message) + require.Equal(t, validator.GetFormmatedWarning(), message) }) } diff --git a/src/pkg/packager/validator/validator.go b/src/pkg/packager/validator/validator.go index 8848d3e67c..60a5eb3f98 100644 --- a/src/pkg/packager/validator/validator.go +++ b/src/pkg/packager/validator/validator.go @@ -7,8 +7,12 @@ package validator import ( "errors" "fmt" + "path/filepath" "strings" + "github.com/defenseunicorns/zarf/src/config" + "github.com/defenseunicorns/zarf/src/pkg/layout" + "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/types" "github.com/xeipuuv/gojsonschema" ) @@ -19,11 +23,12 @@ const ( ) type Validator struct { - warnings []string - errors []string + warnings []string + errors []string + jsonSchema []byte } -func (v Validator) getFormmatedError() error { +func (v Validator) GetFormmatedError() error { if len(v.errors) == 0 { return nil } @@ -34,37 +39,48 @@ func (v Validator) getFormmatedError() error { return errors.New(errorMessage) } -func (v Validator) getFormmatedWarning() string { +func (v Validator) GetFormmatedWarning() string { if len(v.warnings) == 0 { return "" } return fmt.Sprintf("%s %s", validatorWarningPrefix, strings.Join(v.warnings, ", ")) } +func (v Validator) GetFormmatedSuccess() string { + return fmt.Sprintf("Schema validation successful for %q", "init") +} + +func (v Validator) HasWarnings() bool { + return !(len(v.warnings) == 0) +} + +func (v Validator) HasErrors() bool { + return !(len(v.errors) == 0) +} + // ValidateZarfSchema a zarf file against the zarf schema, returns an error if the file is invalid -func ValidateZarfSchema(path string) (err error) { - // var zarfTypedData types.ZarfPackage - // if err := utils.ReadYaml(filepath.Join(path, layout.ZarfYAML), &zarfTypedData); err != nil { - // return err - // } - - // if err := checkForVarInComponentImport(zarfTypedData); err != nil { - // message.Warn("") - // } - - // zarfSchema, _ := config.GetSchemaFile() - - // var zarfData interface{} - // if err := utils.ReadYaml(filepath.Join(path, layout.ZarfYAML), &zarfData); err != nil { - // return err - // } - // var validator Validator - // if validator, err = validateSchema(nil,zarfData, zarfSchema); err != nil { - // return err - // } - - // message.Success(fmt.Sprintf("Schema validation successful for %q", zarfTypedData.Metadata.Name)) - return nil +func ValidateZarfSchema(path string) (Validator, error) { + validator := Validator{} + var err error + var zarfTypedData types.ZarfPackage + if err := utils.ReadYaml(filepath.Join(path, layout.ZarfYAML), &zarfTypedData); err != nil { + return validator, err + } + + validator = checkForVarInComponentImport(validator, zarfTypedData) + + zarfSchema, _ := config.GetSchemaFile() + + var zarfData interface{} + if err := utils.ReadYaml(filepath.Join(path, layout.ZarfYAML), &zarfData); err != nil { + return validator, err + } + + if validator, err = validateSchema(validator, zarfData, zarfSchema); err != nil { + return validator, err + } + + return validator, nil } func checkForVarInComponentImport(validator Validator, zarfPackage types.ZarfPackage) Validator { From a02647a8eee8b8cb8c813d99687c76947ae45648 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Fri, 17 Nov 2023 15:58:58 +0000 Subject: [PATCH 061/192] refactoring --- src/pkg/packager/validator/validater_test.go | 10 ++++------ src/pkg/packager/validator/validator.go | 12 +++++++----- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/pkg/packager/validator/validater_test.go b/src/pkg/packager/validator/validater_test.go index 297b68ea26..4a427e109a 100644 --- a/src/pkg/packager/validator/validater_test.go +++ b/src/pkg/packager/validator/validater_test.go @@ -69,18 +69,16 @@ func TestValidateSchema(t *testing.T) { t.Run("validate schema success", func(t *testing.T) { unmarshalledYaml := readAndUnmarshalYaml[interface{}](t, goodZarfPackage) - zarfSchema := getZarfSchema(t) - validator := Validator{} - validator, err := validateSchema(validator, unmarshalledYaml, zarfSchema) + validator := Validator{jsonSchema: getZarfSchema(t)} + validator, err := validateSchema(validator, unmarshalledYaml) require.NoError(t, err) require.Empty(t, validator.errors) }) t.Run("validate schema fail", func(t *testing.T) { unmarshalledYaml := readAndUnmarshalYaml[interface{}](t, badZarfPackage) - zarfSchema := getZarfSchema(t) - validator := Validator{} - validator, err := validateSchema(validator, unmarshalledYaml, zarfSchema) + validator := Validator{jsonSchema: getZarfSchema(t)} + validator, err := validateSchema(validator, unmarshalledYaml) require.NoError(t, err) require.Equal(t, validator.errors[0], "components.0.import: Additional property not-path is not allowed") require.Equal(t, validator.errors[1], "components.1.import.path: Invalid type. Expected: string, given: integer") diff --git a/src/pkg/packager/validator/validator.go b/src/pkg/packager/validator/validator.go index 60a5eb3f98..dc9da9c2ed 100644 --- a/src/pkg/packager/validator/validator.go +++ b/src/pkg/packager/validator/validator.go @@ -58,7 +58,7 @@ func (v Validator) HasErrors() bool { return !(len(v.errors) == 0) } -// ValidateZarfSchema a zarf file against the zarf schema, returns an error if the file is invalid +// ValidateZarfSchema validates a zarf file against the zarf schema, returns a validator with warnings or errors if they exist func ValidateZarfSchema(path string) (Validator, error) { validator := Validator{} var err error @@ -69,14 +69,16 @@ func ValidateZarfSchema(path string) (Validator, error) { validator = checkForVarInComponentImport(validator, zarfTypedData) - zarfSchema, _ := config.GetSchemaFile() + if validator.jsonSchema, err = config.GetSchemaFile(); err != nil { + return validator, err + } var zarfData interface{} if err := utils.ReadYaml(filepath.Join(path, layout.ZarfYAML), &zarfData); err != nil { return validator, err } - if validator, err = validateSchema(validator, zarfData, zarfSchema); err != nil { + if validator, err = validateSchema(validator, zarfData); err != nil { return validator, err } @@ -96,8 +98,8 @@ func checkForVarInComponentImport(validator Validator, zarfPackage types.ZarfPac return validator } -func validateSchema(validator Validator, unmarshalledYaml interface{}, jsonSchema []byte) (Validator, error) { - schemaLoader := gojsonschema.NewBytesLoader(jsonSchema) +func validateSchema(validator Validator, unmarshalledYaml interface{}) (Validator, error) { + schemaLoader := gojsonschema.NewBytesLoader(validator.jsonSchema) documentLoader := gojsonschema.NewGoLoader(unmarshalledYaml) result, err := gojsonschema.Validate(schemaLoader, documentLoader) From a9eb84f9510e7c3edbc03c44d8583706134a6ee7 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Fri, 17 Nov 2023 16:48:14 +0000 Subject: [PATCH 062/192] validator refactor --- src/cmd/prepare.go | 23 +++++----- src/pkg/packager/validator/validater_test.go | 18 ++++---- src/pkg/packager/validator/validator.go | 46 +++++++++++--------- 3 files changed, 47 insertions(+), 40 deletions(-) diff --git a/src/cmd/prepare.go b/src/cmd/prepare.go index 60db3cf4f4..cb3a476b7f 100644 --- a/src/cmd/prepare.go +++ b/src/cmd/prepare.go @@ -206,6 +206,18 @@ var prepareGenerateConfigFile = &cobra.Command{ }, } +func displayFormattedMessage(validator validator.Validator) { + if validator.HasWarnings() { + message.Warn(validator.GetFormatedWarning()) + } + if validator.HasErrors() { + message.Fatal(validator.GetFormatedError(), validator.GetFormatedError().Error()) + } + if validator.IsSuccess() { + message.Success(validator.GetFormatedSuccess()) + } +} + var lintCmd = &cobra.Command{ Use: "lint [ DIRECTORY ]", Args: cobra.MaximumNArgs(1), @@ -226,16 +238,7 @@ var lintCmd = &cobra.Command{ if err != nil { message.Fatal(err, err.Error()) } - if validator.HasWarnings() { - message.Warn(validator.GetFormmatedWarning()) - } - if validator.HasErrors() { - message.Fatal(validator.GetFormmatedError(), validator.GetFormmatedError().Error()) - } - if !validator.HasWarnings() && !validator.HasErrors() { - message.Success(validator.GetFormmatedSuccess()) - } - + displayFormattedMessage(validator) }, } diff --git a/src/pkg/packager/validator/validater_test.go b/src/pkg/packager/validator/validater_test.go index 4a427e109a..5965dbd0b0 100644 --- a/src/pkg/packager/validator/validater_test.go +++ b/src/pkg/packager/validator/validater_test.go @@ -69,16 +69,16 @@ func TestValidateSchema(t *testing.T) { t.Run("validate schema success", func(t *testing.T) { unmarshalledYaml := readAndUnmarshalYaml[interface{}](t, goodZarfPackage) - validator := Validator{jsonSchema: getZarfSchema(t)} - validator, err := validateSchema(validator, unmarshalledYaml) + validator := Validator{untypedZarfPackage: unmarshalledYaml, jsonSchema: getZarfSchema(t)} + validator, err := validateSchema(validator) require.NoError(t, err) require.Empty(t, validator.errors) }) t.Run("validate schema fail", func(t *testing.T) { unmarshalledYaml := readAndUnmarshalYaml[interface{}](t, badZarfPackage) - validator := Validator{jsonSchema: getZarfSchema(t)} - validator, err := validateSchema(validator, unmarshalledYaml) + validator := Validator{untypedZarfPackage: unmarshalledYaml, jsonSchema: getZarfSchema(t)} + validator, err := validateSchema(validator) require.NoError(t, err) require.Equal(t, validator.errors[0], "components.0.import: Additional property not-path is not allowed") require.Equal(t, validator.errors[1], "components.1.import.path: Invalid type. Expected: string, given: integer") @@ -86,13 +86,13 @@ func TestValidateSchema(t *testing.T) { t.Run("Template in component import success", func(t *testing.T) { unmarshalledYaml := readAndUnmarshalYaml[types.ZarfPackage](t, goodZarfPackage) - validator := checkForVarInComponentImport(Validator{}, unmarshalledYaml) - require.Empty(t, validator) + validator := checkForVarInComponentImport(Validator{typedZarfPackage: unmarshalledYaml}) + require.Empty(t, validator.warnings) }) t.Run("Template in component import failure", func(t *testing.T) { unmarshalledYaml := readAndUnmarshalYaml[types.ZarfPackage](t, badZarfPackage) - validator := checkForVarInComponentImport(Validator{}, unmarshalledYaml) + validator := checkForVarInComponentImport(Validator{typedZarfPackage: unmarshalledYaml}) require.Equal(t, validator.warnings[0], "component.2.import.path will not resolve ZARF_PKG_TMPL_* variables") require.Equal(t, validator.warnings[1], "component.3.import.url will not resolve ZARF_PKG_TMPL_* variables") }) @@ -102,7 +102,7 @@ func TestValidateSchema(t *testing.T) { error2 := "components.1.import.path: Invalid type. Expected: string, given: integer" validator := Validator{errors: []string{error1, error2}} errorMessage := fmt.Sprintf("%s\n - %s\n - %s", validatorInvalidPrefix, error1, error2) - require.EqualError(t, validator.GetFormmatedError(), errorMessage) + require.EqualError(t, validator.GetFormatedError(), errorMessage) }) t.Run("Validator Warning formatting", func(t *testing.T) { @@ -110,6 +110,6 @@ func TestValidateSchema(t *testing.T) { warning2 := "components.1.import.path: Invalid type. Expected: string, given: integer" validator := Validator{warnings: []string{warning1, warning2}} message := fmt.Sprintf("%s %s, %s", validatorWarningPrefix, warning1, warning2) - require.Equal(t, validator.GetFormmatedWarning(), message) + require.Equal(t, validator.GetFormatedWarning(), message) }) } diff --git a/src/pkg/packager/validator/validator.go b/src/pkg/packager/validator/validator.go index dc9da9c2ed..ac823ea2c8 100644 --- a/src/pkg/packager/validator/validator.go +++ b/src/pkg/packager/validator/validator.go @@ -23,13 +23,15 @@ const ( ) type Validator struct { - warnings []string - errors []string - jsonSchema []byte + warnings []string + errors []string + jsonSchema []byte + typedZarfPackage types.ZarfPackage + untypedZarfPackage interface{} } -func (v Validator) GetFormmatedError() error { - if len(v.errors) == 0 { +func (v Validator) GetFormatedError() error { + if !v.HasErrors() { return nil } errorMessage := validatorInvalidPrefix @@ -39,54 +41,56 @@ func (v Validator) GetFormmatedError() error { return errors.New(errorMessage) } -func (v Validator) GetFormmatedWarning() string { - if len(v.warnings) == 0 { +func (v Validator) GetFormatedWarning() string { + if !v.HasWarnings() { return "" } return fmt.Sprintf("%s %s", validatorWarningPrefix, strings.Join(v.warnings, ", ")) } -func (v Validator) GetFormmatedSuccess() string { - return fmt.Sprintf("Schema validation successful for %q", "init") +func (v Validator) GetFormatedSuccess() string { + return fmt.Sprintf("Schema validation successful for %q", v.typedZarfPackage.Metadata.Name) } func (v Validator) HasWarnings() bool { - return !(len(v.warnings) == 0) + return len(v.warnings) > 0 } func (v Validator) HasErrors() bool { - return !(len(v.errors) == 0) + return len(v.errors) > 0 +} + +func (v Validator) IsSuccess() bool { + return !v.HasWarnings() && !v.HasErrors() } // ValidateZarfSchema validates a zarf file against the zarf schema, returns a validator with warnings or errors if they exist func ValidateZarfSchema(path string) (Validator, error) { validator := Validator{} var err error - var zarfTypedData types.ZarfPackage - if err := utils.ReadYaml(filepath.Join(path, layout.ZarfYAML), &zarfTypedData); err != nil { + if err := utils.ReadYaml(filepath.Join(path, layout.ZarfYAML), &validator.typedZarfPackage); err != nil { return validator, err } - validator = checkForVarInComponentImport(validator, zarfTypedData) + validator = checkForVarInComponentImport(validator) if validator.jsonSchema, err = config.GetSchemaFile(); err != nil { return validator, err } - var zarfData interface{} - if err := utils.ReadYaml(filepath.Join(path, layout.ZarfYAML), &zarfData); err != nil { + if err := utils.ReadYaml(filepath.Join(path, layout.ZarfYAML), &validator.untypedZarfPackage); err != nil { return validator, err } - if validator, err = validateSchema(validator, zarfData); err != nil { + if validator, err = validateSchema(validator); err != nil { return validator, err } return validator, nil } -func checkForVarInComponentImport(validator Validator, zarfPackage types.ZarfPackage) Validator { - for i, component := range zarfPackage.Components { +func checkForVarInComponentImport(validator Validator) Validator { + for i, component := range validator.typedZarfPackage.Components { if strings.Contains(component.Import.Path, types.ZarfPackageTemplatePrefix) { validator.warnings = append(validator.warnings, fmt.Sprintf("component.%d.import.path will not resolve ZARF_PKG_TMPL_* variables", i)) } @@ -98,9 +102,9 @@ func checkForVarInComponentImport(validator Validator, zarfPackage types.ZarfPac return validator } -func validateSchema(validator Validator, unmarshalledYaml interface{}) (Validator, error) { +func validateSchema(validator Validator) (Validator, error) { schemaLoader := gojsonschema.NewBytesLoader(validator.jsonSchema) - documentLoader := gojsonschema.NewGoLoader(unmarshalledYaml) + documentLoader := gojsonschema.NewGoLoader(validator.untypedZarfPackage) result, err := gojsonschema.Validate(schemaLoader, documentLoader) if err != nil { From 50e85a02abdf6aa1cec8c4e2f3c0369286d7a5c9 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Fri, 17 Nov 2023 17:02:03 +0000 Subject: [PATCH 063/192] refactoring validator --- src/cmd/prepare.go | 14 +----- src/pkg/packager/validator/validater_test.go | 14 +++--- src/pkg/packager/validator/validator.go | 45 +++++++++++++------- 3 files changed, 38 insertions(+), 35 deletions(-) diff --git a/src/cmd/prepare.go b/src/cmd/prepare.go index cb3a476b7f..c5404e4de8 100644 --- a/src/cmd/prepare.go +++ b/src/cmd/prepare.go @@ -206,18 +206,6 @@ var prepareGenerateConfigFile = &cobra.Command{ }, } -func displayFormattedMessage(validator validator.Validator) { - if validator.HasWarnings() { - message.Warn(validator.GetFormatedWarning()) - } - if validator.HasErrors() { - message.Fatal(validator.GetFormatedError(), validator.GetFormatedError().Error()) - } - if validator.IsSuccess() { - message.Success(validator.GetFormatedSuccess()) - } -} - var lintCmd = &cobra.Command{ Use: "lint [ DIRECTORY ]", Args: cobra.MaximumNArgs(1), @@ -238,7 +226,7 @@ var lintCmd = &cobra.Command{ if err != nil { message.Fatal(err, err.Error()) } - displayFormattedMessage(validator) + validator.DisplayFormattedMessage() }, } diff --git a/src/pkg/packager/validator/validater_test.go b/src/pkg/packager/validator/validater_test.go index 5965dbd0b0..6e0d22ab81 100644 --- a/src/pkg/packager/validator/validater_test.go +++ b/src/pkg/packager/validator/validater_test.go @@ -70,7 +70,7 @@ func TestValidateSchema(t *testing.T) { t.Run("validate schema success", func(t *testing.T) { unmarshalledYaml := readAndUnmarshalYaml[interface{}](t, goodZarfPackage) validator := Validator{untypedZarfPackage: unmarshalledYaml, jsonSchema: getZarfSchema(t)} - validator, err := validateSchema(validator) + err := validateSchema(&validator) require.NoError(t, err) require.Empty(t, validator.errors) }) @@ -78,7 +78,7 @@ func TestValidateSchema(t *testing.T) { t.Run("validate schema fail", func(t *testing.T) { unmarshalledYaml := readAndUnmarshalYaml[interface{}](t, badZarfPackage) validator := Validator{untypedZarfPackage: unmarshalledYaml, jsonSchema: getZarfSchema(t)} - validator, err := validateSchema(validator) + err := validateSchema(&validator) require.NoError(t, err) require.Equal(t, validator.errors[0], "components.0.import: Additional property not-path is not allowed") require.Equal(t, validator.errors[1], "components.1.import.path: Invalid type. Expected: string, given: integer") @@ -86,13 +86,15 @@ func TestValidateSchema(t *testing.T) { t.Run("Template in component import success", func(t *testing.T) { unmarshalledYaml := readAndUnmarshalYaml[types.ZarfPackage](t, goodZarfPackage) - validator := checkForVarInComponentImport(Validator{typedZarfPackage: unmarshalledYaml}) + validator := Validator{typedZarfPackage: unmarshalledYaml} + checkForVarInComponentImport(&validator) require.Empty(t, validator.warnings) }) t.Run("Template in component import failure", func(t *testing.T) { unmarshalledYaml := readAndUnmarshalYaml[types.ZarfPackage](t, badZarfPackage) - validator := checkForVarInComponentImport(Validator{typedZarfPackage: unmarshalledYaml}) + validator := Validator{typedZarfPackage: unmarshalledYaml} + checkForVarInComponentImport(&validator) require.Equal(t, validator.warnings[0], "component.2.import.path will not resolve ZARF_PKG_TMPL_* variables") require.Equal(t, validator.warnings[1], "component.3.import.url will not resolve ZARF_PKG_TMPL_* variables") }) @@ -102,7 +104,7 @@ func TestValidateSchema(t *testing.T) { error2 := "components.1.import.path: Invalid type. Expected: string, given: integer" validator := Validator{errors: []string{error1, error2}} errorMessage := fmt.Sprintf("%s\n - %s\n - %s", validatorInvalidPrefix, error1, error2) - require.EqualError(t, validator.GetFormatedError(), errorMessage) + require.EqualError(t, validator.getFormatedError(), errorMessage) }) t.Run("Validator Warning formatting", func(t *testing.T) { @@ -110,6 +112,6 @@ func TestValidateSchema(t *testing.T) { warning2 := "components.1.import.path: Invalid type. Expected: string, given: integer" validator := Validator{warnings: []string{warning1, warning2}} message := fmt.Sprintf("%s %s, %s", validatorWarningPrefix, warning1, warning2) - require.Equal(t, validator.GetFormatedWarning(), message) + require.Equal(t, validator.getFormatedWarning(), message) }) } diff --git a/src/pkg/packager/validator/validator.go b/src/pkg/packager/validator/validator.go index ac823ea2c8..8d63c79611 100644 --- a/src/pkg/packager/validator/validator.go +++ b/src/pkg/packager/validator/validator.go @@ -12,6 +12,7 @@ import ( "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/pkg/layout" + "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/types" "github.com/xeipuuv/gojsonschema" @@ -22,6 +23,7 @@ const ( validatorWarningPrefix = "zarf schema warning:" ) +// Validator is used by other packages to check their zarf packages for errors/warnings type Validator struct { warnings []string errors []string @@ -30,8 +32,8 @@ type Validator struct { untypedZarfPackage interface{} } -func (v Validator) GetFormatedError() error { - if !v.HasErrors() { +func (v Validator) getFormatedError() error { + if !v.hasErrors() { return nil } errorMessage := validatorInvalidPrefix @@ -41,27 +43,39 @@ func (v Validator) GetFormatedError() error { return errors.New(errorMessage) } -func (v Validator) GetFormatedWarning() string { - if !v.HasWarnings() { +func (v Validator) getFormatedWarning() string { + if !v.hasWarnings() { return "" } return fmt.Sprintf("%s %s", validatorWarningPrefix, strings.Join(v.warnings, ", ")) } -func (v Validator) GetFormatedSuccess() string { +func (v Validator) getFormatedSuccess() string { return fmt.Sprintf("Schema validation successful for %q", v.typedZarfPackage.Metadata.Name) } -func (v Validator) HasWarnings() bool { +func (v Validator) hasWarnings() bool { return len(v.warnings) > 0 } -func (v Validator) HasErrors() bool { +func (v Validator) hasErrors() bool { return len(v.errors) > 0 } -func (v Validator) IsSuccess() bool { - return !v.HasWarnings() && !v.HasErrors() +func (v Validator) isSuccess() bool { + return !v.hasWarnings() && !v.hasErrors() +} + +func (v Validator) DisplayFormattedMessage() { + if v.hasWarnings() { + message.Warn(v.getFormatedWarning()) + } + if v.hasErrors() { + message.Fatal(v.getFormatedError(), v.getFormatedError().Error()) + } + if v.isSuccess() { + message.Success(v.getFormatedSuccess()) + } } // ValidateZarfSchema validates a zarf file against the zarf schema, returns a validator with warnings or errors if they exist @@ -72,7 +86,7 @@ func ValidateZarfSchema(path string) (Validator, error) { return validator, err } - validator = checkForVarInComponentImport(validator) + checkForVarInComponentImport(&validator) if validator.jsonSchema, err = config.GetSchemaFile(); err != nil { return validator, err @@ -82,14 +96,14 @@ func ValidateZarfSchema(path string) (Validator, error) { return validator, err } - if validator, err = validateSchema(validator); err != nil { + if err = validateSchema(&validator); err != nil { return validator, err } return validator, nil } -func checkForVarInComponentImport(validator Validator) Validator { +func checkForVarInComponentImport(validator *Validator) { for i, component := range validator.typedZarfPackage.Components { if strings.Contains(component.Import.Path, types.ZarfPackageTemplatePrefix) { validator.warnings = append(validator.warnings, fmt.Sprintf("component.%d.import.path will not resolve ZARF_PKG_TMPL_* variables", i)) @@ -99,16 +113,15 @@ func checkForVarInComponentImport(validator Validator) Validator { } } - return validator } -func validateSchema(validator Validator) (Validator, error) { +func validateSchema(validator *Validator) error { schemaLoader := gojsonschema.NewBytesLoader(validator.jsonSchema) documentLoader := gojsonschema.NewGoLoader(validator.untypedZarfPackage) result, err := gojsonschema.Validate(schemaLoader, documentLoader) if err != nil { - return validator, err + return err } if !result.Valid() { @@ -117,5 +130,5 @@ func validateSchema(validator Validator) (Validator, error) { } } - return validator, err + return err } From acbf48ea78846ce805867095576415b971c24cc2 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Fri, 17 Nov 2023 17:08:41 +0000 Subject: [PATCH 064/192] change commment --- src/pkg/packager/validator/validator.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pkg/packager/validator/validator.go b/src/pkg/packager/validator/validator.go index 8d63c79611..154405240b 100644 --- a/src/pkg/packager/validator/validator.go +++ b/src/pkg/packager/validator/validator.go @@ -23,7 +23,7 @@ const ( validatorWarningPrefix = "zarf schema warning:" ) -// Validator is used by other packages to check their zarf packages for errors/warnings +// Validator holds the warnings/errors and messaging that we get from validation type Validator struct { warnings []string errors []string From 739aaa9223c2ccfcdec66f98fc5c73fd3ec2e46b Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Fri, 17 Nov 2023 17:16:58 +0000 Subject: [PATCH 065/192] moving errors to type error --- src/pkg/packager/validator/validater_test.go | 13 +++++++------ src/pkg/packager/validator/validator.go | 8 +++++--- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/src/pkg/packager/validator/validater_test.go b/src/pkg/packager/validator/validater_test.go index 6e0d22ab81..dc6acb26d1 100644 --- a/src/pkg/packager/validator/validater_test.go +++ b/src/pkg/packager/validator/validater_test.go @@ -5,6 +5,7 @@ package validator import ( + "errors" "fmt" "os" "testing" @@ -80,8 +81,8 @@ func TestValidateSchema(t *testing.T) { validator := Validator{untypedZarfPackage: unmarshalledYaml, jsonSchema: getZarfSchema(t)} err := validateSchema(&validator) require.NoError(t, err) - require.Equal(t, validator.errors[0], "components.0.import: Additional property not-path is not allowed") - require.Equal(t, validator.errors[1], "components.1.import.path: Invalid type. Expected: string, given: integer") + require.EqualError(t, validator.errors[0], "components.0.import: Additional property not-path is not allowed") + require.EqualError(t, validator.errors[1], "components.1.import.path: Invalid type. Expected: string, given: integer") }) t.Run("Template in component import success", func(t *testing.T) { @@ -100,10 +101,10 @@ func TestValidateSchema(t *testing.T) { }) t.Run("Validator Error formatting", func(t *testing.T) { - error1 := "components.0.import: Additional property not-path is not allowed" - error2 := "components.1.import.path: Invalid type. Expected: string, given: integer" - validator := Validator{errors: []string{error1, error2}} - errorMessage := fmt.Sprintf("%s\n - %s\n - %s", validatorInvalidPrefix, error1, error2) + error1 := errors.New("components.0.import: Additional property not-path is not allowed") + error2 := errors.New("components.1.import.path: Invalid type. Expected: string, given: integer") + validator := Validator{errors: []error{error1, error2}} + errorMessage := fmt.Sprintf("%s\n - %s\n - %s", validatorInvalidPrefix, error1.Error(), error2.Error()) require.EqualError(t, validator.getFormatedError(), errorMessage) }) diff --git a/src/pkg/packager/validator/validator.go b/src/pkg/packager/validator/validator.go index 154405240b..839c887e3e 100644 --- a/src/pkg/packager/validator/validator.go +++ b/src/pkg/packager/validator/validator.go @@ -26,7 +26,7 @@ const ( // Validator holds the warnings/errors and messaging that we get from validation type Validator struct { warnings []string - errors []string + errors []error jsonSchema []byte typedZarfPackage types.ZarfPackage untypedZarfPackage interface{} @@ -38,7 +38,7 @@ func (v Validator) getFormatedError() error { } errorMessage := validatorInvalidPrefix for _, errorStr := range v.errors { - errorMessage = fmt.Sprintf("%s\n - %s", errorMessage, errorStr) + errorMessage = fmt.Sprintf("%s\n - %s", errorMessage, errorStr.Error()) } return errors.New(errorMessage) } @@ -66,6 +66,8 @@ func (v Validator) isSuccess() bool { return !v.hasWarnings() && !v.hasErrors() } +// DisplayFormattedMessage Displays the message to the user with proper warnings, failures, or success +// Will exit if there are errors func (v Validator) DisplayFormattedMessage() { if v.hasWarnings() { message.Warn(v.getFormatedWarning()) @@ -126,7 +128,7 @@ func validateSchema(validator *Validator) error { if !result.Valid() { for _, desc := range result.Errors() { - validator.errors = append(validator.errors, desc.String()) + validator.errors = append(validator.errors, errors.New(desc.String())) } } From 48eaf4ba814b3c3821fff82049fd7b0759c4b459 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Fri, 17 Nov 2023 20:29:25 +0000 Subject: [PATCH 066/192] renaming package to lint to make it easier to different object and package --- src/cmd/prepare.go | 4 ++-- src/pkg/packager/{validator/validator.go => lint/lint.go} | 2 +- .../{validator/validater_test.go => lint/lint_test.go} | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) rename src/pkg/packager/{validator/validator.go => lint/lint.go} (99%) rename src/pkg/packager/{validator/validater_test.go => lint/lint_test.go} (97%) diff --git a/src/cmd/prepare.go b/src/cmd/prepare.go index c5404e4de8..6433af1280 100644 --- a/src/cmd/prepare.go +++ b/src/cmd/prepare.go @@ -17,7 +17,7 @@ import ( "github.com/defenseunicorns/zarf/src/config/lang" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/packager" - "github.com/defenseunicorns/zarf/src/pkg/packager/validator" + "github.com/defenseunicorns/zarf/src/pkg/packager/lint" "github.com/defenseunicorns/zarf/src/pkg/transform" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" @@ -222,7 +222,7 @@ var lintCmd = &cobra.Command{ message.Fatalf(err, lang.CmdPrepareFindImagesErr, err.Error()) } } - validator, err := validator.ValidateZarfSchema(baseDir) + validator, err := lint.ValidateZarfSchema(baseDir) if err != nil { message.Fatal(err, err.Error()) } diff --git a/src/pkg/packager/validator/validator.go b/src/pkg/packager/lint/lint.go similarity index 99% rename from src/pkg/packager/validator/validator.go rename to src/pkg/packager/lint/lint.go index 839c887e3e..100f48523d 100644 --- a/src/pkg/packager/validator/validator.go +++ b/src/pkg/packager/lint/lint.go @@ -2,7 +2,7 @@ // SPDX-FileCopyrightText: 2021-Present The Zarf Authors // Package validator contains functions for verifying zarf yaml files are valid -package validator +package lint import ( "errors" diff --git a/src/pkg/packager/validator/validater_test.go b/src/pkg/packager/lint/lint_test.go similarity index 97% rename from src/pkg/packager/validator/validater_test.go rename to src/pkg/packager/lint/lint_test.go index dc6acb26d1..313dcfcb16 100644 --- a/src/pkg/packager/validator/validater_test.go +++ b/src/pkg/packager/lint/lint_test.go @@ -1,8 +1,8 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: 2021-Present The Zarf Authors -// Package validator contains functions for verifying zarf yaml files are valid -package validator +// Package lint contains functions for verifying zarf yaml files are valid +package lint import ( "errors" From 6b9d3f970c7e237f21f96e17c0f8e8488306e094 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Fri, 17 Nov 2023 20:38:23 +0000 Subject: [PATCH 067/192] moving get schema file inside of lint --- main.go | 3 ++- src/config/config.go | 5 ----- src/pkg/packager/lint/lint.go | 12 ++++++++++-- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/main.go b/main.go index 651ba68a26..b9f58f6e44 100644 --- a/main.go +++ b/main.go @@ -9,6 +9,7 @@ import ( "github.com/defenseunicorns/zarf/src/cmd" "github.com/defenseunicorns/zarf/src/config" + "github.com/defenseunicorns/zarf/src/pkg/packager/lint" ) //go:embed cosign.pub @@ -19,6 +20,6 @@ var zarfSchema embed.FS func main() { config.CosignPublicKey = cosignPublicKey - config.ZarfSchema = zarfSchema + lint.ZarfSchema = zarfSchema cmd.Execute() } diff --git a/src/config/config.go b/src/config/config.go index f5d24b2e05..071d160fe8 100644 --- a/src/config/config.go +++ b/src/config/config.go @@ -102,11 +102,6 @@ var ( ZarfDefaultCachePath = filepath.Join("~", ".zarf-cache") ) -// GetSchemaFile returns the zarf schema file -func GetSchemaFile() ([]byte, error) { - return ZarfSchema.ReadFile("zarf.schema.json") -} - // GetArch returns the arch based on a priority list with options for overriding. func GetArch(archs ...string) string { // List of architecture overrides. diff --git a/src/pkg/packager/lint/lint.go b/src/pkg/packager/lint/lint.go index 100f48523d..2b3a913d24 100644 --- a/src/pkg/packager/lint/lint.go +++ b/src/pkg/packager/lint/lint.go @@ -5,12 +5,12 @@ package lint import ( + "embed" "errors" "fmt" "path/filepath" "strings" - "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/utils" @@ -23,6 +23,14 @@ const ( validatorWarningPrefix = "zarf schema warning:" ) +var ( + ZarfSchema embed.FS +) + +func getSchemaFile() ([]byte, error) { + return ZarfSchema.ReadFile("zarf.schema.json") +} + // Validator holds the warnings/errors and messaging that we get from validation type Validator struct { warnings []string @@ -90,7 +98,7 @@ func ValidateZarfSchema(path string) (Validator, error) { checkForVarInComponentImport(&validator) - if validator.jsonSchema, err = config.GetSchemaFile(); err != nil { + if validator.jsonSchema, err = getSchemaFile(); err != nil { return validator, err } From bf588687ca4ee932b921e35537c2a538b628ab68 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Fri, 17 Nov 2023 20:59:48 +0000 Subject: [PATCH 068/192] making warnings with numbers work with yq --- src/pkg/packager/lint/lint.go | 14 +++++++++++--- src/pkg/packager/lint/lint_test.go | 4 ++-- src/test/e2e/12_lint_test.go | 2 +- 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/src/pkg/packager/lint/lint.go b/src/pkg/packager/lint/lint.go index 2b3a913d24..92f33dfaf0 100644 --- a/src/pkg/packager/lint/lint.go +++ b/src/pkg/packager/lint/lint.go @@ -74,6 +74,14 @@ func (v Validator) isSuccess() bool { return !v.hasWarnings() && !v.hasErrors() } +func (v *Validator) addWarning(warning string) { + v.warnings = append(v.warnings, warning) +} + +func (v *Validator) addError(err error) { + v.errors = append(v.errors, err) +} + // DisplayFormattedMessage Displays the message to the user with proper warnings, failures, or success // Will exit if there are errors func (v Validator) DisplayFormattedMessage() { @@ -116,10 +124,10 @@ func ValidateZarfSchema(path string) (Validator, error) { func checkForVarInComponentImport(validator *Validator) { for i, component := range validator.typedZarfPackage.Components { if strings.Contains(component.Import.Path, types.ZarfPackageTemplatePrefix) { - validator.warnings = append(validator.warnings, fmt.Sprintf("component.%d.import.path will not resolve ZARF_PKG_TMPL_* variables", i)) + validator.addWarning(fmt.Sprintf("component.[%d].import.path will not resolve ZARF_PKG_TMPL_* variables", i)) } if strings.Contains(component.Import.URL, types.ZarfPackageTemplatePrefix) { - validator.warnings = append(validator.warnings, fmt.Sprintf("component.%d.import.url will not resolve ZARF_PKG_TMPL_* variables", i)) + validator.addWarning(fmt.Sprintf("component.[%d].import.url will not resolve ZARF_PKG_TMPL_* variables", i)) } } @@ -136,7 +144,7 @@ func validateSchema(validator *Validator) error { if !result.Valid() { for _, desc := range result.Errors() { - validator.errors = append(validator.errors, errors.New(desc.String())) + validator.addError(errors.New(desc.String())) } } diff --git a/src/pkg/packager/lint/lint_test.go b/src/pkg/packager/lint/lint_test.go index 313dcfcb16..dd668a9449 100644 --- a/src/pkg/packager/lint/lint_test.go +++ b/src/pkg/packager/lint/lint_test.go @@ -96,8 +96,8 @@ func TestValidateSchema(t *testing.T) { unmarshalledYaml := readAndUnmarshalYaml[types.ZarfPackage](t, badZarfPackage) validator := Validator{typedZarfPackage: unmarshalledYaml} checkForVarInComponentImport(&validator) - require.Equal(t, validator.warnings[0], "component.2.import.path will not resolve ZARF_PKG_TMPL_* variables") - require.Equal(t, validator.warnings[1], "component.3.import.url will not resolve ZARF_PKG_TMPL_* variables") + require.Equal(t, validator.warnings[0], "component.[2].import.path will not resolve ZARF_PKG_TMPL_* variables") + require.Equal(t, validator.warnings[1], "component.[3].import.url will not resolve ZARF_PKG_TMPL_* variables") }) t.Run("Validator Error formatting", func(t *testing.T) { diff --git a/src/test/e2e/12_lint_test.go b/src/test/e2e/12_lint_test.go index 0a3c8dc60d..6abd4eb706 100644 --- a/src/test/e2e/12_lint_test.go +++ b/src/test/e2e/12_lint_test.go @@ -17,7 +17,7 @@ func TestLint(t *testing.T) { _, stderr, err := e2e.Zarf("prepare", "lint", path) require.Error(t, err, "Expect error here because the yaml file is not following schema") require.Contains(t, stderr, "- components.0.import: Additional property not-path is not allowed") - require.Contains(t, stderr, "component.2.import.path will not resolve ZARF_PKG_TMPL_* variables") + require.Contains(t, stderr, "component.[2].import.path will not resolve ZARF_PKG_TMPL_* variables") }) t.Run("zarf test lint success", func(t *testing.T) { From bc4eb7b4294e8e8eab3d42507998c8a90354d348 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Mon, 27 Nov 2023 15:24:03 +0000 Subject: [PATCH 069/192] changign validatezarfschema to return a pointer --- src/pkg/packager/lint/lint.go | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/pkg/packager/lint/lint.go b/src/pkg/packager/lint/lint.go index 92f33dfaf0..ef9a36a4c2 100644 --- a/src/pkg/packager/lint/lint.go +++ b/src/pkg/packager/lint/lint.go @@ -96,29 +96,30 @@ func (v Validator) DisplayFormattedMessage() { } } -// ValidateZarfSchema validates a zarf file against the zarf schema, returns a validator with warnings or errors if they exist -func ValidateZarfSchema(path string) (Validator, error) { +// ValidateZarfSchema validates a zarf file against the zarf schema, returns *validator with warnings or errors if they exist +// along with an error if the validation itself failed +func ValidateZarfSchema(path string) (*Validator, error) { validator := Validator{} var err error if err := utils.ReadYaml(filepath.Join(path, layout.ZarfYAML), &validator.typedZarfPackage); err != nil { - return validator, err + return &validator, err } checkForVarInComponentImport(&validator) if validator.jsonSchema, err = getSchemaFile(); err != nil { - return validator, err + return &validator, err } if err := utils.ReadYaml(filepath.Join(path, layout.ZarfYAML), &validator.untypedZarfPackage); err != nil { - return validator, err + return &validator, err } if err = validateSchema(&validator); err != nil { - return validator, err + return &validator, err } - return validator, nil + return &validator, nil } func checkForVarInComponentImport(validator *Validator) { From ee5cce065d0b74ce4a5b7840f2f967d5fe3758cb Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Mon, 27 Nov 2023 15:51:24 +0000 Subject: [PATCH 070/192] separating validator object into it's own file --- src/pkg/packager/lint/lint.go | 53 +------------------- src/pkg/packager/lint/lint_test.go | 17 ------- src/pkg/packager/lint/validator.go | 64 +++++++++++++++++++++++++ src/pkg/packager/lint/validator_test.go | 31 ++++++++++++ 4 files changed, 96 insertions(+), 69 deletions(-) create mode 100644 src/pkg/packager/lint/validator.go create mode 100644 src/pkg/packager/lint/validator_test.go diff --git a/src/pkg/packager/lint/lint.go b/src/pkg/packager/lint/lint.go index ef9a36a4c2..05cda78353 100644 --- a/src/pkg/packager/lint/lint.go +++ b/src/pkg/packager/lint/lint.go @@ -1,7 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: 2021-Present The Zarf Authors -// Package validator contains functions for verifying zarf yaml files are valid +// Package lint contains functions for verifying zarf yaml files are valid package lint import ( @@ -31,57 +31,6 @@ func getSchemaFile() ([]byte, error) { return ZarfSchema.ReadFile("zarf.schema.json") } -// Validator holds the warnings/errors and messaging that we get from validation -type Validator struct { - warnings []string - errors []error - jsonSchema []byte - typedZarfPackage types.ZarfPackage - untypedZarfPackage interface{} -} - -func (v Validator) getFormatedError() error { - if !v.hasErrors() { - return nil - } - errorMessage := validatorInvalidPrefix - for _, errorStr := range v.errors { - errorMessage = fmt.Sprintf("%s\n - %s", errorMessage, errorStr.Error()) - } - return errors.New(errorMessage) -} - -func (v Validator) getFormatedWarning() string { - if !v.hasWarnings() { - return "" - } - return fmt.Sprintf("%s %s", validatorWarningPrefix, strings.Join(v.warnings, ", ")) -} - -func (v Validator) getFormatedSuccess() string { - return fmt.Sprintf("Schema validation successful for %q", v.typedZarfPackage.Metadata.Name) -} - -func (v Validator) hasWarnings() bool { - return len(v.warnings) > 0 -} - -func (v Validator) hasErrors() bool { - return len(v.errors) > 0 -} - -func (v Validator) isSuccess() bool { - return !v.hasWarnings() && !v.hasErrors() -} - -func (v *Validator) addWarning(warning string) { - v.warnings = append(v.warnings, warning) -} - -func (v *Validator) addError(err error) { - v.errors = append(v.errors, err) -} - // DisplayFormattedMessage Displays the message to the user with proper warnings, failures, or success // Will exit if there are errors func (v Validator) DisplayFormattedMessage() { diff --git a/src/pkg/packager/lint/lint_test.go b/src/pkg/packager/lint/lint_test.go index dd668a9449..da8f2c5b36 100644 --- a/src/pkg/packager/lint/lint_test.go +++ b/src/pkg/packager/lint/lint_test.go @@ -5,8 +5,6 @@ package lint import ( - "errors" - "fmt" "os" "testing" @@ -100,19 +98,4 @@ func TestValidateSchema(t *testing.T) { require.Equal(t, validator.warnings[1], "component.[3].import.url will not resolve ZARF_PKG_TMPL_* variables") }) - t.Run("Validator Error formatting", func(t *testing.T) { - error1 := errors.New("components.0.import: Additional property not-path is not allowed") - error2 := errors.New("components.1.import.path: Invalid type. Expected: string, given: integer") - validator := Validator{errors: []error{error1, error2}} - errorMessage := fmt.Sprintf("%s\n - %s\n - %s", validatorInvalidPrefix, error1.Error(), error2.Error()) - require.EqualError(t, validator.getFormatedError(), errorMessage) - }) - - t.Run("Validator Warning formatting", func(t *testing.T) { - warning1 := "components.0.import: Additional property not-path is not allowed" - warning2 := "components.1.import.path: Invalid type. Expected: string, given: integer" - validator := Validator{warnings: []string{warning1, warning2}} - message := fmt.Sprintf("%s %s, %s", validatorWarningPrefix, warning1, warning2) - require.Equal(t, validator.getFormatedWarning(), message) - }) } diff --git a/src/pkg/packager/lint/validator.go b/src/pkg/packager/lint/validator.go new file mode 100644 index 0000000000..850ca7b0eb --- /dev/null +++ b/src/pkg/packager/lint/validator.go @@ -0,0 +1,64 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package lint contains functions for verifying zarf yaml files are valid +package lint + +import ( + "errors" + "fmt" + "strings" + + "github.com/defenseunicorns/zarf/src/types" +) + +// Validator holds the warnings/errors and messaging that we get from validation +type Validator struct { + warnings []string + errors []error + jsonSchema []byte + typedZarfPackage types.ZarfPackage + untypedZarfPackage interface{} +} + +func (v Validator) getFormatedError() error { + if !v.hasErrors() { + return nil + } + errorMessage := validatorInvalidPrefix + for _, errorStr := range v.errors { + errorMessage = fmt.Sprintf("%s\n - %s", errorMessage, errorStr.Error()) + } + return errors.New(errorMessage) +} + +func (v Validator) getFormatedWarning() string { + if !v.hasWarnings() { + return "" + } + return fmt.Sprintf("%s %s", validatorWarningPrefix, strings.Join(v.warnings, ", ")) +} + +func (v Validator) getFormatedSuccess() string { + return fmt.Sprintf("Schema validation successful for %q", v.typedZarfPackage.Metadata.Name) +} + +func (v Validator) hasWarnings() bool { + return len(v.warnings) > 0 +} + +func (v Validator) hasErrors() bool { + return len(v.errors) > 0 +} + +func (v Validator) isSuccess() bool { + return !v.hasWarnings() && !v.hasErrors() +} + +func (v *Validator) addWarning(warning string) { + v.warnings = append(v.warnings, warning) +} + +func (v *Validator) addError(err error) { + v.errors = append(v.errors, err) +} diff --git a/src/pkg/packager/lint/validator_test.go b/src/pkg/packager/lint/validator_test.go new file mode 100644 index 0000000000..c6fa4a037b --- /dev/null +++ b/src/pkg/packager/lint/validator_test.go @@ -0,0 +1,31 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package lint contains functions for verifying zarf yaml files are valid +package lint + +import ( + "errors" + "fmt" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestValidator(t *testing.T) { + t.Run("Validator Error formatting", func(t *testing.T) { + error1 := errors.New("components.0.import: Additional property not-path is not allowed") + error2 := errors.New("components.1.import.path: Invalid type. Expected: string, given: integer") + validator := Validator{errors: []error{error1, error2}} + errorMessage := fmt.Sprintf("%s\n - %s\n - %s", validatorInvalidPrefix, error1.Error(), error2.Error()) + require.EqualError(t, validator.getFormatedError(), errorMessage) + }) + + t.Run("Validator Warning formatting", func(t *testing.T) { + warning1 := "components.0.import: Additional property not-path is not allowed" + warning2 := "components.1.import.path: Invalid type. Expected: string, given: integer" + validator := Validator{warnings: []string{warning1, warning2}} + message := fmt.Sprintf("%s %s, %s", validatorWarningPrefix, warning1, warning2) + require.Equal(t, validator.getFormatedWarning(), message) + }) +} From bce27962c113f7260c458717ee232ce45990d854 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Mon, 27 Nov 2023 17:57:13 +0000 Subject: [PATCH 071/192] lint comment --- src/pkg/packager/lint/lint.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/pkg/packager/lint/lint.go b/src/pkg/packager/lint/lint.go index 05cda78353..8792c582b7 100644 --- a/src/pkg/packager/lint/lint.go +++ b/src/pkg/packager/lint/lint.go @@ -23,9 +23,8 @@ const ( validatorWarningPrefix = "zarf schema warning:" ) -var ( - ZarfSchema embed.FS -) +// ZarfSchema is exported so main.go can embed the schema file +var ZarfSchema embed.FS func getSchemaFile() ([]byte, error) { return ZarfSchema.ReadFile("zarf.schema.json") From 0bd8d4a8c4a84b214ed79972876e00140b9e5910 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Mon, 27 Nov 2023 18:25:46 +0000 Subject: [PATCH 072/192] implementing error in validate --- src/pkg/packager/lint/lint.go | 2 +- src/pkg/packager/lint/validator.go | 7 +++---- src/pkg/packager/lint/validator_test.go | 2 +- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/src/pkg/packager/lint/lint.go b/src/pkg/packager/lint/lint.go index 8792c582b7..6771fce20c 100644 --- a/src/pkg/packager/lint/lint.go +++ b/src/pkg/packager/lint/lint.go @@ -37,7 +37,7 @@ func (v Validator) DisplayFormattedMessage() { message.Warn(v.getFormatedWarning()) } if v.hasErrors() { - message.Fatal(v.getFormatedError(), v.getFormatedError().Error()) + message.Fatal(v, v.Error()) } if v.isSuccess() { message.Success(v.getFormatedSuccess()) diff --git a/src/pkg/packager/lint/validator.go b/src/pkg/packager/lint/validator.go index 850ca7b0eb..02cf15775e 100644 --- a/src/pkg/packager/lint/validator.go +++ b/src/pkg/packager/lint/validator.go @@ -5,7 +5,6 @@ package lint import ( - "errors" "fmt" "strings" @@ -21,15 +20,15 @@ type Validator struct { untypedZarfPackage interface{} } -func (v Validator) getFormatedError() error { +func (v Validator) Error() string { if !v.hasErrors() { - return nil + return "" } errorMessage := validatorInvalidPrefix for _, errorStr := range v.errors { errorMessage = fmt.Sprintf("%s\n - %s", errorMessage, errorStr.Error()) } - return errors.New(errorMessage) + return errorMessage } func (v Validator) getFormatedWarning() string { diff --git a/src/pkg/packager/lint/validator_test.go b/src/pkg/packager/lint/validator_test.go index c6fa4a037b..70adb10931 100644 --- a/src/pkg/packager/lint/validator_test.go +++ b/src/pkg/packager/lint/validator_test.go @@ -18,7 +18,7 @@ func TestValidator(t *testing.T) { error2 := errors.New("components.1.import.path: Invalid type. Expected: string, given: integer") validator := Validator{errors: []error{error1, error2}} errorMessage := fmt.Sprintf("%s\n - %s\n - %s", validatorInvalidPrefix, error1.Error(), error2.Error()) - require.EqualError(t, validator.getFormatedError(), errorMessage) + require.EqualError(t, validator, errorMessage) }) t.Run("Validator Warning formatting", func(t *testing.T) { From 4eb7fc15c67dfb9db2ab35e30a57fedd4def2787 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Mon, 27 Nov 2023 18:46:38 +0000 Subject: [PATCH 073/192] updating zarf docs for lint --- docs/3-create-a-zarf-package/index.md | 3 +++ docs/5-zarf-tutorials/0-creating-a-zarf-package.md | 1 + 2 files changed, 4 insertions(+) diff --git a/docs/3-create-a-zarf-package/index.md b/docs/3-create-a-zarf-package/index.md index 21bc292bef..17746a01b5 100644 --- a/docs/3-create-a-zarf-package/index.md +++ b/docs/3-create-a-zarf-package/index.md @@ -21,6 +21,9 @@ To learn more about creating a Zarf package, you can check out the following res The general flow of a Zarf package deployment on an existing initialized cluster is as follows: ```shell +# Before creating your package you can lint your zarf.yaml +$ zarf package lint + # To create a package run the following: $ zarf package create # - Enter any package templates that have not yet been defined diff --git a/docs/5-zarf-tutorials/0-creating-a-zarf-package.md b/docs/5-zarf-tutorials/0-creating-a-zarf-package.md index d0b53f65dd..f311a4d387 100644 --- a/docs/5-zarf-tutorials/0-creating-a-zarf-package.md +++ b/docs/5-zarf-tutorials/0-creating-a-zarf-package.md @@ -37,6 +37,7 @@ metadata: :::tip If you are using an Integrated Development Environment (such as [VS Code](../3-create-a-zarf-package/8-vscode.md)) to create and edit the `zarf.yaml` file, you can install or reference the [`zarf.schema.json`](https://github.com/defenseunicorns/zarf/blob/main/zarf.schema.json) file to get error checking and autocomplete. +Additionally, you can run `zarf prepare lint ` to validate aginst the [`zarf.schema.json`](https://github.com/defenseunicorns/zarf/blob/main/zarf.schema.json) ::: From 63bcb8c5fc7ca3b5902a153d7c8fd30547fd1d55 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Mon, 27 Nov 2023 21:53:22 +0000 Subject: [PATCH 074/192] refactoring message to use table function and printing a table for warning --- src/pkg/message/connect.go | 10 +++--- src/pkg/message/credentials.go | 33 ++++++++--------- src/pkg/message/message.go | 21 +++++++++++ src/pkg/packager/lint/lint.go | 19 ++-------- src/pkg/packager/lint/validator.go | 47 ++++++++++++++++++++----- src/pkg/packager/lint/validator_test.go | 14 ++++---- src/test/e2e/12_lint_test.go | 2 +- 7 files changed, 89 insertions(+), 57 deletions(-) diff --git a/src/pkg/message/connect.go b/src/pkg/message/connect.go index 0df6f08d38..22b125ed51 100644 --- a/src/pkg/message/connect.go +++ b/src/pkg/message/connect.go @@ -8,7 +8,6 @@ import ( "fmt" "github.com/defenseunicorns/zarf/src/types" - "github.com/pterm/pterm" ) // PrintConnectStringTable prints a table of connect strings. @@ -16,14 +15,15 @@ func PrintConnectStringTable(connectStrings types.ConnectStrings) { Debugf("message.PrintConnectStringTable(%#v)", connectStrings) if len(connectStrings) > 0 { - list := pterm.TableData{{" Connect Command", "Description"}} + connectData := [][]string{} // Loop over each connectStrings and convert to pterm.TableData for name, connect := range connectStrings { - name = fmt.Sprintf(" zarf connect %s", name) - list = append(list, []string{name, connect.Description}) + name = fmt.Sprintf("zarf connect %s", name) + connectData = append(connectData, []string{name, connect.Description}) } // Create the table output with the data - _ = pterm.DefaultTable.WithHasHeader().WithData(list).Render() + header := []string{"Connect Command", "Description"} + Table(header, connectData) } } diff --git a/src/pkg/message/credentials.go b/src/pkg/message/credentials.go index b89e15da74..15e1e87c71 100644 --- a/src/pkg/message/credentials.go +++ b/src/pkg/message/credentials.go @@ -34,37 +34,32 @@ func PrintCredentialTable(state *types.ZarfState, componentsToDeploy []types.Dep // Set output to os.Stderr to avoid creds being printed in logs pterm.SetDefaultOutput(os.Stderr) - pterm.Println() - loginTableHeader := pterm.TableData{ - {" Application", "Username", "Password", "Connect", "Get-Creds Key"}, - } - - loginTable := pterm.TableData{} + loginData := [][]string{} if state.RegistryInfo.InternalRegistry { - loginTable = append(loginTable, pterm.TableData{ - {" Registry", state.RegistryInfo.PushUsername, state.RegistryInfo.PushPassword, "zarf connect registry", RegistryKey}, - {" Registry (read-only)", state.RegistryInfo.PullUsername, state.RegistryInfo.PullPassword, "zarf connect registry", RegistryReadKey}, - }...) + loginData = append(loginData, + []string{"Registry", state.RegistryInfo.PushUsername, state.RegistryInfo.PushPassword, "zarf connect registry", RegistryKey}, + []string{"Registry (read-only)", state.RegistryInfo.PullUsername, state.RegistryInfo.PullPassword, "zarf connect registry", RegistryReadKey}, + ) } for _, component := range componentsToDeploy { // Show message if including logging stack if component.Name == "logging" { - loginTable = append(loginTable, pterm.TableData{{" Logging", config.ZarfLoggingUser, state.LoggingSecret, "zarf connect logging", LoggingKey}}...) + loginData = append(loginData, []string{"Logging", config.ZarfLoggingUser, state.LoggingSecret, "zarf connect logging", LoggingKey}) } // Show message if including git-server if component.Name == "git-server" { - loginTable = append(loginTable, pterm.TableData{ - {" Git", state.GitServer.PushUsername, state.GitServer.PushPassword, "zarf connect git", GitKey}, - {" Git (read-only)", state.GitServer.PullUsername, state.GitServer.PullPassword, "zarf connect git", GitReadKey}, - {" Artifact Token", state.ArtifactServer.PushUsername, state.ArtifactServer.PushToken, "zarf connect git", ArtifactKey}, - }...) + loginData = append(loginData, + []string{"Git", state.GitServer.PushUsername, state.GitServer.PushPassword, "zarf connect git", GitKey}, + []string{"Git (read-only)", state.GitServer.PullUsername, state.GitServer.PullPassword, "zarf connect git", GitReadKey}, + []string{"Artifact Token", state.ArtifactServer.PushUsername, state.ArtifactServer.PushToken, "zarf connect git", ArtifactKey}, + ) } } - if len(loginTable) > 0 { - loginTable = append(loginTableHeader, loginTable...) - _ = pterm.DefaultTable.WithHasHeader().WithData(loginTable).Render() + if len(loginData) > 0 { + header := []string{"Application", "Username", "Password", "Connect", "Get-Creds Key"} + Table(header, loginData) } // Restore the log file if it was specified diff --git a/src/pkg/message/message.go b/src/pkg/message/message.go index 07b3db9983..1a1687a9c0 100644 --- a/src/pkg/message/message.go +++ b/src/pkg/message/message.go @@ -340,3 +340,24 @@ func debugPrinter(offset int, a ...any) { func errorPrinter(offset int) *pterm.PrefixPrinter { return pterm.Error.WithShowLineNumber(logLevel > 2).WithLineNumberOffset(offset) } + +func Table(header []string, data [][]string) { + pterm.Println() + + if len(header) > 0 { + header[0] = fmt.Sprintf(" %s", header[0]) + } + + table := pterm.TableData{ + header, + } + + for _, row := range data { + if len(row) > 0 { + row[0] = fmt.Sprintf(" %s", row[0]) + } + table = append(table, pterm.TableData{row}...) + } + + pterm.DefaultTable.WithHasHeader().WithData(table).Render() +} diff --git a/src/pkg/packager/lint/lint.go b/src/pkg/packager/lint/lint.go index 6771fce20c..b07d1b6b2e 100644 --- a/src/pkg/packager/lint/lint.go +++ b/src/pkg/packager/lint/lint.go @@ -12,7 +12,6 @@ import ( "strings" "github.com/defenseunicorns/zarf/src/pkg/layout" - "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/types" "github.com/xeipuuv/gojsonschema" @@ -30,20 +29,6 @@ func getSchemaFile() ([]byte, error) { return ZarfSchema.ReadFile("zarf.schema.json") } -// DisplayFormattedMessage Displays the message to the user with proper warnings, failures, or success -// Will exit if there are errors -func (v Validator) DisplayFormattedMessage() { - if v.hasWarnings() { - message.Warn(v.getFormatedWarning()) - } - if v.hasErrors() { - message.Fatal(v, v.Error()) - } - if v.isSuccess() { - message.Success(v.getFormatedSuccess()) - } -} - // ValidateZarfSchema validates a zarf file against the zarf schema, returns *validator with warnings or errors if they exist // along with an error if the validation itself failed func ValidateZarfSchema(path string) (*Validator, error) { @@ -73,10 +58,10 @@ func ValidateZarfSchema(path string) (*Validator, error) { func checkForVarInComponentImport(validator *Validator) { for i, component := range validator.typedZarfPackage.Components { if strings.Contains(component.Import.Path, types.ZarfPackageTemplatePrefix) { - validator.addWarning(fmt.Sprintf("component.[%d].import.path will not resolve ZARF_PKG_TMPL_* variables", i)) + validator.addWarning(fmt.Sprintf("component.[%d].import.path", i), "will not resolve ZARF_PKG_TMPL_* variables") } if strings.Contains(component.Import.URL, types.ZarfPackageTemplatePrefix) { - validator.addWarning(fmt.Sprintf("component.[%d].import.url will not resolve ZARF_PKG_TMPL_* variables", i)) + validator.addWarning(fmt.Sprintf("component.[%d].import.url", i), "will not resolve ZARF_PKG_TMPL_* variables") } } diff --git a/src/pkg/packager/lint/validator.go b/src/pkg/packager/lint/validator.go index 02cf15775e..e340855c90 100644 --- a/src/pkg/packager/lint/validator.go +++ b/src/pkg/packager/lint/validator.go @@ -6,14 +6,20 @@ package lint import ( "fmt" - "strings" + "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/types" ) +// Warning holds the location of warning and the message +type Warning struct { + location string + message string +} + // Validator holds the warnings/errors and messaging that we get from validation type Validator struct { - warnings []string + warnings []Warning errors []error jsonSchema []byte typedZarfPackage types.ZarfPackage @@ -31,11 +37,22 @@ func (v Validator) Error() string { return errorMessage } -func (v Validator) getFormatedWarning() string { - if !v.hasWarnings() { - return "" +// func (v Validator) getFormatedWarning() string { +// if !v.hasWarnings() { +// return "" +// } +// return fmt.Sprintf("%s %s", validatorWarningPrefix, strings.Join(v.warnings, ", ")) +// } + +func (v Validator) printWarningTable() { + if v.hasWarnings() { + connectData := [][]string{} + for _, warning := range v.warnings { + connectData = append(connectData, []string{warning.location, warning.message}) + } + header := []string{"Location", "Warning"} + message.Table(header, connectData) } - return fmt.Sprintf("%s %s", validatorWarningPrefix, strings.Join(v.warnings, ", ")) } func (v Validator) getFormatedSuccess() string { @@ -54,10 +71,24 @@ func (v Validator) isSuccess() bool { return !v.hasWarnings() && !v.hasErrors() } -func (v *Validator) addWarning(warning string) { - v.warnings = append(v.warnings, warning) +func (v *Validator) addWarning(location string, message string) { + v.warnings = append(v.warnings, Warning{location, message}) } func (v *Validator) addError(err error) { v.errors = append(v.errors, err) } + +// DisplayFormattedMessage Displays the message to the user with proper warnings, failures, or success +// Will exit if there are errors +func (v Validator) DisplayFormattedMessage() { + if v.hasWarnings() { + v.printWarningTable() + } + if v.hasErrors() { + message.Fatal(v, v.Error()) + } + if v.isSuccess() { + message.Success(v.getFormatedSuccess()) + } +} diff --git a/src/pkg/packager/lint/validator_test.go b/src/pkg/packager/lint/validator_test.go index 70adb10931..deaace5383 100644 --- a/src/pkg/packager/lint/validator_test.go +++ b/src/pkg/packager/lint/validator_test.go @@ -21,11 +21,11 @@ func TestValidator(t *testing.T) { require.EqualError(t, validator, errorMessage) }) - t.Run("Validator Warning formatting", func(t *testing.T) { - warning1 := "components.0.import: Additional property not-path is not allowed" - warning2 := "components.1.import.path: Invalid type. Expected: string, given: integer" - validator := Validator{warnings: []string{warning1, warning2}} - message := fmt.Sprintf("%s %s, %s", validatorWarningPrefix, warning1, warning2) - require.Equal(t, validator.getFormatedWarning(), message) - }) + // t.Run("Validator Warning formatting", func(t *testing.T) { + // warning1 := "components.0.import: Additional property not-path is not allowed" + // warning2 := "components.1.import.path: Invalid type. Expected: string, given: integer" + // validator := Validator{warnings: []string{warning1, warning2}} + // message := fmt.Sprintf("%s %s, %s", validatorWarningPrefix, warning1, warning2) + // require.Equal(t, validator.getFormatedWarning(), message) + // }) } diff --git a/src/test/e2e/12_lint_test.go b/src/test/e2e/12_lint_test.go index 6abd4eb706..ec8b3720b6 100644 --- a/src/test/e2e/12_lint_test.go +++ b/src/test/e2e/12_lint_test.go @@ -17,7 +17,7 @@ func TestLint(t *testing.T) { _, stderr, err := e2e.Zarf("prepare", "lint", path) require.Error(t, err, "Expect error here because the yaml file is not following schema") require.Contains(t, stderr, "- components.0.import: Additional property not-path is not allowed") - require.Contains(t, stderr, "component.[2].import.path will not resolve ZARF_PKG_TMPL_* variables") + require.Contains(t, stderr, "component.[2].import.path") }) t.Run("zarf test lint success", func(t *testing.T) { From 3106628e9f803578168bcc2fc744a9574fb8b785 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Tue, 28 Nov 2023 16:43:52 +0000 Subject: [PATCH 075/192] formmating lint table --- src/pkg/packager/lint/lint.go | 4 +- src/pkg/packager/lint/validator.go | 49 +++++++------------------ src/pkg/packager/lint/validator_test.go | 31 ---------------- src/pkg/utils/random.go | 8 ++++ src/test/e2e/12_lint_test.go | 5 +-- 5 files changed, 26 insertions(+), 71 deletions(-) delete mode 100644 src/pkg/packager/lint/validator_test.go diff --git a/src/pkg/packager/lint/lint.go b/src/pkg/packager/lint/lint.go index b07d1b6b2e..9336d156f0 100644 --- a/src/pkg/packager/lint/lint.go +++ b/src/pkg/packager/lint/lint.go @@ -58,10 +58,10 @@ func ValidateZarfSchema(path string) (*Validator, error) { func checkForVarInComponentImport(validator *Validator) { for i, component := range validator.typedZarfPackage.Components { if strings.Contains(component.Import.Path, types.ZarfPackageTemplatePrefix) { - validator.addWarning(fmt.Sprintf("component.[%d].import.path", i), "will not resolve ZARF_PKG_TMPL_* variables") + validator.addWarning(fmt.Sprintf("component.[%d].import.path will not resolve ZARF_PKG_TMPL_* variables", i)) } if strings.Contains(component.Import.URL, types.ZarfPackageTemplatePrefix) { - validator.addWarning(fmt.Sprintf("component.[%d].import.url", i), "will not resolve ZARF_PKG_TMPL_* variables") + validator.addWarning(fmt.Sprintf("component.[%d].import.url will not resolve ZARF_PKG_TMPL_* variables", i)) } } diff --git a/src/pkg/packager/lint/validator.go b/src/pkg/packager/lint/validator.go index e340855c90..dc7b42dd00 100644 --- a/src/pkg/packager/lint/validator.go +++ b/src/pkg/packager/lint/validator.go @@ -8,50 +8,33 @@ import ( "fmt" "github.com/defenseunicorns/zarf/src/pkg/message" + "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/types" + "github.com/fatih/color" ) -// Warning holds the location of warning and the message -type Warning struct { - location string - message string -} - // Validator holds the warnings/errors and messaging that we get from validation type Validator struct { - warnings []Warning + warnings []string errors []error jsonSchema []byte typedZarfPackage types.ZarfPackage untypedZarfPackage interface{} } -func (v Validator) Error() string { - if !v.hasErrors() { - return "" - } - errorMessage := validatorInvalidPrefix - for _, errorStr := range v.errors { - errorMessage = fmt.Sprintf("%s\n - %s", errorMessage, errorStr.Error()) - } - return errorMessage -} - -// func (v Validator) getFormatedWarning() string { -// if !v.hasWarnings() { -// return "" -// } -// return fmt.Sprintf("%s %s", validatorWarningPrefix, strings.Join(v.warnings, ", ")) -// } - func (v Validator) printWarningTable() { if v.hasWarnings() { + header := []string{"Type", "Message"} connectData := [][]string{} for _, warning := range v.warnings { - connectData = append(connectData, []string{warning.location, warning.message}) + connectData = append(connectData, []string{utils.ColorWrap("Warning", color.FgYellow), warning}) + } + for _, err := range v.errors { + connectData = append(connectData, []string{utils.ColorWrap("Error", color.FgRed), err.Error()}) } - header := []string{"Location", "Warning"} message.Table(header, connectData) + message.Info(fmt.Sprintf("%d warnings and %d errors in %q", + len(v.warnings), len(v.errors), v.typedZarfPackage.Metadata.Name)) } } @@ -71,8 +54,8 @@ func (v Validator) isSuccess() bool { return !v.hasWarnings() && !v.hasErrors() } -func (v *Validator) addWarning(location string, message string) { - v.warnings = append(v.warnings, Warning{location, message}) +func (v *Validator) addWarning(message string) { + v.warnings = append(v.warnings, message) } func (v *Validator) addError(err error) { @@ -82,13 +65,9 @@ func (v *Validator) addError(err error) { // DisplayFormattedMessage Displays the message to the user with proper warnings, failures, or success // Will exit if there are errors func (v Validator) DisplayFormattedMessage() { - if v.hasWarnings() { - v.printWarningTable() - } - if v.hasErrors() { - message.Fatal(v, v.Error()) - } if v.isSuccess() { message.Success(v.getFormatedSuccess()) + } else { + v.printWarningTable() } } diff --git a/src/pkg/packager/lint/validator_test.go b/src/pkg/packager/lint/validator_test.go deleted file mode 100644 index deaace5383..0000000000 --- a/src/pkg/packager/lint/validator_test.go +++ /dev/null @@ -1,31 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: 2021-Present The Zarf Authors - -// Package lint contains functions for verifying zarf yaml files are valid -package lint - -import ( - "errors" - "fmt" - "testing" - - "github.com/stretchr/testify/require" -) - -func TestValidator(t *testing.T) { - t.Run("Validator Error formatting", func(t *testing.T) { - error1 := errors.New("components.0.import: Additional property not-path is not allowed") - error2 := errors.New("components.1.import.path: Invalid type. Expected: string, given: integer") - validator := Validator{errors: []error{error1, error2}} - errorMessage := fmt.Sprintf("%s\n - %s\n - %s", validatorInvalidPrefix, error1.Error(), error2.Error()) - require.EqualError(t, validator, errorMessage) - }) - - // t.Run("Validator Warning formatting", func(t *testing.T) { - // warning1 := "components.0.import: Additional property not-path is not allowed" - // warning2 := "components.1.import.path: Invalid type. Expected: string, given: integer" - // validator := Validator{warnings: []string{warning1, warning2}} - // message := fmt.Sprintf("%s %s, %s", validatorWarningPrefix, warning1, warning2) - // require.Equal(t, validator.getFormatedWarning(), message) - // }) -} diff --git a/src/pkg/utils/random.go b/src/pkg/utils/random.go index 0c074580d2..7966cd765a 100644 --- a/src/pkg/utils/random.go +++ b/src/pkg/utils/random.go @@ -6,8 +6,10 @@ package utils import ( "crypto/rand" + "fmt" "github.com/defenseunicorns/zarf/src/pkg/message" + "github.com/fatih/color" ) // Very limited special chars for git / basic auth @@ -37,3 +39,9 @@ func First30last30(s string) string { return s } + +// ColorWrap changes a string to an ansi color code and appends the default color to the end +// preventing future characters from taking on the given color +func ColorWrap(str string, attr color.Attribute) string { + return fmt.Sprintf("\x1b[%dm%s\x1b[0m", attr, str) +} diff --git a/src/test/e2e/12_lint_test.go b/src/test/e2e/12_lint_test.go index ec8b3720b6..916bda129a 100644 --- a/src/test/e2e/12_lint_test.go +++ b/src/test/e2e/12_lint_test.go @@ -14,9 +14,8 @@ func TestLint(t *testing.T) { t.Log("E2E: Test lint on schema fail") path := filepath.Join("src", "test", "packages", "12-lint") - _, stderr, err := e2e.Zarf("prepare", "lint", path) - require.Error(t, err, "Expect error here because the yaml file is not following schema") - require.Contains(t, stderr, "- components.0.import: Additional property not-path is not allowed") + _, stderr, _ := e2e.Zarf("prepare", "lint", path) + require.Contains(t, stderr, "components.0.import: Additional property not-path is not allowed") require.Contains(t, stderr, "component.[2].import.path") }) From 7443793ac7a2a6e18d214e86e8cafc4ceab7396f Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Tue, 28 Nov 2023 17:13:49 +0000 Subject: [PATCH 076/192] exported function comment --- src/pkg/message/message.go | 1 + 1 file changed, 1 insertion(+) diff --git a/src/pkg/message/message.go b/src/pkg/message/message.go index 1a1687a9c0..c2713250b7 100644 --- a/src/pkg/message/message.go +++ b/src/pkg/message/message.go @@ -341,6 +341,7 @@ func errorPrinter(offset int) *pterm.PrefixPrinter { return pterm.Error.WithShowLineNumber(logLevel > 2).WithLineNumberOffset(offset) } +// Table prints a padded table containing the specified header and data func Table(header []string, data [][]string) { pterm.Println() From 714ba99a1520c57184395261e2b1f1ea914b9e51 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Tue, 28 Nov 2023 18:50:43 +0000 Subject: [PATCH 077/192] exiting when there is a warning or error --- src/cmd/prepare.go | 3 +++ src/pkg/packager/lint/validator.go | 13 +++++++------ src/test/e2e/12_lint_test.go | 3 ++- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/src/cmd/prepare.go b/src/cmd/prepare.go index 6433af1280..2bf8e29d07 100644 --- a/src/cmd/prepare.go +++ b/src/cmd/prepare.go @@ -227,6 +227,9 @@ var lintCmd = &cobra.Command{ message.Fatal(err, err.Error()) } validator.DisplayFormattedMessage() + if !validator.IsSuccess() { + os.Exit(1) + } }, } diff --git a/src/pkg/packager/lint/validator.go b/src/pkg/packager/lint/validator.go index dc7b42dd00..736a54ddcb 100644 --- a/src/pkg/packager/lint/validator.go +++ b/src/pkg/packager/lint/validator.go @@ -23,7 +23,7 @@ type Validator struct { } func (v Validator) printWarningTable() { - if v.hasWarnings() { + if !v.IsSuccess() { header := []string{"Type", "Message"} connectData := [][]string{} for _, warning := range v.warnings { @@ -50,10 +50,6 @@ func (v Validator) hasErrors() bool { return len(v.errors) > 0 } -func (v Validator) isSuccess() bool { - return !v.hasWarnings() && !v.hasErrors() -} - func (v *Validator) addWarning(message string) { v.warnings = append(v.warnings, message) } @@ -62,10 +58,15 @@ func (v *Validator) addError(err error) { v.errors = append(v.errors, err) } +// IsSuccess returns true if there are not any warnings or errors +func (v Validator) IsSuccess() bool { + return !v.hasWarnings() && !v.hasErrors() +} + // DisplayFormattedMessage Displays the message to the user with proper warnings, failures, or success // Will exit if there are errors func (v Validator) DisplayFormattedMessage() { - if v.isSuccess() { + if v.IsSuccess() { message.Success(v.getFormatedSuccess()) } else { v.printWarningTable() diff --git a/src/test/e2e/12_lint_test.go b/src/test/e2e/12_lint_test.go index 916bda129a..e540c92fda 100644 --- a/src/test/e2e/12_lint_test.go +++ b/src/test/e2e/12_lint_test.go @@ -14,7 +14,8 @@ func TestLint(t *testing.T) { t.Log("E2E: Test lint on schema fail") path := filepath.Join("src", "test", "packages", "12-lint") - _, stderr, _ := e2e.Zarf("prepare", "lint", path) + _, stderr, err := e2e.Zarf("prepare", "lint", path) + require.Error(t, err, "Require an exit code since there was warnings / errors") require.Contains(t, stderr, "components.0.import: Additional property not-path is not allowed") require.Contains(t, stderr, "component.[2].import.path") }) From 5b549d2febf394533058647e55d23da555c07594 Mon Sep 17 00:00:00 2001 From: Austin Abro <37223396+AustinAbro321@users.noreply.github.com> Date: Wed, 29 Nov 2023 08:32:24 -0500 Subject: [PATCH 078/192] Apply suggestions from code review Co-authored-by: Lucas Rodriguez --- src/pkg/packager/lint/lint.go | 8 ++++---- src/pkg/packager/lint/lint_test.go | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/pkg/packager/lint/lint.go b/src/pkg/packager/lint/lint.go index 9336d156f0..5d67088e4c 100644 --- a/src/pkg/packager/lint/lint.go +++ b/src/pkg/packager/lint/lint.go @@ -35,21 +35,21 @@ func ValidateZarfSchema(path string) (*Validator, error) { validator := Validator{} var err error if err := utils.ReadYaml(filepath.Join(path, layout.ZarfYAML), &validator.typedZarfPackage); err != nil { - return &validator, err + return nil, err } checkForVarInComponentImport(&validator) if validator.jsonSchema, err = getSchemaFile(); err != nil { - return &validator, err + return nil, err } if err := utils.ReadYaml(filepath.Join(path, layout.ZarfYAML), &validator.untypedZarfPackage); err != nil { - return &validator, err + return nil, err } if err = validateSchema(&validator); err != nil { - return &validator, err + return nil, err } return &validator, nil diff --git a/src/pkg/packager/lint/lint_test.go b/src/pkg/packager/lint/lint_test.go index da8f2c5b36..c02df8f699 100644 --- a/src/pkg/packager/lint/lint_test.go +++ b/src/pkg/packager/lint/lint_test.go @@ -51,7 +51,7 @@ func readAndUnmarshalYaml[T interface{}](t *testing.T, yamlString string) T { var unmarshalledYaml T err := goyaml.Unmarshal([]byte(yamlString), &unmarshalledYaml) if err != nil { - t.Errorf("error unmarshalling yaml %v", err) + t.Errorf("error unmarshalling yaml: %v", err) } return unmarshalledYaml } @@ -61,7 +61,7 @@ func TestValidateSchema(t *testing.T) { t.Helper() file, err := os.ReadFile("../../../../zarf.schema.json") if err != nil { - t.Errorf("error reading file: %s", err) + t.Errorf("error reading file: %v", err) } return file } From da445e53a3763f148a27b88a973ab890ca4e49b8 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Wed, 29 Nov 2023 13:55:16 +0000 Subject: [PATCH 079/192] fixing language to be lint --- src/cmd/prepare.go | 3 ++- src/config/lang/english.go | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/cmd/prepare.go b/src/cmd/prepare.go index 2bf8e29d07..fcd5e113c8 100644 --- a/src/cmd/prepare.go +++ b/src/cmd/prepare.go @@ -211,6 +211,7 @@ var lintCmd = &cobra.Command{ Args: cobra.MaximumNArgs(1), Aliases: []string{"l"}, Short: lang.CmdPrepareLintShort, + Long: lang.CmdPrepareLintLong, Run: func(cmd *cobra.Command, args []string) { baseDir := "" if len(args) > 0 { @@ -219,7 +220,7 @@ var lintCmd = &cobra.Command{ var err error baseDir, err = os.Getwd() if err != nil { - message.Fatalf(err, lang.CmdPrepareFindImagesErr, err.Error()) + message.Fatalf(err, lang.CmdPrepareLintErr, err.Error()) } } validator, err := lint.ValidateZarfSchema(baseDir) diff --git a/src/config/lang/english.go b/src/config/lang/english.go index 0afd52beb1..13f337682a 100644 --- a/src/config/lang/english.go +++ b/src/config/lang/english.go @@ -360,6 +360,8 @@ $ zarf package publish ./path/to/dir oci://my-registry.com/my-namespace CmdPrepareFlagKubeVersion = "Override the default helm template KubeVersion when performing a package chart template" CmdPrepareLintShort = "Verifies the package schema" + CmdPrepareLintLong = "Verifies the package schema and warns the user if they have variables that won't be evaluated" + CmdPrepareLintErr = "Unable to lint package: %s" // zarf tools CmdToolsShort = "Collection of additional tools to make airgap easier" From ca7cc1be3372f81bc9f6ecdc7f2925f4cf87e20a Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Wed, 29 Nov 2023 13:58:32 +0000 Subject: [PATCH 080/192] changing order of functions --- src/pkg/message/message.go | 44 +++++++++++++++--------------- src/pkg/packager/lint/validator.go | 24 +++++++--------- 2 files changed, 32 insertions(+), 36 deletions(-) diff --git a/src/pkg/message/message.go b/src/pkg/message/message.go index c2713250b7..ef1d2e235f 100644 --- a/src/pkg/message/message.go +++ b/src/pkg/message/message.go @@ -318,6 +318,28 @@ func Truncate(text string, length int, invert bool) string { return textEscaped } +// Table prints a padded table containing the specified header and data +func Table(header []string, data [][]string) { + pterm.Println() + + if len(header) > 0 { + header[0] = fmt.Sprintf(" %s", header[0]) + } + + table := pterm.TableData{ + header, + } + + for _, row := range data { + if len(row) > 0 { + row[0] = fmt.Sprintf(" %s", row[0]) + } + table = append(table, pterm.TableData{row}...) + } + + pterm.DefaultTable.WithHasHeader().WithData(table).Render() +} + func debugPrinter(offset int, a ...any) { printer := pterm.Debug.WithShowLineNumber(logLevel > 2).WithLineNumberOffset(offset) now := time.Now().Format(time.RFC3339) @@ -340,25 +362,3 @@ func debugPrinter(offset int, a ...any) { func errorPrinter(offset int) *pterm.PrefixPrinter { return pterm.Error.WithShowLineNumber(logLevel > 2).WithLineNumberOffset(offset) } - -// Table prints a padded table containing the specified header and data -func Table(header []string, data [][]string) { - pterm.Println() - - if len(header) > 0 { - header[0] = fmt.Sprintf(" %s", header[0]) - } - - table := pterm.TableData{ - header, - } - - for _, row := range data { - if len(row) > 0 { - row[0] = fmt.Sprintf(" %s", row[0]) - } - table = append(table, pterm.TableData{row}...) - } - - pterm.DefaultTable.WithHasHeader().WithData(table).Render() -} diff --git a/src/pkg/packager/lint/validator.go b/src/pkg/packager/lint/validator.go index 736a54ddcb..e299e860d3 100644 --- a/src/pkg/packager/lint/validator.go +++ b/src/pkg/packager/lint/validator.go @@ -22,6 +22,16 @@ type Validator struct { untypedZarfPackage interface{} } +// DisplayFormattedMessage Displays the message to the user with proper warnings, failures, or success +// Will exit if there are errors +func (v Validator) DisplayFormattedMessage() { + if v.IsSuccess() { + message.Success(fmt.Sprintf("Schema validation successful for %q", v.typedZarfPackage.Metadata.Name)) + } else { + v.printWarningTable() + } +} + func (v Validator) printWarningTable() { if !v.IsSuccess() { header := []string{"Type", "Message"} @@ -38,10 +48,6 @@ func (v Validator) printWarningTable() { } } -func (v Validator) getFormatedSuccess() string { - return fmt.Sprintf("Schema validation successful for %q", v.typedZarfPackage.Metadata.Name) -} - func (v Validator) hasWarnings() bool { return len(v.warnings) > 0 } @@ -62,13 +68,3 @@ func (v *Validator) addError(err error) { func (v Validator) IsSuccess() bool { return !v.hasWarnings() && !v.hasErrors() } - -// DisplayFormattedMessage Displays the message to the user with proper warnings, failures, or success -// Will exit if there are errors -func (v Validator) DisplayFormattedMessage() { - if v.IsSuccess() { - message.Success(v.getFormatedSuccess()) - } else { - v.printWarningTable() - } -} From 3a95e031d38df37c79361c98049f994eadf715f6 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Wed, 29 Nov 2023 13:59:58 +0000 Subject: [PATCH 081/192] refactor validate schema --- src/pkg/packager/lint/lint.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/pkg/packager/lint/lint.go b/src/pkg/packager/lint/lint.go index 5d67088e4c..26901d3dc3 100644 --- a/src/pkg/packager/lint/lint.go +++ b/src/pkg/packager/lint/lint.go @@ -78,7 +78,8 @@ func validateSchema(validator *Validator) error { if !result.Valid() { for _, desc := range result.Errors() { - validator.addError(errors.New(desc.String())) + err := errors.New(desc.String()) + validator.addError(err) } } From 37df5f7bc80881315e98ab59c43126f9f0c630d7 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Wed, 29 Nov 2023 14:51:25 +0000 Subject: [PATCH 082/192] making error strings yqable --- src/pkg/packager/lint/lint.go | 25 ++++++++++++++++--------- src/pkg/packager/lint/lint_test.go | 15 +++++++++++---- src/test/e2e/12_lint_test.go | 5 +++-- src/test/packages/12-lint/zarf.yaml | 5 ++++- 4 files changed, 34 insertions(+), 16 deletions(-) diff --git a/src/pkg/packager/lint/lint.go b/src/pkg/packager/lint/lint.go index 26901d3dc3..e62a25ab99 100644 --- a/src/pkg/packager/lint/lint.go +++ b/src/pkg/packager/lint/lint.go @@ -6,9 +6,9 @@ package lint import ( "embed" - "errors" "fmt" "path/filepath" + "regexp" "strings" "github.com/defenseunicorns/zarf/src/pkg/layout" @@ -17,11 +17,6 @@ import ( "github.com/xeipuuv/gojsonschema" ) -const ( - validatorInvalidPrefix = "schema is invalid:" - validatorWarningPrefix = "zarf schema warning:" -) - // ZarfSchema is exported so main.go can embed the schema file var ZarfSchema embed.FS @@ -58,15 +53,26 @@ func ValidateZarfSchema(path string) (*Validator, error) { func checkForVarInComponentImport(validator *Validator) { for i, component := range validator.typedZarfPackage.Components { if strings.Contains(component.Import.Path, types.ZarfPackageTemplatePrefix) { - validator.addWarning(fmt.Sprintf("component.[%d].import.path will not resolve ZARF_PKG_TMPL_* variables", i)) + validator.addWarning(fmt.Sprintf(".component.[%d].import.path: Will not resolve ZARF_PKG_TMPL_* variables", i)) } if strings.Contains(component.Import.URL, types.ZarfPackageTemplatePrefix) { - validator.addWarning(fmt.Sprintf("component.[%d].import.url will not resolve ZARF_PKG_TMPL_* variables", i)) + validator.addWarning(fmt.Sprintf(".component.[%d].import.url: Will not resolve ZARF_PKG_TMPL_* variables", i)) } } } +func wrapNumbersInBrackets(input string) string { + // . is a non-word chacter (\b) so this gets digits between two . + re := regexp.MustCompile(`\b\d+\b`) + + wrapped := re.ReplaceAllStringFunc(input, func(match string) string { + return "[" + match + "]" + }) + + return wrapped +} + func validateSchema(validator *Validator) error { schemaLoader := gojsonschema.NewBytesLoader(validator.jsonSchema) documentLoader := gojsonschema.NewGoLoader(validator.untypedZarfPackage) @@ -78,7 +84,8 @@ func validateSchema(validator *Validator) error { if !result.Valid() { for _, desc := range result.Errors() { - err := errors.New(desc.String()) + err := fmt.Errorf( + ".%s: %s", wrapNumbersInBrackets(desc.Field()), desc.Description()) validator.addError(err) } } diff --git a/src/pkg/packager/lint/lint_test.go b/src/pkg/packager/lint/lint_test.go index c02df8f699..c932e7452d 100644 --- a/src/pkg/packager/lint/lint_test.go +++ b/src/pkg/packager/lint/lint_test.go @@ -79,8 +79,8 @@ func TestValidateSchema(t *testing.T) { validator := Validator{untypedZarfPackage: unmarshalledYaml, jsonSchema: getZarfSchema(t)} err := validateSchema(&validator) require.NoError(t, err) - require.EqualError(t, validator.errors[0], "components.0.import: Additional property not-path is not allowed") - require.EqualError(t, validator.errors[1], "components.1.import.path: Invalid type. Expected: string, given: integer") + require.EqualError(t, validator.errors[0], ".components.[0].import: Additional property not-path is not allowed") + require.EqualError(t, validator.errors[1], ".components.[1].import.path: Invalid type. Expected: string, given: integer") }) t.Run("Template in component import success", func(t *testing.T) { @@ -94,8 +94,15 @@ func TestValidateSchema(t *testing.T) { unmarshalledYaml := readAndUnmarshalYaml[types.ZarfPackage](t, badZarfPackage) validator := Validator{typedZarfPackage: unmarshalledYaml} checkForVarInComponentImport(&validator) - require.Equal(t, validator.warnings[0], "component.[2].import.path will not resolve ZARF_PKG_TMPL_* variables") - require.Equal(t, validator.warnings[1], "component.[3].import.url will not resolve ZARF_PKG_TMPL_* variables") + require.Equal(t, validator.warnings[0], ".component.[2].import.path: Will not resolve ZARF_PKG_TMPL_* variables") + require.Equal(t, validator.warnings[1], ".component.[3].import.url: Will not resolve ZARF_PKG_TMPL_* variables") + }) + + t.Run("Wrap standalone numbers in bracket", func(t *testing.T) { + input := "components12.12.import.path" + expected := "components12.[12].import.path" + acutal := wrapNumbersInBrackets(input) + require.Equal(t, expected, acutal) }) } diff --git a/src/test/e2e/12_lint_test.go b/src/test/e2e/12_lint_test.go index e540c92fda..aa59a0dc3c 100644 --- a/src/test/e2e/12_lint_test.go +++ b/src/test/e2e/12_lint_test.go @@ -16,8 +16,9 @@ func TestLint(t *testing.T) { path := filepath.Join("src", "test", "packages", "12-lint") _, stderr, err := e2e.Zarf("prepare", "lint", path) require.Error(t, err, "Require an exit code since there was warnings / errors") - require.Contains(t, stderr, "components.0.import: Additional property not-path is not allowed") - require.Contains(t, stderr, "component.[2].import.path") + require.Contains(t, stderr, ".components.[0].import: Additional property not-path is not allowed") + require.Contains(t, stderr, "component.[2].import.path: Will not resolve ZARF_PKG_TMPL_* variables") + require.Contains(t, stderr, ".variables: Invalid type. Expected: array, given: null") }) t.Run("zarf test lint success", func(t *testing.T) { diff --git a/src/test/packages/12-lint/zarf.yaml b/src/test/packages/12-lint/zarf.yaml index 3c2d75c816..7c026a15b7 100644 --- a/src/test/packages/12-lint/zarf.yaml +++ b/src/test/packages/12-lint/zarf.yaml @@ -1,7 +1,10 @@ kind: ZarfInitConfig metadata: name: init - description: Testing bad yaml + description1: Testing bad yaml + + +variables: components: - name: first-test-component From 47e724d3c9a2545de87153ad172fcd6862005835 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Wed, 29 Nov 2023 15:12:38 +0000 Subject: [PATCH 083/192] fix .(root) exit only on error now --- src/pkg/packager/lint/lint.go | 11 +++++++---- src/pkg/packager/lint/lint_test.go | 9 +++++++-- src/pkg/packager/lint/validator.go | 21 ++++++++++----------- 3 files changed, 24 insertions(+), 17 deletions(-) diff --git a/src/pkg/packager/lint/lint.go b/src/pkg/packager/lint/lint.go index e62a25ab99..7a9001a816 100644 --- a/src/pkg/packager/lint/lint.go +++ b/src/pkg/packager/lint/lint.go @@ -62,15 +62,18 @@ func checkForVarInComponentImport(validator *Validator) { } -func wrapNumbersInBrackets(input string) string { +func makeFieldYqEval(field string) string { + if field == "(root)" { + return field + } // . is a non-word chacter (\b) so this gets digits between two . re := regexp.MustCompile(`\b\d+\b`) - wrapped := re.ReplaceAllStringFunc(input, func(match string) string { + wrappedField := re.ReplaceAllStringFunc(field, func(match string) string { return "[" + match + "]" }) - return wrapped + return fmt.Sprintf(".%s", wrappedField) } func validateSchema(validator *Validator) error { @@ -85,7 +88,7 @@ func validateSchema(validator *Validator) error { if !result.Valid() { for _, desc := range result.Errors() { err := fmt.Errorf( - ".%s: %s", wrapNumbersInBrackets(desc.Field()), desc.Description()) + "%s: %s", makeFieldYqEval(desc.Field()), desc.Description()) validator.addError(err) } } diff --git a/src/pkg/packager/lint/lint_test.go b/src/pkg/packager/lint/lint_test.go index c932e7452d..3e7166db2a 100644 --- a/src/pkg/packager/lint/lint_test.go +++ b/src/pkg/packager/lint/lint_test.go @@ -100,9 +100,14 @@ func TestValidateSchema(t *testing.T) { t.Run("Wrap standalone numbers in bracket", func(t *testing.T) { input := "components12.12.import.path" - expected := "components12.[12].import.path" - acutal := wrapNumbersInBrackets(input) + expected := ".components12.[12].import.path" + acutal := makeFieldYqEval(input) require.Equal(t, expected, acutal) }) + t.Run("root doesn't change", func(t *testing.T) { + input := "(root)" + acutal := makeFieldYqEval(input) + require.Equal(t, input, acutal) + }) } diff --git a/src/pkg/packager/lint/validator.go b/src/pkg/packager/lint/validator.go index e299e860d3..46771eb5b1 100644 --- a/src/pkg/packager/lint/validator.go +++ b/src/pkg/packager/lint/validator.go @@ -22,18 +22,22 @@ type Validator struct { untypedZarfPackage interface{} } -// DisplayFormattedMessage Displays the message to the user with proper warnings, failures, or success -// Will exit if there are errors +// DisplayFormattedMessage message sent to user based on validator results func (v Validator) DisplayFormattedMessage() { - if v.IsSuccess() { + if !v.hasWarnings() && !v.hasErrors() { message.Success(fmt.Sprintf("Schema validation successful for %q", v.typedZarfPackage.Metadata.Name)) } else { - v.printWarningTable() + v.printValidationTable() } } -func (v Validator) printWarningTable() { - if !v.IsSuccess() { +// IsSuccess returns true if there are not any errors +func (v Validator) IsSuccess() bool { + return !v.hasErrors() +} + +func (v Validator) printValidationTable() { + if v.hasWarnings() || v.hasErrors() { header := []string{"Type", "Message"} connectData := [][]string{} for _, warning := range v.warnings { @@ -63,8 +67,3 @@ func (v *Validator) addWarning(message string) { func (v *Validator) addError(err error) { v.errors = append(v.errors, err) } - -// IsSuccess returns true if there are not any warnings or errors -func (v Validator) IsSuccess() bool { - return !v.hasWarnings() && !v.hasErrors() -} From 170938f76684988dd247e0feaff1559724a66dbc Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Wed, 29 Nov 2023 15:37:22 +0000 Subject: [PATCH 084/192] make docs and schema --- docs/2-the-zarf-cli/100-cli-commands/zarf_prepare_lint.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/2-the-zarf-cli/100-cli-commands/zarf_prepare_lint.md b/docs/2-the-zarf-cli/100-cli-commands/zarf_prepare_lint.md index 4de015b12d..bb36206a1d 100644 --- a/docs/2-the-zarf-cli/100-cli-commands/zarf_prepare_lint.md +++ b/docs/2-the-zarf-cli/100-cli-commands/zarf_prepare_lint.md @@ -3,6 +3,10 @@ Verifies the package schema +## Synopsis + +Verifies the package schema and warns the user if they have variables that won't be evaluated + ``` zarf prepare lint [ DIRECTORY ] [flags] ``` From 140a5b8b4e88940a947b33b860c986ac9d97c987 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Wed, 29 Nov 2023 16:20:57 +0000 Subject: [PATCH 085/192] refactor validator --- src/pkg/packager/lint/lint.go | 13 ++++++------- src/pkg/packager/lint/lint_test.go | 4 ++-- src/pkg/packager/lint/validator.go | 3 +-- 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/src/pkg/packager/lint/lint.go b/src/pkg/packager/lint/lint.go index 7a9001a816..82bd52820d 100644 --- a/src/pkg/packager/lint/lint.go +++ b/src/pkg/packager/lint/lint.go @@ -62,16 +62,15 @@ func checkForVarInComponentImport(validator *Validator) { } -func makeFieldYqEval(field string) string { +func makeFieldPathYqCompat(field string) string { if field == "(root)" { return field } - // . is a non-word chacter (\b) so this gets digits between two . - re := regexp.MustCompile(`\b\d+\b`) + // \b is a metacharacter that will stop at the next non-word character (including .) + // https://regex101.com/r/pIRPk0/1 + re := regexp.MustCompile(`(\b\d+\b)`) - wrappedField := re.ReplaceAllStringFunc(field, func(match string) string { - return "[" + match + "]" - }) + wrappedField := re.ReplaceAllString(field, "[$1]") return fmt.Sprintf(".%s", wrappedField) } @@ -88,7 +87,7 @@ func validateSchema(validator *Validator) error { if !result.Valid() { for _, desc := range result.Errors() { err := fmt.Errorf( - "%s: %s", makeFieldYqEval(desc.Field()), desc.Description()) + "%s: %s", makeFieldPathYqCompat(desc.Field()), desc.Description()) validator.addError(err) } } diff --git a/src/pkg/packager/lint/lint_test.go b/src/pkg/packager/lint/lint_test.go index 3e7166db2a..fedbaf49a1 100644 --- a/src/pkg/packager/lint/lint_test.go +++ b/src/pkg/packager/lint/lint_test.go @@ -101,13 +101,13 @@ func TestValidateSchema(t *testing.T) { t.Run("Wrap standalone numbers in bracket", func(t *testing.T) { input := "components12.12.import.path" expected := ".components12.[12].import.path" - acutal := makeFieldYqEval(input) + acutal := makeFieldPathYqCompat(input) require.Equal(t, expected, acutal) }) t.Run("root doesn't change", func(t *testing.T) { input := "(root)" - acutal := makeFieldYqEval(input) + acutal := makeFieldPathYqCompat(input) require.Equal(t, input, acutal) }) } diff --git a/src/pkg/packager/lint/validator.go b/src/pkg/packager/lint/validator.go index 46771eb5b1..d11f0930fb 100644 --- a/src/pkg/packager/lint/validator.go +++ b/src/pkg/packager/lint/validator.go @@ -26,9 +26,8 @@ type Validator struct { func (v Validator) DisplayFormattedMessage() { if !v.hasWarnings() && !v.hasErrors() { message.Success(fmt.Sprintf("Schema validation successful for %q", v.typedZarfPackage.Metadata.Name)) - } else { - v.printValidationTable() } + v.printValidationTable() } // IsSuccess returns true if there are not any errors From b7a7b4c9c92afa3dcc1b11b9fecbe893a8f128c6 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Wed, 29 Nov 2023 19:35:31 +0000 Subject: [PATCH 086/192] make warning actually yq compatiable, refactor message --- src/pkg/packager/lint/lint.go | 4 ++-- src/pkg/packager/lint/lint_test.go | 4 ++-- src/pkg/packager/lint/validator.go | 2 +- src/test/e2e/12_lint_test.go | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/pkg/packager/lint/lint.go b/src/pkg/packager/lint/lint.go index 82bd52820d..b9142cc3fb 100644 --- a/src/pkg/packager/lint/lint.go +++ b/src/pkg/packager/lint/lint.go @@ -53,10 +53,10 @@ func ValidateZarfSchema(path string) (*Validator, error) { func checkForVarInComponentImport(validator *Validator) { for i, component := range validator.typedZarfPackage.Components { if strings.Contains(component.Import.Path, types.ZarfPackageTemplatePrefix) { - validator.addWarning(fmt.Sprintf(".component.[%d].import.path: Will not resolve ZARF_PKG_TMPL_* variables", i)) + validator.addWarning(fmt.Sprintf(".components.[%d].import.path: Will not resolve ZARF_PKG_TMPL_* variables", i)) } if strings.Contains(component.Import.URL, types.ZarfPackageTemplatePrefix) { - validator.addWarning(fmt.Sprintf(".component.[%d].import.url: Will not resolve ZARF_PKG_TMPL_* variables", i)) + validator.addWarning(fmt.Sprintf(".components.[%d].import.url: Will not resolve ZARF_PKG_TMPL_* variables", i)) } } diff --git a/src/pkg/packager/lint/lint_test.go b/src/pkg/packager/lint/lint_test.go index fedbaf49a1..ddc9907e0f 100644 --- a/src/pkg/packager/lint/lint_test.go +++ b/src/pkg/packager/lint/lint_test.go @@ -94,8 +94,8 @@ func TestValidateSchema(t *testing.T) { unmarshalledYaml := readAndUnmarshalYaml[types.ZarfPackage](t, badZarfPackage) validator := Validator{typedZarfPackage: unmarshalledYaml} checkForVarInComponentImport(&validator) - require.Equal(t, validator.warnings[0], ".component.[2].import.path: Will not resolve ZARF_PKG_TMPL_* variables") - require.Equal(t, validator.warnings[1], ".component.[3].import.url: Will not resolve ZARF_PKG_TMPL_* variables") + require.Equal(t, validator.warnings[0], ".components.[2].import.path: Will not resolve ZARF_PKG_TMPL_* variables") + require.Equal(t, validator.warnings[1], ".components.[3].import.url: Will not resolve ZARF_PKG_TMPL_* variables") }) t.Run("Wrap standalone numbers in bracket", func(t *testing.T) { diff --git a/src/pkg/packager/lint/validator.go b/src/pkg/packager/lint/validator.go index d11f0930fb..f88c22de5f 100644 --- a/src/pkg/packager/lint/validator.go +++ b/src/pkg/packager/lint/validator.go @@ -25,7 +25,7 @@ type Validator struct { // DisplayFormattedMessage message sent to user based on validator results func (v Validator) DisplayFormattedMessage() { if !v.hasWarnings() && !v.hasErrors() { - message.Success(fmt.Sprintf("Schema validation successful for %q", v.typedZarfPackage.Metadata.Name)) + message.Successf("Schema validation successful for %q", v.typedZarfPackage.Metadata.Name) } v.printValidationTable() } diff --git a/src/test/e2e/12_lint_test.go b/src/test/e2e/12_lint_test.go index aa59a0dc3c..98257fdc89 100644 --- a/src/test/e2e/12_lint_test.go +++ b/src/test/e2e/12_lint_test.go @@ -17,7 +17,7 @@ func TestLint(t *testing.T) { _, stderr, err := e2e.Zarf("prepare", "lint", path) require.Error(t, err, "Require an exit code since there was warnings / errors") require.Contains(t, stderr, ".components.[0].import: Additional property not-path is not allowed") - require.Contains(t, stderr, "component.[2].import.path: Will not resolve ZARF_PKG_TMPL_* variables") + require.Contains(t, stderr, ".components.[2].import.path: Will not resolve ZARF_PKG_TMPL_* variables") require.Contains(t, stderr, ".variables: Invalid type. Expected: array, given: null") }) From e087e2d5c1fc1b3347957e4467a6a0d50dcd7750 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Wed, 29 Nov 2023 20:37:45 +0000 Subject: [PATCH 087/192] checking for unpinned repos in lint --- src/pkg/packager/lint/lint.go | 22 +++++++++++++++++++++- src/pkg/packager/lint/lint_test.go | 13 +++++++++++++ src/test/packages/12-lint/zarf.yaml | 5 +++++ 3 files changed, 39 insertions(+), 1 deletion(-) diff --git a/src/pkg/packager/lint/lint.go b/src/pkg/packager/lint/lint.go index b9142cc3fb..5abaf3a291 100644 --- a/src/pkg/packager/lint/lint.go +++ b/src/pkg/packager/lint/lint.go @@ -34,6 +34,7 @@ func ValidateZarfSchema(path string) (*Validator, error) { } checkForVarInComponentImport(&validator) + checkforUnpinnedRepos(&validator) if validator.jsonSchema, err = getSchemaFile(); err != nil { return nil, err @@ -50,6 +51,26 @@ func ValidateZarfSchema(path string) (*Validator, error) { return &validator, nil } +func repoIsUnpinned(repo string) bool { + // Pinned github and dev.azure.com repos will have @ + // Pinned gitlab repos will have /-/ + if !strings.Contains(repo, "@") && !strings.Contains(repo, "/-/") { + return true + } + return false +} + +func checkforUnpinnedRepos(validator *Validator) { + for i, component := range validator.typedZarfPackage.Components { + for j, repo := range component.Repos { + if repoIsUnpinned(repo) { + validator.addWarning(fmt.Sprintf(".components.[%d].repos.[%d]: Unpinned repository", i, j)) + } + } + } + +} + func checkForVarInComponentImport(validator *Validator) { for i, component := range validator.typedZarfPackage.Components { if strings.Contains(component.Import.Path, types.ZarfPackageTemplatePrefix) { @@ -59,7 +80,6 @@ func checkForVarInComponentImport(validator *Validator) { validator.addWarning(fmt.Sprintf(".components.[%d].import.url: Will not resolve ZARF_PKG_TMPL_* variables", i)) } } - } func makeFieldPathYqCompat(field string) string { diff --git a/src/pkg/packager/lint/lint_test.go b/src/pkg/packager/lint/lint_test.go index ddc9907e0f..d8e3eb9593 100644 --- a/src/pkg/packager/lint/lint_test.go +++ b/src/pkg/packager/lint/lint_test.go @@ -34,6 +34,11 @@ components: - name: import-url import: url: "oci://###ZARF_PKG_TMPL_ZEBRA###" + +- name: full-repo + repos: + - https://github.com/defenseunicorns/zarf-public-test.git + - https://dev.azure.com/defenseunicorns/zarf-public-test/_git/zarf-public-test@v0.0.1 ` const goodZarfPackage = ` @@ -98,6 +103,14 @@ func TestValidateSchema(t *testing.T) { require.Equal(t, validator.warnings[1], ".components.[3].import.url: Will not resolve ZARF_PKG_TMPL_* variables") }) + t.Run("Unpinnned repo warning", func(t *testing.T) { + unmarshalledYaml := readAndUnmarshalYaml[types.ZarfPackage](t, badZarfPackage) + validator := Validator{typedZarfPackage: unmarshalledYaml} + checkforUnpinnedRepos(&validator) + require.Equal(t, validator.warnings[0], ".components.[4].repos.[0]: Unpinned repository") + require.Equal(t, len(validator.warnings), 1) + }) + t.Run("Wrap standalone numbers in bracket", func(t *testing.T) { input := "components12.12.import.path" expected := ".components12.[12].import.path" diff --git a/src/test/packages/12-lint/zarf.yaml b/src/test/packages/12-lint/zarf.yaml index 7c026a15b7..3d3ab75957 100644 --- a/src/test/packages/12-lint/zarf.yaml +++ b/src/test/packages/12-lint/zarf.yaml @@ -22,3 +22,8 @@ components: - name: import-url import: url: "oci://###ZARF_PKG_TMPL_ZEBRA###" + + - name: full-repo + repos: + - https://github.com/defenseunicorns/zarf-public-test.git + - https://dev.azure.com/defenseunicorns/zarf-public-test/_git/zarf-public-test@v0.0.1 From 3a64288e176bfee7d6eb0bf43dae48e1bf02d1a0 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Thu, 30 Nov 2023 16:45:16 +0000 Subject: [PATCH 088/192] unpinned images and repos --- src/pkg/packager/lint/lint.go | 39 +++++++++++++++++++++++++---- src/pkg/packager/lint/lint_test.go | 40 ++++++++++++++++++++++++++++-- 2 files changed, 72 insertions(+), 7 deletions(-) diff --git a/src/pkg/packager/lint/lint.go b/src/pkg/packager/lint/lint.go index 01bfa53d0d..b5d7214586 100644 --- a/src/pkg/packager/lint/lint.go +++ b/src/pkg/packager/lint/lint.go @@ -35,6 +35,7 @@ func ValidateZarfSchema(path string) (*Validator, error) { checkForVarInComponentImport(&validator) checkforUnpinnedRepos(&validator) + checkForUnpinnedImages(&validator) if validator.jsonSchema, err = getSchemaFile(); err != nil { return nil, err @@ -51,24 +52,52 @@ func ValidateZarfSchema(path string) (*Validator, error) { return &validator, nil } -func repoIsUnpinned(repo string) bool { - // Pinned github and dev.azure.com repos will have @ - // Pinned gitlab repos will have /-/ - if !strings.Contains(repo, "@") && !strings.Contains(repo, "/-/") { +func imageIsPinned(image string) bool { + if strings.HasSuffix(image, ":latest") { + return false + } + // Pinned with shasum or equivalent + if strings.Contains(image, "@") { + return true + } + splitImage := strings.Split(image, ":") + // In this case we have a port and a tag + if len(splitImage) > 2 { return true + } else if len(splitImage) > 1 { + // In this case we either have a port and a "/" afterwards for the path to the image + // Or we have a tag which cannot have the / character in it + if !strings.Contains(splitImage[len(splitImage)-1], "/") { + return true + } } return false } +func repoIsPinned(repo string) bool { + // Pinned github and dev.azure.com repos will have @ + // Pinned gitlab repos will have /-/ + return (strings.Contains(repo, "@") || strings.Contains(repo, "/-/")) +} + func checkforUnpinnedRepos(validator *Validator) { for i, component := range validator.typedZarfPackage.Components { for j, repo := range component.Repos { - if repoIsUnpinned(repo) { + if !repoIsPinned(repo) { validator.addWarning(fmt.Sprintf(".components.[%d].repos.[%d]: Unpinned repository", i, j)) } } } +} +func checkForUnpinnedImages(validator *Validator) { + for i, component := range validator.typedZarfPackage.Components { + for j, repo := range component.Images { + if !imageIsPinned(repo) { + validator.addWarning(fmt.Sprintf(".components.[%d].images.[%d]: Unpinned image", i, j)) + } + } + } } func checkForVarInComponentImport(validator *Validator) { diff --git a/src/pkg/packager/lint/lint_test.go b/src/pkg/packager/lint/lint_test.go index d8e3eb9593..97bcfdcad5 100644 --- a/src/pkg/packager/lint/lint_test.go +++ b/src/pkg/packager/lint/lint_test.go @@ -37,8 +37,14 @@ components: - name: full-repo repos: - - https://github.com/defenseunicorns/zarf-public-test.git - - https://dev.azure.com/defenseunicorns/zarf-public-test/_git/zarf-public-test@v0.0.1 + - https://github.com/defenseunicorns/zarf-public-test.git + - https://dev.azure.com/defenseunicorns/zarf-public-test/_git/zarf-public-test@v0.0.1 + - https://gitlab.com/gitlab-org/build/omnibus-mirror/pcre2/-/tree/vreverse?ref_type=heads + images: + - ghcr.io/kiwix/kiwix-serve:3.5.0-2 + - registry.com:9001/whatever/image:1.0.0 + - busybox@sha256:3fbc632167424a6d997e74f52b878d7cc478225cffac6bc977eedfe51c7f4e79 + - busybox:latest ` const goodZarfPackage = ` @@ -111,6 +117,14 @@ func TestValidateSchema(t *testing.T) { require.Equal(t, len(validator.warnings), 1) }) + t.Run("Unpinnned image warning", func(t *testing.T) { + unmarshalledYaml := readAndUnmarshalYaml[types.ZarfPackage](t, badZarfPackage) + validator := Validator{typedZarfPackage: unmarshalledYaml} + checkForUnpinnedImages(&validator) + require.Equal(t, validator.warnings[0], ".components.[4].images.[3]: Unpinned image") + require.Equal(t, len(validator.warnings), 1) + }) + t.Run("Wrap standalone numbers in bracket", func(t *testing.T) { input := "components12.12.import.path" expected := ".components12.[12].import.path" @@ -123,4 +137,26 @@ func TestValidateSchema(t *testing.T) { acutal := makeFieldPathYqCompat(input) require.Equal(t, input, acutal) }) + + t.Run("image is pinned", func(t *testing.T) { + input := "ghcr.io/defenseunicorns/pepr/controller:v0.15.0" + expcected := true + acutal := imageIsPinned(input) + require.Equal(t, expcected, acutal) + }) + + t.Run("image is unpinned", func(t *testing.T) { + input := "ghcr.io/defenseunicorns/pepr/controller" + expcected := false + acutal := imageIsPinned(input) + require.Equal(t, expcected, acutal) + }) + + t.Run("image is pinned and has port", func(t *testing.T) { + input := "registry.com:8080/defenseunicorns/whatever" + expcected := false + acutal := imageIsPinned(input) + require.Equal(t, expcected, acutal) + }) + //Image signature ghcr.io/stefanprodan/podinfo:sha256-57a654ace69ec02ba8973093b6a786faa15640575fbf0dbb603db55aca2ccec8.sig } From c19721222abb0773715348d43b28efc00ae018fb Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Thu, 30 Nov 2023 17:08:11 +0000 Subject: [PATCH 089/192] adding support for unpinned files --- src/pkg/packager/lint/lint.go | 15 +++++++++++++-- src/pkg/packager/lint/lint_test.go | 12 ++++++++++++ src/test/packages/12-lint/zarf.yaml | 14 ++++++++++++-- 3 files changed, 37 insertions(+), 4 deletions(-) diff --git a/src/pkg/packager/lint/lint.go b/src/pkg/packager/lint/lint.go index b5d7214586..83e2ba7f05 100644 --- a/src/pkg/packager/lint/lint.go +++ b/src/pkg/packager/lint/lint.go @@ -36,6 +36,7 @@ func ValidateZarfSchema(path string) (*Validator, error) { checkForVarInComponentImport(&validator) checkforUnpinnedRepos(&validator) checkForUnpinnedImages(&validator) + checkForUnpinnedFiles(&validator) if validator.jsonSchema, err = getSchemaFile(); err != nil { return nil, err @@ -92,14 +93,24 @@ func checkforUnpinnedRepos(validator *Validator) { func checkForUnpinnedImages(validator *Validator) { for i, component := range validator.typedZarfPackage.Components { - for j, repo := range component.Images { - if !imageIsPinned(repo) { + for j, image := range component.Images { + if !imageIsPinned(image) { validator.addWarning(fmt.Sprintf(".components.[%d].images.[%d]: Unpinned image", i, j)) } } } } +func checkForUnpinnedFiles(validator *Validator) { + for i, component := range validator.typedZarfPackage.Components { + for j, file := range component.Files { + if file.Shasum == "" { + validator.addWarning(fmt.Sprintf(".components.[%d].files.[%d]: Unpinned file", i, j)) + } + } + } +} + func checkForVarInComponentImport(validator *Validator) { for i, component := range validator.typedZarfPackage.Components { if strings.Contains(component.Import.Path, types.ZarfPackageTemplatePrefix) { diff --git a/src/pkg/packager/lint/lint_test.go b/src/pkg/packager/lint/lint_test.go index 97bcfdcad5..0936fa93b3 100644 --- a/src/pkg/packager/lint/lint_test.go +++ b/src/pkg/packager/lint/lint_test.go @@ -45,6 +45,10 @@ components: - registry.com:9001/whatever/image:1.0.0 - busybox@sha256:3fbc632167424a6d997e74f52b878d7cc478225cffac6bc977eedfe51c7f4e79 - busybox:latest + files: + - source: https://github.com/k3s-io/k3s/releases/download/v1.28.2+k3s1/k3s + shasum: 2f041d37a2c6d54d53e106e1c7713bc48f806f3919b0d9e092f5fcbdc55b41cf + - source: file-without-shasum.txt ` const goodZarfPackage = ` @@ -125,6 +129,14 @@ func TestValidateSchema(t *testing.T) { require.Equal(t, len(validator.warnings), 1) }) + t.Run("Unpinnned file warning", func(t *testing.T) { + unmarshalledYaml := readAndUnmarshalYaml[types.ZarfPackage](t, badZarfPackage) + validator := Validator{typedZarfPackage: unmarshalledYaml} + checkForUnpinnedFiles(&validator) + require.Equal(t, validator.warnings[0], ".components.[4].files.[1]: Unpinned file") + require.Equal(t, len(validator.warnings), 1) + }) + t.Run("Wrap standalone numbers in bracket", func(t *testing.T) { input := "components12.12.import.path" expected := ".components12.[12].import.path" diff --git a/src/test/packages/12-lint/zarf.yaml b/src/test/packages/12-lint/zarf.yaml index 3d3ab75957..c71d329b22 100644 --- a/src/test/packages/12-lint/zarf.yaml +++ b/src/test/packages/12-lint/zarf.yaml @@ -25,5 +25,15 @@ components: - name: full-repo repos: - - https://github.com/defenseunicorns/zarf-public-test.git - - https://dev.azure.com/defenseunicorns/zarf-public-test/_git/zarf-public-test@v0.0.1 + - https://github.com/defenseunicorns/zarf-public-test.git + - https://dev.azure.com/defenseunicorns/zarf-public-test/_git/zarf-public-test@v0.0.1 + - https://gitlab.com/gitlab-org/build/omnibus-mirror/pcre2/-/tree/vreverse?ref_type=heads + images: + - ghcr.io/kiwix/kiwix-serve:3.5.0-2 + - registry.com:9001/whatever/image:1.0.0 + - busybox@sha256:3fbc632167424a6d997e74f52b878d7cc478225cffac6bc977eedfe51c7f4e79 + - busybox:latest + files: + - source: https://github.com/k3s-io/k3s/releases/download/v1.28.2+k3s1/k3s + shasum: 2f041d37a2c6d54d53e106e1c7713bc48f806f3919b0d9e092f5fcbdc55b41cf + - source: file-without-shasum.txt From 8da1d9a9f04b9ed0ec77ad25921cb6c8fff35282 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Thu, 30 Nov 2023 17:11:58 +0000 Subject: [PATCH 090/192] update docs --- docs/2-the-zarf-cli/100-cli-commands/zarf_prepare_lint.md | 2 +- src/config/lang/english.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/2-the-zarf-cli/100-cli-commands/zarf_prepare_lint.md b/docs/2-the-zarf-cli/100-cli-commands/zarf_prepare_lint.md index bb36206a1d..98efd7b4a0 100644 --- a/docs/2-the-zarf-cli/100-cli-commands/zarf_prepare_lint.md +++ b/docs/2-the-zarf-cli/100-cli-commands/zarf_prepare_lint.md @@ -5,7 +5,7 @@ Verifies the package schema ## Synopsis -Verifies the package schema and warns the user if they have variables that won't be evaluated +Verifies the package schema, warns the user if variables that won't be evaluated, and checks for unpinned images/repos/files ``` zarf prepare lint [ DIRECTORY ] [flags] diff --git a/src/config/lang/english.go b/src/config/lang/english.go index 6c14bafa6a..e09965e0cb 100644 --- a/src/config/lang/english.go +++ b/src/config/lang/english.go @@ -361,7 +361,7 @@ $ zarf package publish ./path/to/dir oci://my-registry.com/my-namespace CmdPrepareFlagKubeVersion = "Override the default helm template KubeVersion when performing a package chart template" CmdPrepareLintShort = "Verifies the package schema" - CmdPrepareLintLong = "Verifies the package schema and warns the user if they have variables that won't be evaluated" + CmdPrepareLintLong = "Verifies the package schema, warns the user if variables that won't be evaluated, and checks for unpinned images/repos/files" CmdPrepareLintErr = "Unable to lint package: %s" // zarf tools From fb20de69af400ec212cc3dc267e623bf77fc45e2 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Thu, 30 Nov 2023 17:18:41 +0000 Subject: [PATCH 091/192] update docs --- docs/2-the-zarf-cli/100-cli-commands/zarf_prepare.md | 2 +- docs/2-the-zarf-cli/100-cli-commands/zarf_prepare_lint.md | 4 ++-- src/config/lang/english.go | 4 ++-- src/pkg/packager/lint/lint.go | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/2-the-zarf-cli/100-cli-commands/zarf_prepare.md b/docs/2-the-zarf-cli/100-cli-commands/zarf_prepare.md index 837e4c03ba..a34778f876 100644 --- a/docs/2-the-zarf-cli/100-cli-commands/zarf_prepare.md +++ b/docs/2-the-zarf-cli/100-cli-commands/zarf_prepare.md @@ -27,7 +27,7 @@ Tools to help prepare assets for packaging * [zarf](zarf.md) - DevSecOps for Airgap * [zarf prepare find-images](zarf_prepare_find-images.md) - Evaluates components in a zarf file to identify images specified in their helm charts and manifests * [zarf prepare generate-config](zarf_prepare_generate-config.md) - Generates a config file for Zarf -* [zarf prepare lint](zarf_prepare_lint.md) - Verifies the package schema +* [zarf prepare lint](zarf_prepare_lint.md) - Lints package for schema and reccomended practices * [zarf prepare patch-git](zarf_prepare_patch-git.md) - Converts all .git URLs to the specified Zarf HOST and with the Zarf URL pattern in a given FILE. NOTE: This should only be used for manifests that are not mutated by the Zarf Agent Mutating Webhook. * [zarf prepare sha256sum](zarf_prepare_sha256sum.md) - Generates a SHA256SUM for the given file diff --git a/docs/2-the-zarf-cli/100-cli-commands/zarf_prepare_lint.md b/docs/2-the-zarf-cli/100-cli-commands/zarf_prepare_lint.md index 98efd7b4a0..4a6ca20aa6 100644 --- a/docs/2-the-zarf-cli/100-cli-commands/zarf_prepare_lint.md +++ b/docs/2-the-zarf-cli/100-cli-commands/zarf_prepare_lint.md @@ -1,11 +1,11 @@ # zarf prepare lint -Verifies the package schema +Lints package for schema and reccomended practices ## Synopsis -Verifies the package schema, warns the user if variables that won't be evaluated, and checks for unpinned images/repos/files +Verifies the package schema, warns the user if any variables won't be evaluated, and checks for unpinned images/repos/files ``` zarf prepare lint [ DIRECTORY ] [flags] diff --git a/src/config/lang/english.go b/src/config/lang/english.go index e09965e0cb..33e0f2fda1 100644 --- a/src/config/lang/english.go +++ b/src/config/lang/english.go @@ -360,8 +360,8 @@ $ zarf package publish ./path/to/dir oci://my-registry.com/my-namespace CmdPrepareFlagGitAccount = "User or organization name for the git account that the repos are created under." CmdPrepareFlagKubeVersion = "Override the default helm template KubeVersion when performing a package chart template" - CmdPrepareLintShort = "Verifies the package schema" - CmdPrepareLintLong = "Verifies the package schema, warns the user if variables that won't be evaluated, and checks for unpinned images/repos/files" + CmdPrepareLintShort = "Lints package for schema and reccomended practices" + CmdPrepareLintLong = "Verifies the package schema, warns the user if any variables won't be evaluated, and checks for unpinned images/repos/files" CmdPrepareLintErr = "Unable to lint package: %s" // zarf tools diff --git a/src/pkg/packager/lint/lint.go b/src/pkg/packager/lint/lint.go index 83e2ba7f05..aabbe6a0ba 100644 --- a/src/pkg/packager/lint/lint.go +++ b/src/pkg/packager/lint/lint.go @@ -57,7 +57,7 @@ func imageIsPinned(image string) bool { if strings.HasSuffix(image, ":latest") { return false } - // Pinned with shasum or equivalent + // Pinned with image digest if strings.Contains(image, "@") { return true } From dd2b03cf0672b5d53f321dd57794d6ac76c14b21 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Thu, 30 Nov 2023 17:23:57 +0000 Subject: [PATCH 092/192] refactor isPinnedImage --- src/pkg/packager/lint/lint.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/pkg/packager/lint/lint.go b/src/pkg/packager/lint/lint.go index aabbe6a0ba..d90928bd0d 100644 --- a/src/pkg/packager/lint/lint.go +++ b/src/pkg/packager/lint/lint.go @@ -62,10 +62,7 @@ func imageIsPinned(image string) bool { return true } splitImage := strings.Split(image, ":") - // In this case we have a port and a tag - if len(splitImage) > 2 { - return true - } else if len(splitImage) > 1 { + if len(splitImage) > 1 { // In this case we either have a port and a "/" afterwards for the path to the image // Or we have a tag which cannot have the / character in it if !strings.Contains(splitImage[len(splitImage)-1], "/") { From 596292a1207928cf2cb53297163a6d41873b0720 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Thu, 30 Nov 2023 17:25:02 +0000 Subject: [PATCH 093/192] refactor lint --- src/pkg/packager/lint/lint.go | 8 ++++---- src/pkg/packager/lint/lint_test.go | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/pkg/packager/lint/lint.go b/src/pkg/packager/lint/lint.go index d90928bd0d..4808fff529 100644 --- a/src/pkg/packager/lint/lint.go +++ b/src/pkg/packager/lint/lint.go @@ -53,7 +53,7 @@ func ValidateZarfSchema(path string) (*Validator, error) { return &validator, nil } -func imageIsPinned(image string) bool { +func isPinnedImage(image string) bool { if strings.HasSuffix(image, ":latest") { return false } @@ -72,7 +72,7 @@ func imageIsPinned(image string) bool { return false } -func repoIsPinned(repo string) bool { +func isPinnedRepo(repo string) bool { // Pinned github and dev.azure.com repos will have @ // Pinned gitlab repos will have /-/ return (strings.Contains(repo, "@") || strings.Contains(repo, "/-/")) @@ -81,7 +81,7 @@ func repoIsPinned(repo string) bool { func checkforUnpinnedRepos(validator *Validator) { for i, component := range validator.typedZarfPackage.Components { for j, repo := range component.Repos { - if !repoIsPinned(repo) { + if !isPinnedRepo(repo) { validator.addWarning(fmt.Sprintf(".components.[%d].repos.[%d]: Unpinned repository", i, j)) } } @@ -91,7 +91,7 @@ func checkforUnpinnedRepos(validator *Validator) { func checkForUnpinnedImages(validator *Validator) { for i, component := range validator.typedZarfPackage.Components { for j, image := range component.Images { - if !imageIsPinned(image) { + if !isPinnedImage(image) { validator.addWarning(fmt.Sprintf(".components.[%d].images.[%d]: Unpinned image", i, j)) } } diff --git a/src/pkg/packager/lint/lint_test.go b/src/pkg/packager/lint/lint_test.go index 0936fa93b3..59820184ec 100644 --- a/src/pkg/packager/lint/lint_test.go +++ b/src/pkg/packager/lint/lint_test.go @@ -153,21 +153,21 @@ func TestValidateSchema(t *testing.T) { t.Run("image is pinned", func(t *testing.T) { input := "ghcr.io/defenseunicorns/pepr/controller:v0.15.0" expcected := true - acutal := imageIsPinned(input) + acutal := isPinnedImage(input) require.Equal(t, expcected, acutal) }) t.Run("image is unpinned", func(t *testing.T) { input := "ghcr.io/defenseunicorns/pepr/controller" expcected := false - acutal := imageIsPinned(input) + acutal := isPinnedImage(input) require.Equal(t, expcected, acutal) }) t.Run("image is pinned and has port", func(t *testing.T) { input := "registry.com:8080/defenseunicorns/whatever" expcected := false - acutal := imageIsPinned(input) + acutal := isPinnedImage(input) require.Equal(t, expcected, acutal) }) //Image signature ghcr.io/stefanprodan/podinfo:sha256-57a654ace69ec02ba8973093b6a786faa15640575fbf0dbb603db55aca2ccec8.sig From 4ec54a95b63a8ad3eef2d54af6e1188334bc70c3 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Thu, 30 Nov 2023 17:27:59 +0000 Subject: [PATCH 094/192] removed unnecessary comment --- src/pkg/packager/lint/lint_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/src/pkg/packager/lint/lint_test.go b/src/pkg/packager/lint/lint_test.go index 59820184ec..34bdd411e6 100644 --- a/src/pkg/packager/lint/lint_test.go +++ b/src/pkg/packager/lint/lint_test.go @@ -170,5 +170,4 @@ func TestValidateSchema(t *testing.T) { acutal := isPinnedImage(input) require.Equal(t, expcected, acutal) }) - //Image signature ghcr.io/stefanprodan/podinfo:sha256-57a654ace69ec02ba8973093b6a786faa15640575fbf0dbb603db55aca2ccec8.sig } From 03054f296383ecee7bf5dc6fc19433f882e59972 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Thu, 30 Nov 2023 17:29:32 +0000 Subject: [PATCH 095/192] spelling fix --- docs/2-the-zarf-cli/100-cli-commands/zarf_prepare.md | 2 +- docs/2-the-zarf-cli/100-cli-commands/zarf_prepare_lint.md | 2 +- src/config/lang/english.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/2-the-zarf-cli/100-cli-commands/zarf_prepare.md b/docs/2-the-zarf-cli/100-cli-commands/zarf_prepare.md index a34778f876..9af49e3303 100644 --- a/docs/2-the-zarf-cli/100-cli-commands/zarf_prepare.md +++ b/docs/2-the-zarf-cli/100-cli-commands/zarf_prepare.md @@ -27,7 +27,7 @@ Tools to help prepare assets for packaging * [zarf](zarf.md) - DevSecOps for Airgap * [zarf prepare find-images](zarf_prepare_find-images.md) - Evaluates components in a zarf file to identify images specified in their helm charts and manifests * [zarf prepare generate-config](zarf_prepare_generate-config.md) - Generates a config file for Zarf -* [zarf prepare lint](zarf_prepare_lint.md) - Lints package for schema and reccomended practices +* [zarf prepare lint](zarf_prepare_lint.md) - Lints package for schema and recommended practices * [zarf prepare patch-git](zarf_prepare_patch-git.md) - Converts all .git URLs to the specified Zarf HOST and with the Zarf URL pattern in a given FILE. NOTE: This should only be used for manifests that are not mutated by the Zarf Agent Mutating Webhook. * [zarf prepare sha256sum](zarf_prepare_sha256sum.md) - Generates a SHA256SUM for the given file diff --git a/docs/2-the-zarf-cli/100-cli-commands/zarf_prepare_lint.md b/docs/2-the-zarf-cli/100-cli-commands/zarf_prepare_lint.md index 4a6ca20aa6..0f948cccf7 100644 --- a/docs/2-the-zarf-cli/100-cli-commands/zarf_prepare_lint.md +++ b/docs/2-the-zarf-cli/100-cli-commands/zarf_prepare_lint.md @@ -1,7 +1,7 @@ # zarf prepare lint -Lints package for schema and reccomended practices +Lints package for schema and recommended practices ## Synopsis diff --git a/src/config/lang/english.go b/src/config/lang/english.go index 33e0f2fda1..ed03eded0e 100644 --- a/src/config/lang/english.go +++ b/src/config/lang/english.go @@ -360,7 +360,7 @@ $ zarf package publish ./path/to/dir oci://my-registry.com/my-namespace CmdPrepareFlagGitAccount = "User or organization name for the git account that the repos are created under." CmdPrepareFlagKubeVersion = "Override the default helm template KubeVersion when performing a package chart template" - CmdPrepareLintShort = "Lints package for schema and reccomended practices" + CmdPrepareLintShort = "Lints package for schema and recommended practices" CmdPrepareLintLong = "Verifies the package schema, warns the user if any variables won't be evaluated, and checks for unpinned images/repos/files" CmdPrepareLintErr = "Unable to lint package: %s" From 05da414a0f6ee88c984540011e3c6e09aef695f5 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Thu, 30 Nov 2023 17:33:11 +0000 Subject: [PATCH 096/192] doc changes --- docs/2-the-zarf-cli/100-cli-commands/zarf_prepare_lint.md | 2 +- src/config/lang/english.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/2-the-zarf-cli/100-cli-commands/zarf_prepare_lint.md b/docs/2-the-zarf-cli/100-cli-commands/zarf_prepare_lint.md index 0f948cccf7..9db10cc901 100644 --- a/docs/2-the-zarf-cli/100-cli-commands/zarf_prepare_lint.md +++ b/docs/2-the-zarf-cli/100-cli-commands/zarf_prepare_lint.md @@ -5,7 +5,7 @@ Lints package for schema and recommended practices ## Synopsis -Verifies the package schema, warns the user if any variables won't be evaluated, and checks for unpinned images/repos/files +Verifies the package schema, checks if any variables won't be evaluated, and checks for unpinned images/repos/files ``` zarf prepare lint [ DIRECTORY ] [flags] diff --git a/src/config/lang/english.go b/src/config/lang/english.go index ed03eded0e..6b5d591878 100644 --- a/src/config/lang/english.go +++ b/src/config/lang/english.go @@ -361,7 +361,7 @@ $ zarf package publish ./path/to/dir oci://my-registry.com/my-namespace CmdPrepareFlagKubeVersion = "Override the default helm template KubeVersion when performing a package chart template" CmdPrepareLintShort = "Lints package for schema and recommended practices" - CmdPrepareLintLong = "Verifies the package schema, warns the user if any variables won't be evaluated, and checks for unpinned images/repos/files" + CmdPrepareLintLong = "Verifies the package schema, checks if any variables won't be evaluated, and checks for unpinned images/repos/files" CmdPrepareLintErr = "Unable to lint package: %s" // zarf tools From 70b26ed64620cc713fcfc702f514f965c185b349 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Thu, 30 Nov 2023 20:51:37 +0000 Subject: [PATCH 097/192] using transformer.parseimageref now --- src/pkg/packager/lint/lint.go | 29 ++++++------- src/pkg/packager/lint/lint_test.go | 67 +++++++++++++++++++++--------- 2 files changed, 59 insertions(+), 37 deletions(-) diff --git a/src/pkg/packager/lint/lint.go b/src/pkg/packager/lint/lint.go index 4808fff529..3640ab4765 100644 --- a/src/pkg/packager/lint/lint.go +++ b/src/pkg/packager/lint/lint.go @@ -12,6 +12,7 @@ import ( "strings" "github.com/defenseunicorns/zarf/src/pkg/layout" + "github.com/defenseunicorns/zarf/src/pkg/transform" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/types" "github.com/xeipuuv/gojsonschema" @@ -53,23 +54,12 @@ func ValidateZarfSchema(path string) (*Validator, error) { return &validator, nil } -func isPinnedImage(image string) bool { - if strings.HasSuffix(image, ":latest") { - return false - } - // Pinned with image digest - if strings.Contains(image, "@") { - return true - } - splitImage := strings.Split(image, ":") - if len(splitImage) > 1 { - // In this case we either have a port and a "/" afterwards for the path to the image - // Or we have a tag which cannot have the / character in it - if !strings.Contains(splitImage[len(splitImage)-1], "/") { - return true - } +func isPinnedImage(image string) (bool, error) { + transformedImage, err := transform.ParseImageRef(image) + if err != nil { + return false, err } - return false + return !(transformedImage.TagOrDigest == ":latest"), err } func isPinnedRepo(repo string) bool { @@ -91,7 +81,12 @@ func checkforUnpinnedRepos(validator *Validator) { func checkForUnpinnedImages(validator *Validator) { for i, component := range validator.typedZarfPackage.Components { for j, image := range component.Images { - if !isPinnedImage(image) { + pinnedImage, err := isPinnedImage(image) + if err != nil { + validator.addError(fmt.Errorf(".components.[%d].images.[%d]: Invalid image format", i, j)) + continue + } + if !pinnedImage { validator.addWarning(fmt.Sprintf(".components.[%d].images.[%d]: Unpinned image", i, j)) } } diff --git a/src/pkg/packager/lint/lint_test.go b/src/pkg/packager/lint/lint_test.go index 34bdd411e6..90afbd2154 100644 --- a/src/pkg/packager/lint/lint_test.go +++ b/src/pkg/packager/lint/lint_test.go @@ -5,6 +5,7 @@ package lint import ( + "errors" "os" "testing" @@ -43,8 +44,9 @@ components: images: - ghcr.io/kiwix/kiwix-serve:3.5.0-2 - registry.com:9001/whatever/image:1.0.0 - - busybox@sha256:3fbc632167424a6d997e74f52b878d7cc478225cffac6bc977eedfe51c7f4e79 + - busybox:latest@sha256:3fbc632167424a6d997e74f52b878d7cc478225cffac6bc977eedfe51c7f4e79 - busybox:latest + - badimage:badimage@@sha256:3fbc632167424a6d997e74f5 files: - source: https://github.com/k3s-io/k3s/releases/download/v1.28.2+k3s1/k3s shasum: 2f041d37a2c6d54d53e106e1c7713bc48f806f3919b0d9e092f5fcbdc55b41cf @@ -127,6 +129,8 @@ func TestValidateSchema(t *testing.T) { checkForUnpinnedImages(&validator) require.Equal(t, validator.warnings[0], ".components.[4].images.[3]: Unpinned image") require.Equal(t, len(validator.warnings), 1) + require.EqualError(t, validator.errors[0], ".components.[4].images.[4]: Invalid image format") + require.Equal(t, len(validator.errors), 1) }) t.Run("Unpinnned file warning", func(t *testing.T) { @@ -150,24 +154,47 @@ func TestValidateSchema(t *testing.T) { require.Equal(t, input, acutal) }) - t.Run("image is pinned", func(t *testing.T) { - input := "ghcr.io/defenseunicorns/pepr/controller:v0.15.0" - expcected := true - acutal := isPinnedImage(input) - require.Equal(t, expcected, acutal) - }) - - t.Run("image is unpinned", func(t *testing.T) { - input := "ghcr.io/defenseunicorns/pepr/controller" - expcected := false - acutal := isPinnedImage(input) - require.Equal(t, expcected, acutal) - }) - - t.Run("image is pinned and has port", func(t *testing.T) { - input := "registry.com:8080/defenseunicorns/whatever" - expcected := false - acutal := isPinnedImage(input) - require.Equal(t, expcected, acutal) + t.Run("isImagePinned", func(t *testing.T) { + t.Parallel() + tests := []struct { + input string + expected bool + err error + }{ + { + input: "registry.com:8080/defenseunicorns/whatever", + expected: false, + err: nil, + }, + { + input: "ghcr.io/defenseunicorns/pepr/controller", + expected: false, + err: nil, + }, + { + input: "ghcr.io/defenseunicorns/pepr/controller:v0.15.0", + expected: true, + err: nil, + }, + { + input: "busybox:latest@sha256:3fbc632167424a6d997e74f52b878d7cc478225cffac6bc977eedfe51c7f4e79", + expected: true, + err: nil, + }, + { + input: "busybox:bad/image", + expected: false, + err: errors.New("invalid reference format"), + }, + } + for _, tc := range tests { + t.Run(tc.input, func(t *testing.T) { + acutal, err := isPinnedImage(tc.input) + if err != nil { + require.EqualError(t, err, tc.err.Error()) + } + require.Equal(t, tc.expected, acutal) + }) + } }) } From 56f95aea8a3cb8ea411c84ec0d78ae8bd36b47ec Mon Sep 17 00:00:00 2001 From: Austin Abro <37223396+AustinAbro321@users.noreply.github.com> Date: Mon, 4 Dec 2023 09:26:16 -0500 Subject: [PATCH 098/192] Update src/config/lang/english.go Co-authored-by: Wayne Starr --- src/config/lang/english.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/config/lang/english.go b/src/config/lang/english.go index 6b5d591878..5351705b13 100644 --- a/src/config/lang/english.go +++ b/src/config/lang/english.go @@ -360,7 +360,7 @@ $ zarf package publish ./path/to/dir oci://my-registry.com/my-namespace CmdPrepareFlagGitAccount = "User or organization name for the git account that the repos are created under." CmdPrepareFlagKubeVersion = "Override the default helm template KubeVersion when performing a package chart template" - CmdPrepareLintShort = "Lints package for schema and recommended practices" + CmdPrepareLintShort = "Lints the given package for valid schema and recommended practices" CmdPrepareLintLong = "Verifies the package schema, checks if any variables won't be evaluated, and checks for unpinned images/repos/files" CmdPrepareLintErr = "Unable to lint package: %s" From 2821adf9357ab37086dc4cc983bf840db4985626 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Mon, 4 Dec 2023 19:34:35 +0000 Subject: [PATCH 099/192] wip --- src/cmd/prepare.go | 10 +- src/pkg/packager/compose.go | 4 +- src/pkg/packager/composer/compose.go | 63 ++++++++++ src/pkg/packager/composer/list.go | 68 ++++++---- src/pkg/packager/composer/list_test.go | 8 +- src/pkg/packager/composer/oci.go | 6 +- src/pkg/packager/lint/lint.go | 119 ++++++++++++++++-- src/pkg/packager/lint/lint_test.go | 11 ++ src/pkg/packager/variables.go | 32 ++--- src/test/e2e/12_lint_test.go | 33 +++-- .../packages/12-lint/linted-import/zarf.yaml | 17 +++ src/test/packages/12-lint/zarf-config.toml | 2 + src/test/packages/12-lint/zarf.yaml | 18 ++- 13 files changed, 307 insertions(+), 84 deletions(-) create mode 100644 src/pkg/packager/composer/compose.go create mode 100644 src/test/packages/12-lint/linted-import/zarf.yaml create mode 100644 src/test/packages/12-lint/zarf-config.toml diff --git a/src/cmd/prepare.go b/src/cmd/prepare.go index fcd5e113c8..37aaedab3a 100644 --- a/src/cmd/prepare.go +++ b/src/cmd/prepare.go @@ -213,17 +213,19 @@ var lintCmd = &cobra.Command{ Short: lang.CmdPrepareLintShort, Long: lang.CmdPrepareLintLong, Run: func(cmd *cobra.Command, args []string) { - baseDir := "" if len(args) > 0 { - baseDir = args[0] + pkgConfig.CreateOpts.BaseDir = args[0] } else { var err error - baseDir, err = os.Getwd() + pkgConfig.CreateOpts.BaseDir, err = os.Getwd() if err != nil { message.Fatalf(err, lang.CmdPrepareLintErr, err.Error()) } } - validator, err := lint.ValidateZarfSchema(baseDir) + v := common.GetViper() + pkgConfig.CreateOpts.SetVariables = helpers.TransformAndMergeMap( + v.GetStringMapString(common.VPkgCreateSet), pkgConfig.CreateOpts.SetVariables, strings.ToUpper) + validator, err := lint.ValidateZarfSchema(pkgConfig.CreateOpts) if err != nil { message.Fatal(err, err.Error()) } diff --git a/src/pkg/packager/compose.go b/src/pkg/packager/compose.go index 6961e8b876..b298c404cf 100644 --- a/src/pkg/packager/compose.go +++ b/src/pkg/packager/compose.go @@ -17,7 +17,7 @@ func (p *Packager) composeComponents() error { pkgVars := p.cfg.Pkg.Variables pkgConsts := p.cfg.Pkg.Constants - for _, component := range p.cfg.Pkg.Components { + for i, component := range p.cfg.Pkg.Components { arch := p.arch // filter by architecture if !composer.CompatibleComponent(component, arch, p.cfg.CreateOpts.Flavor) { @@ -29,7 +29,7 @@ func (p *Packager) composeComponents() error { component.Only.Flavor = "" // build the import chain - chain, err := composer.NewImportChain(component, arch, p.cfg.CreateOpts.Flavor) + chain, err := composer.NewImportChain(component, i, arch, p.cfg.CreateOpts.Flavor) if err != nil { return err } diff --git a/src/pkg/packager/composer/compose.go b/src/pkg/packager/composer/compose.go new file mode 100644 index 0000000000..479b1f369a --- /dev/null +++ b/src/pkg/packager/composer/compose.go @@ -0,0 +1,63 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package composer contains functions for composing components within Zarf packages. +package composer + +import ( + "github.com/defenseunicorns/zarf/src/config" + "github.com/defenseunicorns/zarf/src/pkg/message" + "github.com/defenseunicorns/zarf/src/types" +) + +func ComposeComponents(zarfPackage *types.ZarfPackage, createOpts types.ZarfCreateOptions, + warnings []string) ([]string, error) { + components := []types.ZarfComponent{} + + pkgVars := zarfPackage.Variables + pkgConsts := zarfPackage.Constants + + for i, component := range zarfPackage.Components { + //TODO allow this to be a CLI option + arch := config.GetArch(zarfPackage.Metadata.Architecture) + + // filter by architecture + if !CompatibleComponent(component, arch, createOpts.Flavor) { + continue + } + + // if a match was found, strip flavor and architecture to reduce bloat in the package definition + component.Only.Cluster.Architecture = "" + component.Only.Flavor = "" + + // build the import chain + chain, err := NewImportChain(component, i, arch, createOpts.Flavor) + if err != nil { + return warnings, err + } + message.Debugf("%s", chain) + + // migrate any deprecated component configurations now + warnings := chain.Migrate(zarfPackage.Build) + warnings = append(warnings, warnings...) + + // get the composed component + composed, err := chain.Compose() + if err != nil { + return warnings, err + } + components = append(components, composed) + + // merge variables and constants + pkgVars = chain.MergeVariables(pkgVars) + pkgConsts = chain.MergeConstants(pkgConsts) + } + + // set the filtered + composed components + zarfPackage.Components = components + + zarfPackage.Variables = pkgVars + zarfPackage.Constants = pkgConsts + + return warnings, nil +} diff --git a/src/pkg/packager/composer/list.go b/src/pkg/packager/composer/list.go index 89da2380c0..939d85da88 100644 --- a/src/pkg/packager/composer/list.go +++ b/src/pkg/packager/composer/list.go @@ -7,6 +7,7 @@ package composer import ( "fmt" "path/filepath" + "reflect" "strings" "github.com/defenseunicorns/zarf/src/internal/packager/validate" @@ -22,6 +23,11 @@ import ( type Node struct { types.ZarfComponent + // We are going to use this in lint so we can give the correct path + // We can give the index of the zarf component within the file that's being composed on + // So users can have the yaml path + index int + vars []types.ZarfPackageVariable consts []types.ZarfPackageConstant @@ -45,45 +51,46 @@ func (n *Node) ImportName() string { // ImportChain is a doubly linked list of component import definitions type ImportChain struct { - head *Node - tail *Node + Head *Node + Tail *Node remote *oci.OrasRemote } -func (ic *ImportChain) append(c types.ZarfComponent, relativeToHead string, vars []types.ZarfPackageVariable, consts []types.ZarfPackageConstant) { +func (ic *ImportChain) append(c types.ZarfComponent, index int, relativeToHead string, vars []types.ZarfPackageVariable, consts []types.ZarfPackageConstant) { node := &Node{ ZarfComponent: c, + index: index, relativeToHead: relativeToHead, vars: vars, consts: consts, prev: nil, next: nil, } - if ic.head == nil { - ic.head = node - ic.tail = node + if ic.Head == nil { + ic.Head = node + ic.Tail = node } else { - p := ic.tail + p := ic.Tail node.prev = p p.next = node - ic.tail = node + ic.Tail = node } } // NewImportChain creates a new import chain from a component -func NewImportChain(head types.ZarfComponent, arch, flavor string) (*ImportChain, error) { +func NewImportChain(head types.ZarfComponent, index int, arch, flavor string) (*ImportChain, error) { if arch == "" { return nil, fmt.Errorf("cannot build import chain: architecture must be provided") } ic := &ImportChain{} - ic.append(head, ".", nil, nil) + ic.append(head, index, ".", nil, nil) history := []string{} - node := ic.head + node := ic.Head for node != nil { isLocal := node.Import.Path != "" isRemote := node.Import.URL != "" @@ -160,7 +167,14 @@ func NewImportChain(head types.ZarfComponent, arch, flavor string) (*ImportChain } } - ic.append(found[0], filepath.Join(history...), pkg.Variables, pkg.Constants) + var index int + //Probably can do this better, maybe have filter also give an index + for i, component := range pkg.Components { + if reflect.DeepEqual(found[0], component) { + index = i + } + } + ic.append(found[0], index, filepath.Join(history...), pkg.Variables, pkg.Constants) node = node.next } return ic, nil @@ -168,22 +182,22 @@ func NewImportChain(head types.ZarfComponent, arch, flavor string) (*ImportChain // String returns a string representation of the import chain func (ic *ImportChain) String() string { - if ic.head.next == nil { - return fmt.Sprintf("component %q imports nothing", ic.head.Name) + if ic.Head.next == nil { + return fmt.Sprintf("component %q imports nothing", ic.Head.Name) } s := strings.Builder{} - name := ic.head.ImportName() + name := ic.Head.ImportName() - if ic.head.Import.Path != "" { - s.WriteString(fmt.Sprintf("component %q imports %q in %s", ic.head.Name, name, ic.head.Import.Path)) + if ic.Head.Import.Path != "" { + s.WriteString(fmt.Sprintf("component %q imports %q in %s", ic.Head.Name, name, ic.Head.Import.Path)) } else { - s.WriteString(fmt.Sprintf("component %q imports %q in %s", ic.head.Name, name, ic.head.Import.URL)) + s.WriteString(fmt.Sprintf("component %q imports %q in %s", ic.Head.Name, name, ic.Head.Import.URL)) } - node := ic.head.next - for node != ic.tail { + node := ic.Head.next + for node != ic.Tail { name := node.ImportName() s.WriteString(", which imports ") if node.Import.Path != "" { @@ -200,7 +214,7 @@ func (ic *ImportChain) String() string { // Migrate performs migrations on the import chain func (ic *ImportChain) Migrate(build types.ZarfBuildData) (warnings []string) { - node := ic.head + node := ic.Head for node != nil { migrated, w := deprecated.MigrateComponent(build, node.ZarfComponent) node.ZarfComponent = migrated @@ -208,7 +222,7 @@ func (ic *ImportChain) Migrate(build types.ZarfBuildData) (warnings []string) { node = node.next } if len(warnings) > 0 { - final := fmt.Sprintf("migrations were performed on the import chain of: %q", ic.head.Name) + final := fmt.Sprintf("migrations were performed on the import chain of: %q", ic.Head.Name) warnings = append(warnings, final) } return warnings @@ -217,9 +231,9 @@ func (ic *ImportChain) Migrate(build types.ZarfBuildData) (warnings []string) { // Compose merges the import chain into a single component // fixing paths, overriding metadata, etc func (ic *ImportChain) Compose() (composed types.ZarfComponent, err error) { - composed = ic.tail.ZarfComponent + composed = ic.Tail.ZarfComponent - if ic.tail.prev == nil { + if ic.Tail.prev == nil { // only had one component in the import chain return composed, nil } @@ -232,7 +246,7 @@ func (ic *ImportChain) Compose() (composed types.ZarfComponent, err error) { composed = types.ZarfComponent{} // start overriding with the tail node - node := ic.tail + node := ic.Tail for node != nil { fixPaths(&node.ZarfComponent, node.relativeToHead) @@ -256,7 +270,7 @@ func (ic *ImportChain) MergeVariables(existing []types.ZarfPackageVariable) (mer return v1.Name == v2.Name } - node := ic.tail + node := ic.Tail for node != nil { // merge the vars merged = helpers.MergeSlices(node.vars, merged, exists) @@ -273,7 +287,7 @@ func (ic *ImportChain) MergeConstants(existing []types.ZarfPackageConstant) (mer return c1.Name == c2.Name } - node := ic.tail + node := ic.Tail for node != nil { // merge the consts merged = helpers.MergeSlices(node.consts, merged, exists) diff --git a/src/pkg/packager/composer/list_test.go b/src/pkg/packager/composer/list_test.go index bca4cd2fd5..41d2cd62bc 100644 --- a/src/pkg/packager/composer/list_test.go +++ b/src/pkg/packager/composer/list_test.go @@ -50,7 +50,7 @@ func TestNewImportChain(t *testing.T) { t.Run(testCase.name, func(t *testing.T) { t.Parallel() - _, err := NewImportChain(testCase.head, testCase.arch, testCase.flavor) + _, err := NewImportChain(testCase.head, 0, testCase.arch, testCase.flavor) require.Contains(t, err.Error(), testCase.expectedErrorMessage) }) } @@ -313,7 +313,7 @@ func TestMerging(t *testing.T) { } head.next = &tail tail.prev = &head - testIC := &ImportChain{head: &head, tail: &tail} + testIC := &ImportChain{Head: &head, Tail: &tail} testCases := []testCase{ { @@ -446,12 +446,12 @@ func createChainFromSlice(components []types.ZarfComponent) (ic *ImportChain) { return ic } - ic.append(components[0], ".", nil, nil) + ic.append(components[0], 0, ".", nil, nil) history := []string{} for idx := 1; idx < len(components); idx++ { history = append(history, components[idx-1].Import.Path) - ic.append(components[idx], filepath.Join(history...), nil, nil) + ic.append(components[idx], idx, filepath.Join(history...), nil, nil) } return ic diff --git a/src/pkg/packager/composer/oci.go b/src/pkg/packager/composer/oci.go index 5a7b4637ed..b2eea815aa 100644 --- a/src/pkg/packager/composer/oci.go +++ b/src/pkg/packager/composer/oci.go @@ -37,14 +37,14 @@ func (ic *ImportChain) getRemote(url string) (*oci.OrasRemote, error) { // ContainsOCIImport returns true if the import chain contains a remote import func (ic *ImportChain) ContainsOCIImport() bool { // only the 2nd to last node may have a remote import - return ic.tail.prev != nil && ic.tail.prev.Import.URL != "" + return ic.Tail.prev != nil && ic.Tail.prev.Import.URL != "" } func (ic *ImportChain) fetchOCISkeleton() error { if !ic.ContainsOCIImport() { return nil } - node := ic.tail.prev + node := ic.Tail.prev remote, err := ic.getRemote(node.Import.URL) if err != nil { return err @@ -114,7 +114,7 @@ func (ic *ImportChain) fetchOCISkeleton() error { } // the tail node is the only node whose relativeToHead is based solely upon cwd<->cache // contrary to the other nodes, which are based upon the previous node - ic.tail.relativeToHead = rel + ic.Tail.relativeToHead = rel if oci.IsEmptyDescriptor(componentDesc) { // nothing was fetched, nothing to extract diff --git a/src/pkg/packager/lint/lint.go b/src/pkg/packager/lint/lint.go index 3640ab4765..f8d2ed9c5b 100644 --- a/src/pkg/packager/lint/lint.go +++ b/src/pkg/packager/lint/lint.go @@ -7,13 +7,19 @@ package lint import ( "embed" "fmt" + "os" "path/filepath" "regexp" "strings" + "github.com/defenseunicorns/zarf/src/config" + "github.com/defenseunicorns/zarf/src/config/lang" "github.com/defenseunicorns/zarf/src/pkg/layout" + "github.com/defenseunicorns/zarf/src/pkg/packager" + "github.com/defenseunicorns/zarf/src/pkg/packager/composer" "github.com/defenseunicorns/zarf/src/pkg/transform" "github.com/defenseunicorns/zarf/src/pkg/utils" + "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "github.com/defenseunicorns/zarf/src/types" "github.com/xeipuuv/gojsonschema" ) @@ -27,14 +33,33 @@ func getSchemaFile() ([]byte, error) { // ValidateZarfSchema validates a zarf file against the zarf schema, returns *validator with warnings or errors if they exist // along with an error if the validation itself failed -func ValidateZarfSchema(path string) (*Validator, error) { +func ValidateZarfSchema(createOpts types.ZarfCreateOptions) (*Validator, error) { validator := Validator{} var err error - if err := utils.ReadYaml(filepath.Join(path, layout.ZarfYAML), &validator.typedZarfPackage); err != nil { + + if err := utils.ReadYaml(filepath.Join(createOpts.BaseDir, layout.ZarfYAML), &validator.typedZarfPackage); err != nil { + return nil, err + } + + if err := utils.ReadYaml(filepath.Join(createOpts.BaseDir, layout.ZarfYAML), &validator.untypedZarfPackage); err != nil { return nil, err } + if err := os.Chdir(createOpts.BaseDir); err != nil { + return nil, fmt.Errorf("unable to access directory '%s': %w", createOpts.BaseDir, err) + } + checkForVarInComponentImport(&validator) + + _, err = composer.ComposeComponents(&validator.typedZarfPackage, createOpts, []string{}) + if err != nil { + return nil, err + } + + if err := fillActiveTemplate(&validator, createOpts); err != nil { + return nil, err + } + checkforUnpinnedRepos(&validator) checkForUnpinnedImages(&validator) checkForUnpinnedFiles(&validator) @@ -43,10 +68,6 @@ func ValidateZarfSchema(path string) (*Validator, error) { return nil, err } - if err := utils.ReadYaml(filepath.Join(path, layout.ZarfYAML), &validator.untypedZarfPackage); err != nil { - return nil, err - } - if err = validateSchema(&validator); err != nil { return nil, err } @@ -54,12 +75,94 @@ func ValidateZarfSchema(path string) (*Validator, error) { return &validator, nil } +func ValidateComposableComponenets(validator Validator, createOpts types.ZarfCreateOptions) error { + for i, component := range validator.typedZarfPackage.Components { + //TODO allow this to be a CLI option + arch := config.GetArch(validator.typedZarfPackage.Metadata.Architecture) + + // filter by architecture + if !composer.CompatibleComponent(component, arch, createOpts.Flavor) { + continue + } + + // if a match was found, strip flavor and architecture to reduce bloat in the package definition + component.Only.Cluster.Architecture = "" + component.Only.Flavor = "" + + // build the import chain + chain, err := composer.NewImportChain(component, i, arch, createOpts.Flavor) + if err != nil { + return err + } + node := chain.Tail + for node != nil { + + } + } + return nil +} + +func fillActiveTemplate(validator *Validator, createOpts types.ZarfCreateOptions) error { + templateMap := map[string]string{} + unsetVarWarning := false + + promptAndSetTemplate := func(templatePrefix string, deprecated bool) error { + yamlTemplates, err := utils.FindYamlTemplates(validator.typedZarfPackage, templatePrefix, "###") + if err != nil { + return err + } + + for key := range yamlTemplates { + if deprecated { + validator.warnings = append(validator.warnings, fmt.Sprintf(lang.PkgValidateTemplateDeprecation, key, key, key)) + } + _, present := createOpts.SetVariables[key] + if !present { + unsetVarWarning = true + } + } + + for key, value := range createOpts.SetVariables { + templateMap[fmt.Sprintf("%s%s###", templatePrefix, key)] = value + } + return nil + } + + // update the component templates on the package + err := packager.FindComponentTemplatesAndReload(&validator.typedZarfPackage) + if err != nil { + return err + } + + if err := promptAndSetTemplate(types.ZarfPackageTemplatePrefix, false); err != nil { + return err + } + + // [DEPRECATION] Set the Package Variable syntax as well for backward compatibility + if err := promptAndSetTemplate(types.ZarfPackageVariablePrefix, true); err != nil { + return err + } + + // Add special variable for the current package architecture + templateMap[types.ZarfPackageArch] = config.GetArch(validator.typedZarfPackage.Metadata.Architecture, validator.typedZarfPackage.Build.Architecture) + + if unsetVarWarning { + validator.warnings = append([]string{"There are variables that are unset and won't be evaluated during lint"}, validator.warnings...) + } + + return utils.ReloadYamlTemplate(&validator.typedZarfPackage, templateMap) +} + func isPinnedImage(image string) (bool, error) { transformedImage, err := transform.ParseImageRef(image) if err != nil { + if strings.Contains(image, types.ZarfPackageTemplatePrefix) || + strings.Contains(image, types.ZarfPackageVariablePrefix) { + return true, nil + } return false, err } - return !(transformedImage.TagOrDigest == ":latest"), err + return !(transformedImage.Digest == ":latest"), err } func isPinnedRepo(repo string) bool { @@ -96,7 +199,7 @@ func checkForUnpinnedImages(validator *Validator) { func checkForUnpinnedFiles(validator *Validator) { for i, component := range validator.typedZarfPackage.Components { for j, file := range component.Files { - if file.Shasum == "" { + if file.Shasum == "" && helpers.IsURL(file.Source) { validator.addWarning(fmt.Sprintf(".components.[%d].files.[%d]: Unpinned file", i, j)) } } diff --git a/src/pkg/packager/lint/lint_test.go b/src/pkg/packager/lint/lint_test.go index 90afbd2154..ffc43a6fb0 100644 --- a/src/pkg/packager/lint/lint_test.go +++ b/src/pkg/packager/lint/lint_test.go @@ -154,6 +154,12 @@ func TestValidateSchema(t *testing.T) { require.Equal(t, input, acutal) }) + // t.Run("Validate composible components", func(t *testing.T) { + // input := "" + // acutal := makeFieldPathYqCompat(input) + // require.Equal(t, input, acutal) + // }) + t.Run("isImagePinned", func(t *testing.T) { t.Parallel() tests := []struct { @@ -186,6 +192,11 @@ func TestValidateSchema(t *testing.T) { expected: false, err: errors.New("invalid reference format"), }, + { + input: "busybox:###ZARF_PKG_TMPL_BUSYBOX_IMAGE###", + expected: true, + err: nil, + }, } for _, tc := range tests { t.Run(tc.input, func(t *testing.T) { diff --git a/src/pkg/packager/variables.go b/src/pkg/packager/variables.go index 2caa135907..80efcbc3d1 100644 --- a/src/pkg/packager/variables.go +++ b/src/pkg/packager/variables.go @@ -15,6 +15,21 @@ import ( "github.com/defenseunicorns/zarf/src/types" ) +// FindComponentTemplatesAndReload appends ###ZARF_COMPONENT_NAME### for each component, assigns value, and reloads +func FindComponentTemplatesAndReload(zarfPackage *types.ZarfPackage) error { + // iterate through components to and find all ###ZARF_COMPONENT_NAME, assign to component Name and value + for i, component := range zarfPackage.Components { + mappings := map[string]string{} + mappings[types.ZarfComponentName] = component.Name + err := utils.ReloadYamlTemplate(&zarfPackage.Components[i], mappings) + if err != nil { + return err + } + } + + return nil +} + // fillActiveTemplate handles setting the active variables and reloading the base template. func (p *Packager) fillActiveTemplate() error { templateMap := map[string]string{} @@ -54,7 +69,7 @@ func (p *Packager) fillActiveTemplate() error { } // update the component templates on the package - err := p.findComponentTemplatesAndReload() + err := FindComponentTemplatesAndReload(&p.cfg.Pkg) if err != nil { return err } @@ -126,21 +141,6 @@ func (p *Packager) setVariableInConfig(name, value string, sensitive bool, autoI } } -// findComponentTemplatesAndReload appends ###ZARF_COMPONENT_NAME### for each component, assigns value, and reloads -func (p *Packager) findComponentTemplatesAndReload() error { - // iterate through components to and find all ###ZARF_COMPONENT_NAME, assign to component Name and value - for i, component := range p.cfg.Pkg.Components { - mappings := map[string]string{} - mappings[types.ZarfComponentName] = component.Name - err := utils.ReloadYamlTemplate(&p.cfg.Pkg.Components[i], mappings) - if err != nil { - return err - } - } - - return nil -} - // checkVariablePattern checks to see if a current variable is set to a value that matches its pattern func (p *Packager) checkVariablePattern(name, pattern string) error { if regexp.MustCompile(pattern).MatchString(p.cfg.SetVariableMap[name].Value) { diff --git a/src/test/e2e/12_lint_test.go b/src/test/e2e/12_lint_test.go index 98257fdc89..1f0a6ec4ce 100644 --- a/src/test/e2e/12_lint_test.go +++ b/src/test/e2e/12_lint_test.go @@ -1,31 +1,44 @@ package test import ( + "fmt" "path/filepath" "testing" + "github.com/defenseunicorns/zarf/src/config/lang" "github.com/stretchr/testify/require" ) func TestLint(t *testing.T) { t.Log("E2E: Lint") + t.Run("zarf test lint success", func(t *testing.T) { + t.Log("E2E: Test lint on schema success") + + // This runs lint on the zarf.yaml in the base directory of the repo + _, _, err := e2e.Zarf("prepare", "lint") + require.NoError(t, err, "Expect no error here because the yaml file is following schema") + }) + t.Run("zarf test lint fail", func(t *testing.T) { t.Log("E2E: Test lint on schema fail") path := filepath.Join("src", "test", "packages", "12-lint") - _, stderr, err := e2e.Zarf("prepare", "lint", path) + // In this case I'm guessing we should also remove color from the table? + _, stderr, err := e2e.Zarf("prepare", "lint", path, "--no-color") require.Error(t, err, "Require an exit code since there was warnings / errors") - require.Contains(t, stderr, ".components.[0].import: Additional property not-path is not allowed") - require.Contains(t, stderr, ".components.[2].import.path: Will not resolve ZARF_PKG_TMPL_* variables") - require.Contains(t, stderr, ".variables: Invalid type. Expected: array, given: null") - }) - t.Run("zarf test lint success", func(t *testing.T) { - t.Log("E2E: Test lint on schema success") + // It's a bit weird to have a period here and not in the other warnings + key := "WHATEVER_IMAGE" + require.Contains(t, stderr, fmt.Sprintf(lang.PkgValidateTemplateDeprecation, key, key, key)) + require.Contains(t, stderr, "Package template \"WHATEVER_IMAGE\" is using the deprecated syntax ###ZARF_PKG_VAR_WHATEVER_IMAGE###. This will be removed in Zarf v1.0.0. Please update to ###ZARF_PKG_TMPL_WHATEVER_IMAGE###.") + require.Contains(t, stderr, ".components.[2].repos.[0]: Unpinned repository") + require.Contains(t, stderr, ".metadata: Additional property description1 is not allowed") + require.Contains(t, stderr, ".components.[0].import: Additional property not-path is not allowed") + require.Contains(t, stderr, "There are variables that are unset and won't be evaluated during lint") - // This runs lint on the zarf.yaml in the base directory of the repo - _, _, err := e2e.Zarf("prepare", "lint") - require.NoError(t, err, "Expect no error here because the yaml file is following schema") + // require.Contains(t, stderr, ".components.[1].images.[0] in linted-import: Unpinned image") + // require.Contains(t, stderr, ".components.[1].images.[2] in linted-import: Unpinned image") }) + } diff --git a/src/test/packages/12-lint/linted-import/zarf.yaml b/src/test/packages/12-lint/linted-import/zarf.yaml new file mode 100644 index 0000000000..07380e1997 --- /dev/null +++ b/src/test/packages/12-lint/linted-import/zarf.yaml @@ -0,0 +1,17 @@ +kind: ZarfInitConfig +metadata: + name: init + description: Testing bad yaml imported + +variables: + - name: BUSYBOX_IMAGE + description: "whatever" + +components: + - name: dont-care + + - name: import-test + images: + - registry.com:9001/whatever/image:latest + - busybox@sha256:3fbc632167424a6d997e74f52b878d7cc478225cffac6bc977eedfe51c7f4e79 + - busybox:###ZARF_PKG_TMPL_BUSYBOX_IMAGE### diff --git a/src/test/packages/12-lint/zarf-config.toml b/src/test/packages/12-lint/zarf-config.toml new file mode 100644 index 0000000000..5f869ded0a --- /dev/null +++ b/src/test/packages/12-lint/zarf-config.toml @@ -0,0 +1,2 @@ +[package.create.set] +BUSYBOX_IMAGE = 'latest' diff --git a/src/test/packages/12-lint/zarf.yaml b/src/test/packages/12-lint/zarf.yaml index c71d329b22..af4343fc1d 100644 --- a/src/test/packages/12-lint/zarf.yaml +++ b/src/test/packages/12-lint/zarf.yaml @@ -5,6 +5,8 @@ metadata: variables: + - name: BUSYBOX_IMAGE + description: "whatever" components: - name: first-test-component @@ -13,15 +15,7 @@ components: - name: import-test import: - path: 123123 - - - name: import-test - import: - path: "###ZARF_PKG_TMPL_ZEBRA###" - - - name: import-url - import: - url: "oci://###ZARF_PKG_TMPL_ZEBRA###" + path: linted-import - name: full-repo repos: @@ -32,8 +26,12 @@ components: - ghcr.io/kiwix/kiwix-serve:3.5.0-2 - registry.com:9001/whatever/image:1.0.0 - busybox@sha256:3fbc632167424a6d997e74f52b878d7cc478225cffac6bc977eedfe51c7f4e79 - - busybox:latest + - busybox:###ZARF_PKG_VAR_WHATEVER_IMAGE### + - busybox:###ZARF_PKG_TMPL_BUSYBOX_IMAGE### + - ubuntu:###ZARF_PKG_TMPL_UBUNTU_IMAGE### files: - source: https://github.com/k3s-io/k3s/releases/download/v1.28.2+k3s1/k3s shasum: 2f041d37a2c6d54d53e106e1c7713bc48f806f3919b0d9e092f5fcbdc55b41cf + target: src/ - source: file-without-shasum.txt + target: src/ From 601bebf0e7adac1c1afb746411c5a82332171f76 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Mon, 4 Dec 2023 20:25:16 +0000 Subject: [PATCH 100/192] refactor lint --- src/pkg/packager/lint/lint.go | 72 ++++++++++++++---------------- src/pkg/packager/lint/lint_test.go | 51 +++++++++++++-------- 2 files changed, 67 insertions(+), 56 deletions(-) diff --git a/src/pkg/packager/lint/lint.go b/src/pkg/packager/lint/lint.go index f8d2ed9c5b..d1366ff8c6 100644 --- a/src/pkg/packager/lint/lint.go +++ b/src/pkg/packager/lint/lint.go @@ -49,8 +49,6 @@ func ValidateZarfSchema(createOpts types.ZarfCreateOptions) (*Validator, error) return nil, fmt.Errorf("unable to access directory '%s': %w", createOpts.BaseDir, err) } - checkForVarInComponentImport(&validator) - _, err = composer.ComposeComponents(&validator.typedZarfPackage, createOpts, []string{}) if err != nil { return nil, err @@ -60,9 +58,7 @@ func ValidateZarfSchema(createOpts types.ZarfCreateOptions) (*Validator, error) return nil, err } - checkforUnpinnedRepos(&validator) - checkForUnpinnedImages(&validator) - checkForUnpinnedFiles(&validator) + lintComponents(&validator) if validator.jsonSchema, err = getSchemaFile(); err != nil { return nil, err @@ -75,7 +71,7 @@ func ValidateZarfSchema(createOpts types.ZarfCreateOptions) (*Validator, error) return &validator, nil } -func ValidateComposableComponenets(validator Validator, createOpts types.ZarfCreateOptions) error { +func ValidateComposableComponenets(validator *Validator, createOpts types.ZarfCreateOptions) error { for i, component := range validator.typedZarfPackage.Components { //TODO allow this to be a CLI option arch := config.GetArch(validator.typedZarfPackage.Metadata.Architecture) @@ -96,7 +92,7 @@ func ValidateComposableComponenets(validator Validator, createOpts types.ZarfCre } node := chain.Tail for node != nil { - + // lintComponent(validator, node.ZarfComponent) } } return nil @@ -162,7 +158,7 @@ func isPinnedImage(image string) (bool, error) { } return false, err } - return !(transformedImage.Digest == ":latest"), err + return (transformedImage.Digest != ""), err } func isPinnedRepo(repo string) bool { @@ -171,51 +167,51 @@ func isPinnedRepo(repo string) bool { return (strings.Contains(repo, "@") || strings.Contains(repo, "/-/")) } -func checkforUnpinnedRepos(validator *Validator) { +func lintComponents(validator *Validator) { for i, component := range validator.typedZarfPackage.Components { - for j, repo := range component.Repos { - if !isPinnedRepo(repo) { - validator.addWarning(fmt.Sprintf(".components.[%d].repos.[%d]: Unpinned repository", i, j)) - } - } + checkforUnpinnedRepos(validator, i, component) + checkForUnpinnedImages(validator, i, component) + checkForUnpinnedFiles(validator, i, component) + checkForVarInComponentImport(validator, i, component) } } -func checkForUnpinnedImages(validator *Validator) { - for i, component := range validator.typedZarfPackage.Components { - for j, image := range component.Images { - pinnedImage, err := isPinnedImage(image) - if err != nil { - validator.addError(fmt.Errorf(".components.[%d].images.[%d]: Invalid image format", i, j)) - continue - } - if !pinnedImage { - validator.addWarning(fmt.Sprintf(".components.[%d].images.[%d]: Unpinned image", i, j)) - } +func checkforUnpinnedRepos(validator *Validator, index int, component types.ZarfComponent) { + for j, repo := range component.Repos { + if !isPinnedRepo(repo) { + validator.addWarning(fmt.Sprintf(".components.[%d].repos.[%d]: Unpinned repository", index, j)) } } } -func checkForUnpinnedFiles(validator *Validator) { - for i, component := range validator.typedZarfPackage.Components { - for j, file := range component.Files { - if file.Shasum == "" && helpers.IsURL(file.Source) { - validator.addWarning(fmt.Sprintf(".components.[%d].files.[%d]: Unpinned file", i, j)) - } +func checkForUnpinnedImages(validator *Validator, index int, component types.ZarfComponent) { + for j, image := range component.Images { + pinnedImage, err := isPinnedImage(image) + if err != nil { + validator.addError(fmt.Errorf(".components.[%d].images.[%d]: Invalid image format", index, j)) + continue + } + if !pinnedImage { + validator.addWarning(fmt.Sprintf(".components.[%d].images.[%d]: Unpinned image", index, j)) } } } -func checkForVarInComponentImport(validator *Validator) { - for i, component := range validator.typedZarfPackage.Components { - if strings.Contains(component.Import.Path, types.ZarfPackageTemplatePrefix) { - validator.addWarning(fmt.Sprintf(".components.[%d].import.path: Will not resolve ZARF_PKG_TMPL_* variables", i)) - } - if strings.Contains(component.Import.URL, types.ZarfPackageTemplatePrefix) { - validator.addWarning(fmt.Sprintf(".components.[%d].import.url: Will not resolve ZARF_PKG_TMPL_* variables", i)) +func checkForUnpinnedFiles(validator *Validator, index int, component types.ZarfComponent) { + for j, file := range component.Files { + if file.Shasum == "" && helpers.IsURL(file.Source) { + validator.addWarning(fmt.Sprintf(".components.[%d].files.[%d]: Unpinned file", index, j)) } } +} +func checkForVarInComponentImport(validator *Validator, index int, component types.ZarfComponent) { + if strings.Contains(component.Import.Path, types.ZarfPackageTemplatePrefix) { + validator.addWarning(fmt.Sprintf(".components.[%d].import.path: Will not resolve ZARF_PKG_TMPL_* variables", index)) + } + if strings.Contains(component.Import.URL, types.ZarfPackageTemplatePrefix) { + validator.addWarning(fmt.Sprintf(".components.[%d].import.url: Will not resolve ZARF_PKG_TMPL_* variables", index)) + } } func makeFieldPathYqCompat(field string) string { diff --git a/src/pkg/packager/lint/lint_test.go b/src/pkg/packager/lint/lint_test.go index ffc43a6fb0..2fa64f4155 100644 --- a/src/pkg/packager/lint/lint_test.go +++ b/src/pkg/packager/lint/lint_test.go @@ -103,41 +103,56 @@ func TestValidateSchema(t *testing.T) { t.Run("Template in component import success", func(t *testing.T) { unmarshalledYaml := readAndUnmarshalYaml[types.ZarfPackage](t, goodZarfPackage) validator := Validator{typedZarfPackage: unmarshalledYaml} - checkForVarInComponentImport(&validator) + lintComponents(&validator) require.Empty(t, validator.warnings) + require.Empty(t, validator.errors) }) t.Run("Template in component import failure", func(t *testing.T) { - unmarshalledYaml := readAndUnmarshalYaml[types.ZarfPackage](t, badZarfPackage) - validator := Validator{typedZarfPackage: unmarshalledYaml} - checkForVarInComponentImport(&validator) + validator := Validator{} + pathComponent := types.ZarfComponent{Import: types.ZarfComponentImport{Path: "###ZARF_PKG_TMPL_ZEBRA###"}} + URLComponent := types.ZarfComponent{Import: types.ZarfComponentImport{URL: "oci://###ZARF_PKG_TMPL_ZEBRA###"}} + checkForVarInComponentImport(&validator, 2, pathComponent) + checkForVarInComponentImport(&validator, 3, URLComponent) require.Equal(t, validator.warnings[0], ".components.[2].import.path: Will not resolve ZARF_PKG_TMPL_* variables") require.Equal(t, validator.warnings[1], ".components.[3].import.url: Will not resolve ZARF_PKG_TMPL_* variables") }) t.Run("Unpinnned repo warning", func(t *testing.T) { - unmarshalledYaml := readAndUnmarshalYaml[types.ZarfPackage](t, badZarfPackage) - validator := Validator{typedZarfPackage: unmarshalledYaml} - checkforUnpinnedRepos(&validator) - require.Equal(t, validator.warnings[0], ".components.[4].repos.[0]: Unpinned repository") + validator := Validator{} + component := types.ZarfComponent{Repos: []string{ + "https://github.com/defenseunicorns/zarf-public-test.git", + "https://dev.azure.com/defenseunicorns/zarf-public-test/_git/zarf-public-test@v0.0.1", + "https://gitlab.com/gitlab-org/build/omnibus-mirror/pcre2/-/tree/vreverse?ref_type=heads"}} + checkforUnpinnedRepos(&validator, 0, component) + require.Equal(t, validator.warnings[0], ".components.[0].repos.[0]: Unpinned repository") require.Equal(t, len(validator.warnings), 1) }) t.Run("Unpinnned image warning", func(t *testing.T) { - unmarshalledYaml := readAndUnmarshalYaml[types.ZarfPackage](t, badZarfPackage) - validator := Validator{typedZarfPackage: unmarshalledYaml} - checkForUnpinnedImages(&validator) - require.Equal(t, validator.warnings[0], ".components.[4].images.[3]: Unpinned image") + validator := Validator{} + component := types.ZarfComponent{Images: []string{ + "registry.com:9001/whatever/image:1.0.0", + "busybox:latest@sha256:3fbc632167424a6d997e74f52b878d7cc478225cffac6bc977eedfe51c7f4e79", + "badimage:badimage@@sha256:3fbc632167424a6d997e74f5"}} + checkForUnpinnedImages(&validator, 0, component) + require.Equal(t, validator.warnings[0], ".components.[0].images.[0]: Unpinned image") require.Equal(t, len(validator.warnings), 1) - require.EqualError(t, validator.errors[0], ".components.[4].images.[4]: Invalid image format") + require.EqualError(t, validator.errors[0], ".components.[0].images.[2]: Invalid image format") require.Equal(t, len(validator.errors), 1) }) t.Run("Unpinnned file warning", func(t *testing.T) { - unmarshalledYaml := readAndUnmarshalYaml[types.ZarfPackage](t, badZarfPackage) - validator := Validator{typedZarfPackage: unmarshalledYaml} - checkForUnpinnedFiles(&validator) - require.Equal(t, validator.warnings[0], ".components.[4].files.[1]: Unpinned file") + validator := Validator{} + zarfFiles := []types.ZarfFile{ + { + Source: "http://example.com/file.zip", + Target: "/path/to/target", + }, + } + component := types.ZarfComponent{Files: zarfFiles} + checkForUnpinnedFiles(&validator, 0, component) + require.Equal(t, validator.warnings[0], ".components.[0].files.[0]: Unpinned file") require.Equal(t, len(validator.warnings), 1) }) @@ -179,7 +194,7 @@ func TestValidateSchema(t *testing.T) { }, { input: "ghcr.io/defenseunicorns/pepr/controller:v0.15.0", - expected: true, + expected: false, err: nil, }, { From 62bf8a3ca038be966f96c492909067cdadc2c1dd Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Tue, 5 Dec 2023 14:48:16 +0000 Subject: [PATCH 101/192] WIP, closer --- src/cmd/prepare.go | 2 + src/pkg/packager/composer/list.go | 52 ++++++++++---------- src/pkg/packager/composer/list_test.go | 4 +- src/pkg/packager/composer/oci.go | 6 +-- src/pkg/packager/lint/lint.go | 55 ++++++++++++---------- src/pkg/packager/lint/lint_test.go | 10 ++-- src/test/e2e/12_lint_test.go | 12 +++-- src/test/packages/12-lint/zarf-config.toml | 2 +- 8 files changed, 77 insertions(+), 66 deletions(-) diff --git a/src/cmd/prepare.go b/src/cmd/prepare.go index 37aaedab3a..14c0348a40 100644 --- a/src/cmd/prepare.go +++ b/src/cmd/prepare.go @@ -254,5 +254,7 @@ func init() { // allow for the override of the default helm KubeVersion prepareFindImages.Flags().StringVar(&pkgConfig.FindImagesOpts.KubeVersionOverride, "kube-version", "", lang.CmdPrepareFlagKubeVersion) + lintCmd.Flags().StringToStringVar(&pkgConfig.CreateOpts.SetVariables, "set", v.GetStringMapString(common.VPkgCreateSet), lang.CmdPackageCreateFlagSet) + prepareTransformGitLinks.Flags().StringVar(&pkgConfig.InitOpts.GitServer.PushUsername, "git-account", config.ZarfGitPushUser, lang.CmdPrepareFlagGitAccount) } diff --git a/src/pkg/packager/composer/list.go b/src/pkg/packager/composer/list.go index 939d85da88..bbe5c2c97c 100644 --- a/src/pkg/packager/composer/list.go +++ b/src/pkg/packager/composer/list.go @@ -24,17 +24,17 @@ type Node struct { types.ZarfComponent // We are going to use this in lint so we can give the correct path - // We can give the index of the zarf component within the file that's being composed on + // We can give the Index of the zarf component within the file that's being composed on // So users can have the yaml path - index int + Index int vars []types.ZarfPackageVariable consts []types.ZarfPackageConstant - relativeToHead string + RelativeToHead string - prev *Node - next *Node + Prev *Node + Next *Node } // ImportName returns the name of the component to import @@ -60,20 +60,20 @@ type ImportChain struct { func (ic *ImportChain) append(c types.ZarfComponent, index int, relativeToHead string, vars []types.ZarfPackageVariable, consts []types.ZarfPackageConstant) { node := &Node{ ZarfComponent: c, - index: index, - relativeToHead: relativeToHead, + Index: index, + RelativeToHead: relativeToHead, vars: vars, consts: consts, - prev: nil, - next: nil, + Prev: nil, + Next: nil, } if ic.Head == nil { ic.Head = node ic.Tail = node } else { p := ic.Tail - node.prev = p - p.next = node + node.Prev = p + p.Next = node ic.Tail = node } } @@ -107,11 +107,11 @@ func NewImportChain(head types.ZarfComponent, index int, arch, flavor string) (* } // ensure that remote components are not importing other remote components - if node.prev != nil && node.prev.Import.URL != "" && isRemote { + if node.Prev != nil && node.Prev.Import.URL != "" && isRemote { return ic, fmt.Errorf("detected malformed import chain, cannot import remote components from remote components") } // ensure that remote components are not importing local components - if node.prev != nil && node.prev.Import.URL != "" && isLocal { + if node.Prev != nil && node.Prev.Import.URL != "" && isLocal { return ic, fmt.Errorf("detected malformed import chain, cannot import local components from remote components") } @@ -125,10 +125,10 @@ func NewImportChain(head types.ZarfComponent, index int, arch, flavor string) (* // this is O(n^2) but the import chain should be small prev := node for prev != nil { - if prev.relativeToHead == relativeToHead { + if prev.RelativeToHead == relativeToHead { return ic, fmt.Errorf("detected circular import chain: %s", strings.Join(history, " -> ")) } - prev = prev.prev + prev = prev.Prev } // this assumes the composed package is following the zarf layout @@ -175,14 +175,14 @@ func NewImportChain(head types.ZarfComponent, index int, arch, flavor string) (* } } ic.append(found[0], index, filepath.Join(history...), pkg.Variables, pkg.Constants) - node = node.next + node = node.Next } return ic, nil } // String returns a string representation of the import chain func (ic *ImportChain) String() string { - if ic.Head.next == nil { + if ic.Head.Next == nil { return fmt.Sprintf("component %q imports nothing", ic.Head.Name) } @@ -196,7 +196,7 @@ func (ic *ImportChain) String() string { s.WriteString(fmt.Sprintf("component %q imports %q in %s", ic.Head.Name, name, ic.Head.Import.URL)) } - node := ic.Head.next + node := ic.Head.Next for node != ic.Tail { name := node.ImportName() s.WriteString(", which imports ") @@ -206,7 +206,7 @@ func (ic *ImportChain) String() string { s.WriteString(fmt.Sprintf("%q in %s", name, node.Import.URL)) } - node = node.next + node = node.Next } return s.String() @@ -219,7 +219,7 @@ func (ic *ImportChain) Migrate(build types.ZarfBuildData) (warnings []string) { migrated, w := deprecated.MigrateComponent(build, node.ZarfComponent) node.ZarfComponent = migrated warnings = append(warnings, w...) - node = node.next + node = node.Next } if len(warnings) > 0 { final := fmt.Sprintf("migrations were performed on the import chain of: %q", ic.Head.Name) @@ -233,7 +233,7 @@ func (ic *ImportChain) Migrate(build types.ZarfBuildData) (warnings []string) { func (ic *ImportChain) Compose() (composed types.ZarfComponent, err error) { composed = ic.Tail.ZarfComponent - if ic.Tail.prev == nil { + if ic.Tail.Prev == nil { // only had one component in the import chain return composed, nil } @@ -248,7 +248,7 @@ func (ic *ImportChain) Compose() (composed types.ZarfComponent, err error) { // start overriding with the tail node node := ic.Tail for node != nil { - fixPaths(&node.ZarfComponent, node.relativeToHead) + fixPaths(&node.ZarfComponent, node.RelativeToHead) // perform overrides here overrideMetadata(&composed, node.ZarfComponent) @@ -256,9 +256,9 @@ func (ic *ImportChain) Compose() (composed types.ZarfComponent, err error) { overrideResources(&composed, node.ZarfComponent) overrideActions(&composed, node.ZarfComponent) - composeExtensions(&composed, node.ZarfComponent, node.relativeToHead) + composeExtensions(&composed, node.ZarfComponent, node.RelativeToHead) - node = node.prev + node = node.Prev } return composed, nil @@ -274,7 +274,7 @@ func (ic *ImportChain) MergeVariables(existing []types.ZarfPackageVariable) (mer for node != nil { // merge the vars merged = helpers.MergeSlices(node.vars, merged, exists) - node = node.prev + node = node.Prev } merged = helpers.MergeSlices(existing, merged, exists) @@ -291,7 +291,7 @@ func (ic *ImportChain) MergeConstants(existing []types.ZarfPackageConstant) (mer for node != nil { // merge the consts merged = helpers.MergeSlices(node.consts, merged, exists) - node = node.prev + node = node.Prev } merged = helpers.MergeSlices(existing, merged, exists) diff --git a/src/pkg/packager/composer/list_test.go b/src/pkg/packager/composer/list_test.go index 41d2cd62bc..e2aa1db018 100644 --- a/src/pkg/packager/composer/list_test.go +++ b/src/pkg/packager/composer/list_test.go @@ -311,8 +311,8 @@ func TestMerging(t *testing.T) { }, }, } - head.next = &tail - tail.prev = &head + head.Next = &tail + tail.Prev = &head testIC := &ImportChain{Head: &head, Tail: &tail} testCases := []testCase{ diff --git a/src/pkg/packager/composer/oci.go b/src/pkg/packager/composer/oci.go index b2eea815aa..7c10b279ab 100644 --- a/src/pkg/packager/composer/oci.go +++ b/src/pkg/packager/composer/oci.go @@ -37,14 +37,14 @@ func (ic *ImportChain) getRemote(url string) (*oci.OrasRemote, error) { // ContainsOCIImport returns true if the import chain contains a remote import func (ic *ImportChain) ContainsOCIImport() bool { // only the 2nd to last node may have a remote import - return ic.Tail.prev != nil && ic.Tail.prev.Import.URL != "" + return ic.Tail.Prev != nil && ic.Tail.Prev.Import.URL != "" } func (ic *ImportChain) fetchOCISkeleton() error { if !ic.ContainsOCIImport() { return nil } - node := ic.Tail.prev + node := ic.Tail.Prev remote, err := ic.getRemote(node.Import.URL) if err != nil { return err @@ -114,7 +114,7 @@ func (ic *ImportChain) fetchOCISkeleton() error { } // the tail node is the only node whose relativeToHead is based solely upon cwd<->cache // contrary to the other nodes, which are based upon the previous node - ic.Tail.relativeToHead = rel + ic.Tail.RelativeToHead = rel if oci.IsEmptyDescriptor(componentDesc) { // nothing was fetched, nothing to extract diff --git a/src/pkg/packager/lint/lint.go b/src/pkg/packager/lint/lint.go index d1366ff8c6..ef76ec1878 100644 --- a/src/pkg/packager/lint/lint.go +++ b/src/pkg/packager/lint/lint.go @@ -45,21 +45,20 @@ func ValidateZarfSchema(createOpts types.ZarfCreateOptions) (*Validator, error) return nil, err } - if err := os.Chdir(createOpts.BaseDir); err != nil { - return nil, fmt.Errorf("unable to access directory '%s': %w", createOpts.BaseDir, err) + if err := fillActiveTemplate(&validator, createOpts); err != nil { + return nil, err } - _, err = composer.ComposeComponents(&validator.typedZarfPackage, createOpts, []string{}) - if err != nil { - return nil, err + lintComponents(&validator) + + if err := os.Chdir(createOpts.BaseDir); err != nil { + return nil, fmt.Errorf("unable to access directory '%s': %w", createOpts.BaseDir, err) } - if err := fillActiveTemplate(&validator, createOpts); err != nil { + if err := ValidateComposableComponenets(&validator, createOpts); err != nil { return nil, err } - lintComponents(&validator) - if validator.jsonSchema, err = getSchemaFile(); err != nil { return nil, err } @@ -90,10 +89,13 @@ func ValidateComposableComponenets(validator *Validator, createOpts types.ZarfCr if err != nil { return err } - node := chain.Tail + // Skipping initial component + node := chain.Head.Next for node != nil { - // lintComponent(validator, node.ZarfComponent) + lintComponent(validator, node.Index, node.ZarfComponent, fmt.Sprintf(" %s", node.RelativeToHead)) + node = node.Next } + } return nil } @@ -167,50 +169,55 @@ func isPinnedRepo(repo string) bool { return (strings.Contains(repo, "@") || strings.Contains(repo, "/-/")) } +// Feels like validator may have too much with both the zarf package and the warnings func lintComponents(validator *Validator) { for i, component := range validator.typedZarfPackage.Components { - checkforUnpinnedRepos(validator, i, component) - checkForUnpinnedImages(validator, i, component) - checkForUnpinnedFiles(validator, i, component) - checkForVarInComponentImport(validator, i, component) + lintComponent(validator, i, component, "") } } -func checkforUnpinnedRepos(validator *Validator, index int, component types.ZarfComponent) { +func lintComponent(validator *Validator, index int, component types.ZarfComponent, path string) { + checkforUnpinnedRepos(validator, index, component, path) + checkForUnpinnedImages(validator, index, component, path) + checkForUnpinnedFiles(validator, index, component, path) + checkForVarInComponentImport(validator, index, component, path) +} + +func checkforUnpinnedRepos(validator *Validator, index int, component types.ZarfComponent, path string) { for j, repo := range component.Repos { if !isPinnedRepo(repo) { - validator.addWarning(fmt.Sprintf(".components.[%d].repos.[%d]: Unpinned repository", index, j)) + validator.addWarning(fmt.Sprintf(".components.[%d].repos.[%d]%s: Unpinned repository", index, j, path)) } } } -func checkForUnpinnedImages(validator *Validator, index int, component types.ZarfComponent) { +func checkForUnpinnedImages(validator *Validator, index int, component types.ZarfComponent, path string) { for j, image := range component.Images { pinnedImage, err := isPinnedImage(image) if err != nil { - validator.addError(fmt.Errorf(".components.[%d].images.[%d]: Invalid image format", index, j)) + validator.addError(fmt.Errorf(".components.[%d].images.[%d]%s: Invalid image format", index, j, path)) continue } if !pinnedImage { - validator.addWarning(fmt.Sprintf(".components.[%d].images.[%d]: Unpinned image", index, j)) + validator.addWarning(fmt.Sprintf(".components.[%d].images.[%d]%s: Unpinned image", index, j, path)) } } } -func checkForUnpinnedFiles(validator *Validator, index int, component types.ZarfComponent) { +func checkForUnpinnedFiles(validator *Validator, index int, component types.ZarfComponent, path string) { for j, file := range component.Files { if file.Shasum == "" && helpers.IsURL(file.Source) { - validator.addWarning(fmt.Sprintf(".components.[%d].files.[%d]: Unpinned file", index, j)) + validator.addWarning(fmt.Sprintf(".components.[%d].files.[%d]%s: Unpinned file", index, j, path)) } } } -func checkForVarInComponentImport(validator *Validator, index int, component types.ZarfComponent) { +func checkForVarInComponentImport(validator *Validator, index int, component types.ZarfComponent, path string) { if strings.Contains(component.Import.Path, types.ZarfPackageTemplatePrefix) { - validator.addWarning(fmt.Sprintf(".components.[%d].import.path: Will not resolve ZARF_PKG_TMPL_* variables", index)) + validator.addWarning(fmt.Sprintf(".components.[%d].import.path%s: Will not resolve ZARF_PKG_TMPL_* variables", index, path)) } if strings.Contains(component.Import.URL, types.ZarfPackageTemplatePrefix) { - validator.addWarning(fmt.Sprintf(".components.[%d].import.url: Will not resolve ZARF_PKG_TMPL_* variables", index)) + validator.addWarning(fmt.Sprintf(".components.[%d].import.url%s: Will not resolve ZARF_PKG_TMPL_* variables", index, path)) } } diff --git a/src/pkg/packager/lint/lint_test.go b/src/pkg/packager/lint/lint_test.go index 2fa64f4155..603f13e12a 100644 --- a/src/pkg/packager/lint/lint_test.go +++ b/src/pkg/packager/lint/lint_test.go @@ -112,8 +112,8 @@ func TestValidateSchema(t *testing.T) { validator := Validator{} pathComponent := types.ZarfComponent{Import: types.ZarfComponentImport{Path: "###ZARF_PKG_TMPL_ZEBRA###"}} URLComponent := types.ZarfComponent{Import: types.ZarfComponentImport{URL: "oci://###ZARF_PKG_TMPL_ZEBRA###"}} - checkForVarInComponentImport(&validator, 2, pathComponent) - checkForVarInComponentImport(&validator, 3, URLComponent) + checkForVarInComponentImport(&validator, 2, pathComponent, "") + checkForVarInComponentImport(&validator, 3, URLComponent, "") require.Equal(t, validator.warnings[0], ".components.[2].import.path: Will not resolve ZARF_PKG_TMPL_* variables") require.Equal(t, validator.warnings[1], ".components.[3].import.url: Will not resolve ZARF_PKG_TMPL_* variables") }) @@ -124,7 +124,7 @@ func TestValidateSchema(t *testing.T) { "https://github.com/defenseunicorns/zarf-public-test.git", "https://dev.azure.com/defenseunicorns/zarf-public-test/_git/zarf-public-test@v0.0.1", "https://gitlab.com/gitlab-org/build/omnibus-mirror/pcre2/-/tree/vreverse?ref_type=heads"}} - checkforUnpinnedRepos(&validator, 0, component) + checkforUnpinnedRepos(&validator, 0, component, "") require.Equal(t, validator.warnings[0], ".components.[0].repos.[0]: Unpinned repository") require.Equal(t, len(validator.warnings), 1) }) @@ -135,7 +135,7 @@ func TestValidateSchema(t *testing.T) { "registry.com:9001/whatever/image:1.0.0", "busybox:latest@sha256:3fbc632167424a6d997e74f52b878d7cc478225cffac6bc977eedfe51c7f4e79", "badimage:badimage@@sha256:3fbc632167424a6d997e74f5"}} - checkForUnpinnedImages(&validator, 0, component) + checkForUnpinnedImages(&validator, 0, component, "") require.Equal(t, validator.warnings[0], ".components.[0].images.[0]: Unpinned image") require.Equal(t, len(validator.warnings), 1) require.EqualError(t, validator.errors[0], ".components.[0].images.[2]: Invalid image format") @@ -151,7 +151,7 @@ func TestValidateSchema(t *testing.T) { }, } component := types.ZarfComponent{Files: zarfFiles} - checkForUnpinnedFiles(&validator, 0, component) + checkForUnpinnedFiles(&validator, 0, component, "") require.Equal(t, validator.warnings[0], ".components.[0].files.[0]: Unpinned file") require.Equal(t, len(validator.warnings), 1) }) diff --git a/src/test/e2e/12_lint_test.go b/src/test/e2e/12_lint_test.go index 1f0a6ec4ce..7734dffe52 100644 --- a/src/test/e2e/12_lint_test.go +++ b/src/test/e2e/12_lint_test.go @@ -2,6 +2,7 @@ package test import ( "fmt" + "os" "path/filepath" "testing" @@ -24,21 +25,22 @@ func TestLint(t *testing.T) { t.Log("E2E: Test lint on schema fail") path := filepath.Join("src", "test", "packages", "12-lint") + configPath := filepath.Join(path, "zarf-config.toml") + os.Setenv("ZARF_CONFIG", configPath) // In this case I'm guessing we should also remove color from the table? _, stderr, err := e2e.Zarf("prepare", "lint", path, "--no-color") require.Error(t, err, "Require an exit code since there was warnings / errors") // It's a bit weird to have a period here and not in the other warnings key := "WHATEVER_IMAGE" + require.Contains(t, stderr, "There are variables that are unset and won't be evaluated during lint") require.Contains(t, stderr, fmt.Sprintf(lang.PkgValidateTemplateDeprecation, key, key, key)) - require.Contains(t, stderr, "Package template \"WHATEVER_IMAGE\" is using the deprecated syntax ###ZARF_PKG_VAR_WHATEVER_IMAGE###. This will be removed in Zarf v1.0.0. Please update to ###ZARF_PKG_TMPL_WHATEVER_IMAGE###.") require.Contains(t, stderr, ".components.[2].repos.[0]: Unpinned repository") require.Contains(t, stderr, ".metadata: Additional property description1 is not allowed") require.Contains(t, stderr, ".components.[0].import: Additional property not-path is not allowed") - require.Contains(t, stderr, "There are variables that are unset and won't be evaluated during lint") - - // require.Contains(t, stderr, ".components.[1].images.[0] in linted-import: Unpinned image") - // require.Contains(t, stderr, ".components.[1].images.[2] in linted-import: Unpinned image") + require.Contains(t, stderr, ".components.[1].images.[0] linted-import: Unpinned image") + require.Contains(t, stderr, ".components.[2].images.[4]: Unpinned image") + //require.Contains(t, stderr, ".components.[1].images.[2] in linted-import: Unpinned image") }) } diff --git a/src/test/packages/12-lint/zarf-config.toml b/src/test/packages/12-lint/zarf-config.toml index 5f869ded0a..0b026e5642 100644 --- a/src/test/packages/12-lint/zarf-config.toml +++ b/src/test/packages/12-lint/zarf-config.toml @@ -1,2 +1,2 @@ [package.create.set] -BUSYBOX_IMAGE = 'latest' +BUSYBOX_IMAGE = "latest" From def023d7e1181a524e7af49889e9d27f3d5cbb82 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Tue, 5 Dec 2023 15:17:06 +0000 Subject: [PATCH 102/192] things are working seemingly --- src/pkg/packager/lint/lint.go | 6 +++++- src/test/e2e/12_lint_test.go | 4 ++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/pkg/packager/lint/lint.go b/src/pkg/packager/lint/lint.go index ef76ec1878..a3f4ce8d4f 100644 --- a/src/pkg/packager/lint/lint.go +++ b/src/pkg/packager/lint/lint.go @@ -89,10 +89,14 @@ func ValidateComposableComponenets(validator *Validator, createOpts types.ZarfCr if err != nil { return err } + originalPackage := validator.typedZarfPackage // Skipping initial component node := chain.Head.Next for node != nil { - lintComponent(validator, node.Index, node.ZarfComponent, fmt.Sprintf(" %s", node.RelativeToHead)) + validator.typedZarfPackage.Components = []types.ZarfComponent{node.ZarfComponent} + fillActiveTemplate(validator, createOpts) + lintComponent(validator, node.Index, validator.typedZarfPackage.Components[0], fmt.Sprintf(" %s", node.RelativeToHead)) + validator.typedZarfPackage = originalPackage node = node.Next } diff --git a/src/test/e2e/12_lint_test.go b/src/test/e2e/12_lint_test.go index 7734dffe52..de6398db6b 100644 --- a/src/test/e2e/12_lint_test.go +++ b/src/test/e2e/12_lint_test.go @@ -38,9 +38,9 @@ func TestLint(t *testing.T) { require.Contains(t, stderr, ".components.[2].repos.[0]: Unpinned repository") require.Contains(t, stderr, ".metadata: Additional property description1 is not allowed") require.Contains(t, stderr, ".components.[0].import: Additional property not-path is not allowed") - require.Contains(t, stderr, ".components.[1].images.[0] linted-import: Unpinned image") require.Contains(t, stderr, ".components.[2].images.[4]: Unpinned image") - //require.Contains(t, stderr, ".components.[1].images.[2] in linted-import: Unpinned image") + require.Contains(t, stderr, ".components.[1].images.[0] linted-import: Unpinned image") + require.Contains(t, stderr, ".components.[1].images.[2] linted-import: Unpinned image") }) } From e4ad437df6c5362c5fdb1eb973c37e28abe400d1 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Tue, 5 Dec 2023 15:19:56 +0000 Subject: [PATCH 103/192] deleting composer function --- src/pkg/packager/composer/compose.go | 63 ---------------------------- 1 file changed, 63 deletions(-) delete mode 100644 src/pkg/packager/composer/compose.go diff --git a/src/pkg/packager/composer/compose.go b/src/pkg/packager/composer/compose.go deleted file mode 100644 index 479b1f369a..0000000000 --- a/src/pkg/packager/composer/compose.go +++ /dev/null @@ -1,63 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: 2021-Present The Zarf Authors - -// Package composer contains functions for composing components within Zarf packages. -package composer - -import ( - "github.com/defenseunicorns/zarf/src/config" - "github.com/defenseunicorns/zarf/src/pkg/message" - "github.com/defenseunicorns/zarf/src/types" -) - -func ComposeComponents(zarfPackage *types.ZarfPackage, createOpts types.ZarfCreateOptions, - warnings []string) ([]string, error) { - components := []types.ZarfComponent{} - - pkgVars := zarfPackage.Variables - pkgConsts := zarfPackage.Constants - - for i, component := range zarfPackage.Components { - //TODO allow this to be a CLI option - arch := config.GetArch(zarfPackage.Metadata.Architecture) - - // filter by architecture - if !CompatibleComponent(component, arch, createOpts.Flavor) { - continue - } - - // if a match was found, strip flavor and architecture to reduce bloat in the package definition - component.Only.Cluster.Architecture = "" - component.Only.Flavor = "" - - // build the import chain - chain, err := NewImportChain(component, i, arch, createOpts.Flavor) - if err != nil { - return warnings, err - } - message.Debugf("%s", chain) - - // migrate any deprecated component configurations now - warnings := chain.Migrate(zarfPackage.Build) - warnings = append(warnings, warnings...) - - // get the composed component - composed, err := chain.Compose() - if err != nil { - return warnings, err - } - components = append(components, composed) - - // merge variables and constants - pkgVars = chain.MergeVariables(pkgVars) - pkgConsts = chain.MergeConstants(pkgConsts) - } - - // set the filtered + composed components - zarfPackage.Components = components - - zarfPackage.Variables = pkgVars - zarfPackage.Constants = pkgConsts - - return warnings, nil -} From 31a8ff2dc23689363d101d5ae075c7657ae773f8 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Tue, 5 Dec 2023 15:34:33 +0000 Subject: [PATCH 104/192] fix color wrap --- src/pkg/packager/lint/lint.go | 2 +- src/pkg/utils/random.go | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/pkg/packager/lint/lint.go b/src/pkg/packager/lint/lint.go index a3f4ce8d4f..942dde16f5 100644 --- a/src/pkg/packager/lint/lint.go +++ b/src/pkg/packager/lint/lint.go @@ -90,7 +90,7 @@ func ValidateComposableComponenets(validator *Validator, createOpts types.ZarfCr return err } originalPackage := validator.typedZarfPackage - // Skipping initial component + // Skipping initial component since it will be linted the usual way node := chain.Head.Next for node != nil { validator.typedZarfPackage.Components = []types.ZarfComponent{node.ZarfComponent} diff --git a/src/pkg/utils/random.go b/src/pkg/utils/random.go index 7966cd765a..0cc7b77bad 100644 --- a/src/pkg/utils/random.go +++ b/src/pkg/utils/random.go @@ -8,6 +8,7 @@ import ( "crypto/rand" "fmt" + "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/fatih/color" ) @@ -42,6 +43,10 @@ func First30last30(s string) string { // ColorWrap changes a string to an ansi color code and appends the default color to the end // preventing future characters from taking on the given color +// returns string as normal if color is disabled func ColorWrap(str string, attr color.Attribute) string { + if config.NoColor { + return str + } return fmt.Sprintf("\x1b[%dm%s\x1b[0m", attr, str) } From d6682ba9bd6dae67feaf46c9c8ed9490db0223ed Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Tue, 5 Dec 2023 15:49:09 +0000 Subject: [PATCH 105/192] remove duplication --- src/cmd/package.go | 23 ++++++++++++++--------- src/cmd/prepare.go | 10 +--------- 2 files changed, 15 insertions(+), 18 deletions(-) diff --git a/src/cmd/package.go b/src/cmd/package.go index 42fc9fbc34..9cc0464b56 100644 --- a/src/cmd/package.go +++ b/src/cmd/package.go @@ -16,6 +16,7 @@ import ( "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/packager/sources" "github.com/defenseunicorns/zarf/src/pkg/utils" + "github.com/defenseunicorns/zarf/src/types" "oras.land/oras-go/v2/registry" @@ -34,6 +35,18 @@ var packageCmd = &cobra.Command{ Short: lang.CmdPackageShort, } +func setBaseDirectory(args []string, pkgConfig *types.PackagerConfig) { + if len(args) > 0 { + pkgConfig.CreateOpts.BaseDir = args[0] + } else { + var err error + pkgConfig.CreateOpts.BaseDir, err = os.Getwd() + if err != nil { + message.Fatalf(err, lang.CmdPackageCreateErr, err.Error()) + } + } +} + var packageCreateCmd = &cobra.Command{ Use: "create [ DIRECTORY ]", Aliases: []string{"c"}, @@ -43,15 +56,7 @@ var packageCreateCmd = &cobra.Command{ Run: func(cmd *cobra.Command, args []string) { // If a directory was provided, use that as the base directory - if len(args) > 0 { - pkgConfig.CreateOpts.BaseDir = args[0] - } else { - var err error - pkgConfig.CreateOpts.BaseDir, err = os.Getwd() - if err != nil { - message.Fatalf(err, lang.CmdPackageCreateErr, err.Error()) - } - } + setBaseDirectory(args, &pkgConfig) var isCleanPathRegex = regexp.MustCompile(`^[a-zA-Z0-9\_\-\/\.\~\\:]+$`) if !isCleanPathRegex.MatchString(config.CommonOptions.CachePath) { diff --git a/src/cmd/prepare.go b/src/cmd/prepare.go index 14c0348a40..20edca7d8e 100644 --- a/src/cmd/prepare.go +++ b/src/cmd/prepare.go @@ -213,15 +213,7 @@ var lintCmd = &cobra.Command{ Short: lang.CmdPrepareLintShort, Long: lang.CmdPrepareLintLong, Run: func(cmd *cobra.Command, args []string) { - if len(args) > 0 { - pkgConfig.CreateOpts.BaseDir = args[0] - } else { - var err error - pkgConfig.CreateOpts.BaseDir, err = os.Getwd() - if err != nil { - message.Fatalf(err, lang.CmdPrepareLintErr, err.Error()) - } - } + setBaseDirectory(args, &pkgConfig) v := common.GetViper() pkgConfig.CreateOpts.SetVariables = helpers.TransformAndMergeMap( v.GetStringMapString(common.VPkgCreateSet), pkgConfig.CreateOpts.SetVariables, strings.ToUpper) From 09d0132b483f580d62f7fb992ab5e20061a8ce7a Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Tue, 5 Dec 2023 16:29:41 +0000 Subject: [PATCH 106/192] make docs + refactor --- .../100-cli-commands/zarf_prepare.md | 2 +- .../100-cli-commands/zarf_prepare_lint.md | 5 +++-- src/pkg/packager/lint/lint.go | 14 ++++++-------- src/pkg/packager/lint/lint_test.go | 15 +++++++++------ src/test/e2e/12_lint_test.go | 2 +- 5 files changed, 20 insertions(+), 18 deletions(-) diff --git a/docs/2-the-zarf-cli/100-cli-commands/zarf_prepare.md b/docs/2-the-zarf-cli/100-cli-commands/zarf_prepare.md index 9af49e3303..efc82e121b 100644 --- a/docs/2-the-zarf-cli/100-cli-commands/zarf_prepare.md +++ b/docs/2-the-zarf-cli/100-cli-commands/zarf_prepare.md @@ -27,7 +27,7 @@ Tools to help prepare assets for packaging * [zarf](zarf.md) - DevSecOps for Airgap * [zarf prepare find-images](zarf_prepare_find-images.md) - Evaluates components in a zarf file to identify images specified in their helm charts and manifests * [zarf prepare generate-config](zarf_prepare_generate-config.md) - Generates a config file for Zarf -* [zarf prepare lint](zarf_prepare_lint.md) - Lints package for schema and recommended practices +* [zarf prepare lint](zarf_prepare_lint.md) - Lints the given package for valid schema and recommended practices * [zarf prepare patch-git](zarf_prepare_patch-git.md) - Converts all .git URLs to the specified Zarf HOST and with the Zarf URL pattern in a given FILE. NOTE: This should only be used for manifests that are not mutated by the Zarf Agent Mutating Webhook. * [zarf prepare sha256sum](zarf_prepare_sha256sum.md) - Generates a SHA256SUM for the given file diff --git a/docs/2-the-zarf-cli/100-cli-commands/zarf_prepare_lint.md b/docs/2-the-zarf-cli/100-cli-commands/zarf_prepare_lint.md index 9db10cc901..648956db33 100644 --- a/docs/2-the-zarf-cli/100-cli-commands/zarf_prepare_lint.md +++ b/docs/2-the-zarf-cli/100-cli-commands/zarf_prepare_lint.md @@ -1,7 +1,7 @@ # zarf prepare lint -Lints package for schema and recommended practices +Lints the given package for valid schema and recommended practices ## Synopsis @@ -14,7 +14,8 @@ zarf prepare lint [ DIRECTORY ] [flags] ## Options ``` - -h, --help help for lint + -h, --help help for lint + --set stringToString Specify package variables to set on the command line (KEY=value) (default []) ``` ## Options inherited from parent commands diff --git a/src/pkg/packager/lint/lint.go b/src/pkg/packager/lint/lint.go index 942dde16f5..6b122406a1 100644 --- a/src/pkg/packager/lint/lint.go +++ b/src/pkg/packager/lint/lint.go @@ -75,7 +75,6 @@ func ValidateComposableComponenets(validator *Validator, createOpts types.ZarfCr //TODO allow this to be a CLI option arch := config.GetArch(validator.typedZarfPackage.Metadata.Architecture) - // filter by architecture if !composer.CompatibleComponent(component, arch, createOpts.Flavor) { continue } @@ -84,11 +83,11 @@ func ValidateComposableComponenets(validator *Validator, createOpts types.ZarfCr component.Only.Cluster.Architecture = "" component.Only.Flavor = "" - // build the import chain chain, err := composer.NewImportChain(component, i, arch, createOpts.Flavor) if err != nil { return err } + originalPackage := validator.typedZarfPackage // Skipping initial component since it will be linted the usual way node := chain.Head.Next @@ -99,7 +98,6 @@ func ValidateComposableComponenets(validator *Validator, createOpts types.ZarfCr validator.typedZarfPackage = originalPackage node = node.Next } - } return nil } @@ -146,7 +144,7 @@ func fillActiveTemplate(validator *Validator, createOpts types.ZarfCreateOptions } // Add special variable for the current package architecture - templateMap[types.ZarfPackageArch] = config.GetArch(validator.typedZarfPackage.Metadata.Architecture, validator.typedZarfPackage.Build.Architecture) + templateMap[types.ZarfPackageArch] = config.GetArch(validator.typedZarfPackage.Metadata.Architecture) if unsetVarWarning { validator.warnings = append([]string{"There are variables that are unset and won't be evaluated during lint"}, validator.warnings...) @@ -181,13 +179,13 @@ func lintComponents(validator *Validator) { } func lintComponent(validator *Validator, index int, component types.ZarfComponent, path string) { - checkforUnpinnedRepos(validator, index, component, path) + checkForUnpinnedRepos(validator, index, component, path) checkForUnpinnedImages(validator, index, component, path) checkForUnpinnedFiles(validator, index, component, path) checkForVarInComponentImport(validator, index, component, path) } -func checkforUnpinnedRepos(validator *Validator, index int, component types.ZarfComponent, path string) { +func checkForUnpinnedRepos(validator *Validator, index int, component types.ZarfComponent, path string) { for j, repo := range component.Repos { if !isPinnedRepo(repo) { validator.addWarning(fmt.Sprintf(".components.[%d].repos.[%d]%s: Unpinned repository", index, j, path)) @@ -199,11 +197,11 @@ func checkForUnpinnedImages(validator *Validator, index int, component types.Zar for j, image := range component.Images { pinnedImage, err := isPinnedImage(image) if err != nil { - validator.addError(fmt.Errorf(".components.[%d].images.[%d]%s: Invalid image format", index, j, path)) + validator.addError(fmt.Errorf(".components.[%d].images.[%d]%s: Invalid image format %s", index, j, path, image)) continue } if !pinnedImage { - validator.addWarning(fmt.Sprintf(".components.[%d].images.[%d]%s: Unpinned image", index, j, path)) + validator.addWarning(fmt.Sprintf(".components.[%d].images.[%d]%s: Unpinned image %s", index, j, path, image)) } } } diff --git a/src/pkg/packager/lint/lint_test.go b/src/pkg/packager/lint/lint_test.go index 603f13e12a..e87f9db200 100644 --- a/src/pkg/packager/lint/lint_test.go +++ b/src/pkg/packager/lint/lint_test.go @@ -6,6 +6,7 @@ package lint import ( "errors" + "fmt" "os" "testing" @@ -124,21 +125,24 @@ func TestValidateSchema(t *testing.T) { "https://github.com/defenseunicorns/zarf-public-test.git", "https://dev.azure.com/defenseunicorns/zarf-public-test/_git/zarf-public-test@v0.0.1", "https://gitlab.com/gitlab-org/build/omnibus-mirror/pcre2/-/tree/vreverse?ref_type=heads"}} - checkforUnpinnedRepos(&validator, 0, component, "") + checkForUnpinnedRepos(&validator, 0, component, "") require.Equal(t, validator.warnings[0], ".components.[0].repos.[0]: Unpinned repository") require.Equal(t, len(validator.warnings), 1) }) t.Run("Unpinnned image warning", func(t *testing.T) { validator := Validator{} + unpinnedImage := "registry.com:9001/whatever/image:1.0.0" + badImage := "badimage:badimage@@sha256:3fbc632167424a6d997e74f5" component := types.ZarfComponent{Images: []string{ - "registry.com:9001/whatever/image:1.0.0", + unpinnedImage, "busybox:latest@sha256:3fbc632167424a6d997e74f52b878d7cc478225cffac6bc977eedfe51c7f4e79", - "badimage:badimage@@sha256:3fbc632167424a6d997e74f5"}} + badImage}} checkForUnpinnedImages(&validator, 0, component, "") - require.Equal(t, validator.warnings[0], ".components.[0].images.[0]: Unpinned image") + require.Equal(t, fmt.Sprintf(".components.[0].images.[0]: Unpinned image %s", unpinnedImage), validator.warnings[0]) require.Equal(t, len(validator.warnings), 1) - require.EqualError(t, validator.errors[0], ".components.[0].images.[2]: Invalid image format") + expectedErr := fmt.Sprintf(".components.[0].images.[2]: Invalid image format %s", badImage) + require.EqualError(t, validator.errors[0], expectedErr) require.Equal(t, len(validator.errors), 1) }) @@ -147,7 +151,6 @@ func TestValidateSchema(t *testing.T) { zarfFiles := []types.ZarfFile{ { Source: "http://example.com/file.zip", - Target: "/path/to/target", }, } component := types.ZarfComponent{Files: zarfFiles} diff --git a/src/test/e2e/12_lint_test.go b/src/test/e2e/12_lint_test.go index de6398db6b..cabee40d0d 100644 --- a/src/test/e2e/12_lint_test.go +++ b/src/test/e2e/12_lint_test.go @@ -28,7 +28,7 @@ func TestLint(t *testing.T) { configPath := filepath.Join(path, "zarf-config.toml") os.Setenv("ZARF_CONFIG", configPath) // In this case I'm guessing we should also remove color from the table? - _, stderr, err := e2e.Zarf("prepare", "lint", path, "--no-color") + _, stderr, err := e2e.Zarf("prepare", "lint", path) require.Error(t, err, "Require an exit code since there was warnings / errors") // It's a bit weird to have a period here and not in the other warnings From f841f75e93095a7fdac7286cdd22c690ed8fc7a2 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Tue, 5 Dec 2023 17:15:28 +0000 Subject: [PATCH 107/192] WIP, changing message output --- src/pkg/packager/lint/lint.go | 15 ++++++++++++- src/pkg/packager/lint/validator.go | 36 ++++++++++++++++++++++++++---- 2 files changed, 46 insertions(+), 5 deletions(-) diff --git a/src/pkg/packager/lint/lint.go b/src/pkg/packager/lint/lint.go index 6b122406a1..ca3bd005d1 100644 --- a/src/pkg/packager/lint/lint.go +++ b/src/pkg/packager/lint/lint.go @@ -94,7 +94,7 @@ func ValidateComposableComponenets(validator *Validator, createOpts types.ZarfCr for node != nil { validator.typedZarfPackage.Components = []types.ZarfComponent{node.ZarfComponent} fillActiveTemplate(validator, createOpts) - lintComponent(validator, node.Index, validator.typedZarfPackage.Components[0], fmt.Sprintf(" %s", node.RelativeToHead)) + lintComponent(validator, node.Index, validator.typedZarfPackage.Components[0], node.RelativeToHead) validator.typedZarfPackage = originalPackage node = node.Next } @@ -195,13 +195,26 @@ func checkForUnpinnedRepos(validator *Validator, index int, component types.Zarf func checkForUnpinnedImages(validator *Validator, index int, component types.ZarfComponent, path string) { for j, image := range component.Images { + imageYqPath := ".components.[%d].images.[%d]" pinnedImage, err := isPinnedImage(image) if err != nil { validator.addError(fmt.Errorf(".components.[%d].images.[%d]%s: Invalid image format %s", index, j, path, image)) + validator.addError2(ValidatorMessage{ + yqPath: fmt.Sprintf(imageYqPath, index, j), + filePath: path, + description: "Invalid Image format", + item: image, + }) continue } if !pinnedImage { validator.addWarning(fmt.Sprintf(".components.[%d].images.[%d]%s: Unpinned image %s", index, j, path, image)) + validator.addWarning2(ValidatorMessage{ + yqPath: fmt.Sprintf(imageYqPath, index, j), + filePath: path, + description: "Unpinned image", + item: image, + }) } } } diff --git a/src/pkg/packager/lint/validator.go b/src/pkg/packager/lint/validator.go index f88c22de5f..572daad802 100644 --- a/src/pkg/packager/lint/validator.go +++ b/src/pkg/packager/lint/validator.go @@ -13,10 +13,30 @@ import ( "github.com/fatih/color" ) +type ValidatorMessage struct { + yqPath string + filePath string + description string + item string +} + +func (v ValidatorMessage) String() string { + if v.filePath != "" { + v.filePath = fmt.Sprintf(" %s", v.filePath) + } + if v.item != "" { + v.item = fmt.Sprintf(" %s", v.item) + } + return fmt.Sprintf("%s%s: %s%s", + v.yqPath, v.filePath, v.description, v.item) +} + // Validator holds the warnings/errors and messaging that we get from validation type Validator struct { warnings []string + warnings2 []ValidatorMessage errors []error + errors2 []ValidatorMessage jsonSchema []byte typedZarfPackage types.ZarfPackage untypedZarfPackage interface{} @@ -39,11 +59,11 @@ func (v Validator) printValidationTable() { if v.hasWarnings() || v.hasErrors() { header := []string{"Type", "Message"} connectData := [][]string{} - for _, warning := range v.warnings { - connectData = append(connectData, []string{utils.ColorWrap("Warning", color.FgYellow), warning}) + for _, warning := range v.warnings2 { + connectData = append(connectData, []string{utils.ColorWrap("Warning", color.FgYellow), warning.String()}) } - for _, err := range v.errors { - connectData = append(connectData, []string{utils.ColorWrap("Error", color.FgRed), err.Error()}) + for _, err := range v.errors2 { + connectData = append(connectData, []string{utils.ColorWrap("Error", color.FgRed), err.String()}) } message.Table(header, connectData) message.Info(fmt.Sprintf("%d warnings and %d errors in %q", @@ -63,6 +83,14 @@ func (v *Validator) addWarning(message string) { v.warnings = append(v.warnings, message) } +func (v *Validator) addWarning2(vmessage ValidatorMessage) { + v.warnings2 = append(v.warnings2, vmessage) +} + func (v *Validator) addError(err error) { v.errors = append(v.errors, err) } + +func (v *Validator) addError2(vMessage ValidatorMessage) { + v.errors2 = append(v.errors2, vMessage) +} From f1fd85653b609c041666fd29cfa0ca00ed8cbd80 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Tue, 5 Dec 2023 17:46:00 +0000 Subject: [PATCH 108/192] refactor message --- src/pkg/packager/lint/lint.go | 55 ++++++++++++++++++++--------- src/pkg/packager/lint/validator.go | 29 ++++++--------- src/test/e2e/12_lint_test.go | 2 +- src/test/packages/12-lint/zarf.yaml | 1 - 4 files changed, 51 insertions(+), 36 deletions(-) diff --git a/src/pkg/packager/lint/lint.go b/src/pkg/packager/lint/lint.go index ca3bd005d1..2650b6677b 100644 --- a/src/pkg/packager/lint/lint.go +++ b/src/pkg/packager/lint/lint.go @@ -114,7 +114,9 @@ func fillActiveTemplate(validator *Validator, createOpts types.ZarfCreateOptions for key := range yamlTemplates { if deprecated { - validator.warnings = append(validator.warnings, fmt.Sprintf(lang.PkgValidateTemplateDeprecation, key, key, key)) + validator.addWarning(ValidatorMessage{ + description: fmt.Sprintf(lang.PkgValidateTemplateDeprecation, key, key, key), + }) } _, present := createOpts.SetVariables[key] if !present { @@ -147,7 +149,9 @@ func fillActiveTemplate(validator *Validator, createOpts types.ZarfCreateOptions templateMap[types.ZarfPackageArch] = config.GetArch(validator.typedZarfPackage.Metadata.Architecture) if unsetVarWarning { - validator.warnings = append([]string{"There are variables that are unset and won't be evaluated during lint"}, validator.warnings...) + validator.warnings = append([]ValidatorMessage{{ + description: "There are variables that are unset and won't be evaluated during lint", + }}, validator.warnings...) } return utils.ReloadYamlTemplate(&validator.typedZarfPackage, templateMap) @@ -187,20 +191,25 @@ func lintComponent(validator *Validator, index int, component types.ZarfComponen func checkForUnpinnedRepos(validator *Validator, index int, component types.ZarfComponent, path string) { for j, repo := range component.Repos { + repoYqPath := fmt.Sprintf(".components.[%d].repos.[%d]", index, j) if !isPinnedRepo(repo) { - validator.addWarning(fmt.Sprintf(".components.[%d].repos.[%d]%s: Unpinned repository", index, j, path)) + validator.addWarning(ValidatorMessage{ + yqPath: repoYqPath, + filePath: path, + description: "Unpinned repository", + item: repo, + }) } } } func checkForUnpinnedImages(validator *Validator, index int, component types.ZarfComponent, path string) { for j, image := range component.Images { - imageYqPath := ".components.[%d].images.[%d]" + imageYqPath := fmt.Sprintf(".components.[%d].images.[%d]", index, j) pinnedImage, err := isPinnedImage(image) if err != nil { - validator.addError(fmt.Errorf(".components.[%d].images.[%d]%s: Invalid image format %s", index, j, path, image)) - validator.addError2(ValidatorMessage{ - yqPath: fmt.Sprintf(imageYqPath, index, j), + validator.addError(ValidatorMessage{ + yqPath: imageYqPath, filePath: path, description: "Invalid Image format", item: image, @@ -208,9 +217,8 @@ func checkForUnpinnedImages(validator *Validator, index int, component types.Zar continue } if !pinnedImage { - validator.addWarning(fmt.Sprintf(".components.[%d].images.[%d]%s: Unpinned image %s", index, j, path, image)) - validator.addWarning2(ValidatorMessage{ - yqPath: fmt.Sprintf(imageYqPath, index, j), + validator.addWarning(ValidatorMessage{ + yqPath: imageYqPath, filePath: path, description: "Unpinned image", item: image, @@ -221,18 +229,32 @@ func checkForUnpinnedImages(validator *Validator, index int, component types.Zar func checkForUnpinnedFiles(validator *Validator, index int, component types.ZarfComponent, path string) { for j, file := range component.Files { + fileYqPath := fmt.Sprintf(".components.[%d].files.[%d]%s: Unpinned file", index, j) if file.Shasum == "" && helpers.IsURL(file.Source) { - validator.addWarning(fmt.Sprintf(".components.[%d].files.[%d]%s: Unpinned file", index, j, path)) + validator.addWarning(ValidatorMessage{ + yqPath: fileYqPath, + filePath: path, + description: "Unpinned image", + item: file.Source, + }) } } } func checkForVarInComponentImport(validator *Validator, index int, component types.ZarfComponent, path string) { if strings.Contains(component.Import.Path, types.ZarfPackageTemplatePrefix) { - validator.addWarning(fmt.Sprintf(".components.[%d].import.path%s: Will not resolve ZARF_PKG_TMPL_* variables", index, path)) + validator.addWarning(ValidatorMessage{ + yqPath: fmt.Sprintf(".components.[%d].import.path", index), + filePath: path, + description: "Will not resolve ZARF_PKG_TMPL_* variables", + }) } if strings.Contains(component.Import.URL, types.ZarfPackageTemplatePrefix) { - validator.addWarning(fmt.Sprintf(".components.[%d].import.url%s: Will not resolve ZARF_PKG_TMPL_* variables", index, path)) + validator.addWarning(ValidatorMessage{ + yqPath: fmt.Sprintf(".components.[%d].import.url", index), + filePath: path, + description: "Will not resolve ZARF_PKG_TMPL_* variables", + }) } } @@ -260,9 +282,10 @@ func validateSchema(validator *Validator) error { if !result.Valid() { for _, desc := range result.Errors() { - err := fmt.Errorf( - "%s: %s", makeFieldPathYqCompat(desc.Field()), desc.Description()) - validator.addError(err) + validator.addError(ValidatorMessage{ + yqPath: makeFieldPathYqCompat(desc.Field()), + description: desc.Description(), + }) } } diff --git a/src/pkg/packager/lint/validator.go b/src/pkg/packager/lint/validator.go index 572daad802..34b5ea9ff0 100644 --- a/src/pkg/packager/lint/validator.go +++ b/src/pkg/packager/lint/validator.go @@ -27,16 +27,17 @@ func (v ValidatorMessage) String() string { if v.item != "" { v.item = fmt.Sprintf(" %s", v.item) } + if v.filePath == "" && v.yqPath == "" && v.item == "" { + return v.description + } return fmt.Sprintf("%s%s: %s%s", v.yqPath, v.filePath, v.description, v.item) } // Validator holds the warnings/errors and messaging that we get from validation type Validator struct { - warnings []string - warnings2 []ValidatorMessage - errors []error - errors2 []ValidatorMessage + warnings []ValidatorMessage + errors []ValidatorMessage jsonSchema []byte typedZarfPackage types.ZarfPackage untypedZarfPackage interface{} @@ -59,10 +60,10 @@ func (v Validator) printValidationTable() { if v.hasWarnings() || v.hasErrors() { header := []string{"Type", "Message"} connectData := [][]string{} - for _, warning := range v.warnings2 { + for _, warning := range v.warnings { connectData = append(connectData, []string{utils.ColorWrap("Warning", color.FgYellow), warning.String()}) } - for _, err := range v.errors2 { + for _, err := range v.errors { connectData = append(connectData, []string{utils.ColorWrap("Error", color.FgRed), err.String()}) } message.Table(header, connectData) @@ -79,18 +80,10 @@ func (v Validator) hasErrors() bool { return len(v.errors) > 0 } -func (v *Validator) addWarning(message string) { - v.warnings = append(v.warnings, message) -} - -func (v *Validator) addWarning2(vmessage ValidatorMessage) { - v.warnings2 = append(v.warnings2, vmessage) -} - -func (v *Validator) addError(err error) { - v.errors = append(v.errors, err) +func (v *Validator) addWarning(vmessage ValidatorMessage) { + v.warnings = append(v.warnings, vmessage) } -func (v *Validator) addError2(vMessage ValidatorMessage) { - v.errors2 = append(v.errors2, vMessage) +func (v *Validator) addError(vMessage ValidatorMessage) { + v.errors = append(v.errors, vMessage) } diff --git a/src/test/e2e/12_lint_test.go b/src/test/e2e/12_lint_test.go index cabee40d0d..daecdac79f 100644 --- a/src/test/e2e/12_lint_test.go +++ b/src/test/e2e/12_lint_test.go @@ -38,7 +38,7 @@ func TestLint(t *testing.T) { require.Contains(t, stderr, ".components.[2].repos.[0]: Unpinned repository") require.Contains(t, stderr, ".metadata: Additional property description1 is not allowed") require.Contains(t, stderr, ".components.[0].import: Additional property not-path is not allowed") - require.Contains(t, stderr, ".components.[2].images.[4]: Unpinned image") + require.Contains(t, stderr, ".components.[2].images.[3]: Unpinned image") require.Contains(t, stderr, ".components.[1].images.[0] linted-import: Unpinned image") require.Contains(t, stderr, ".components.[1].images.[2] linted-import: Unpinned image") }) diff --git a/src/test/packages/12-lint/zarf.yaml b/src/test/packages/12-lint/zarf.yaml index af4343fc1d..9a26dd42d0 100644 --- a/src/test/packages/12-lint/zarf.yaml +++ b/src/test/packages/12-lint/zarf.yaml @@ -23,7 +23,6 @@ components: - https://dev.azure.com/defenseunicorns/zarf-public-test/_git/zarf-public-test@v0.0.1 - https://gitlab.com/gitlab-org/build/omnibus-mirror/pcre2/-/tree/vreverse?ref_type=heads images: - - ghcr.io/kiwix/kiwix-serve:3.5.0-2 - registry.com:9001/whatever/image:1.0.0 - busybox@sha256:3fbc632167424a6d997e74f52b878d7cc478225cffac6bc977eedfe51c7f4e79 - busybox:###ZARF_PKG_VAR_WHATEVER_IMAGE### From f03eb235287943ae39ebd0ec8b072b6d6c834115 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Tue, 5 Dec 2023 19:20:51 +0000 Subject: [PATCH 109/192] WIP --- src/pkg/packager/lint/lint.go | 4 ++-- src/pkg/packager/lint/lint_test.go | 35 ++++++++++++------------------ 2 files changed, 16 insertions(+), 23 deletions(-) diff --git a/src/pkg/packager/lint/lint.go b/src/pkg/packager/lint/lint.go index 2650b6677b..6de0278e2a 100644 --- a/src/pkg/packager/lint/lint.go +++ b/src/pkg/packager/lint/lint.go @@ -229,12 +229,12 @@ func checkForUnpinnedImages(validator *Validator, index int, component types.Zar func checkForUnpinnedFiles(validator *Validator, index int, component types.ZarfComponent, path string) { for j, file := range component.Files { - fileYqPath := fmt.Sprintf(".components.[%d].files.[%d]%s: Unpinned file", index, j) + fileYqPath := fmt.Sprintf(".components.[%d].files.[%d]", index, j) if file.Shasum == "" && helpers.IsURL(file.Source) { validator.addWarning(ValidatorMessage{ yqPath: fileYqPath, filePath: path, - description: "Unpinned image", + description: "Unpinned file", item: file.Source, }) } diff --git a/src/pkg/packager/lint/lint_test.go b/src/pkg/packager/lint/lint_test.go index e87f9db200..dd4d6dd287 100644 --- a/src/pkg/packager/lint/lint_test.go +++ b/src/pkg/packager/lint/lint_test.go @@ -97,8 +97,8 @@ func TestValidateSchema(t *testing.T) { validator := Validator{untypedZarfPackage: unmarshalledYaml, jsonSchema: getZarfSchema(t)} err := validateSchema(&validator) require.NoError(t, err) - require.EqualError(t, validator.errors[0], ".components.[0].import: Additional property not-path is not allowed") - require.EqualError(t, validator.errors[1], ".components.[1].import.path: Invalid type. Expected: string, given: integer") + require.Equal(t, validator.errors[0].String(), ".components.[0].import: Additional property not-path is not allowed") + require.Equal(t, validator.errors[1].String(), ".components.[1].import.path: Invalid type. Expected: string, given: integer") }) t.Run("Template in component import success", func(t *testing.T) { @@ -115,8 +115,10 @@ func TestValidateSchema(t *testing.T) { URLComponent := types.ZarfComponent{Import: types.ZarfComponentImport{URL: "oci://###ZARF_PKG_TMPL_ZEBRA###"}} checkForVarInComponentImport(&validator, 2, pathComponent, "") checkForVarInComponentImport(&validator, 3, URLComponent, "") - require.Equal(t, validator.warnings[0], ".components.[2].import.path: Will not resolve ZARF_PKG_TMPL_* variables") - require.Equal(t, validator.warnings[1], ".components.[3].import.url: Will not resolve ZARF_PKG_TMPL_* variables") + require.Equal(t, validator.warnings[0].String(), + ".components.[2].import.path: Will not resolve ZARF_PKG_TMPL_* variables") + require.Equal(t, validator.warnings[1].String(), + ".components.[3].import.url: Will not resolve ZARF_PKG_TMPL_* variables") }) t.Run("Unpinnned repo warning", func(t *testing.T) { @@ -126,7 +128,7 @@ func TestValidateSchema(t *testing.T) { "https://dev.azure.com/defenseunicorns/zarf-public-test/_git/zarf-public-test@v0.0.1", "https://gitlab.com/gitlab-org/build/omnibus-mirror/pcre2/-/tree/vreverse?ref_type=heads"}} checkForUnpinnedRepos(&validator, 0, component, "") - require.Equal(t, validator.warnings[0], ".components.[0].repos.[0]: Unpinned repository") + require.Equal(t, validator.warnings[0].String(), ".components.[0].repos.[0]: Unpinned repository") require.Equal(t, len(validator.warnings), 1) }) @@ -139,24 +141,26 @@ func TestValidateSchema(t *testing.T) { "busybox:latest@sha256:3fbc632167424a6d997e74f52b878d7cc478225cffac6bc977eedfe51c7f4e79", badImage}} checkForUnpinnedImages(&validator, 0, component, "") - require.Equal(t, fmt.Sprintf(".components.[0].images.[0]: Unpinned image %s", unpinnedImage), validator.warnings[0]) + require.Equal(t, fmt.Sprintf(".components.[0].images.[0]: Unpinned image %s", unpinnedImage), validator.warnings[0].String()) require.Equal(t, len(validator.warnings), 1) expectedErr := fmt.Sprintf(".components.[0].images.[2]: Invalid image format %s", badImage) - require.EqualError(t, validator.errors[0], expectedErr) + require.Equal(t, validator.errors[0].String(), expectedErr) require.Equal(t, len(validator.errors), 1) }) t.Run("Unpinnned file warning", func(t *testing.T) { validator := Validator{} + filename := "http://example.com/file.zip" zarfFiles := []types.ZarfFile{ { - Source: "http://example.com/file.zip", + Source: filename, }, } component := types.ZarfComponent{Files: zarfFiles} checkForUnpinnedFiles(&validator, 0, component, "") - require.Equal(t, validator.warnings[0], ".components.[0].files.[0]: Unpinned file") - require.Equal(t, len(validator.warnings), 1) + expected := fmt.Sprintf(".components.[0].files.[0]: Unpinned file %s", filename) + require.Equal(t, expected, validator.warnings[0].String()) + require.Equal(t, 1, len(validator.warnings)) }) t.Run("Wrap standalone numbers in bracket", func(t *testing.T) { @@ -172,12 +176,6 @@ func TestValidateSchema(t *testing.T) { require.Equal(t, input, acutal) }) - // t.Run("Validate composible components", func(t *testing.T) { - // input := "" - // acutal := makeFieldPathYqCompat(input) - // require.Equal(t, input, acutal) - // }) - t.Run("isImagePinned", func(t *testing.T) { t.Parallel() tests := []struct { @@ -190,11 +188,6 @@ func TestValidateSchema(t *testing.T) { expected: false, err: nil, }, - { - input: "ghcr.io/defenseunicorns/pepr/controller", - expected: false, - err: nil, - }, { input: "ghcr.io/defenseunicorns/pepr/controller:v0.15.0", expected: false, From dc76d861211770dd6157166a9d3bef27ebe79fc7 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Wed, 6 Dec 2023 16:53:55 +0000 Subject: [PATCH 110/192] adding getters for node --- src/pkg/packager/composer/list.go | 68 ++++++++++++++++---------- src/pkg/packager/composer/list_test.go | 4 +- src/pkg/packager/composer/oci.go | 6 +-- src/pkg/packager/lint/lint.go | 22 +++++---- 4 files changed, 60 insertions(+), 40 deletions(-) diff --git a/src/pkg/packager/composer/list.go b/src/pkg/packager/composer/list.go index bbe5c2c97c..5b2dade9f3 100644 --- a/src/pkg/packager/composer/list.go +++ b/src/pkg/packager/composer/list.go @@ -24,17 +24,33 @@ type Node struct { types.ZarfComponent // We are going to use this in lint so we can give the correct path - // We can give the Index of the zarf component within the file that's being composed on + // We can give the index of the zarf component within the file that's being composed on // So users can have the yaml path - Index int + index int vars []types.ZarfPackageVariable consts []types.ZarfPackageConstant - RelativeToHead string + relativeToHead string - Prev *Node - Next *Node + prev *Node + next *Node +} + +func (n *Node) GetIndex() int { + return n.index +} + +func (n *Node) GetRelativeToHead() string { + return n.relativeToHead +} + +func (n *Node) Next() *Node { + return n.next +} + +func (n *Node) Prev() *Node { + return n.prev } // ImportName returns the name of the component to import @@ -60,20 +76,20 @@ type ImportChain struct { func (ic *ImportChain) append(c types.ZarfComponent, index int, relativeToHead string, vars []types.ZarfPackageVariable, consts []types.ZarfPackageConstant) { node := &Node{ ZarfComponent: c, - Index: index, - RelativeToHead: relativeToHead, + index: index, + relativeToHead: relativeToHead, vars: vars, consts: consts, - Prev: nil, - Next: nil, + prev: nil, + next: nil, } if ic.Head == nil { ic.Head = node ic.Tail = node } else { p := ic.Tail - node.Prev = p - p.Next = node + node.prev = p + p.next = node ic.Tail = node } } @@ -107,11 +123,11 @@ func NewImportChain(head types.ZarfComponent, index int, arch, flavor string) (* } // ensure that remote components are not importing other remote components - if node.Prev != nil && node.Prev.Import.URL != "" && isRemote { + if node.prev != nil && node.prev.Import.URL != "" && isRemote { return ic, fmt.Errorf("detected malformed import chain, cannot import remote components from remote components") } // ensure that remote components are not importing local components - if node.Prev != nil && node.Prev.Import.URL != "" && isLocal { + if node.prev != nil && node.prev.Import.URL != "" && isLocal { return ic, fmt.Errorf("detected malformed import chain, cannot import local components from remote components") } @@ -125,10 +141,10 @@ func NewImportChain(head types.ZarfComponent, index int, arch, flavor string) (* // this is O(n^2) but the import chain should be small prev := node for prev != nil { - if prev.RelativeToHead == relativeToHead { + if prev.relativeToHead == relativeToHead { return ic, fmt.Errorf("detected circular import chain: %s", strings.Join(history, " -> ")) } - prev = prev.Prev + prev = prev.prev } // this assumes the composed package is following the zarf layout @@ -175,14 +191,14 @@ func NewImportChain(head types.ZarfComponent, index int, arch, flavor string) (* } } ic.append(found[0], index, filepath.Join(history...), pkg.Variables, pkg.Constants) - node = node.Next + node = node.next } return ic, nil } // String returns a string representation of the import chain func (ic *ImportChain) String() string { - if ic.Head.Next == nil { + if ic.Head.next == nil { return fmt.Sprintf("component %q imports nothing", ic.Head.Name) } @@ -196,7 +212,7 @@ func (ic *ImportChain) String() string { s.WriteString(fmt.Sprintf("component %q imports %q in %s", ic.Head.Name, name, ic.Head.Import.URL)) } - node := ic.Head.Next + node := ic.Head.next for node != ic.Tail { name := node.ImportName() s.WriteString(", which imports ") @@ -206,7 +222,7 @@ func (ic *ImportChain) String() string { s.WriteString(fmt.Sprintf("%q in %s", name, node.Import.URL)) } - node = node.Next + node = node.next } return s.String() @@ -219,7 +235,7 @@ func (ic *ImportChain) Migrate(build types.ZarfBuildData) (warnings []string) { migrated, w := deprecated.MigrateComponent(build, node.ZarfComponent) node.ZarfComponent = migrated warnings = append(warnings, w...) - node = node.Next + node = node.next } if len(warnings) > 0 { final := fmt.Sprintf("migrations were performed on the import chain of: %q", ic.Head.Name) @@ -233,7 +249,7 @@ func (ic *ImportChain) Migrate(build types.ZarfBuildData) (warnings []string) { func (ic *ImportChain) Compose() (composed types.ZarfComponent, err error) { composed = ic.Tail.ZarfComponent - if ic.Tail.Prev == nil { + if ic.Tail.prev == nil { // only had one component in the import chain return composed, nil } @@ -248,7 +264,7 @@ func (ic *ImportChain) Compose() (composed types.ZarfComponent, err error) { // start overriding with the tail node node := ic.Tail for node != nil { - fixPaths(&node.ZarfComponent, node.RelativeToHead) + fixPaths(&node.ZarfComponent, node.relativeToHead) // perform overrides here overrideMetadata(&composed, node.ZarfComponent) @@ -256,9 +272,9 @@ func (ic *ImportChain) Compose() (composed types.ZarfComponent, err error) { overrideResources(&composed, node.ZarfComponent) overrideActions(&composed, node.ZarfComponent) - composeExtensions(&composed, node.ZarfComponent, node.RelativeToHead) + composeExtensions(&composed, node.ZarfComponent, node.relativeToHead) - node = node.Prev + node = node.prev } return composed, nil @@ -274,7 +290,7 @@ func (ic *ImportChain) MergeVariables(existing []types.ZarfPackageVariable) (mer for node != nil { // merge the vars merged = helpers.MergeSlices(node.vars, merged, exists) - node = node.Prev + node = node.prev } merged = helpers.MergeSlices(existing, merged, exists) @@ -291,7 +307,7 @@ func (ic *ImportChain) MergeConstants(existing []types.ZarfPackageConstant) (mer for node != nil { // merge the consts merged = helpers.MergeSlices(node.consts, merged, exists) - node = node.Prev + node = node.prev } merged = helpers.MergeSlices(existing, merged, exists) diff --git a/src/pkg/packager/composer/list_test.go b/src/pkg/packager/composer/list_test.go index e2aa1db018..41d2cd62bc 100644 --- a/src/pkg/packager/composer/list_test.go +++ b/src/pkg/packager/composer/list_test.go @@ -311,8 +311,8 @@ func TestMerging(t *testing.T) { }, }, } - head.Next = &tail - tail.Prev = &head + head.next = &tail + tail.prev = &head testIC := &ImportChain{Head: &head, Tail: &tail} testCases := []testCase{ diff --git a/src/pkg/packager/composer/oci.go b/src/pkg/packager/composer/oci.go index 7c10b279ab..b2eea815aa 100644 --- a/src/pkg/packager/composer/oci.go +++ b/src/pkg/packager/composer/oci.go @@ -37,14 +37,14 @@ func (ic *ImportChain) getRemote(url string) (*oci.OrasRemote, error) { // ContainsOCIImport returns true if the import chain contains a remote import func (ic *ImportChain) ContainsOCIImport() bool { // only the 2nd to last node may have a remote import - return ic.Tail.Prev != nil && ic.Tail.Prev.Import.URL != "" + return ic.Tail.prev != nil && ic.Tail.prev.Import.URL != "" } func (ic *ImportChain) fetchOCISkeleton() error { if !ic.ContainsOCIImport() { return nil } - node := ic.Tail.Prev + node := ic.Tail.prev remote, err := ic.getRemote(node.Import.URL) if err != nil { return err @@ -114,7 +114,7 @@ func (ic *ImportChain) fetchOCISkeleton() error { } // the tail node is the only node whose relativeToHead is based solely upon cwd<->cache // contrary to the other nodes, which are based upon the previous node - ic.Tail.RelativeToHead = rel + ic.Tail.relativeToHead = rel if oci.IsEmptyDescriptor(componentDesc) { // nothing was fetched, nothing to extract diff --git a/src/pkg/packager/lint/lint.go b/src/pkg/packager/lint/lint.go index 6de0278e2a..ce4649978b 100644 --- a/src/pkg/packager/lint/lint.go +++ b/src/pkg/packager/lint/lint.go @@ -45,20 +45,21 @@ func ValidateZarfSchema(createOpts types.ZarfCreateOptions) (*Validator, error) return nil, err } + if err := LintComposableComponenets(&validator, createOpts); err != nil { + return nil, err + } + if err := fillActiveTemplate(&validator, createOpts); err != nil { return nil, err } + // Currently I won't be able to find the ZARF_PKG_TMPL_* in import path lintComponents(&validator) if err := os.Chdir(createOpts.BaseDir); err != nil { return nil, fmt.Errorf("unable to access directory '%s': %w", createOpts.BaseDir, err) } - if err := ValidateComposableComponenets(&validator, createOpts); err != nil { - return nil, err - } - if validator.jsonSchema, err = getSchemaFile(); err != nil { return nil, err } @@ -70,7 +71,7 @@ func ValidateZarfSchema(createOpts types.ZarfCreateOptions) (*Validator, error) return &validator, nil } -func ValidateComposableComponenets(validator *Validator, createOpts types.ZarfCreateOptions) error { +func LintComposableComponenets(validator *Validator, createOpts types.ZarfCreateOptions) error { for i, component := range validator.typedZarfPackage.Components { //TODO allow this to be a CLI option arch := config.GetArch(validator.typedZarfPackage.Metadata.Architecture) @@ -90,18 +91,21 @@ func ValidateComposableComponenets(validator *Validator, createOpts types.ZarfCr originalPackage := validator.typedZarfPackage // Skipping initial component since it will be linted the usual way - node := chain.Head.Next + node := chain.Head.Next() for node != nil { validator.typedZarfPackage.Components = []types.ZarfComponent{node.ZarfComponent} fillActiveTemplate(validator, createOpts) - lintComponent(validator, node.Index, validator.typedZarfPackage.Components[0], node.RelativeToHead) - validator.typedZarfPackage = originalPackage - node = node.Next + lintComponent(validator, node.GetIndex(), validator.typedZarfPackage.Components[0], node.GetRelativeToHead()) + node = node.Next() } + validator.typedZarfPackage = originalPackage } return nil } +// Look into breaking apart this function to allow for passing in a component +// Look into returning warnings so I don't have to pass in validator +// Look into removing package from validator func fillActiveTemplate(validator *Validator, createOpts types.ZarfCreateOptions) error { templateMap := map[string]string{} unsetVarWarning := false From 603b8cad279ac72a2e156ac5e41bf58134c27e20 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Wed, 6 Dec 2023 17:29:24 +0000 Subject: [PATCH 111/192] added text for import path and changed messaging --- src/pkg/packager/lint/lint.go | 20 +++++++++++--------- src/test/e2e/12_lint_test.go | 1 + src/test/packages/12-lint/zarf.yaml | 4 ++++ 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/src/pkg/packager/lint/lint.go b/src/pkg/packager/lint/lint.go index ce4649978b..f452e09edd 100644 --- a/src/pkg/packager/lint/lint.go +++ b/src/pkg/packager/lint/lint.go @@ -45,6 +45,10 @@ func ValidateZarfSchema(createOpts types.ZarfCreateOptions) (*Validator, error) return nil, err } + if err := os.Chdir(createOpts.BaseDir); err != nil { + return nil, fmt.Errorf("unable to access directory '%s': %w", createOpts.BaseDir, err) + } + if err := LintComposableComponenets(&validator, createOpts); err != nil { return nil, err } @@ -56,10 +60,6 @@ func ValidateZarfSchema(createOpts types.ZarfCreateOptions) (*Validator, error) // Currently I won't be able to find the ZARF_PKG_TMPL_* in import path lintComponents(&validator) - if err := os.Chdir(createOpts.BaseDir); err != nil { - return nil, fmt.Errorf("unable to access directory '%s': %w", createOpts.BaseDir, err) - } - if validator.jsonSchema, err = getSchemaFile(); err != nil { return nil, err } @@ -110,7 +110,7 @@ func fillActiveTemplate(validator *Validator, createOpts types.ZarfCreateOptions templateMap := map[string]string{} unsetVarWarning := false - promptAndSetTemplate := func(templatePrefix string, deprecated bool) error { + setVarsAndWarn := func(templatePrefix string, deprecated bool) error { yamlTemplates, err := utils.FindYamlTemplates(validator.typedZarfPackage, templatePrefix, "###") if err != nil { return err @@ -140,12 +140,12 @@ func fillActiveTemplate(validator *Validator, createOpts types.ZarfCreateOptions return err } - if err := promptAndSetTemplate(types.ZarfPackageTemplatePrefix, false); err != nil { + if err := setVarsAndWarn(types.ZarfPackageTemplatePrefix, false); err != nil { return err } // [DEPRECATION] Set the Package Variable syntax as well for backward compatibility - if err := promptAndSetTemplate(types.ZarfPackageVariablePrefix, true); err != nil { + if err := setVarsAndWarn(types.ZarfPackageVariablePrefix, true); err != nil { return err } @@ -250,14 +250,16 @@ func checkForVarInComponentImport(validator *Validator, index int, component typ validator.addWarning(ValidatorMessage{ yqPath: fmt.Sprintf(".components.[%d].import.path", index), filePath: path, - description: "Will not resolve ZARF_PKG_TMPL_* variables", + description: "Component import path variables are not resolved by zarf", + item: component.Import.Path, }) } if strings.Contains(component.Import.URL, types.ZarfPackageTemplatePrefix) { validator.addWarning(ValidatorMessage{ yqPath: fmt.Sprintf(".components.[%d].import.url", index), filePath: path, - description: "Will not resolve ZARF_PKG_TMPL_* variables", + description: "Component import URL variables are not resolved by zarf", + item: component.Import.URL, }) } } diff --git a/src/test/e2e/12_lint_test.go b/src/test/e2e/12_lint_test.go index daecdac79f..c7c1301927 100644 --- a/src/test/e2e/12_lint_test.go +++ b/src/test/e2e/12_lint_test.go @@ -41,6 +41,7 @@ func TestLint(t *testing.T) { require.Contains(t, stderr, ".components.[2].images.[3]: Unpinned image") require.Contains(t, stderr, ".components.[1].images.[0] linted-import: Unpinned image") require.Contains(t, stderr, ".components.[1].images.[2] linted-import: Unpinned image") + require.Contains(t, stderr, ".components.[3].import.path: Component import path variables are not resolved by zarf ###ZARF_PKG_TMPL_PATH###") }) } diff --git a/src/test/packages/12-lint/zarf.yaml b/src/test/packages/12-lint/zarf.yaml index 9a26dd42d0..fc6f84207b 100644 --- a/src/test/packages/12-lint/zarf.yaml +++ b/src/test/packages/12-lint/zarf.yaml @@ -34,3 +34,7 @@ components: target: src/ - source: file-without-shasum.txt target: src/ + + - name: import + import: + path: "###ZARF_PKG_TMPL_PATH###" From e88bb309679adc2a3717967cce39f58527bdaa56 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Wed, 6 Dec 2023 19:16:55 +0000 Subject: [PATCH 112/192] fixed logic to allow for detection of zarf vars in component imports --- src/pkg/packager/lint/lint.go | 71 +++++++++++-------- src/pkg/packager/lint/validator.go | 1 + src/pkg/packager/variables.go | 19 +++-- .../packages/12-lint/linted-import/zarf.yaml | 1 + src/test/packages/12-lint/zarf-config.toml | 1 + 5 files changed, 57 insertions(+), 36 deletions(-) diff --git a/src/pkg/packager/lint/lint.go b/src/pkg/packager/lint/lint.go index f452e09edd..006e7239c0 100644 --- a/src/pkg/packager/lint/lint.go +++ b/src/pkg/packager/lint/lint.go @@ -53,11 +53,12 @@ func ValidateZarfSchema(createOpts types.ZarfCreateOptions) (*Validator, error) return nil, err } + preVarEvalLintComponents(&validator) + if err := fillActiveTemplate(&validator, createOpts); err != nil { return nil, err } - // Currently I won't be able to find the ZARF_PKG_TMPL_* in import path lintComponents(&validator) if validator.jsonSchema, err = getSchemaFile(); err != nil { @@ -73,16 +74,15 @@ func ValidateZarfSchema(createOpts types.ZarfCreateOptions) (*Validator, error) func LintComposableComponenets(validator *Validator, createOpts types.ZarfCreateOptions) error { for i, component := range validator.typedZarfPackage.Components { - //TODO allow this to be a CLI option arch := config.GetArch(validator.typedZarfPackage.Metadata.Architecture) if !composer.CompatibleComponent(component, arch, createOpts.Flavor) { continue } - // if a match was found, strip flavor and architecture to reduce bloat in the package definition - component.Only.Cluster.Architecture = "" - component.Only.Flavor = "" + // This is done in composer, should I just delete this or move it into newImportChain + // component.Only.Cluster.Architecture = "" + // component.Only.Flavor = "" chain, err := composer.NewImportChain(component, i, arch, createOpts.Flavor) if err != nil { @@ -93,9 +93,9 @@ func LintComposableComponenets(validator *Validator, createOpts types.ZarfCreate // Skipping initial component since it will be linted the usual way node := chain.Head.Next() for node != nil { - validator.typedZarfPackage.Components = []types.ZarfComponent{node.ZarfComponent} - fillActiveTemplate(validator, createOpts) - lintComponent(validator, node.GetIndex(), validator.typedZarfPackage.Components[0], node.GetRelativeToHead()) + checkForVarInComponentImport(validator, node.GetIndex(), node.ZarfComponent, node.GetRelativeToHead()) + fillComponentTemplate(validator, &node.ZarfComponent, createOpts) + lintComponent(validator, node.GetIndex(), node.ZarfComponent, node.GetRelativeToHead()) node = node.Next() } validator.typedZarfPackage = originalPackage @@ -103,15 +103,35 @@ func LintComposableComponenets(validator *Validator, createOpts types.ZarfCreate return nil } +func fillComponentTemplate(validator *Validator, component *types.ZarfComponent, createOpts types.ZarfCreateOptions) error { + // update the component templates on the package + err := packager.ReloadComponentTemplate(component) + if err != nil { + return err + } + return fillYamlTemplate(validator, component, createOpts) +} + // Look into breaking apart this function to allow for passing in a component // Look into returning warnings so I don't have to pass in validator // Look into removing package from validator func fillActiveTemplate(validator *Validator, createOpts types.ZarfCreateOptions) error { + + // update the component templates on the package + // TODO add test to make sure this is covered still + err := packager.FindComponentTemplatesAndReload(&validator.typedZarfPackage) + if err != nil { + return err + } + //Does typed zarf package need to be addressed here or is it already addresssed + return fillYamlTemplate(validator, &validator.typedZarfPackage, createOpts) +} + +func fillYamlTemplate(validator *Validator, yamlObj any, createOpts types.ZarfCreateOptions) error { templateMap := map[string]string{} - unsetVarWarning := false setVarsAndWarn := func(templatePrefix string, deprecated bool) error { - yamlTemplates, err := utils.FindYamlTemplates(validator.typedZarfPackage, templatePrefix, "###") + yamlTemplates, err := utils.FindYamlTemplates(yamlObj, templatePrefix, "###") if err != nil { return err } @@ -123,8 +143,11 @@ func fillActiveTemplate(validator *Validator, createOpts types.ZarfCreateOptions }) } _, present := createOpts.SetVariables[key] - if !present { - unsetVarWarning = true + if !present && !validator.hasUnSetVarWarning { + validator.warnings = append([]ValidatorMessage{{ + description: "There are variables that are unset and won't be evaluated during lint", + }}, validator.warnings...) + validator.hasUnSetVarWarning = true } } @@ -134,12 +157,6 @@ func fillActiveTemplate(validator *Validator, createOpts types.ZarfCreateOptions return nil } - // update the component templates on the package - err := packager.FindComponentTemplatesAndReload(&validator.typedZarfPackage) - if err != nil { - return err - } - if err := setVarsAndWarn(types.ZarfPackageTemplatePrefix, false); err != nil { return err } @@ -149,16 +166,7 @@ func fillActiveTemplate(validator *Validator, createOpts types.ZarfCreateOptions return err } - // Add special variable for the current package architecture - templateMap[types.ZarfPackageArch] = config.GetArch(validator.typedZarfPackage.Metadata.Architecture) - - if unsetVarWarning { - validator.warnings = append([]ValidatorMessage{{ - description: "There are variables that are unset and won't be evaluated during lint", - }}, validator.warnings...) - } - - return utils.ReloadYamlTemplate(&validator.typedZarfPackage, templateMap) + return utils.ReloadYamlTemplate(yamlObj, templateMap) } func isPinnedImage(image string) (bool, error) { @@ -190,7 +198,6 @@ func lintComponent(validator *Validator, index int, component types.ZarfComponen checkForUnpinnedRepos(validator, index, component, path) checkForUnpinnedImages(validator, index, component, path) checkForUnpinnedFiles(validator, index, component, path) - checkForVarInComponentImport(validator, index, component, path) } func checkForUnpinnedRepos(validator *Validator, index int, component types.ZarfComponent, path string) { @@ -245,6 +252,12 @@ func checkForUnpinnedFiles(validator *Validator, index int, component types.Zarf } } +func preVarEvalLintComponents(validator *Validator) { + for i, component := range validator.typedZarfPackage.Components { + checkForVarInComponentImport(validator, i, component, "") + } +} + func checkForVarInComponentImport(validator *Validator, index int, component types.ZarfComponent, path string) { if strings.Contains(component.Import.Path, types.ZarfPackageTemplatePrefix) { validator.addWarning(ValidatorMessage{ diff --git a/src/pkg/packager/lint/validator.go b/src/pkg/packager/lint/validator.go index 34b5ea9ff0..27419b2cc2 100644 --- a/src/pkg/packager/lint/validator.go +++ b/src/pkg/packager/lint/validator.go @@ -41,6 +41,7 @@ type Validator struct { jsonSchema []byte typedZarfPackage types.ZarfPackage untypedZarfPackage interface{} + hasUnSetVarWarning bool } // DisplayFormattedMessage message sent to user based on validator results diff --git a/src/pkg/packager/variables.go b/src/pkg/packager/variables.go index 80efcbc3d1..2ca6f553bb 100644 --- a/src/pkg/packager/variables.go +++ b/src/pkg/packager/variables.go @@ -15,16 +15,21 @@ import ( "github.com/defenseunicorns/zarf/src/types" ) +func ReloadComponentTemplate(component *types.ZarfComponent) error { + mappings := map[string]string{} + mappings[types.ZarfComponentName] = component.Name + err := utils.ReloadYamlTemplate(component, mappings) + if err != nil { + return err + } + return nil +} + // FindComponentTemplatesAndReload appends ###ZARF_COMPONENT_NAME### for each component, assigns value, and reloads func FindComponentTemplatesAndReload(zarfPackage *types.ZarfPackage) error { // iterate through components to and find all ###ZARF_COMPONENT_NAME, assign to component Name and value - for i, component := range zarfPackage.Components { - mappings := map[string]string{} - mappings[types.ZarfComponentName] = component.Name - err := utils.ReloadYamlTemplate(&zarfPackage.Components[i], mappings) - if err != nil { - return err - } + for i := range zarfPackage.Components { + ReloadComponentTemplate(&zarfPackage.Components[i]) } return nil diff --git a/src/test/packages/12-lint/linted-import/zarf.yaml b/src/test/packages/12-lint/linted-import/zarf.yaml index 07380e1997..db2d92415d 100644 --- a/src/test/packages/12-lint/linted-import/zarf.yaml +++ b/src/test/packages/12-lint/linted-import/zarf.yaml @@ -15,3 +15,4 @@ components: - registry.com:9001/whatever/image:latest - busybox@sha256:3fbc632167424a6d997e74f52b878d7cc478225cffac6bc977eedfe51c7f4e79 - busybox:###ZARF_PKG_TMPL_BUSYBOX_IMAGE### + - busybox:###ZARF_PKG_TMPL_UNSET### diff --git a/src/test/packages/12-lint/zarf-config.toml b/src/test/packages/12-lint/zarf-config.toml index 0b026e5642..1d1c57f33e 100644 --- a/src/test/packages/12-lint/zarf-config.toml +++ b/src/test/packages/12-lint/zarf-config.toml @@ -1,2 +1,3 @@ [package.create.set] BUSYBOX_IMAGE = "latest" +PATH = "linted-import" From 774cad581271e77e52b18eadc00f90822883b06e Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Wed, 6 Dec 2023 19:33:31 +0000 Subject: [PATCH 113/192] fix unit tests --- src/pkg/packager/lint/lint.go | 6 +++--- src/pkg/packager/lint/lint_test.go | 20 ++++++++++++-------- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/src/pkg/packager/lint/lint.go b/src/pkg/packager/lint/lint.go index 006e7239c0..eee2489376 100644 --- a/src/pkg/packager/lint/lint.go +++ b/src/pkg/packager/lint/lint.go @@ -222,7 +222,7 @@ func checkForUnpinnedImages(validator *Validator, index int, component types.Zar validator.addError(ValidatorMessage{ yqPath: imageYqPath, filePath: path, - description: "Invalid Image format", + description: "Invalid image format", item: image, }) continue @@ -263,7 +263,7 @@ func checkForVarInComponentImport(validator *Validator, index int, component typ validator.addWarning(ValidatorMessage{ yqPath: fmt.Sprintf(".components.[%d].import.path", index), filePath: path, - description: "Component import path variables are not resolved by zarf", + description: "Zarf does not evaluate variables at component.x.import.path", item: component.Import.Path, }) } @@ -271,7 +271,7 @@ func checkForVarInComponentImport(validator *Validator, index int, component typ validator.addWarning(ValidatorMessage{ yqPath: fmt.Sprintf(".components.[%d].import.url", index), filePath: path, - description: "Component import URL variables are not resolved by zarf", + description: "Zarf does not evaluate variables at component.x.import.url", item: component.Import.URL, }) } diff --git a/src/pkg/packager/lint/lint_test.go b/src/pkg/packager/lint/lint_test.go index dd4d6dd287..b2a53f2057 100644 --- a/src/pkg/packager/lint/lint_test.go +++ b/src/pkg/packager/lint/lint_test.go @@ -115,20 +115,24 @@ func TestValidateSchema(t *testing.T) { URLComponent := types.ZarfComponent{Import: types.ZarfComponentImport{URL: "oci://###ZARF_PKG_TMPL_ZEBRA###"}} checkForVarInComponentImport(&validator, 2, pathComponent, "") checkForVarInComponentImport(&validator, 3, URLComponent, "") - require.Equal(t, validator.warnings[0].String(), - ".components.[2].import.path: Will not resolve ZARF_PKG_TMPL_* variables") - require.Equal(t, validator.warnings[1].String(), - ".components.[3].import.url: Will not resolve ZARF_PKG_TMPL_* variables") + require.Equal(t, + ".components.[2].import.path: Zarf does not evaluate variables at component.x.import.path ###ZARF_PKG_TMPL_ZEBRA###", + validator.warnings[0].String()) + require.Equal(t, + ".components.[3].import.url: Zarf does not evaluate variables at component.x.import.url oci://###ZARF_PKG_TMPL_ZEBRA###", + validator.warnings[1].String()) }) t.Run("Unpinnned repo warning", func(t *testing.T) { validator := Validator{} + unpinnedRepo := "https://github.com/defenseunicorns/zarf-public-test.git" component := types.ZarfComponent{Repos: []string{ - "https://github.com/defenseunicorns/zarf-public-test.git", - "https://dev.azure.com/defenseunicorns/zarf-public-test/_git/zarf-public-test@v0.0.1", - "https://gitlab.com/gitlab-org/build/omnibus-mirror/pcre2/-/tree/vreverse?ref_type=heads"}} + unpinnedRepo, + "https://dev.azure.com/defenseunicorns/zarf-public-test/_git/zarf-public-test@v0.0.1"}} checkForUnpinnedRepos(&validator, 0, component, "") - require.Equal(t, validator.warnings[0].String(), ".components.[0].repos.[0]: Unpinned repository") + require.Equal(t, + fmt.Sprintf(".components.[0].repos.[0]: Unpinned repository %s", unpinnedRepo), + validator.warnings[0].String()) require.Equal(t, len(validator.warnings), 1) }) From 14af47a6c2cab1c88bf22e090bd7da69714153d2 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Wed, 6 Dec 2023 19:54:51 +0000 Subject: [PATCH 114/192] refactor tests --- src/pkg/packager/lint/lint.go | 8 +++----- src/pkg/packager/lint/lint_test.go | 30 ++++++++++++------------------ src/test/e2e/12_lint_test.go | 2 +- 3 files changed, 16 insertions(+), 24 deletions(-) diff --git a/src/pkg/packager/lint/lint.go b/src/pkg/packager/lint/lint.go index eee2489376..43c281f9d0 100644 --- a/src/pkg/packager/lint/lint.go +++ b/src/pkg/packager/lint/lint.go @@ -182,9 +182,7 @@ func isPinnedImage(image string) (bool, error) { } func isPinnedRepo(repo string) bool { - // Pinned github and dev.azure.com repos will have @ - // Pinned gitlab repos will have /-/ - return (strings.Contains(repo, "@") || strings.Contains(repo, "/-/")) + return (strings.Contains(repo, "@")) } // Feels like validator may have too much with both the zarf package and the warnings @@ -231,7 +229,7 @@ func checkForUnpinnedImages(validator *Validator, index int, component types.Zar validator.addWarning(ValidatorMessage{ yqPath: imageYqPath, filePath: path, - description: "Unpinned image", + description: "Image not pinned with digest", item: image, }) } @@ -245,7 +243,7 @@ func checkForUnpinnedFiles(validator *Validator, index int, component types.Zarf validator.addWarning(ValidatorMessage{ yqPath: fileYqPath, filePath: path, - description: "Unpinned file", + description: "No shasum for remote file", item: file.Source, }) } diff --git a/src/pkg/packager/lint/lint_test.go b/src/pkg/packager/lint/lint_test.go index b2a53f2057..d02d61d484 100644 --- a/src/pkg/packager/lint/lint_test.go +++ b/src/pkg/packager/lint/lint_test.go @@ -97,8 +97,8 @@ func TestValidateSchema(t *testing.T) { validator := Validator{untypedZarfPackage: unmarshalledYaml, jsonSchema: getZarfSchema(t)} err := validateSchema(&validator) require.NoError(t, err) - require.Equal(t, validator.errors[0].String(), ".components.[0].import: Additional property not-path is not allowed") - require.Equal(t, validator.errors[1].String(), ".components.[1].import.path: Invalid type. Expected: string, given: integer") + require.Equal(t, ".components.[0].import: Additional property not-path is not allowed", validator.errors[0].String()) + require.Equal(t, ".components.[1].import.path: Invalid type. Expected: string, given: integer", validator.errors[1].String()) }) t.Run("Template in component import success", func(t *testing.T) { @@ -111,16 +111,14 @@ func TestValidateSchema(t *testing.T) { t.Run("Template in component import failure", func(t *testing.T) { validator := Validator{} - pathComponent := types.ZarfComponent{Import: types.ZarfComponentImport{Path: "###ZARF_PKG_TMPL_ZEBRA###"}} - URLComponent := types.ZarfComponent{Import: types.ZarfComponentImport{URL: "oci://###ZARF_PKG_TMPL_ZEBRA###"}} + pathVar := "###ZARF_PKG_TMPL_PATH###" + ociPathVar := fmt.Sprintf("oci://%s", pathVar) + pathComponent := types.ZarfComponent{Import: types.ZarfComponentImport{Path: pathVar}} + URLComponent := types.ZarfComponent{Import: types.ZarfComponentImport{URL: ociPathVar}} checkForVarInComponentImport(&validator, 2, pathComponent, "") checkForVarInComponentImport(&validator, 3, URLComponent, "") - require.Equal(t, - ".components.[2].import.path: Zarf does not evaluate variables at component.x.import.path ###ZARF_PKG_TMPL_ZEBRA###", - validator.warnings[0].String()) - require.Equal(t, - ".components.[3].import.url: Zarf does not evaluate variables at component.x.import.url oci://###ZARF_PKG_TMPL_ZEBRA###", - validator.warnings[1].String()) + require.Equal(t, pathVar, validator.warnings[0].item) + require.Equal(t, ociPathVar, validator.warnings[1].item) }) t.Run("Unpinnned repo warning", func(t *testing.T) { @@ -130,9 +128,7 @@ func TestValidateSchema(t *testing.T) { unpinnedRepo, "https://dev.azure.com/defenseunicorns/zarf-public-test/_git/zarf-public-test@v0.0.1"}} checkForUnpinnedRepos(&validator, 0, component, "") - require.Equal(t, - fmt.Sprintf(".components.[0].repos.[0]: Unpinned repository %s", unpinnedRepo), - validator.warnings[0].String()) + require.Equal(t, unpinnedRepo, validator.warnings[0].item) require.Equal(t, len(validator.warnings), 1) }) @@ -145,10 +141,9 @@ func TestValidateSchema(t *testing.T) { "busybox:latest@sha256:3fbc632167424a6d997e74f52b878d7cc478225cffac6bc977eedfe51c7f4e79", badImage}} checkForUnpinnedImages(&validator, 0, component, "") - require.Equal(t, fmt.Sprintf(".components.[0].images.[0]: Unpinned image %s", unpinnedImage), validator.warnings[0].String()) + require.Equal(t, unpinnedImage, validator.warnings[0].item) require.Equal(t, len(validator.warnings), 1) - expectedErr := fmt.Sprintf(".components.[0].images.[2]: Invalid image format %s", badImage) - require.Equal(t, validator.errors[0].String(), expectedErr) + require.Equal(t, badImage, validator.errors[0].item) require.Equal(t, len(validator.errors), 1) }) @@ -162,8 +157,7 @@ func TestValidateSchema(t *testing.T) { } component := types.ZarfComponent{Files: zarfFiles} checkForUnpinnedFiles(&validator, 0, component, "") - expected := fmt.Sprintf(".components.[0].files.[0]: Unpinned file %s", filename) - require.Equal(t, expected, validator.warnings[0].String()) + require.Equal(t, filename, validator.warnings[0].item) require.Equal(t, 1, len(validator.warnings)) }) diff --git a/src/test/e2e/12_lint_test.go b/src/test/e2e/12_lint_test.go index c7c1301927..dfd3b69b33 100644 --- a/src/test/e2e/12_lint_test.go +++ b/src/test/e2e/12_lint_test.go @@ -41,7 +41,7 @@ func TestLint(t *testing.T) { require.Contains(t, stderr, ".components.[2].images.[3]: Unpinned image") require.Contains(t, stderr, ".components.[1].images.[0] linted-import: Unpinned image") require.Contains(t, stderr, ".components.[1].images.[2] linted-import: Unpinned image") - require.Contains(t, stderr, ".components.[3].import.path: Component import path variables are not resolved by zarf ###ZARF_PKG_TMPL_PATH###") + require.Contains(t, stderr, ".components.[3].import.path: Zarf does not evaluate variables at component.x.import.path ###ZARF_PKG_TMPL_PATH###") }) } From c0e96b8185e9b5edc4e3e845da2524758c68b0d7 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Wed, 6 Dec 2023 19:55:25 +0000 Subject: [PATCH 115/192] refactor tests --- src/pkg/packager/lint/lint_test.go | 24 ------------------------ 1 file changed, 24 deletions(-) diff --git a/src/pkg/packager/lint/lint_test.go b/src/pkg/packager/lint/lint_test.go index d02d61d484..22ebf4b049 100644 --- a/src/pkg/packager/lint/lint_test.go +++ b/src/pkg/packager/lint/lint_test.go @@ -28,30 +28,6 @@ components: - name: import-test import: path: 123123 - -- name: import-test - import: - path: "###ZARF_PKG_TMPL_ZEBRA###" - -- name: import-url - import: - url: "oci://###ZARF_PKG_TMPL_ZEBRA###" - -- name: full-repo - repos: - - https://github.com/defenseunicorns/zarf-public-test.git - - https://dev.azure.com/defenseunicorns/zarf-public-test/_git/zarf-public-test@v0.0.1 - - https://gitlab.com/gitlab-org/build/omnibus-mirror/pcre2/-/tree/vreverse?ref_type=heads - images: - - ghcr.io/kiwix/kiwix-serve:3.5.0-2 - - registry.com:9001/whatever/image:1.0.0 - - busybox:latest@sha256:3fbc632167424a6d997e74f52b878d7cc478225cffac6bc977eedfe51c7f4e79 - - busybox:latest - - badimage:badimage@@sha256:3fbc632167424a6d997e74f5 - files: - - source: https://github.com/k3s-io/k3s/releases/download/v1.28.2+k3s1/k3s - shasum: 2f041d37a2c6d54d53e106e1c7713bc48f806f3919b0d9e092f5fcbdc55b41cf - - source: file-without-shasum.txt ` const goodZarfPackage = ` From f1fdee5eb9f774ab3d0f0dffbe677e529e56d31d Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Wed, 6 Dec 2023 19:58:10 +0000 Subject: [PATCH 116/192] fix e2e tests --- src/test/e2e/12_lint_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/test/e2e/12_lint_test.go b/src/test/e2e/12_lint_test.go index dfd3b69b33..7495f17c9e 100644 --- a/src/test/e2e/12_lint_test.go +++ b/src/test/e2e/12_lint_test.go @@ -38,9 +38,9 @@ func TestLint(t *testing.T) { require.Contains(t, stderr, ".components.[2].repos.[0]: Unpinned repository") require.Contains(t, stderr, ".metadata: Additional property description1 is not allowed") require.Contains(t, stderr, ".components.[0].import: Additional property not-path is not allowed") - require.Contains(t, stderr, ".components.[2].images.[3]: Unpinned image") - require.Contains(t, stderr, ".components.[1].images.[0] linted-import: Unpinned image") - require.Contains(t, stderr, ".components.[1].images.[2] linted-import: Unpinned image") + require.Contains(t, stderr, ".components.[2].images.[3]: Image not pinned with digest") + require.Contains(t, stderr, ".components.[1].images.[0] linted-import: Image not pinned with digest") + require.Contains(t, stderr, ".components.[1].images.[2] linted-import: Image not pinned with digest") require.Contains(t, stderr, ".components.[3].import.path: Zarf does not evaluate variables at component.x.import.path ###ZARF_PKG_TMPL_PATH###") }) From 7b94f74b98c8e930bd6332ab66a758c265733b2a Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Wed, 6 Dec 2023 19:58:36 +0000 Subject: [PATCH 117/192] add test file --- .../packages/12-lint/###ZARF_PKG_TMPL_PATH###/zarf.yaml | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 src/test/packages/12-lint/###ZARF_PKG_TMPL_PATH###/zarf.yaml diff --git a/src/test/packages/12-lint/###ZARF_PKG_TMPL_PATH###/zarf.yaml b/src/test/packages/12-lint/###ZARF_PKG_TMPL_PATH###/zarf.yaml new file mode 100644 index 0000000000..db60f4897d --- /dev/null +++ b/src/test/packages/12-lint/###ZARF_PKG_TMPL_PATH###/zarf.yaml @@ -0,0 +1,7 @@ +kind: ZarfInitConfig +metadata: + name: init + description: Testing bad yaml imported + +components: + - name: import From 588d6070510d2cf22030b317ee4fae5be50f0f42 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Wed, 6 Dec 2023 20:00:10 +0000 Subject: [PATCH 118/192] remove unnecessary comments --- src/pkg/packager/composer/list.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/pkg/packager/composer/list.go b/src/pkg/packager/composer/list.go index 5b2dade9f3..2ac2467891 100644 --- a/src/pkg/packager/composer/list.go +++ b/src/pkg/packager/composer/list.go @@ -23,9 +23,6 @@ import ( type Node struct { types.ZarfComponent - // We are going to use this in lint so we can give the correct path - // We can give the index of the zarf component within the file that's being composed on - // So users can have the yaml path index int vars []types.ZarfPackageVariable From a9934184b4020919ede96b3f4f4bb392e979a6a6 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Wed, 6 Dec 2023 20:24:34 +0000 Subject: [PATCH 119/192] add color to yq path --- src/pkg/packager/lint/validator.go | 3 ++- src/test/e2e/12_lint_test.go | 20 +++++++++---------- .../packages/12-lint/linted-import/zarf.yaml | 3 +++ src/test/packages/12-lint/zarf.yaml | 4 ++++ 4 files changed, 19 insertions(+), 11 deletions(-) diff --git a/src/pkg/packager/lint/validator.go b/src/pkg/packager/lint/validator.go index 27419b2cc2..0253adf874 100644 --- a/src/pkg/packager/lint/validator.go +++ b/src/pkg/packager/lint/validator.go @@ -31,7 +31,8 @@ func (v ValidatorMessage) String() string { return v.description } return fmt.Sprintf("%s%s: %s%s", - v.yqPath, v.filePath, v.description, v.item) + utils.ColorWrap(v.yqPath, color.FgCyan), utils.ColorWrap(v.filePath, color.FgCyan), + v.description, v.item) } // Validator holds the warnings/errors and messaging that we get from validation diff --git a/src/test/e2e/12_lint_test.go b/src/test/e2e/12_lint_test.go index 7495f17c9e..7bf79e1d4b 100644 --- a/src/test/e2e/12_lint_test.go +++ b/src/test/e2e/12_lint_test.go @@ -30,18 +30,18 @@ func TestLint(t *testing.T) { // In this case I'm guessing we should also remove color from the table? _, stderr, err := e2e.Zarf("prepare", "lint", path) require.Error(t, err, "Require an exit code since there was warnings / errors") - + strippedStderr := e2e.StripANSICodes(stderr) // It's a bit weird to have a period here and not in the other warnings key := "WHATEVER_IMAGE" - require.Contains(t, stderr, "There are variables that are unset and won't be evaluated during lint") - require.Contains(t, stderr, fmt.Sprintf(lang.PkgValidateTemplateDeprecation, key, key, key)) - require.Contains(t, stderr, ".components.[2].repos.[0]: Unpinned repository") - require.Contains(t, stderr, ".metadata: Additional property description1 is not allowed") - require.Contains(t, stderr, ".components.[0].import: Additional property not-path is not allowed") - require.Contains(t, stderr, ".components.[2].images.[3]: Image not pinned with digest") - require.Contains(t, stderr, ".components.[1].images.[0] linted-import: Image not pinned with digest") - require.Contains(t, stderr, ".components.[1].images.[2] linted-import: Image not pinned with digest") - require.Contains(t, stderr, ".components.[3].import.path: Zarf does not evaluate variables at component.x.import.path ###ZARF_PKG_TMPL_PATH###") + require.Contains(t, strippedStderr, "There are variables that are unset and won't be evaluated during lint") + require.Contains(t, strippedStderr, fmt.Sprintf(lang.PkgValidateTemplateDeprecation, key, key, key)) + require.Contains(t, strippedStderr, ".components.[2].repos.[0]: Unpinned repository") + require.Contains(t, strippedStderr, ".metadata: Additional property description1 is not allowed") + require.Contains(t, strippedStderr, ".components.[0].import: Additional property not-path is not allowed") + require.Contains(t, strippedStderr, ".components.[2].images.[3]: Image not pinned with digest") + require.Contains(t, strippedStderr, ".components.[1].images.[0] linted-import: Image not pinned with digest") + require.Contains(t, strippedStderr, ".components.[1].images.[2] linted-import: Image not pinned with digest") + require.Contains(t, strippedStderr, ".components.[3].import.path: Zarf does not evaluate variables at component.x.import.path ###ZARF_PKG_TMPL_PATH###") }) } diff --git a/src/test/packages/12-lint/linted-import/zarf.yaml b/src/test/packages/12-lint/linted-import/zarf.yaml index db2d92415d..1f6b2b818d 100644 --- a/src/test/packages/12-lint/linted-import/zarf.yaml +++ b/src/test/packages/12-lint/linted-import/zarf.yaml @@ -16,3 +16,6 @@ components: - busybox@sha256:3fbc632167424a6d997e74f52b878d7cc478225cffac6bc977eedfe51c7f4e79 - busybox:###ZARF_PKG_TMPL_BUSYBOX_IMAGE### - busybox:###ZARF_PKG_TMPL_UNSET### + files: + - source: https://github.com/k3s-io/k3s/releases/download/v1.28.2+k3s1/k3s + target: src/ diff --git a/src/test/packages/12-lint/zarf.yaml b/src/test/packages/12-lint/zarf.yaml index fc6f84207b..2eec2597c7 100644 --- a/src/test/packages/12-lint/zarf.yaml +++ b/src/test/packages/12-lint/zarf.yaml @@ -16,6 +16,10 @@ components: - name: import-test import: path: linted-import + files: + - source: https://github.com/k3s-io/k3s/releases/download/v1.28.2+k3s1/k3s + shasum: 2f041d37a2c6d54d53e106e1c7713bc48f806f3919b0d9e092f5fcbdc55b41cf + target: src/ - name: full-repo repos: From 0a94706c5063bce21a263d75a33acc45da2c5e31 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Wed, 6 Dec 2023 20:27:47 +0000 Subject: [PATCH 120/192] fix merge and remove doublespaces --- src/config/lang/english.go | 2 +- src/test/e2e/12_lint_test.go | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/config/lang/english.go b/src/config/lang/english.go index dd9c5ca4cc..e8f2e4a3d4 100644 --- a/src/config/lang/english.go +++ b/src/config/lang/english.go @@ -599,7 +599,7 @@ const ( // src/internal/packager/validate. const ( - PkgValidateTemplateDeprecation = "Package template %q is using the deprecated syntax ###ZARF_PKG_VAR_%s###. This will be removed in Zarf v1.0.0. Please update to ###ZARF_PKG_TMPL_%s###." + PkgValidateTemplateDeprecation = "Package template %q is using the deprecated syntax ###ZARF_PKG_VAR_%s###. This will be removed in Zarf v1.0.0. Please update to ###ZARF_PKG_TMPL_%s###." PkgValidateMustBeUppercase = "variable name %q must be all uppercase and contain no special characters except _" PkgValidateErrAction = "invalid action: %w" PkgValidateErrActionVariables = "component %q cannot contain setVariables outside of onDeploy in actions" diff --git a/src/test/e2e/12_lint_test.go b/src/test/e2e/12_lint_test.go index 7bf79e1d4b..9db1f5904f 100644 --- a/src/test/e2e/12_lint_test.go +++ b/src/test/e2e/12_lint_test.go @@ -30,10 +30,11 @@ func TestLint(t *testing.T) { // In this case I'm guessing we should also remove color from the table? _, stderr, err := e2e.Zarf("prepare", "lint", path) require.Error(t, err, "Require an exit code since there was warnings / errors") - strippedStderr := e2e.StripANSICodes(stderr) + strippedStderr := e2e.StripMessageFormatting(stderr) // It's a bit weird to have a period here and not in the other warnings key := "WHATEVER_IMAGE" require.Contains(t, strippedStderr, "There are variables that are unset and won't be evaluated during lint") + // Multiple spaces in lang require.Contains(t, strippedStderr, fmt.Sprintf(lang.PkgValidateTemplateDeprecation, key, key, key)) require.Contains(t, strippedStderr, ".components.[2].repos.[0]: Unpinned repository") require.Contains(t, strippedStderr, ".metadata: Additional property description1 is not allowed") From ba6e500fbee1353a26ce9f6dea1b7afbef3eff13 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Wed, 6 Dec 2023 21:30:00 +0000 Subject: [PATCH 121/192] now getting the oci name and outtping it --- src/pkg/packager/composer/list.go | 7 +++++++ src/pkg/packager/lint/lint.go | 10 ++++++---- src/test/e2e/12_lint_test.go | 1 + .../12-lint/###ZARF_PKG_TMPL_PATH###/zarf.yaml | 3 +++ src/test/packages/12-lint/zarf.yaml | 7 +++++++ 5 files changed, 24 insertions(+), 4 deletions(-) diff --git a/src/pkg/packager/composer/list.go b/src/pkg/packager/composer/list.go index 2ac2467891..fc953c9f29 100644 --- a/src/pkg/packager/composer/list.go +++ b/src/pkg/packager/composer/list.go @@ -70,6 +70,13 @@ type ImportChain struct { remote *oci.OrasRemote } +func (ic *ImportChain) GetRemoteName() string { + if ic.remote == nil { + return "" + } + return ic.remote.Repo().Reference.String() +} + func (ic *ImportChain) append(c types.ZarfComponent, index int, relativeToHead string, vars []types.ZarfPackageVariable, consts []types.ZarfPackageConstant) { node := &Node{ ZarfComponent: c, diff --git a/src/pkg/packager/lint/lint.go b/src/pkg/packager/lint/lint.go index 43c281f9d0..64dbd5216e 100644 --- a/src/pkg/packager/lint/lint.go +++ b/src/pkg/packager/lint/lint.go @@ -89,16 +89,18 @@ func LintComposableComponenets(validator *Validator, createOpts types.ZarfCreate return err } - originalPackage := validator.typedZarfPackage // Skipping initial component since it will be linted the usual way node := chain.Head.Next() + path := chain.GetRemoteName() for node != nil { - checkForVarInComponentImport(validator, node.GetIndex(), node.ZarfComponent, node.GetRelativeToHead()) + if path == "" { + path = node.GetRelativeToHead() + } + checkForVarInComponentImport(validator, node.GetIndex(), node.ZarfComponent, path) fillComponentTemplate(validator, &node.ZarfComponent, createOpts) - lintComponent(validator, node.GetIndex(), node.ZarfComponent, node.GetRelativeToHead()) + lintComponent(validator, node.GetIndex(), node.ZarfComponent, path) node = node.Next() } - validator.typedZarfPackage = originalPackage } return nil } diff --git a/src/test/e2e/12_lint_test.go b/src/test/e2e/12_lint_test.go index 9db1f5904f..d1da5ba88b 100644 --- a/src/test/e2e/12_lint_test.go +++ b/src/test/e2e/12_lint_test.go @@ -43,6 +43,7 @@ func TestLint(t *testing.T) { require.Contains(t, strippedStderr, ".components.[1].images.[0] linted-import: Image not pinned with digest") require.Contains(t, strippedStderr, ".components.[1].images.[2] linted-import: Image not pinned with digest") require.Contains(t, strippedStderr, ".components.[3].import.path: Zarf does not evaluate variables at component.x.import.path ###ZARF_PKG_TMPL_PATH###") + require.Contains(t, strippedStderr, ".components.[0].images.[0] ghcr.io/defenseunicorns/packages/dos-games:1.0.0-skeleton: Image not pinned with digest defenseunicorns/zarf-game:multi-tile-dark") }) } diff --git a/src/test/packages/12-lint/###ZARF_PKG_TMPL_PATH###/zarf.yaml b/src/test/packages/12-lint/###ZARF_PKG_TMPL_PATH###/zarf.yaml index db60f4897d..9c95401914 100644 --- a/src/test/packages/12-lint/###ZARF_PKG_TMPL_PATH###/zarf.yaml +++ b/src/test/packages/12-lint/###ZARF_PKG_TMPL_PATH###/zarf.yaml @@ -5,3 +5,6 @@ metadata: components: - name: import + files: + - source: https://github.com/k3s-io/k3s/releases/download/v1.28.2+k3s1/k3s + target: src/ diff --git a/src/test/packages/12-lint/zarf.yaml b/src/test/packages/12-lint/zarf.yaml index 2eec2597c7..940cfbe197 100644 --- a/src/test/packages/12-lint/zarf.yaml +++ b/src/test/packages/12-lint/zarf.yaml @@ -17,6 +17,8 @@ components: import: path: linted-import files: + # Is this a scenario we care about. The downstream overrides with a shasum. + # Is this even overridding or are they downloading the same file twice - source: https://github.com/k3s-io/k3s/releases/download/v1.28.2+k3s1/k3s shasum: 2f041d37a2c6d54d53e106e1c7713bc48f806f3919b0d9e092f5fcbdc55b41cf target: src/ @@ -42,3 +44,8 @@ components: - name: import import: path: "###ZARF_PKG_TMPL_PATH###" + + - name: oci-games-url + import: + url: oci://🦄/dos-games:1.0.0-skeleton + name: baseline From 5f370d9dc08e76ac762a54655973ca326e2f0fd1 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Wed, 6 Dec 2023 21:38:03 +0000 Subject: [PATCH 122/192] fix lints --- src/pkg/packager/composer/list.go | 4 ++++ src/pkg/packager/lint/lint.go | 22 +++++++++++----------- src/pkg/packager/lint/validator.go | 12 ++++++------ src/pkg/packager/variables.go | 2 ++ 4 files changed, 23 insertions(+), 17 deletions(-) diff --git a/src/pkg/packager/composer/list.go b/src/pkg/packager/composer/list.go index fc953c9f29..e0e59c05fb 100644 --- a/src/pkg/packager/composer/list.go +++ b/src/pkg/packager/composer/list.go @@ -34,18 +34,22 @@ type Node struct { next *Node } +// GetIndex gives the component index of the node on it's original zarf file func (n *Node) GetIndex() int { return n.index } +// Path from downstream zarf file to upstream imported zarf file func (n *Node) GetRelativeToHead() string { return n.relativeToHead } +// Next node in the chain func (n *Node) Next() *Node { return n.next } +// Prev node in the chain func (n *Node) Prev() *Node { return n.prev } diff --git a/src/pkg/packager/lint/lint.go b/src/pkg/packager/lint/lint.go index 64dbd5216e..173dd71f8a 100644 --- a/src/pkg/packager/lint/lint.go +++ b/src/pkg/packager/lint/lint.go @@ -49,7 +49,7 @@ func ValidateZarfSchema(createOpts types.ZarfCreateOptions) (*Validator, error) return nil, fmt.Errorf("unable to access directory '%s': %w", createOpts.BaseDir, err) } - if err := LintComposableComponenets(&validator, createOpts); err != nil { + if err := lintComposableComponenets(&validator, createOpts); err != nil { return nil, err } @@ -72,7 +72,7 @@ func ValidateZarfSchema(createOpts types.ZarfCreateOptions) (*Validator, error) return &validator, nil } -func LintComposableComponenets(validator *Validator, createOpts types.ZarfCreateOptions) error { +func lintComposableComponenets(validator *Validator, createOpts types.ZarfCreateOptions) error { for i, component := range validator.typedZarfPackage.Components { arch := config.GetArch(validator.typedZarfPackage.Metadata.Architecture) @@ -140,13 +140,13 @@ func fillYamlTemplate(validator *Validator, yamlObj any, createOpts types.ZarfCr for key := range yamlTemplates { if deprecated { - validator.addWarning(ValidatorMessage{ + validator.addWarning(validatorMessage{ description: fmt.Sprintf(lang.PkgValidateTemplateDeprecation, key, key, key), }) } _, present := createOpts.SetVariables[key] if !present && !validator.hasUnSetVarWarning { - validator.warnings = append([]ValidatorMessage{{ + validator.warnings = append([]validatorMessage{{ description: "There are variables that are unset and won't be evaluated during lint", }}, validator.warnings...) validator.hasUnSetVarWarning = true @@ -204,7 +204,7 @@ func checkForUnpinnedRepos(validator *Validator, index int, component types.Zarf for j, repo := range component.Repos { repoYqPath := fmt.Sprintf(".components.[%d].repos.[%d]", index, j) if !isPinnedRepo(repo) { - validator.addWarning(ValidatorMessage{ + validator.addWarning(validatorMessage{ yqPath: repoYqPath, filePath: path, description: "Unpinned repository", @@ -219,7 +219,7 @@ func checkForUnpinnedImages(validator *Validator, index int, component types.Zar imageYqPath := fmt.Sprintf(".components.[%d].images.[%d]", index, j) pinnedImage, err := isPinnedImage(image) if err != nil { - validator.addError(ValidatorMessage{ + validator.addError(validatorMessage{ yqPath: imageYqPath, filePath: path, description: "Invalid image format", @@ -228,7 +228,7 @@ func checkForUnpinnedImages(validator *Validator, index int, component types.Zar continue } if !pinnedImage { - validator.addWarning(ValidatorMessage{ + validator.addWarning(validatorMessage{ yqPath: imageYqPath, filePath: path, description: "Image not pinned with digest", @@ -242,7 +242,7 @@ func checkForUnpinnedFiles(validator *Validator, index int, component types.Zarf for j, file := range component.Files { fileYqPath := fmt.Sprintf(".components.[%d].files.[%d]", index, j) if file.Shasum == "" && helpers.IsURL(file.Source) { - validator.addWarning(ValidatorMessage{ + validator.addWarning(validatorMessage{ yqPath: fileYqPath, filePath: path, description: "No shasum for remote file", @@ -260,7 +260,7 @@ func preVarEvalLintComponents(validator *Validator) { func checkForVarInComponentImport(validator *Validator, index int, component types.ZarfComponent, path string) { if strings.Contains(component.Import.Path, types.ZarfPackageTemplatePrefix) { - validator.addWarning(ValidatorMessage{ + validator.addWarning(validatorMessage{ yqPath: fmt.Sprintf(".components.[%d].import.path", index), filePath: path, description: "Zarf does not evaluate variables at component.x.import.path", @@ -268,7 +268,7 @@ func checkForVarInComponentImport(validator *Validator, index int, component typ }) } if strings.Contains(component.Import.URL, types.ZarfPackageTemplatePrefix) { - validator.addWarning(ValidatorMessage{ + validator.addWarning(validatorMessage{ yqPath: fmt.Sprintf(".components.[%d].import.url", index), filePath: path, description: "Zarf does not evaluate variables at component.x.import.url", @@ -301,7 +301,7 @@ func validateSchema(validator *Validator) error { if !result.Valid() { for _, desc := range result.Errors() { - validator.addError(ValidatorMessage{ + validator.addError(validatorMessage{ yqPath: makeFieldPathYqCompat(desc.Field()), description: desc.Description(), }) diff --git a/src/pkg/packager/lint/validator.go b/src/pkg/packager/lint/validator.go index 0253adf874..5df4af273b 100644 --- a/src/pkg/packager/lint/validator.go +++ b/src/pkg/packager/lint/validator.go @@ -13,14 +13,14 @@ import ( "github.com/fatih/color" ) -type ValidatorMessage struct { +type validatorMessage struct { yqPath string filePath string description string item string } -func (v ValidatorMessage) String() string { +func (v validatorMessage) String() string { if v.filePath != "" { v.filePath = fmt.Sprintf(" %s", v.filePath) } @@ -37,8 +37,8 @@ func (v ValidatorMessage) String() string { // Validator holds the warnings/errors and messaging that we get from validation type Validator struct { - warnings []ValidatorMessage - errors []ValidatorMessage + warnings []validatorMessage + errors []validatorMessage jsonSchema []byte typedZarfPackage types.ZarfPackage untypedZarfPackage interface{} @@ -82,10 +82,10 @@ func (v Validator) hasErrors() bool { return len(v.errors) > 0 } -func (v *Validator) addWarning(vmessage ValidatorMessage) { +func (v *Validator) addWarning(vmessage validatorMessage) { v.warnings = append(v.warnings, vmessage) } -func (v *Validator) addError(vMessage ValidatorMessage) { +func (v *Validator) addError(vMessage validatorMessage) { v.errors = append(v.errors, vMessage) } diff --git a/src/pkg/packager/variables.go b/src/pkg/packager/variables.go index 2ca6f553bb..1532629019 100644 --- a/src/pkg/packager/variables.go +++ b/src/pkg/packager/variables.go @@ -15,6 +15,8 @@ import ( "github.com/defenseunicorns/zarf/src/types" ) +// ReloadComponentTemplate appends ###ZARF_COMPONENT_NAME### for the component, assigns value, and reloads +// Any instance of ###ZARF_COMPONENT_NAME### within a component will be replaced with that components name func ReloadComponentTemplate(component *types.ZarfComponent) error { mappings := map[string]string{} mappings[types.ZarfComponentName] = component.Name From 162f8fae5114da3ba1516765282d1270d595b315 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Thu, 7 Dec 2023 14:45:18 +0000 Subject: [PATCH 123/192] refactoring list and lint --- src/pkg/packager/composer/list.go | 27 ++++++++------------------- src/pkg/packager/lint/lint.go | 6 +++--- src/test/e2e/12_lint_test.go | 2 +- 3 files changed, 12 insertions(+), 23 deletions(-) diff --git a/src/pkg/packager/composer/list.go b/src/pkg/packager/composer/list.go index e0e59c05fb..606c398e68 100644 --- a/src/pkg/packager/composer/list.go +++ b/src/pkg/packager/composer/list.go @@ -7,7 +7,6 @@ package composer import ( "fmt" "path/filepath" - "reflect" "strings" "github.com/defenseunicorns/zarf/src/internal/packager/validate" @@ -74,13 +73,6 @@ type ImportChain struct { remote *oci.OrasRemote } -func (ic *ImportChain) GetRemoteName() string { - if ic.remote == nil { - return "" - } - return ic.remote.Repo().Reference.String() -} - func (ic *ImportChain) append(c types.ZarfComponent, index int, relativeToHead string, vars []types.ZarfPackageVariable, consts []types.ZarfPackageConstant) { node := &Node{ ZarfComponent: c, @@ -172,10 +164,14 @@ func NewImportChain(head types.ZarfComponent, index int, arch, flavor string) (* name := node.ImportName() - found := helpers.Filter(pkg.Components, func(c types.ZarfComponent) bool { - matchesName := c.Name == name - return matchesName && CompatibleComponent(c, arch, flavor) - }) + found := []types.ZarfComponent{} + var index int + for i, component := range pkg.Components { + if component.Name == name && CompatibleComponent(component, arch, flavor) { + found = append(found, component) + index = i + } + } if len(found) == 0 { if isLocal { @@ -191,13 +187,6 @@ func NewImportChain(head types.ZarfComponent, index int, arch, flavor string) (* } } - var index int - //Probably can do this better, maybe have filter also give an index - for i, component := range pkg.Components { - if reflect.DeepEqual(found[0], component) { - index = i - } - } ic.append(found[0], index, filepath.Join(history...), pkg.Variables, pkg.Constants) node = node.next } diff --git a/src/pkg/packager/lint/lint.go b/src/pkg/packager/lint/lint.go index 173dd71f8a..78b7691a05 100644 --- a/src/pkg/packager/lint/lint.go +++ b/src/pkg/packager/lint/lint.go @@ -53,7 +53,7 @@ func ValidateZarfSchema(createOpts types.ZarfCreateOptions) (*Validator, error) return nil, err } - preVarEvalLintComponents(&validator) + lintUnEvaledVariables(&validator) if err := fillActiveTemplate(&validator, createOpts); err != nil { return nil, err @@ -90,8 +90,8 @@ func lintComposableComponenets(validator *Validator, createOpts types.ZarfCreate } // Skipping initial component since it will be linted the usual way + path := chain.Head.Import.URL node := chain.Head.Next() - path := chain.GetRemoteName() for node != nil { if path == "" { path = node.GetRelativeToHead() @@ -252,7 +252,7 @@ func checkForUnpinnedFiles(validator *Validator, index int, component types.Zarf } } -func preVarEvalLintComponents(validator *Validator) { +func lintUnEvaledVariables(validator *Validator) { for i, component := range validator.typedZarfPackage.Components { checkForVarInComponentImport(validator, i, component, "") } diff --git a/src/test/e2e/12_lint_test.go b/src/test/e2e/12_lint_test.go index d1da5ba88b..b4b72f425e 100644 --- a/src/test/e2e/12_lint_test.go +++ b/src/test/e2e/12_lint_test.go @@ -43,7 +43,7 @@ func TestLint(t *testing.T) { require.Contains(t, strippedStderr, ".components.[1].images.[0] linted-import: Image not pinned with digest") require.Contains(t, strippedStderr, ".components.[1].images.[2] linted-import: Image not pinned with digest") require.Contains(t, strippedStderr, ".components.[3].import.path: Zarf does not evaluate variables at component.x.import.path ###ZARF_PKG_TMPL_PATH###") - require.Contains(t, strippedStderr, ".components.[0].images.[0] ghcr.io/defenseunicorns/packages/dos-games:1.0.0-skeleton: Image not pinned with digest defenseunicorns/zarf-game:multi-tile-dark") + require.Contains(t, strippedStderr, ".components.[0].images.[0] oci://🦄/dos-games:1.0.0-skeleton: Image not pinned with digest defenseunicorns/zarf-game:multi-tile-dark") }) } From bc27ab0a43a06af51283ac71fd7ed52b9231f1a2 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Thu, 7 Dec 2023 14:49:55 +0000 Subject: [PATCH 124/192] refactoring list and lint --- src/pkg/packager/composer/list.go | 60 +++++++++++++++----------- src/pkg/packager/composer/list_test.go | 2 +- src/pkg/packager/composer/oci.go | 6 +-- src/pkg/packager/lint/lint.go | 4 +- 4 files changed, 40 insertions(+), 32 deletions(-) diff --git a/src/pkg/packager/composer/list.go b/src/pkg/packager/composer/list.go index 606c398e68..83da97a1c8 100644 --- a/src/pkg/packager/composer/list.go +++ b/src/pkg/packager/composer/list.go @@ -38,17 +38,17 @@ func (n *Node) GetIndex() int { return n.index } -// Path from downstream zarf file to upstream imported zarf file +// GetRelativeToHead gets the path from downstream zarf file to upstream imported zarf file func (n *Node) GetRelativeToHead() string { return n.relativeToHead } -// Next node in the chain +// Next returns next node in the chain func (n *Node) Next() *Node { return n.next } -// Prev node in the chain +// Prev returns previous node in the chain func (n *Node) Prev() *Node { return n.prev } @@ -67,12 +67,20 @@ func (n *Node) ImportName() string { // ImportChain is a doubly linked list of component import definitions type ImportChain struct { - Head *Node - Tail *Node + head *Node + tail *Node remote *oci.OrasRemote } +func (ic *ImportChain) Head() *Node { + return ic.head +} + +func (ic *ImportChain) Tail() *Node { + return ic.tail +} + func (ic *ImportChain) append(c types.ZarfComponent, index int, relativeToHead string, vars []types.ZarfPackageVariable, consts []types.ZarfPackageConstant) { node := &Node{ ZarfComponent: c, @@ -83,14 +91,14 @@ func (ic *ImportChain) append(c types.ZarfComponent, index int, relativeToHead s prev: nil, next: nil, } - if ic.Head == nil { - ic.Head = node - ic.Tail = node + if ic.head == nil { + ic.head = node + ic.tail = node } else { - p := ic.Tail + p := ic.tail node.prev = p p.next = node - ic.Tail = node + ic.tail = node } } @@ -106,7 +114,7 @@ func NewImportChain(head types.ZarfComponent, index int, arch, flavor string) (* history := []string{} - node := ic.Head + node := ic.head for node != nil { isLocal := node.Import.Path != "" isRemote := node.Import.URL != "" @@ -195,22 +203,22 @@ func NewImportChain(head types.ZarfComponent, index int, arch, flavor string) (* // String returns a string representation of the import chain func (ic *ImportChain) String() string { - if ic.Head.next == nil { - return fmt.Sprintf("component %q imports nothing", ic.Head.Name) + if ic.head.next == nil { + return fmt.Sprintf("component %q imports nothing", ic.head.Name) } s := strings.Builder{} - name := ic.Head.ImportName() + name := ic.head.ImportName() - if ic.Head.Import.Path != "" { - s.WriteString(fmt.Sprintf("component %q imports %q in %s", ic.Head.Name, name, ic.Head.Import.Path)) + if ic.head.Import.Path != "" { + s.WriteString(fmt.Sprintf("component %q imports %q in %s", ic.head.Name, name, ic.head.Import.Path)) } else { - s.WriteString(fmt.Sprintf("component %q imports %q in %s", ic.Head.Name, name, ic.Head.Import.URL)) + s.WriteString(fmt.Sprintf("component %q imports %q in %s", ic.head.Name, name, ic.head.Import.URL)) } - node := ic.Head.next - for node != ic.Tail { + node := ic.head.next + for node != ic.tail { name := node.ImportName() s.WriteString(", which imports ") if node.Import.Path != "" { @@ -227,7 +235,7 @@ func (ic *ImportChain) String() string { // Migrate performs migrations on the import chain func (ic *ImportChain) Migrate(build types.ZarfBuildData) (warnings []string) { - node := ic.Head + node := ic.head for node != nil { migrated, w := deprecated.MigrateComponent(build, node.ZarfComponent) node.ZarfComponent = migrated @@ -235,7 +243,7 @@ func (ic *ImportChain) Migrate(build types.ZarfBuildData) (warnings []string) { node = node.next } if len(warnings) > 0 { - final := fmt.Sprintf("migrations were performed on the import chain of: %q", ic.Head.Name) + final := fmt.Sprintf("migrations were performed on the import chain of: %q", ic.head.Name) warnings = append(warnings, final) } return warnings @@ -244,9 +252,9 @@ func (ic *ImportChain) Migrate(build types.ZarfBuildData) (warnings []string) { // Compose merges the import chain into a single component // fixing paths, overriding metadata, etc func (ic *ImportChain) Compose() (composed types.ZarfComponent, err error) { - composed = ic.Tail.ZarfComponent + composed = ic.tail.ZarfComponent - if ic.Tail.prev == nil { + if ic.tail.prev == nil { // only had one component in the import chain return composed, nil } @@ -259,7 +267,7 @@ func (ic *ImportChain) Compose() (composed types.ZarfComponent, err error) { composed = types.ZarfComponent{} // start overriding with the tail node - node := ic.Tail + node := ic.tail for node != nil { fixPaths(&node.ZarfComponent, node.relativeToHead) @@ -283,7 +291,7 @@ func (ic *ImportChain) MergeVariables(existing []types.ZarfPackageVariable) (mer return v1.Name == v2.Name } - node := ic.Tail + node := ic.tail for node != nil { // merge the vars merged = helpers.MergeSlices(node.vars, merged, exists) @@ -300,7 +308,7 @@ func (ic *ImportChain) MergeConstants(existing []types.ZarfPackageConstant) (mer return c1.Name == c2.Name } - node := ic.Tail + node := ic.tail for node != nil { // merge the consts merged = helpers.MergeSlices(node.consts, merged, exists) diff --git a/src/pkg/packager/composer/list_test.go b/src/pkg/packager/composer/list_test.go index 41d2cd62bc..4320f501d2 100644 --- a/src/pkg/packager/composer/list_test.go +++ b/src/pkg/packager/composer/list_test.go @@ -313,7 +313,7 @@ func TestMerging(t *testing.T) { } head.next = &tail tail.prev = &head - testIC := &ImportChain{Head: &head, Tail: &tail} + testIC := &ImportChain{head: &head, tail: &tail} testCases := []testCase{ { diff --git a/src/pkg/packager/composer/oci.go b/src/pkg/packager/composer/oci.go index b2eea815aa..5a7b4637ed 100644 --- a/src/pkg/packager/composer/oci.go +++ b/src/pkg/packager/composer/oci.go @@ -37,14 +37,14 @@ func (ic *ImportChain) getRemote(url string) (*oci.OrasRemote, error) { // ContainsOCIImport returns true if the import chain contains a remote import func (ic *ImportChain) ContainsOCIImport() bool { // only the 2nd to last node may have a remote import - return ic.Tail.prev != nil && ic.Tail.prev.Import.URL != "" + return ic.tail.prev != nil && ic.tail.prev.Import.URL != "" } func (ic *ImportChain) fetchOCISkeleton() error { if !ic.ContainsOCIImport() { return nil } - node := ic.Tail.prev + node := ic.tail.prev remote, err := ic.getRemote(node.Import.URL) if err != nil { return err @@ -114,7 +114,7 @@ func (ic *ImportChain) fetchOCISkeleton() error { } // the tail node is the only node whose relativeToHead is based solely upon cwd<->cache // contrary to the other nodes, which are based upon the previous node - ic.Tail.relativeToHead = rel + ic.tail.relativeToHead = rel if oci.IsEmptyDescriptor(componentDesc) { // nothing was fetched, nothing to extract diff --git a/src/pkg/packager/lint/lint.go b/src/pkg/packager/lint/lint.go index 78b7691a05..f7b6c18317 100644 --- a/src/pkg/packager/lint/lint.go +++ b/src/pkg/packager/lint/lint.go @@ -90,8 +90,8 @@ func lintComposableComponenets(validator *Validator, createOpts types.ZarfCreate } // Skipping initial component since it will be linted the usual way - path := chain.Head.Import.URL - node := chain.Head.Next() + path := chain.Head().Import.URL + node := chain.Head().Next() for node != nil { if path == "" { path = node.GetRelativeToHead() From 4fc27827d06e4bbf6cc74e621c29a42ceec5b798 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Thu, 7 Dec 2023 16:11:24 +0000 Subject: [PATCH 125/192] changed wording, added file to lint all packages --- lint_all_zarf_packages.sh | 8 ++++++++ src/pkg/packager/composer/list.go | 3 +++ src/pkg/packager/lint/lint.go | 9 +++------ src/pkg/packager/lint/validator.go | 18 +++++++++++++++--- 4 files changed, 29 insertions(+), 9 deletions(-) create mode 100755 lint_all_zarf_packages.sh diff --git a/lint_all_zarf_packages.sh b/lint_all_zarf_packages.sh new file mode 100755 index 0000000000..40d14aa941 --- /dev/null +++ b/lint_all_zarf_packages.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +find "." -type f -name 'zarf.yaml' | while read -r yaml_file; do + dir=$(dirname "$yaml_file") + echo "Running 'zarf prepare lint' in directory: $dir" + (cd "$dir" && ~/code/zarf/build/zarf prepare lint) + echo "---" +done diff --git a/src/pkg/packager/composer/list.go b/src/pkg/packager/composer/list.go index 83da97a1c8..6c306745c2 100644 --- a/src/pkg/packager/composer/list.go +++ b/src/pkg/packager/composer/list.go @@ -164,6 +164,9 @@ func NewImportChain(head types.ZarfComponent, index int, arch, flavor string) (* if err != nil { return ic, err } + // When it's a bad file, the error gets tracked here + // Maybe we add something like couldn't fetch zarf yaml to this + // So it's clearer to the user the pull failed pkg, err = remote.FetchZarfYAML() if err != nil { return ic, err diff --git a/src/pkg/packager/lint/lint.go b/src/pkg/packager/lint/lint.go index f7b6c18317..53b2da3bf5 100644 --- a/src/pkg/packager/lint/lint.go +++ b/src/pkg/packager/lint/lint.go @@ -49,9 +49,7 @@ func ValidateZarfSchema(createOpts types.ZarfCreateOptions) (*Validator, error) return nil, fmt.Errorf("unable to access directory '%s': %w", createOpts.BaseDir, err) } - if err := lintComposableComponenets(&validator, createOpts); err != nil { - return nil, err - } + lintComposableComponenets(&validator, createOpts) lintUnEvaledVariables(&validator) @@ -72,7 +70,7 @@ func ValidateZarfSchema(createOpts types.ZarfCreateOptions) (*Validator, error) return &validator, nil } -func lintComposableComponenets(validator *Validator, createOpts types.ZarfCreateOptions) error { +func lintComposableComponenets(validator *Validator, createOpts types.ZarfCreateOptions) { for i, component := range validator.typedZarfPackage.Components { arch := config.GetArch(validator.typedZarfPackage.Metadata.Architecture) @@ -86,7 +84,7 @@ func lintComposableComponenets(validator *Validator, createOpts types.ZarfCreate chain, err := composer.NewImportChain(component, i, arch, createOpts.Flavor) if err != nil { - return err + validator.addError(validatorMessage{description: err.Error()}) } // Skipping initial component since it will be linted the usual way @@ -102,7 +100,6 @@ func lintComposableComponenets(validator *Validator, createOpts types.ZarfCreate node = node.Next() } } - return nil } func fillComponentTemplate(validator *Validator, component *types.ZarfComponent, createOpts types.ZarfCreateOptions) error { diff --git a/src/pkg/packager/lint/validator.go b/src/pkg/packager/lint/validator.go index 5df4af273b..64257dc686 100644 --- a/src/pkg/packager/lint/validator.go +++ b/src/pkg/packager/lint/validator.go @@ -48,7 +48,7 @@ type Validator struct { // DisplayFormattedMessage message sent to user based on validator results func (v Validator) DisplayFormattedMessage() { if !v.hasWarnings() && !v.hasErrors() { - message.Successf("Schema validation successful for %q", v.typedZarfPackage.Metadata.Name) + message.Successf("0 findings for %q", v.typedZarfPackage.Metadata.Name) } v.printValidationTable() } @@ -69,11 +69,23 @@ func (v Validator) printValidationTable() { connectData = append(connectData, []string{utils.ColorWrap("Error", color.FgRed), err.String()}) } message.Table(header, connectData) - message.Info(fmt.Sprintf("%d warnings and %d errors in %q", - len(v.warnings), len(v.errors), v.typedZarfPackage.Metadata.Name)) + message.Info(v.getWarningAndErrorCount()) } } +func (v Validator) getWarningAndErrorCount() string { + wordWarning := "warnings" + if len(v.warnings) == 1 { + wordWarning = "warning" + } + wordError := "errors" + if len(v.errors) == 1 { + wordError = "error" + } + return fmt.Sprintf("%d %s and %d %s in %q", + len(v.warnings), wordWarning, len(v.errors), wordError, v.typedZarfPackage.Metadata.Name) +} + func (v Validator) hasWarnings() bool { return len(v.warnings) > 0 } From c2e56576f19d241d8034ce834ba4d4775c5d798a Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Thu, 7 Dec 2023 16:15:57 +0000 Subject: [PATCH 126/192] removing comments --- src/pkg/packager/lint/lint.go | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/src/pkg/packager/lint/lint.go b/src/pkg/packager/lint/lint.go index 53b2da3bf5..30d97da24d 100644 --- a/src/pkg/packager/lint/lint.go +++ b/src/pkg/packager/lint/lint.go @@ -111,18 +111,13 @@ func fillComponentTemplate(validator *Validator, component *types.ZarfComponent, return fillYamlTemplate(validator, component, createOpts) } -// Look into breaking apart this function to allow for passing in a component -// Look into returning warnings so I don't have to pass in validator -// Look into removing package from validator func fillActiveTemplate(validator *Validator, createOpts types.ZarfCreateOptions) error { - // update the component templates on the package - // TODO add test to make sure this is covered still err := packager.FindComponentTemplatesAndReload(&validator.typedZarfPackage) if err != nil { return err } - //Does typed zarf package need to be addressed here or is it already addresssed + return fillYamlTemplate(validator, &validator.typedZarfPackage, createOpts) } @@ -184,7 +179,6 @@ func isPinnedRepo(repo string) bool { return (strings.Contains(repo, "@")) } -// Feels like validator may have too much with both the zarf package and the warnings func lintComponents(validator *Validator) { for i, component := range validator.typedZarfPackage.Components { lintComponent(validator, i, component, "") From cb4695338b81415a29a3c24a2cf2e1ed2c87a6e7 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Thu, 7 Dec 2023 16:35:38 +0000 Subject: [PATCH 127/192] change lint to not return an error on variable failure --- src/pkg/packager/lint/lint.go | 33 +++++++++++++-------------------- 1 file changed, 13 insertions(+), 20 deletions(-) diff --git a/src/pkg/packager/lint/lint.go b/src/pkg/packager/lint/lint.go index 30d97da24d..60786a0de0 100644 --- a/src/pkg/packager/lint/lint.go +++ b/src/pkg/packager/lint/lint.go @@ -53,9 +53,7 @@ func ValidateZarfSchema(createOpts types.ZarfCreateOptions) (*Validator, error) lintUnEvaledVariables(&validator) - if err := fillActiveTemplate(&validator, createOpts); err != nil { - return nil, err - } + fillActiveTemplate(&validator, createOpts) lintComponents(&validator) @@ -102,32 +100,32 @@ func lintComposableComponenets(validator *Validator, createOpts types.ZarfCreate } } -func fillComponentTemplate(validator *Validator, component *types.ZarfComponent, createOpts types.ZarfCreateOptions) error { +func fillComponentTemplate(validator *Validator, component *types.ZarfComponent, createOpts types.ZarfCreateOptions) { // update the component templates on the package err := packager.ReloadComponentTemplate(component) if err != nil { - return err + validator.addWarning(validatorMessage{description: fmt.Sprintf("unable to find components %s", err)}) } - return fillYamlTemplate(validator, component, createOpts) + fillYamlTemplate(validator, component, createOpts) } -func fillActiveTemplate(validator *Validator, createOpts types.ZarfCreateOptions) error { +func fillActiveTemplate(validator *Validator, createOpts types.ZarfCreateOptions) { err := packager.FindComponentTemplatesAndReload(&validator.typedZarfPackage) if err != nil { - return err + validator.addWarning(validatorMessage{description: fmt.Sprintf("unable to find components %s", err)}) } - return fillYamlTemplate(validator, &validator.typedZarfPackage, createOpts) + fillYamlTemplate(validator, &validator.typedZarfPackage, createOpts) } -func fillYamlTemplate(validator *Validator, yamlObj any, createOpts types.ZarfCreateOptions) error { +func fillYamlTemplate(validator *Validator, yamlObj any, createOpts types.ZarfCreateOptions) { templateMap := map[string]string{} - setVarsAndWarn := func(templatePrefix string, deprecated bool) error { + setVarsAndWarn := func(templatePrefix string, deprecated bool) { yamlTemplates, err := utils.FindYamlTemplates(yamlObj, templatePrefix, "###") if err != nil { - return err + validator.addWarning(validatorMessage{description: fmt.Sprintf("unable to find variables %s", err)}) } for key := range yamlTemplates { @@ -148,19 +146,14 @@ func fillYamlTemplate(validator *Validator, yamlObj any, createOpts types.ZarfCr for key, value := range createOpts.SetVariables { templateMap[fmt.Sprintf("%s%s###", templatePrefix, key)] = value } - return nil } - if err := setVarsAndWarn(types.ZarfPackageTemplatePrefix, false); err != nil { - return err - } + setVarsAndWarn(types.ZarfPackageTemplatePrefix, false) // [DEPRECATION] Set the Package Variable syntax as well for backward compatibility - if err := setVarsAndWarn(types.ZarfPackageVariablePrefix, true); err != nil { - return err - } + setVarsAndWarn(types.ZarfPackageVariablePrefix, true) - return utils.ReloadYamlTemplate(yamlObj, templateMap) + utils.ReloadYamlTemplate(yamlObj, templateMap) } func isPinnedImage(image string) (bool, error) { From b825184baccbdbc6674360ded49030ab6004a930 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Thu, 7 Dec 2023 16:36:21 +0000 Subject: [PATCH 128/192] refactor --- src/pkg/packager/lint/lint.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pkg/packager/lint/lint.go b/src/pkg/packager/lint/lint.go index 60786a0de0..11cfc6cd28 100644 --- a/src/pkg/packager/lint/lint.go +++ b/src/pkg/packager/lint/lint.go @@ -85,8 +85,8 @@ func lintComposableComponenets(validator *Validator, createOpts types.ZarfCreate validator.addError(validatorMessage{description: err.Error()}) } - // Skipping initial component since it will be linted the usual way path := chain.Head().Import.URL + // Skipping initial component since it will be linted the usual way node := chain.Head().Next() for node != nil { if path == "" { From a620c936f02aca9c49ec4b6b9a8a405bf711c1e1 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Thu, 7 Dec 2023 16:38:39 +0000 Subject: [PATCH 129/192] refactor tests --- src/pkg/packager/lint/lint_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pkg/packager/lint/lint_test.go b/src/pkg/packager/lint/lint_test.go index 22ebf4b049..175219514e 100644 --- a/src/pkg/packager/lint/lint_test.go +++ b/src/pkg/packager/lint/lint_test.go @@ -118,9 +118,9 @@ func TestValidateSchema(t *testing.T) { badImage}} checkForUnpinnedImages(&validator, 0, component, "") require.Equal(t, unpinnedImage, validator.warnings[0].item) - require.Equal(t, len(validator.warnings), 1) + require.Equal(t, 1, len(validator.warnings)) require.Equal(t, badImage, validator.errors[0].item) - require.Equal(t, len(validator.errors), 1) + require.Equal(t, 1, len(validator.errors)) }) t.Run("Unpinnned file warning", func(t *testing.T) { From 4cfaa3dac80a8a87e57a9f23b90ac6287b163ff8 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Thu, 7 Dec 2023 16:43:20 +0000 Subject: [PATCH 130/192] fix tests --- src/pkg/packager/lint/lint.go | 2 +- src/pkg/packager/lint/lint_test.go | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/pkg/packager/lint/lint.go b/src/pkg/packager/lint/lint.go index 11cfc6cd28..1a49c75393 100644 --- a/src/pkg/packager/lint/lint.go +++ b/src/pkg/packager/lint/lint.go @@ -101,7 +101,7 @@ func lintComposableComponenets(validator *Validator, createOpts types.ZarfCreate } func fillComponentTemplate(validator *Validator, component *types.ZarfComponent, createOpts types.ZarfCreateOptions) { - // update the component templates on the package + err := packager.ReloadComponentTemplate(component) if err != nil { validator.addWarning(validatorMessage{description: fmt.Sprintf("unable to find components %s", err)}) diff --git a/src/pkg/packager/lint/lint_test.go b/src/pkg/packager/lint/lint_test.go index 175219514e..00b120c77f 100644 --- a/src/pkg/packager/lint/lint_test.go +++ b/src/pkg/packager/lint/lint_test.go @@ -10,6 +10,7 @@ import ( "os" "testing" + "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/types" goyaml "github.com/goccy/go-yaml" "github.com/stretchr/testify/require" @@ -73,6 +74,7 @@ func TestValidateSchema(t *testing.T) { validator := Validator{untypedZarfPackage: unmarshalledYaml, jsonSchema: getZarfSchema(t)} err := validateSchema(&validator) require.NoError(t, err) + config.NoColor = true require.Equal(t, ".components.[0].import: Additional property not-path is not allowed", validator.errors[0].String()) require.Equal(t, ".components.[1].import.path: Invalid type. Expected: string, given: integer", validator.errors[1].String()) }) From cf9059bacc40087817fe005321632e09489d5506 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Thu, 7 Dec 2023 16:46:23 +0000 Subject: [PATCH 131/192] mod tidy --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 2b7271b1c3..b2d84b7f02 100644 --- a/go.mod +++ b/go.mod @@ -54,7 +54,6 @@ require ( k8s.io/component-base v0.28.2 k8s.io/klog/v2 v2.100.1 k8s.io/kubectl v0.28.2 - k8s.io/utils v0.0.0-20231127182322-b307cd553661 oras.land/oras-go/v2 v2.3.1 sigs.k8s.io/kustomize/api v0.14.0 sigs.k8s.io/kustomize/kyaml v0.14.3 @@ -465,6 +464,7 @@ require ( k8s.io/component-helpers v0.28.2 // indirect k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect k8s.io/metrics v0.28.2 // indirect + k8s.io/utils v0.0.0-20231127182322-b307cd553661 // indirect oras.land/oras-go v1.2.4 // indirect sigs.k8s.io/controller-runtime v0.16.0 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect From 8b1d34a43c0870ac7fea5e79e1675ccce1c716bc Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Thu, 7 Dec 2023 16:50:00 +0000 Subject: [PATCH 132/192] refactor tests --- src/pkg/packager/lint/lint_test.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/pkg/packager/lint/lint_test.go b/src/pkg/packager/lint/lint_test.go index 00b120c77f..73a79982fa 100644 --- a/src/pkg/packager/lint/lint_test.go +++ b/src/pkg/packager/lint/lint_test.go @@ -88,13 +88,12 @@ func TestValidateSchema(t *testing.T) { }) t.Run("Template in component import failure", func(t *testing.T) { - validator := Validator{} pathVar := "###ZARF_PKG_TMPL_PATH###" ociPathVar := fmt.Sprintf("oci://%s", pathVar) pathComponent := types.ZarfComponent{Import: types.ZarfComponentImport{Path: pathVar}} URLComponent := types.ZarfComponent{Import: types.ZarfComponentImport{URL: ociPathVar}} - checkForVarInComponentImport(&validator, 2, pathComponent, "") - checkForVarInComponentImport(&validator, 3, URLComponent, "") + validator := Validator{typedZarfPackage: types.ZarfPackage{Components: []types.ZarfComponent{pathComponent, URLComponent}}} + lintUnEvaledVariables(&validator) require.Equal(t, pathVar, validator.warnings[0].item) require.Equal(t, ociPathVar, validator.warnings[1].item) }) From 308f84d220af4aac6047fe922ef5a8d36432c8a3 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Thu, 7 Dec 2023 18:02:29 +0000 Subject: [PATCH 133/192] lint --- src/pkg/packager/composer/list.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/pkg/packager/composer/list.go b/src/pkg/packager/composer/list.go index 9058b719ee..e95da84b40 100644 --- a/src/pkg/packager/composer/list.go +++ b/src/pkg/packager/composer/list.go @@ -73,10 +73,12 @@ type ImportChain struct { remote *oci.OrasRemote } +// Head returns the first node in the import chain func (ic *ImportChain) Head() *Node { return ic.head } +// Returns the last node in the import chain func (ic *ImportChain) Tail() *Node { return ic.tail } From 09de058c005ff6c38b7327e2aaed738e5267d9b9 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Thu, 7 Dec 2023 18:14:57 +0000 Subject: [PATCH 134/192] list returns nil on error + lint --- src/pkg/packager/composer/list.go | 24 ++++++++++++------------ src/pkg/packager/lint/lint.go | 1 + 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/src/pkg/packager/composer/list.go b/src/pkg/packager/composer/list.go index e95da84b40..a4572d48a8 100644 --- a/src/pkg/packager/composer/list.go +++ b/src/pkg/packager/composer/list.go @@ -78,7 +78,7 @@ func (ic *ImportChain) Head() *Node { return ic.head } -// Returns the last node in the import chain +// Tail Returns the last node in the import chain func (ic *ImportChain) Tail() *Node { return ic.tail } @@ -129,16 +129,16 @@ func NewImportChain(head types.ZarfComponent, index int, arch, flavor string) (* // TODO: stuff like this should also happen in linting if err := validate.ImportDefinition(&node.ZarfComponent); err != nil { - return ic, err + return nil, err } // ensure that remote components are not importing other remote components if node.prev != nil && node.prev.Import.URL != "" && isRemote { - return ic, fmt.Errorf("detected malformed import chain, cannot import remote components from remote components") + return nil, fmt.Errorf("detected malformed import chain, cannot import remote components from remote components") } // ensure that remote components are not importing local components if node.prev != nil && node.prev.Import.URL != "" && isLocal { - return ic, fmt.Errorf("detected malformed import chain, cannot import local components from remote components") + return nil, fmt.Errorf("detected malformed import chain, cannot import local components from remote components") } var pkg types.ZarfPackage @@ -152,26 +152,26 @@ func NewImportChain(head types.ZarfComponent, index int, arch, flavor string) (* prev := node for prev != nil { if prev.relativeToHead == relativeToHead { - return ic, fmt.Errorf("detected circular import chain: %s", strings.Join(history, " -> ")) + return nil, fmt.Errorf("detected circular import chain: %s", strings.Join(history, " -> ")) } prev = prev.prev } // this assumes the composed package is following the zarf layout if err := utils.ReadYaml(filepath.Join(relativeToHead, layout.ZarfYAML), &pkg); err != nil { - return ic, err + return nil, err } } else if isRemote { remote, err := ic.getRemote(node.Import.URL) if err != nil { - return ic, err + return nil, err } // When it's a bad file, the error gets tracked here // Maybe we add something like couldn't fetch zarf yaml to this // So it's clearer to the user the pull failed pkg, err = remote.FetchZarfYAML() if err != nil { - return ic, err + return nil, err } } @@ -188,15 +188,15 @@ func NewImportChain(head types.ZarfComponent, index int, arch, flavor string) (* if len(found) == 0 { if isLocal { - return ic, fmt.Errorf("component %q not found in %q", name, filepath.Join(history...)) + return nil, fmt.Errorf("component %q not found in %q", name, filepath.Join(history...)) } else if isRemote { - return ic, fmt.Errorf("component %q not found in %q", name, node.Import.URL) + return nil, fmt.Errorf("component %q not found in %q", name, node.Import.URL) } } else if len(found) > 1 { if isLocal { - return ic, fmt.Errorf("multiple components named %q found in %q satisfying %q", name, filepath.Join(history...), arch) + return nil, fmt.Errorf("multiple components named %q found in %q satisfying %q", name, filepath.Join(history...), arch) } else if isRemote { - return ic, fmt.Errorf("multiple components named %q found in %q satisfying %q", name, node.Import.URL, arch) + return nil, fmt.Errorf("multiple components named %q found in %q satisfying %q", name, node.Import.URL, arch) } } diff --git a/src/pkg/packager/lint/lint.go b/src/pkg/packager/lint/lint.go index 1a49c75393..ac60a64899 100644 --- a/src/pkg/packager/lint/lint.go +++ b/src/pkg/packager/lint/lint.go @@ -83,6 +83,7 @@ func lintComposableComponenets(validator *Validator, createOpts types.ZarfCreate chain, err := composer.NewImportChain(component, i, arch, createOpts.Flavor) if err != nil { validator.addError(validatorMessage{description: err.Error()}) + continue } path := chain.Head().Import.URL From 74ac3db70603bac1e0a218f90d77d3ba55048aff Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Thu, 7 Dec 2023 19:15:50 +0000 Subject: [PATCH 135/192] moving lint all packages into hack + adding make command --- Makefile | 3 +++ .../lint_all_zarf_packages.sh | 9 ++++++++- 2 files changed, 11 insertions(+), 1 deletion(-) rename lint_all_zarf_packages.sh => hack/lint_all_zarf_packages.sh (53%) diff --git a/Makefile b/Makefile index 3c4d044cf6..b970c1cbf8 100644 --- a/Makefile +++ b/Makefile @@ -76,6 +76,9 @@ docs-and-schema: ## Generate the Zarf Documentation and Schema hack/gen-cli-docs.sh ZARF_CONFIG=hack/empty-config.toml hack/create-zarf-schema.sh +lint-all-packages: ## Recurisvely lint all zarf.yaml files + hack/lint_all_zarf_packages.sh + # INTERNAL: a shim used to build the agent image only if needed on Windows using the `test` command init-package-local-agent: @test "$(AGENT_IMAGE_TAG)" != "local" || $(MAKE) build-local-agent-image diff --git a/lint_all_zarf_packages.sh b/hack/lint_all_zarf_packages.sh similarity index 53% rename from lint_all_zarf_packages.sh rename to hack/lint_all_zarf_packages.sh index 40d14aa941..51c61c6fa6 100755 --- a/lint_all_zarf_packages.sh +++ b/hack/lint_all_zarf_packages.sh @@ -1,8 +1,15 @@ #!/bin/bash +use_build=$1 +BASEDIR=$(dirname "$0") +cd ../$BASEDIR find "." -type f -name 'zarf.yaml' | while read -r yaml_file; do dir=$(dirname "$yaml_file") echo "Running 'zarf prepare lint' in directory: $dir" - (cd "$dir" && ~/code/zarf/build/zarf prepare lint) + if [ "$use_build" = true ]; then + ./build/zarf prepare lint $dir + else + zarf prepare lint $dir + fi echo "---" done From 82913ff1cab908a245389f3e3884f09299a40561 Mon Sep 17 00:00:00 2001 From: Austin Abro <37223396+AustinAbro321@users.noreply.github.com> Date: Thu, 7 Dec 2023 15:40:45 -0500 Subject: [PATCH 136/192] Update src/pkg/packager/variables.go Co-authored-by: Lucas Rodriguez --- src/pkg/packager/variables.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/pkg/packager/variables.go b/src/pkg/packager/variables.go index 1532629019..cb223c68bb 100644 --- a/src/pkg/packager/variables.go +++ b/src/pkg/packager/variables.go @@ -31,7 +31,9 @@ func ReloadComponentTemplate(component *types.ZarfComponent) error { func FindComponentTemplatesAndReload(zarfPackage *types.ZarfPackage) error { // iterate through components to and find all ###ZARF_COMPONENT_NAME, assign to component Name and value for i := range zarfPackage.Components { - ReloadComponentTemplate(&zarfPackage.Components[i]) + if err := ReloadComponentTemplate(&zarfPackage.Components[i]); err != nil { + return err + } } return nil From 1a46639a7bc352fc82e87d8f631b3eb59910580f Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Thu, 7 Dec 2023 21:16:18 +0000 Subject: [PATCH 137/192] remove unneeded code --- src/pkg/packager/lint/lint.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/pkg/packager/lint/lint.go b/src/pkg/packager/lint/lint.go index ac60a64899..caad269022 100644 --- a/src/pkg/packager/lint/lint.go +++ b/src/pkg/packager/lint/lint.go @@ -76,10 +76,6 @@ func lintComposableComponenets(validator *Validator, createOpts types.ZarfCreate continue } - // This is done in composer, should I just delete this or move it into newImportChain - // component.Only.Cluster.Architecture = "" - // component.Only.Flavor = "" - chain, err := composer.NewImportChain(component, i, arch, createOpts.Flavor) if err != nil { validator.addError(validatorMessage{description: err.Error()}) From 6b810aa558872708a84373ee0b27d6cbade796f2 Mon Sep 17 00:00:00 2001 From: Austin Abro <37223396+AustinAbro321@users.noreply.github.com> Date: Thu, 7 Dec 2023 16:35:49 -0500 Subject: [PATCH 138/192] Update src/pkg/packager/composer/list.go Co-authored-by: razzle --- src/pkg/packager/composer/list.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pkg/packager/composer/list.go b/src/pkg/packager/composer/list.go index a4572d48a8..83fe06ec8a 100644 --- a/src/pkg/packager/composer/list.go +++ b/src/pkg/packager/composer/list.go @@ -78,7 +78,7 @@ func (ic *ImportChain) Head() *Node { return ic.head } -// Tail Returns the last node in the import chain +// Tail returns the last node in the import chain func (ic *ImportChain) Tail() *Node { return ic.tail } From 3ac81c5219c3e626bfb8494436ba37f9d2dc7200 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Fri, 8 Dec 2023 15:03:35 +0000 Subject: [PATCH 139/192] super ugly wip and testing things out --- hack/lint_all_zarf_packages.sh | 3 + src/pkg/packager/composer/list.go | 10 +- src/pkg/packager/lint/validator.go | 101 +++++++++++++++--- src/test/e2e/12_lint_test.go | 4 +- .../packages/12-lint/linted-import/zarf.yaml | 3 - src/test/packages/12-lint/zarf.yaml | 6 -- 6 files changed, 99 insertions(+), 28 deletions(-) diff --git a/hack/lint_all_zarf_packages.sh b/hack/lint_all_zarf_packages.sh index 51c61c6fa6..94d0df4f9f 100755 --- a/hack/lint_all_zarf_packages.sh +++ b/hack/lint_all_zarf_packages.sh @@ -5,6 +5,9 @@ BASEDIR=$(dirname "$0") cd ../$BASEDIR find "." -type f -name 'zarf.yaml' | while read -r yaml_file; do dir=$(dirname "$yaml_file") + if [[ "$dir" == *src/test/* ]] && [ "$use_build" != true ]; then + continue + fi echo "Running 'zarf prepare lint' in directory: $dir" if [ "$use_build" = true ]; then ./build/zarf prepare lint $dir diff --git a/src/pkg/packager/composer/list.go b/src/pkg/packager/composer/list.go index a4572d48a8..cb394ae8dd 100644 --- a/src/pkg/packager/composer/list.go +++ b/src/pkg/packager/composer/list.go @@ -33,7 +33,7 @@ type Node struct { next *Node } -// GetIndex gives the component index of the node on it's original zarf file +// GetIndex returns the .components index location for this node's source `zarf.yaml` func (n *Node) GetIndex() int { return n.index } @@ -73,7 +73,7 @@ type ImportChain struct { remote *oci.OrasRemote } -// Head returns the first node in the import chain +// Head eturns the first node in the import chain func (ic *ImportChain) Head() *Node { return ic.head } @@ -178,11 +178,11 @@ func NewImportChain(head types.ZarfComponent, index int, arch, flavor string) (* name := node.ImportName() found := []types.ZarfComponent{} - var index int + index := []int{} for i, component := range pkg.Components { if component.Name == name && CompatibleComponent(component, arch, flavor) { found = append(found, component) - index = i + index = append(index, i) } } @@ -200,7 +200,7 @@ func NewImportChain(head types.ZarfComponent, index int, arch, flavor string) (* } } - ic.append(found[0], index, filepath.Join(history...), pkg.Variables, pkg.Constants) + ic.append(found[0], index[0], filepath.Join(history...), pkg.Variables, pkg.Constants) node = node.next } return ic, nil diff --git a/src/pkg/packager/lint/validator.go b/src/pkg/packager/lint/validator.go index 64257dc686..833c97a0fd 100644 --- a/src/pkg/packager/lint/validator.go +++ b/src/pkg/packager/lint/validator.go @@ -21,18 +21,16 @@ type validatorMessage struct { } func (v validatorMessage) String() string { - if v.filePath != "" { - v.filePath = fmt.Sprintf(" %s", v.filePath) - } + // if v.filePath != "" { + // v.filePath = fmt.Sprintf(" %s", v.filePath) + // } if v.item != "" { - v.item = fmt.Sprintf(" %s", v.item) + v.item = fmt.Sprintf(" - %s", v.item) } if v.filePath == "" && v.yqPath == "" && v.item == "" { return v.description } - return fmt.Sprintf("%s%s: %s%s", - utils.ColorWrap(v.yqPath, color.FgCyan), utils.ColorWrap(v.filePath, color.FgCyan), - v.description, v.item) + return fmt.Sprintf("%s%s", v.description, v.item) } // Validator holds the warnings/errors and messaging that we get from validation @@ -50,7 +48,7 @@ func (v Validator) DisplayFormattedMessage() { if !v.hasWarnings() && !v.hasErrors() { message.Successf("0 findings for %q", v.typedZarfPackage.Metadata.Name) } - v.printValidationTable() + v.printValidationTable2() } // IsSuccess returns true if there are not any errors @@ -60,19 +58,98 @@ func (v Validator) IsSuccess() bool { func (v Validator) printValidationTable() { if v.hasWarnings() || v.hasErrors() { - header := []string{"Type", "Message"} + header := []string{"Type", "Path", "Message"} connectData := [][]string{} for _, warning := range v.warnings { - connectData = append(connectData, []string{utils.ColorWrap("Warning", color.FgYellow), warning.String()}) + connectData = append(connectData, + []string{utils.ColorWrap("Warning", color.FgYellow), warning.getPath(), warning.String()}) } - for _, err := range v.errors { - connectData = append(connectData, []string{utils.ColorWrap("Error", color.FgRed), err.String()}) + for _, validatorError := range v.errors { + connectData = append(connectData, + []string{utils.ColorWrap("Error", color.FgRed), validatorError.getPath(), validatorError.String()}) } message.Table(header, connectData) message.Info(v.getWarningAndErrorCount()) } } +func (v Validator) printValidationTable2() { + differentPaths := v.getUniquePaths() + if v.hasWarnings() || v.hasErrors() { + for _, path := range differentPaths { + header := []string{"Type", "Path", "Message"} + connectData := make(map[string][][]string) + item := path + for _, warning := range v.warnings { + if warning.filePath == path { + if item == "" { + item = "original" + } + connectData[item] = append(connectData[item], + []string{utils.ColorWrap("Warning", color.FgYellow), warning.getPath2(), warning.String()}) + } + } + for _, validatorError := range v.errors { + if validatorError.filePath == path { + if item == "" { + item = "original" + } + connectData[item] = append(connectData[item], + []string{utils.ColorWrap("Error", color.FgRed), validatorError.getPath2(), validatorError.String()}) + } + } + + message.Infof("Component at path: %s", item) + message.Table(header, connectData[item]) + //message.Info(v.getWarningAndErrorCount()) + } + } +} + +func contains(slice []string, item string) bool { + for _, v := range slice { + if v == item { + return true + } + } + return false +} + +func (v Validator) getUniquePaths() []string { + paths := []string{} + for _, warning := range v.warnings { + if !contains(paths, warning.filePath) { + paths = append(paths, warning.filePath) + } + } + for _, validatorError := range v.errors { + if !contains(paths, validatorError.filePath) { + paths = append(paths, validatorError.filePath) + } + } + return paths +} + +func (vm validatorMessage) getPath() string { + if vm.yqPath == "" { + return "" + } + if vm.filePath != "" { + return utils.ColorWrap(fmt.Sprintf("%s %s", vm.yqPath, vm.filePath), color.FgCyan) + } + return utils.ColorWrap(vm.yqPath, color.FgCyan) +} + +func (vm validatorMessage) getPath2() string { + if vm.yqPath == "" { + return "" + } + // if vm.filePath != "" { + // return utils.ColorWrap(fmt.Sprintf("%s %s", vm.yqPath, vm.filePath), color.FgCyan) + // } + return utils.ColorWrap(vm.yqPath, color.FgCyan) +} + func (v Validator) getWarningAndErrorCount() string { wordWarning := "warnings" if len(v.warnings) == 1 { diff --git a/src/test/e2e/12_lint_test.go b/src/test/e2e/12_lint_test.go index b4b72f425e..597fa3070b 100644 --- a/src/test/e2e/12_lint_test.go +++ b/src/test/e2e/12_lint_test.go @@ -42,8 +42,8 @@ func TestLint(t *testing.T) { require.Contains(t, strippedStderr, ".components.[2].images.[3]: Image not pinned with digest") require.Contains(t, strippedStderr, ".components.[1].images.[0] linted-import: Image not pinned with digest") require.Contains(t, strippedStderr, ".components.[1].images.[2] linted-import: Image not pinned with digest") - require.Contains(t, strippedStderr, ".components.[3].import.path: Zarf does not evaluate variables at component.x.import.path ###ZARF_PKG_TMPL_PATH###") - require.Contains(t, strippedStderr, ".components.[0].images.[0] oci://🦄/dos-games:1.0.0-skeleton: Image not pinned with digest defenseunicorns/zarf-game:multi-tile-dark") + require.Contains(t, strippedStderr, ".components.[3].import.path: Zarf does not evaluate variables at component.x.import.path - ###ZARF_PKG_TMPL_PATH###") + require.Contains(t, strippedStderr, ".components.[0].images.[0] oci://🦄/dos-games:1.0.0-skeleton: Image not pinned with digest - defenseunicorns/zarf-game:multi-tile-dark") }) } diff --git a/src/test/packages/12-lint/linted-import/zarf.yaml b/src/test/packages/12-lint/linted-import/zarf.yaml index 1f6b2b818d..db2d92415d 100644 --- a/src/test/packages/12-lint/linted-import/zarf.yaml +++ b/src/test/packages/12-lint/linted-import/zarf.yaml @@ -16,6 +16,3 @@ components: - busybox@sha256:3fbc632167424a6d997e74f52b878d7cc478225cffac6bc977eedfe51c7f4e79 - busybox:###ZARF_PKG_TMPL_BUSYBOX_IMAGE### - busybox:###ZARF_PKG_TMPL_UNSET### - files: - - source: https://github.com/k3s-io/k3s/releases/download/v1.28.2+k3s1/k3s - target: src/ diff --git a/src/test/packages/12-lint/zarf.yaml b/src/test/packages/12-lint/zarf.yaml index 940cfbe197..36f762ba0d 100644 --- a/src/test/packages/12-lint/zarf.yaml +++ b/src/test/packages/12-lint/zarf.yaml @@ -16,12 +16,6 @@ components: - name: import-test import: path: linted-import - files: - # Is this a scenario we care about. The downstream overrides with a shasum. - # Is this even overridding or are they downloading the same file twice - - source: https://github.com/k3s-io/k3s/releases/download/v1.28.2+k3s1/k3s - shasum: 2f041d37a2c6d54d53e106e1c7713bc48f806f3919b0d9e092f5fcbdc55b41cf - target: src/ - name: full-repo repos: From 8a93bb3c6529cea6c1124e52cd76772fdae1a39f Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Fri, 8 Dec 2023 17:23:52 +0000 Subject: [PATCH 140/192] WIP, things working somewhat --- src/pkg/packager/lint/lint.go | 4 +- src/pkg/packager/lint/lint_test.go | 29 +++--- src/pkg/packager/lint/validator.go | 158 ++++++++++++++--------------- 3 files changed, 93 insertions(+), 98 deletions(-) diff --git a/src/pkg/packager/lint/lint.go b/src/pkg/packager/lint/lint.go index caad269022..cdf2f78ec0 100644 --- a/src/pkg/packager/lint/lint.go +++ b/src/pkg/packager/lint/lint.go @@ -133,9 +133,9 @@ func fillYamlTemplate(validator *Validator, yamlObj any, createOpts types.ZarfCr } _, present := createOpts.SetVariables[key] if !present && !validator.hasUnSetVarWarning { - validator.warnings = append([]validatorMessage{{ + validator.findings = append([]validatorMessage{{ description: "There are variables that are unset and won't be evaluated during lint", - }}, validator.warnings...) + }}, validator.findings...) validator.hasUnSetVarWarning = true } } diff --git a/src/pkg/packager/lint/lint_test.go b/src/pkg/packager/lint/lint_test.go index 73a79982fa..dc13186205 100644 --- a/src/pkg/packager/lint/lint_test.go +++ b/src/pkg/packager/lint/lint_test.go @@ -66,7 +66,7 @@ func TestValidateSchema(t *testing.T) { validator := Validator{untypedZarfPackage: unmarshalledYaml, jsonSchema: getZarfSchema(t)} err := validateSchema(&validator) require.NoError(t, err) - require.Empty(t, validator.errors) + require.Empty(t, validator.findings) }) t.Run("validate schema fail", func(t *testing.T) { @@ -75,16 +75,15 @@ func TestValidateSchema(t *testing.T) { err := validateSchema(&validator) require.NoError(t, err) config.NoColor = true - require.Equal(t, ".components.[0].import: Additional property not-path is not allowed", validator.errors[0].String()) - require.Equal(t, ".components.[1].import.path: Invalid type. Expected: string, given: integer", validator.errors[1].String()) + require.Equal(t, ".components.[0].import: Additional property not-path is not allowed", validator.findings[0].String()) + require.Equal(t, ".components.[1].import.path: Invalid type. Expected: string, given: integer", validator.findings[1].String()) }) t.Run("Template in component import success", func(t *testing.T) { unmarshalledYaml := readAndUnmarshalYaml[types.ZarfPackage](t, goodZarfPackage) validator := Validator{typedZarfPackage: unmarshalledYaml} lintComponents(&validator) - require.Empty(t, validator.warnings) - require.Empty(t, validator.errors) + require.Empty(t, validator.findings) }) t.Run("Template in component import failure", func(t *testing.T) { @@ -94,8 +93,8 @@ func TestValidateSchema(t *testing.T) { URLComponent := types.ZarfComponent{Import: types.ZarfComponentImport{URL: ociPathVar}} validator := Validator{typedZarfPackage: types.ZarfPackage{Components: []types.ZarfComponent{pathComponent, URLComponent}}} lintUnEvaledVariables(&validator) - require.Equal(t, pathVar, validator.warnings[0].item) - require.Equal(t, ociPathVar, validator.warnings[1].item) + require.Equal(t, pathVar, validator.findings[0].item) + require.Equal(t, ociPathVar, validator.findings[1].item) }) t.Run("Unpinnned repo warning", func(t *testing.T) { @@ -105,8 +104,8 @@ func TestValidateSchema(t *testing.T) { unpinnedRepo, "https://dev.azure.com/defenseunicorns/zarf-public-test/_git/zarf-public-test@v0.0.1"}} checkForUnpinnedRepos(&validator, 0, component, "") - require.Equal(t, unpinnedRepo, validator.warnings[0].item) - require.Equal(t, len(validator.warnings), 1) + require.Equal(t, unpinnedRepo, validator.findings[0].item) + require.Equal(t, len(validator.findings), 1) }) t.Run("Unpinnned image warning", func(t *testing.T) { @@ -118,10 +117,10 @@ func TestValidateSchema(t *testing.T) { "busybox:latest@sha256:3fbc632167424a6d997e74f52b878d7cc478225cffac6bc977eedfe51c7f4e79", badImage}} checkForUnpinnedImages(&validator, 0, component, "") - require.Equal(t, unpinnedImage, validator.warnings[0].item) - require.Equal(t, 1, len(validator.warnings)) - require.Equal(t, badImage, validator.errors[0].item) - require.Equal(t, 1, len(validator.errors)) + require.Equal(t, unpinnedImage, validator.findings[0].item) + require.Equal(t, 1, len(validator.findings)) + require.Equal(t, badImage, validator.findings[1].item) + require.Equal(t, 1, len(validator.findings)) }) t.Run("Unpinnned file warning", func(t *testing.T) { @@ -134,8 +133,8 @@ func TestValidateSchema(t *testing.T) { } component := types.ZarfComponent{Files: zarfFiles} checkForUnpinnedFiles(&validator, 0, component, "") - require.Equal(t, filename, validator.warnings[0].item) - require.Equal(t, 1, len(validator.warnings)) + require.Equal(t, filename, validator.findings[0].item) + require.Equal(t, 1, len(validator.findings)) }) t.Run("Wrap standalone numbers in bracket", func(t *testing.T) { diff --git a/src/pkg/packager/lint/validator.go b/src/pkg/packager/lint/validator.go index 833c97a0fd..a1ead77e0b 100644 --- a/src/pkg/packager/lint/validator.go +++ b/src/pkg/packager/lint/validator.go @@ -13,17 +13,33 @@ import ( "github.com/fatih/color" ) +type ValidationType int + +const ( + validationError ValidationType = 1 + validationWarning ValidationType = 2 +) + type validatorMessage struct { - yqPath string - filePath string - description string - item string + yqPath string + filePath string + description string + item string + packageName string + validationType ValidationType +} + +func (vt ValidationType) String() string { + if vt == validationError { + return utils.ColorWrap("Error", color.FgRed) + } else if vt == validationWarning { + return utils.ColorWrap("Warning", color.FgYellow) + } else { + return "" + } } func (v validatorMessage) String() string { - // if v.filePath != "" { - // v.filePath = fmt.Sprintf(" %s", v.filePath) - // } if v.item != "" { v.item = fmt.Sprintf(" - %s", v.item) } @@ -35,8 +51,7 @@ func (v validatorMessage) String() string { // Validator holds the warnings/errors and messaging that we get from validation type Validator struct { - warnings []validatorMessage - errors []validatorMessage + findings []validatorMessage jsonSchema []byte typedZarfPackage types.ZarfPackage untypedZarfPackage interface{} @@ -45,7 +60,7 @@ type Validator struct { // DisplayFormattedMessage message sent to user based on validator results func (v Validator) DisplayFormattedMessage() { - if !v.hasWarnings() && !v.hasErrors() { + if !v.hasFindings() { message.Successf("0 findings for %q", v.typedZarfPackage.Metadata.Name) } v.printValidationTable2() @@ -53,51 +68,52 @@ func (v Validator) DisplayFormattedMessage() { // IsSuccess returns true if there are not any errors func (v Validator) IsSuccess() bool { - return !v.hasErrors() -} - -func (v Validator) printValidationTable() { - if v.hasWarnings() || v.hasErrors() { - header := []string{"Type", "Path", "Message"} - connectData := [][]string{} - for _, warning := range v.warnings { - connectData = append(connectData, - []string{utils.ColorWrap("Warning", color.FgYellow), warning.getPath(), warning.String()}) - } - for _, validatorError := range v.errors { - connectData = append(connectData, - []string{utils.ColorWrap("Error", color.FgRed), validatorError.getPath(), validatorError.String()}) - } - message.Table(header, connectData) - message.Info(v.getWarningAndErrorCount()) - } -} + return !v.hasFindings() +} + +// func (v Validator) printValidationTable() { +// if v.hasWarnings() || v.hasErrors() { +// header := []string{"Type", "Path", "Message"} +// connectData := [][]string{} +// for _, warning := range v.findings { +// connectData = append(connectData, +// []string{utils.ColorWrap("Warning", color.FgYellow), warning.getPath(), warning.String()}) +// } +// for _, validatorError := range v.errors { +// connectData = append(connectData, +// []string{utils.ColorWrap("Error", color.FgRed), validatorError.getPath(), validatorError.String()}) +// } +// message.Table(header, connectData) +// message.Info(v.getWarningAndErrorCount()) +// } +// } func (v Validator) printValidationTable2() { differentPaths := v.getUniquePaths() - if v.hasWarnings() || v.hasErrors() { + if v.hasFindings() { for _, path := range differentPaths { header := []string{"Type", "Path", "Message"} connectData := make(map[string][][]string) item := path - for _, warning := range v.warnings { - if warning.filePath == path { + for _, finding := range v.findings { + if finding.filePath == path { if item == "" { item = "original" } connectData[item] = append(connectData[item], - []string{utils.ColorWrap("Warning", color.FgYellow), warning.getPath2(), warning.String()}) - } - } - for _, validatorError := range v.errors { - if validatorError.filePath == path { - if item == "" { - item = "original" - } - connectData[item] = append(connectData[item], - []string{utils.ColorWrap("Error", color.FgRed), validatorError.getPath2(), validatorError.String()}) + []string{utils.ColorWrap(finding.validationType.String(), color.FgYellow), + finding.getPath(), finding.String()}) } } + // for _, validatorError := range v.errors { + // if validatorError.filePath == path { + // if item == "" { + // item = "original" + // } + // connectData[item] = append(connectData[item], + // []string{utils.ColorWrap("Error", color.FgRed), validatorError.getPath(), validatorError.String()}) + // } + // } message.Infof("Component at path: %s", item) message.Table(header, connectData[item]) @@ -117,16 +133,11 @@ func contains(slice []string, item string) bool { func (v Validator) getUniquePaths() []string { paths := []string{} - for _, warning := range v.warnings { + for _, warning := range v.findings { if !contains(paths, warning.filePath) { paths = append(paths, warning.filePath) } } - for _, validatorError := range v.errors { - if !contains(paths, validatorError.filePath) { - paths = append(paths, validatorError.filePath) - } - } return paths } @@ -134,47 +145,32 @@ func (vm validatorMessage) getPath() string { if vm.yqPath == "" { return "" } - if vm.filePath != "" { - return utils.ColorWrap(fmt.Sprintf("%s %s", vm.yqPath, vm.filePath), color.FgCyan) - } return utils.ColorWrap(vm.yqPath, color.FgCyan) } -func (vm validatorMessage) getPath2() string { - if vm.yqPath == "" { - return "" - } - // if vm.filePath != "" { - // return utils.ColorWrap(fmt.Sprintf("%s %s", vm.yqPath, vm.filePath), color.FgCyan) - // } - return utils.ColorWrap(vm.yqPath, color.FgCyan) -} - -func (v Validator) getWarningAndErrorCount() string { - wordWarning := "warnings" - if len(v.warnings) == 1 { - wordWarning = "warning" - } - wordError := "errors" - if len(v.errors) == 1 { - wordError = "error" - } - return fmt.Sprintf("%d %s and %d %s in %q", - len(v.warnings), wordWarning, len(v.errors), wordError, v.typedZarfPackage.Metadata.Name) -} - -func (v Validator) hasWarnings() bool { - return len(v.warnings) > 0 -} +// func (v Validator) getWarningAndErrorCount() string { +// wordWarning := "warnings" +// if len(v.findings) == 1 { +// wordWarning = "warning" +// } +// wordError := "errors" +// if len(v.errors) == 1 { +// wordError = "error" +// } +// return fmt.Sprintf("%d %s and %d %s in %q", +// len(v.findings), wordWarning, len(v.errors), wordError, v.typedZarfPackage.Metadata.Name) +// } -func (v Validator) hasErrors() bool { - return len(v.errors) > 0 +func (v Validator) hasFindings() bool { + return len(v.findings) > 0 } func (v *Validator) addWarning(vmessage validatorMessage) { - v.warnings = append(v.warnings, vmessage) + vmessage.validationType = validationWarning + v.findings = append(v.findings, vmessage) } func (v *Validator) addError(vMessage validatorMessage) { - v.errors = append(v.errors, vMessage) + vMessage.validationType = validationError + v.findings = append(v.findings, vMessage) } From 02b5eb800bbc9985bc65569de37e3fcf878c96de Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Fri, 8 Dec 2023 19:37:55 +0000 Subject: [PATCH 141/192] separating tables mostly working --- src/pkg/packager/compose.go | 2 +- src/pkg/packager/composer/list.go | 32 ++++--- src/pkg/packager/composer/list_test.go | 9 +- src/pkg/packager/lint/lint.go | 69 ++++++++++------ src/pkg/packager/lint/lint_test.go | 6 +- src/pkg/packager/lint/validator.go | 110 +++++++++++++------------ 6 files changed, 133 insertions(+), 95 deletions(-) diff --git a/src/pkg/packager/compose.go b/src/pkg/packager/compose.go index a141fcc562..6bbbf22fff 100644 --- a/src/pkg/packager/compose.go +++ b/src/pkg/packager/compose.go @@ -29,7 +29,7 @@ func (p *Packager) composeComponents() error { component.Only.Flavor = "" // build the import chain - chain, err := composer.NewImportChain(component, i, arch, p.cfg.CreateOpts.Flavor) + chain, err := composer.NewImportChain(component, i, p.cfg.Pkg.Metadata.Name, arch, p.cfg.CreateOpts.Flavor) if err != nil { return err } diff --git a/src/pkg/packager/composer/list.go b/src/pkg/packager/composer/list.go index 35dcbad4cc..9b01131178 100644 --- a/src/pkg/packager/composer/list.go +++ b/src/pkg/packager/composer/list.go @@ -27,7 +27,8 @@ type Node struct { vars []types.ZarfPackageVariable consts []types.ZarfPackageConstant - relativeToHead string + relativeToHead string + originalPackageName string prev *Node next *Node @@ -38,6 +39,11 @@ func (n *Node) GetIndex() int { return n.index } +// OriginalPackageName returns the .metadata.name of the zarf package the component originated from +func (n *Node) GetOriginalPackageName() string { + return n.originalPackageName +} + // GetRelativeToHead gets the path from downstream zarf file to upstream imported zarf file func (n *Node) GetRelativeToHead() string { return n.relativeToHead @@ -83,15 +89,17 @@ func (ic *ImportChain) Tail() *Node { return ic.tail } -func (ic *ImportChain) append(c types.ZarfComponent, index int, relativeToHead string, vars []types.ZarfPackageVariable, consts []types.ZarfPackageConstant) { +func (ic *ImportChain) append(c types.ZarfComponent, index int, originalPackageName string, + relativeToHead string, vars []types.ZarfPackageVariable, consts []types.ZarfPackageConstant) { node := &Node{ - ZarfComponent: c, - index: index, - relativeToHead: relativeToHead, - vars: vars, - consts: consts, - prev: nil, - next: nil, + ZarfComponent: c, + index: index, + originalPackageName: originalPackageName, + relativeToHead: relativeToHead, + vars: vars, + consts: consts, + prev: nil, + next: nil, } if ic.head == nil { ic.head = node @@ -105,14 +113,14 @@ func (ic *ImportChain) append(c types.ZarfComponent, index int, relativeToHead s } // NewImportChain creates a new import chain from a component -func NewImportChain(head types.ZarfComponent, index int, arch, flavor string) (*ImportChain, error) { +func NewImportChain(head types.ZarfComponent, index int, originalPackageName, arch, flavor string) (*ImportChain, error) { if arch == "" { return nil, fmt.Errorf("cannot build import chain: architecture must be provided") } ic := &ImportChain{} - ic.append(head, index, ".", nil, nil) + ic.append(head, index, originalPackageName, ".", nil, nil) history := []string{} @@ -200,7 +208,7 @@ func NewImportChain(head types.ZarfComponent, index int, arch, flavor string) (* } } - ic.append(found[0], index[0], filepath.Join(history...), pkg.Variables, pkg.Constants) + ic.append(found[0], index[0], pkg.Metadata.Name, filepath.Join(history...), pkg.Variables, pkg.Constants) node = node.next } return ic, nil diff --git a/src/pkg/packager/composer/list_test.go b/src/pkg/packager/composer/list_test.go index ba6b387b15..d80f3633c3 100644 --- a/src/pkg/packager/composer/list_test.go +++ b/src/pkg/packager/composer/list_test.go @@ -43,14 +43,14 @@ func TestNewImportChain(t *testing.T) { expectedErrorMessage: "detected circular import chain", }, } - + testPackageName := "test-package" for _, testCase := range testCases { testCase := testCase t.Run(testCase.name, func(t *testing.T) { t.Parallel() - _, err := NewImportChain(testCase.head, 0, testCase.arch, testCase.flavor) + _, err := NewImportChain(testCase.head, 0, testPackageName, testCase.arch, testCase.flavor) require.Contains(t, err.Error(), testCase.expectedErrorMessage) }) } @@ -441,17 +441,18 @@ func TestMerging(t *testing.T) { func createChainFromSlice(components []types.ZarfComponent) (ic *ImportChain) { ic = &ImportChain{} + testPackageName := "test-package" if len(components) == 0 { return ic } - ic.append(components[0], 0, ".", nil, nil) + ic.append(components[0], 0, testPackageName, ".", nil, nil) history := []string{} for idx := 1; idx < len(components); idx++ { history = append(history, components[idx-1].Import.Path) - ic.append(components[idx], idx, filepath.Join(history...), nil, nil) + ic.append(components[idx], idx, testPackageName, filepath.Join(history...), nil, nil) } return ic diff --git a/src/pkg/packager/lint/lint.go b/src/pkg/packager/lint/lint.go index cdf2f78ec0..0794b96e87 100644 --- a/src/pkg/packager/lint/lint.go +++ b/src/pkg/packager/lint/lint.go @@ -76,9 +76,12 @@ func lintComposableComponenets(validator *Validator, createOpts types.ZarfCreate continue } - chain, err := composer.NewImportChain(component, i, arch, createOpts.Flavor) + chain, err := composer.NewImportChain(component, i, validator.typedZarfPackage.Metadata.Name, arch, createOpts.Flavor) if err != nil { - validator.addError(validatorMessage{description: err.Error()}) + validator.addError(validatorMessage{ + description: err.Error(), + packageName: validator.typedZarfPackage.Metadata.Name, + }) continue } @@ -89,52 +92,64 @@ func lintComposableComponenets(validator *Validator, createOpts types.ZarfCreate if path == "" { path = node.GetRelativeToHead() } - checkForVarInComponentImport(validator, node.GetIndex(), node.ZarfComponent, path) - fillComponentTemplate(validator, &node.ZarfComponent, createOpts) - lintComponent(validator, node.GetIndex(), node.ZarfComponent, path) + checkForVarInComponentImport(validator, node.GetIndex(), node.ZarfComponent, path, node.GetOriginalPackageName()) + fillComponentTemplate(validator, node, createOpts) + lintComponent(validator, node.GetIndex(), node.ZarfComponent, path, node.GetOriginalPackageName()) node = node.Next() } } } -func fillComponentTemplate(validator *Validator, component *types.ZarfComponent, createOpts types.ZarfCreateOptions) { +func fillComponentTemplate(validator *Validator, node *composer.Node, createOpts types.ZarfCreateOptions) { - err := packager.ReloadComponentTemplate(component) + err := packager.ReloadComponentTemplate(&node.ZarfComponent) if err != nil { - validator.addWarning(validatorMessage{description: fmt.Sprintf("unable to find components %s", err)}) + validator.addWarning(validatorMessage{ + description: fmt.Sprintf("unable to find components %s", err), + filePath: node.GetRelativeToHead(), + packageName: node.GetOriginalPackageName(), + }) } - fillYamlTemplate(validator, component, createOpts) + fillYamlTemplate(validator, node, createOpts, node.GetOriginalPackageName()) } func fillActiveTemplate(validator *Validator, createOpts types.ZarfCreateOptions) { err := packager.FindComponentTemplatesAndReload(&validator.typedZarfPackage) if err != nil { - validator.addWarning(validatorMessage{description: fmt.Sprintf("unable to find components %s", err)}) + validator.addWarning(validatorMessage{ + description: fmt.Sprintf("unable to find components %s", err), + packageName: validator.typedZarfPackage.Metadata.Name, + }) } - fillYamlTemplate(validator, &validator.typedZarfPackage, createOpts) + fillYamlTemplate(validator, &validator.typedZarfPackage, createOpts, validator.typedZarfPackage.Metadata.Name) } -func fillYamlTemplate(validator *Validator, yamlObj any, createOpts types.ZarfCreateOptions) { +func fillYamlTemplate(validator *Validator, yamlObj any, createOpts types.ZarfCreateOptions, pkgName string) { templateMap := map[string]string{} setVarsAndWarn := func(templatePrefix string, deprecated bool) { yamlTemplates, err := utils.FindYamlTemplates(yamlObj, templatePrefix, "###") if err != nil { - validator.addWarning(validatorMessage{description: fmt.Sprintf("unable to find variables %s", err)}) + validator.addWarning(validatorMessage{ + description: fmt.Sprintf("unable to find variables %s", err), + packageName: pkgName, + }) } for key := range yamlTemplates { if deprecated { validator.addWarning(validatorMessage{ description: fmt.Sprintf(lang.PkgValidateTemplateDeprecation, key, key, key), + packageName: pkgName, }) } _, present := createOpts.SetVariables[key] if !present && !validator.hasUnSetVarWarning { validator.findings = append([]validatorMessage{{ description: "There are variables that are unset and won't be evaluated during lint", + packageName: pkgName, }}, validator.findings...) validator.hasUnSetVarWarning = true } @@ -171,17 +186,17 @@ func isPinnedRepo(repo string) bool { func lintComponents(validator *Validator) { for i, component := range validator.typedZarfPackage.Components { - lintComponent(validator, i, component, "") + lintComponent(validator, i, component, "", validator.typedZarfPackage.Metadata.Name) } } -func lintComponent(validator *Validator, index int, component types.ZarfComponent, path string) { - checkForUnpinnedRepos(validator, index, component, path) - checkForUnpinnedImages(validator, index, component, path) - checkForUnpinnedFiles(validator, index, component, path) +func lintComponent(validator *Validator, index int, component types.ZarfComponent, path string, packageName string) { + checkForUnpinnedRepos(validator, index, component, path, packageName) + checkForUnpinnedImages(validator, index, component, path, packageName) + checkForUnpinnedFiles(validator, index, component, path, packageName) } -func checkForUnpinnedRepos(validator *Validator, index int, component types.ZarfComponent, path string) { +func checkForUnpinnedRepos(validator *Validator, index int, component types.ZarfComponent, path string, pkgName string) { for j, repo := range component.Repos { repoYqPath := fmt.Sprintf(".components.[%d].repos.[%d]", index, j) if !isPinnedRepo(repo) { @@ -190,12 +205,13 @@ func checkForUnpinnedRepos(validator *Validator, index int, component types.Zarf filePath: path, description: "Unpinned repository", item: repo, + packageName: pkgName, }) } } } -func checkForUnpinnedImages(validator *Validator, index int, component types.ZarfComponent, path string) { +func checkForUnpinnedImages(validator *Validator, index int, component types.ZarfComponent, path string, pkgName string) { for j, image := range component.Images { imageYqPath := fmt.Sprintf(".components.[%d].images.[%d]", index, j) pinnedImage, err := isPinnedImage(image) @@ -205,6 +221,7 @@ func checkForUnpinnedImages(validator *Validator, index int, component types.Zar filePath: path, description: "Invalid image format", item: image, + packageName: pkgName, }) continue } @@ -214,12 +231,14 @@ func checkForUnpinnedImages(validator *Validator, index int, component types.Zar filePath: path, description: "Image not pinned with digest", item: image, + packageName: pkgName, }) } } } -func checkForUnpinnedFiles(validator *Validator, index int, component types.ZarfComponent, path string) { +func checkForUnpinnedFiles(validator *Validator, index int, + component types.ZarfComponent, path string, pkgName string) { for j, file := range component.Files { fileYqPath := fmt.Sprintf(".components.[%d].files.[%d]", index, j) if file.Shasum == "" && helpers.IsURL(file.Source) { @@ -228,6 +247,7 @@ func checkForUnpinnedFiles(validator *Validator, index int, component types.Zarf filePath: path, description: "No shasum for remote file", item: file.Source, + packageName: pkgName, }) } } @@ -235,17 +255,18 @@ func checkForUnpinnedFiles(validator *Validator, index int, component types.Zarf func lintUnEvaledVariables(validator *Validator) { for i, component := range validator.typedZarfPackage.Components { - checkForVarInComponentImport(validator, i, component, "") + checkForVarInComponentImport(validator, i, component, "", validator.typedZarfPackage.Metadata.Name) } } -func checkForVarInComponentImport(validator *Validator, index int, component types.ZarfComponent, path string) { +func checkForVarInComponentImport(validator *Validator, index int, component types.ZarfComponent, path string, pkgName string) { if strings.Contains(component.Import.Path, types.ZarfPackageTemplatePrefix) { validator.addWarning(validatorMessage{ yqPath: fmt.Sprintf(".components.[%d].import.path", index), filePath: path, description: "Zarf does not evaluate variables at component.x.import.path", item: component.Import.Path, + packageName: pkgName, }) } if strings.Contains(component.Import.URL, types.ZarfPackageTemplatePrefix) { @@ -254,6 +275,7 @@ func checkForVarInComponentImport(validator *Validator, index int, component typ filePath: path, description: "Zarf does not evaluate variables at component.x.import.url", item: component.Import.URL, + packageName: pkgName, }) } } @@ -285,6 +307,7 @@ func validateSchema(validator *Validator) error { validator.addError(validatorMessage{ yqPath: makeFieldPathYqCompat(desc.Field()), description: desc.Description(), + packageName: validator.typedZarfPackage.Metadata.Name, }) } } diff --git a/src/pkg/packager/lint/lint_test.go b/src/pkg/packager/lint/lint_test.go index dc13186205..97a0c68557 100644 --- a/src/pkg/packager/lint/lint_test.go +++ b/src/pkg/packager/lint/lint_test.go @@ -103,7 +103,7 @@ func TestValidateSchema(t *testing.T) { component := types.ZarfComponent{Repos: []string{ unpinnedRepo, "https://dev.azure.com/defenseunicorns/zarf-public-test/_git/zarf-public-test@v0.0.1"}} - checkForUnpinnedRepos(&validator, 0, component, "") + checkForUnpinnedRepos(&validator, 0, component, "", "") require.Equal(t, unpinnedRepo, validator.findings[0].item) require.Equal(t, len(validator.findings), 1) }) @@ -116,7 +116,7 @@ func TestValidateSchema(t *testing.T) { unpinnedImage, "busybox:latest@sha256:3fbc632167424a6d997e74f52b878d7cc478225cffac6bc977eedfe51c7f4e79", badImage}} - checkForUnpinnedImages(&validator, 0, component, "") + checkForUnpinnedImages(&validator, 0, component, "", "") require.Equal(t, unpinnedImage, validator.findings[0].item) require.Equal(t, 1, len(validator.findings)) require.Equal(t, badImage, validator.findings[1].item) @@ -132,7 +132,7 @@ func TestValidateSchema(t *testing.T) { }, } component := types.ZarfComponent{Files: zarfFiles} - checkForUnpinnedFiles(&validator, 0, component, "") + checkForUnpinnedFiles(&validator, 0, component, "", "") require.Equal(t, filename, validator.findings[0].item) require.Equal(t, 1, len(validator.findings)) }) diff --git a/src/pkg/packager/lint/validator.go b/src/pkg/packager/lint/validator.go index a1ead77e0b..3665e139ef 100644 --- a/src/pkg/packager/lint/validator.go +++ b/src/pkg/packager/lint/validator.go @@ -11,6 +11,7 @@ import ( "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/types" "github.com/fatih/color" + "github.com/pterm/pterm" ) type ValidationType int @@ -63,7 +64,7 @@ func (v Validator) DisplayFormattedMessage() { if !v.hasFindings() { message.Successf("0 findings for %q", v.typedZarfPackage.Metadata.Name) } - v.printValidationTable2() + v.printValidationTable() } // IsSuccess returns true if there are not any errors @@ -71,55 +72,44 @@ func (v Validator) IsSuccess() bool { return !v.hasFindings() } -// func (v Validator) printValidationTable() { -// if v.hasWarnings() || v.hasErrors() { -// header := []string{"Type", "Path", "Message"} -// connectData := [][]string{} -// for _, warning := range v.findings { -// connectData = append(connectData, -// []string{utils.ColorWrap("Warning", color.FgYellow), warning.getPath(), warning.String()}) -// } -// for _, validatorError := range v.errors { -// connectData = append(connectData, -// []string{utils.ColorWrap("Error", color.FgRed), validatorError.getPath(), validatorError.String()}) -// } -// message.Table(header, connectData) -// message.Info(v.getWarningAndErrorCount()) -// } -// } +func (v Validator) printValidationTable() { + + if !v.hasFindings() { + return + } + packageKeys := v.getUniquePackageKeys() + connectData := make(map[string][][]string) -func (v Validator) printValidationTable2() { - differentPaths := v.getUniquePaths() - if v.hasFindings() { - for _, path := range differentPaths { - header := []string{"Type", "Path", "Message"} - connectData := make(map[string][][]string) - item := path - for _, finding := range v.findings { - if finding.filePath == path { - if item == "" { - item = "original" - } - connectData[item] = append(connectData[item], - []string{utils.ColorWrap(finding.validationType.String(), color.FgYellow), - finding.getPath(), finding.String()}) - } + for _, packageKey := range packageKeys { + header := []string{"Type", "Path", "Message"} + + for _, finding := range v.findings { + if finding.getPackageKey() == packageKey { + connectData[packageKey] = append(connectData[packageKey], + []string{finding.validationType.String(), finding.getPath(), finding.String()}) } - // for _, validatorError := range v.errors { - // if validatorError.filePath == path { - // if item == "" { - // item = "original" - // } - // connectData[item] = append(connectData[item], - // []string{utils.ColorWrap("Error", color.FgRed), validatorError.getPath(), validatorError.String()}) - // } - // } - - message.Infof("Component at path: %s", item) - message.Table(header, connectData[item]) - //message.Info(v.getWarningAndErrorCount()) + } + //We should probably move this println into info + pterm.Println() + message.Infof("Lint for package: %s", packageKey) + message.Table(header, connectData[packageKey]) + message.Info(v.getFormattedFindingCount(packageKey)) + } +} + +func (vm validatorMessage) getPackageKey() string { + return fmt.Sprintf("%s %s", vm.packageName, vm.filePath) +} + +func (v Validator) getUniquePackageKeys() []string { + paths := []string{} + for _, finding := range v.findings { + packageKey := finding.getPackageKey() + if !contains(paths, packageKey) { + paths = append(paths, packageKey) } } + return paths } func contains(slice []string, item string) bool { @@ -131,14 +121,30 @@ func contains(slice []string, item string) bool { return false } -func (v Validator) getUniquePaths() []string { - paths := []string{} - for _, warning := range v.findings { - if !contains(paths, warning.filePath) { - paths = append(paths, warning.filePath) +func (v Validator) getFormattedFindingCount(packageKey string) string { + warningCount := 0 + errorCount := 0 + for _, finding := range v.findings { + if finding.getPackageKey() != packageKey { + continue + } + if finding.validationType == validationWarning { + warningCount += 1 + } + if finding.validationType == validationError { + errorCount += 1 } } - return paths + wordWarning := "warnings" + if warningCount == 1 { + wordWarning = "warning" + } + wordError := "errors" + if errorCount == 1 { + wordError = "error" + } + return fmt.Sprintf("%d %s and %d %s in %q", + warningCount, wordWarning, errorCount, wordError, packageKey) } func (vm validatorMessage) getPath() string { From 63ee08bbfa991f736bd643b70d817543247e44b5 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Fri, 8 Dec 2023 20:20:50 +0000 Subject: [PATCH 142/192] refactoring --- src/pkg/packager/lint/lint.go | 33 ++++++-------- src/pkg/packager/lint/lint_test.go | 8 ++-- src/pkg/packager/lint/validator.go | 72 +++++++++++++++--------------- src/test/e2e/12_lint_test.go | 16 +++---- 4 files changed, 61 insertions(+), 68 deletions(-) diff --git a/src/pkg/packager/lint/lint.go b/src/pkg/packager/lint/lint.go index 0794b96e87..a287f40230 100644 --- a/src/pkg/packager/lint/lint.go +++ b/src/pkg/packager/lint/lint.go @@ -80,7 +80,7 @@ func lintComposableComponenets(validator *Validator, createOpts types.ZarfCreate if err != nil { validator.addError(validatorMessage{ description: err.Error(), - packageName: validator.typedZarfPackage.Metadata.Name, + packageKey: packageKey{name: validator.typedZarfPackage.Metadata.Name}, }) continue } @@ -106,8 +106,7 @@ func fillComponentTemplate(validator *Validator, node *composer.Node, createOpts if err != nil { validator.addWarning(validatorMessage{ description: fmt.Sprintf("unable to find components %s", err), - filePath: node.GetRelativeToHead(), - packageName: node.GetOriginalPackageName(), + packageKey: packageKey{filePath: node.GetRelativeToHead(), name: node.GetOriginalPackageName()}, }) } fillYamlTemplate(validator, node, createOpts, node.GetOriginalPackageName()) @@ -119,7 +118,7 @@ func fillActiveTemplate(validator *Validator, createOpts types.ZarfCreateOptions if err != nil { validator.addWarning(validatorMessage{ description: fmt.Sprintf("unable to find components %s", err), - packageName: validator.typedZarfPackage.Metadata.Name, + packageKey: packageKey{name: validator.typedZarfPackage.Metadata.Name}, }) } @@ -134,7 +133,7 @@ func fillYamlTemplate(validator *Validator, yamlObj any, createOpts types.ZarfCr if err != nil { validator.addWarning(validatorMessage{ description: fmt.Sprintf("unable to find variables %s", err), - packageName: pkgName, + packageKey: packageKey{name: pkgName}, }) } @@ -142,14 +141,14 @@ func fillYamlTemplate(validator *Validator, yamlObj any, createOpts types.ZarfCr if deprecated { validator.addWarning(validatorMessage{ description: fmt.Sprintf(lang.PkgValidateTemplateDeprecation, key, key, key), - packageName: pkgName, + packageKey: packageKey{name: pkgName}, }) } _, present := createOpts.SetVariables[key] if !present && !validator.hasUnSetVarWarning { validator.findings = append([]validatorMessage{{ description: "There are variables that are unset and won't be evaluated during lint", - packageName: pkgName, + packageKey: packageKey{name: pkgName}, }}, validator.findings...) validator.hasUnSetVarWarning = true } @@ -202,10 +201,9 @@ func checkForUnpinnedRepos(validator *Validator, index int, component types.Zarf if !isPinnedRepo(repo) { validator.addWarning(validatorMessage{ yqPath: repoYqPath, - filePath: path, + packageKey: packageKey{filePath: path, name: pkgName}, description: "Unpinned repository", item: repo, - packageName: pkgName, }) } } @@ -218,20 +216,18 @@ func checkForUnpinnedImages(validator *Validator, index int, component types.Zar if err != nil { validator.addError(validatorMessage{ yqPath: imageYqPath, - filePath: path, + packageKey: packageKey{filePath: path, name: pkgName}, description: "Invalid image format", item: image, - packageName: pkgName, }) continue } if !pinnedImage { validator.addWarning(validatorMessage{ yqPath: imageYqPath, - filePath: path, + packageKey: packageKey{filePath: path, name: pkgName}, description: "Image not pinned with digest", item: image, - packageName: pkgName, }) } } @@ -244,10 +240,9 @@ func checkForUnpinnedFiles(validator *Validator, index int, if file.Shasum == "" && helpers.IsURL(file.Source) { validator.addWarning(validatorMessage{ yqPath: fileYqPath, - filePath: path, + packageKey: packageKey{filePath: path, name: pkgName}, description: "No shasum for remote file", item: file.Source, - packageName: pkgName, }) } } @@ -263,19 +258,17 @@ func checkForVarInComponentImport(validator *Validator, index int, component typ if strings.Contains(component.Import.Path, types.ZarfPackageTemplatePrefix) { validator.addWarning(validatorMessage{ yqPath: fmt.Sprintf(".components.[%d].import.path", index), - filePath: path, + packageKey: packageKey{filePath: path, name: pkgName}, description: "Zarf does not evaluate variables at component.x.import.path", item: component.Import.Path, - packageName: pkgName, }) } if strings.Contains(component.Import.URL, types.ZarfPackageTemplatePrefix) { validator.addWarning(validatorMessage{ yqPath: fmt.Sprintf(".components.[%d].import.url", index), - filePath: path, + packageKey: packageKey{filePath: path, name: pkgName}, description: "Zarf does not evaluate variables at component.x.import.url", item: component.Import.URL, - packageName: pkgName, }) } } @@ -307,7 +300,7 @@ func validateSchema(validator *Validator) error { validator.addError(validatorMessage{ yqPath: makeFieldPathYqCompat(desc.Field()), description: desc.Description(), - packageName: validator.typedZarfPackage.Metadata.Name, + packageKey: packageKey{name: validator.typedZarfPackage.Metadata.Name}, }) } } diff --git a/src/pkg/packager/lint/lint_test.go b/src/pkg/packager/lint/lint_test.go index 97a0c68557..4ba03f2955 100644 --- a/src/pkg/packager/lint/lint_test.go +++ b/src/pkg/packager/lint/lint_test.go @@ -75,8 +75,8 @@ func TestValidateSchema(t *testing.T) { err := validateSchema(&validator) require.NoError(t, err) config.NoColor = true - require.Equal(t, ".components.[0].import: Additional property not-path is not allowed", validator.findings[0].String()) - require.Equal(t, ".components.[1].import.path: Invalid type. Expected: string, given: integer", validator.findings[1].String()) + require.Equal(t, "Additional property not-path is not allowed", validator.findings[0].String()) + require.Equal(t, "Invalid type. Expected: string, given: integer", validator.findings[1].String()) }) t.Run("Template in component import success", func(t *testing.T) { @@ -118,9 +118,9 @@ func TestValidateSchema(t *testing.T) { badImage}} checkForUnpinnedImages(&validator, 0, component, "", "") require.Equal(t, unpinnedImage, validator.findings[0].item) - require.Equal(t, 1, len(validator.findings)) require.Equal(t, badImage, validator.findings[1].item) - require.Equal(t, 1, len(validator.findings)) + require.Equal(t, 2, len(validator.findings)) + }) t.Run("Unpinnned file warning", func(t *testing.T) { diff --git a/src/pkg/packager/lint/validator.go b/src/pkg/packager/lint/validator.go index 3665e139ef..7435bffaf5 100644 --- a/src/pkg/packager/lint/validator.go +++ b/src/pkg/packager/lint/validator.go @@ -6,6 +6,7 @@ package lint import ( "fmt" + "reflect" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/utils" @@ -21,12 +22,16 @@ const ( validationWarning ValidationType = 2 ) +type packageKey struct { + filePath string + name string +} + type validatorMessage struct { yqPath string - filePath string description string item string - packageName string + packageKey packageKey validationType ValidationType } @@ -44,7 +49,7 @@ func (v validatorMessage) String() string { if v.item != "" { v.item = fmt.Sprintf(" - %s", v.item) } - if v.filePath == "" && v.yqPath == "" && v.item == "" { + if v.packageKey.filePath == "" && v.yqPath == "" && v.item == "" { return v.description } return fmt.Sprintf("%s%s", v.description, v.item) @@ -69,63 +74,71 @@ func (v Validator) DisplayFormattedMessage() { // IsSuccess returns true if there are not any errors func (v Validator) IsSuccess() bool { - return !v.hasFindings() + for _, finding := range v.findings { + if finding.validationType == validationError { + return false + } + } + return true } func (v Validator) printValidationTable() { - if !v.hasFindings() { return } - packageKeys := v.getUniquePackageKeys() - connectData := make(map[string][][]string) + packageKeys := v.getUniquePackageKeys() + connectData := make(map[packageKey][][]string) for _, packageKey := range packageKeys { header := []string{"Type", "Path", "Message"} for _, finding := range v.findings { - if finding.getPackageKey() == packageKey { + if finding.packageKey == packageKey { connectData[packageKey] = append(connectData[packageKey], []string{finding.validationType.String(), finding.getPath(), finding.String()}) } } //We should probably move this println into info pterm.Println() - message.Infof("Lint for package: %s", packageKey) + if packageKey.filePath != "" { + message.Infof("Linting package %q at %s", packageKey.name, packageKey.filePath) + } else { + message.Infof("Linting package %q", packageKey.name) + } + message.Table(header, connectData[packageKey]) message.Info(v.getFormattedFindingCount(packageKey)) } } -func (vm validatorMessage) getPackageKey() string { - return fmt.Sprintf("%s %s", vm.packageName, vm.filePath) -} +func (v Validator) getUniquePackageKeys() []packageKey { + uniqueKeys := make(map[packageKey]bool) + var pks []packageKey -func (v Validator) getUniquePackageKeys() []string { - paths := []string{} for _, finding := range v.findings { - packageKey := finding.getPackageKey() - if !contains(paths, packageKey) { - paths = append(paths, packageKey) + if _, exists := uniqueKeys[finding.packageKey]; !exists { + uniqueKeys[finding.packageKey] = true + pks = append(pks, finding.packageKey) } } - return paths + + return pks } -func contains(slice []string, item string) bool { +func contains(slice []any, item any) bool { for _, v := range slice { - if v == item { + if reflect.DeepEqual(v, item) { return true } } return false } -func (v Validator) getFormattedFindingCount(packageKey string) string { +func (v Validator) getFormattedFindingCount(pk packageKey) string { warningCount := 0 errorCount := 0 for _, finding := range v.findings { - if finding.getPackageKey() != packageKey { + if finding.packageKey != pk { continue } if finding.validationType == validationWarning { @@ -144,7 +157,7 @@ func (v Validator) getFormattedFindingCount(packageKey string) string { wordError = "error" } return fmt.Sprintf("%d %s and %d %s in %q", - warningCount, wordWarning, errorCount, wordError, packageKey) + warningCount, wordWarning, errorCount, wordError, pk.name) } func (vm validatorMessage) getPath() string { @@ -154,19 +167,6 @@ func (vm validatorMessage) getPath() string { return utils.ColorWrap(vm.yqPath, color.FgCyan) } -// func (v Validator) getWarningAndErrorCount() string { -// wordWarning := "warnings" -// if len(v.findings) == 1 { -// wordWarning = "warning" -// } -// wordError := "errors" -// if len(v.errors) == 1 { -// wordError = "error" -// } -// return fmt.Sprintf("%d %s and %d %s in %q", -// len(v.findings), wordWarning, len(v.errors), wordError, v.typedZarfPackage.Metadata.Name) -// } - func (v Validator) hasFindings() bool { return len(v.findings) > 0 } diff --git a/src/test/e2e/12_lint_test.go b/src/test/e2e/12_lint_test.go index 597fa3070b..161d540bf7 100644 --- a/src/test/e2e/12_lint_test.go +++ b/src/test/e2e/12_lint_test.go @@ -36,14 +36,14 @@ func TestLint(t *testing.T) { require.Contains(t, strippedStderr, "There are variables that are unset and won't be evaluated during lint") // Multiple spaces in lang require.Contains(t, strippedStderr, fmt.Sprintf(lang.PkgValidateTemplateDeprecation, key, key, key)) - require.Contains(t, strippedStderr, ".components.[2].repos.[0]: Unpinned repository") - require.Contains(t, strippedStderr, ".metadata: Additional property description1 is not allowed") - require.Contains(t, strippedStderr, ".components.[0].import: Additional property not-path is not allowed") - require.Contains(t, strippedStderr, ".components.[2].images.[3]: Image not pinned with digest") - require.Contains(t, strippedStderr, ".components.[1].images.[0] linted-import: Image not pinned with digest") - require.Contains(t, strippedStderr, ".components.[1].images.[2] linted-import: Image not pinned with digest") - require.Contains(t, strippedStderr, ".components.[3].import.path: Zarf does not evaluate variables at component.x.import.path - ###ZARF_PKG_TMPL_PATH###") - require.Contains(t, strippedStderr, ".components.[0].images.[0] oci://🦄/dos-games:1.0.0-skeleton: Image not pinned with digest - defenseunicorns/zarf-game:multi-tile-dark") + require.Contains(t, strippedStderr, ".components.[2].repos.[0] | Unpinned repository") + require.Contains(t, strippedStderr, ".metadata | Additional property description1 is not allowed") + require.Contains(t, strippedStderr, ".components.[0].import | Additional property not-path is not allowed") + require.Contains(t, strippedStderr, ".components.[2].images.[3] | Image not pinned with digest") + require.Contains(t, strippedStderr, ".components.[1].images.[0] | Image not pinned with digest") + require.Contains(t, strippedStderr, ".components.[1].images.[2] | Image not pinned with digest") + require.Contains(t, strippedStderr, ".components.[3].import.path | Zarf does not evaluate variables at component.x.import.path - ###ZARF_PKG_TMPL_PATH###") + require.Contains(t, strippedStderr, ".components.[0].images.[0] | Image not pinned with digest - defenseunicorns/zarf-game:multi-tile-dark") }) } From a58eac86f265cbf7647ffb318f8a3a818b3fb6b6 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Fri, 8 Dec 2023 20:56:01 +0000 Subject: [PATCH 143/192] have unset var message for all packages --- src/config/lang/english.go | 5 +++++ src/pkg/packager/lint/lint.go | 20 ++++++++++---------- src/pkg/packager/lint/validator.go | 21 ++++++++++----------- 3 files changed, 25 insertions(+), 21 deletions(-) diff --git a/src/config/lang/english.go b/src/config/lang/english.go index e8f2e4a3d4..957a2188e2 100644 --- a/src/config/lang/english.go +++ b/src/config/lang/english.go @@ -31,6 +31,11 @@ const ( ErrFileNameExtract = "failed to extract filename from URL %s: %s" ) +// Lint messages +const ( + UnsetVarWarning = "There are variables that are unset and won't be evaluated during lint" +) + // Zarf CLI commands. const ( // common command language diff --git a/src/pkg/packager/lint/lint.go b/src/pkg/packager/lint/lint.go index a287f40230..31592666bc 100644 --- a/src/pkg/packager/lint/lint.go +++ b/src/pkg/packager/lint/lint.go @@ -109,7 +109,7 @@ func fillComponentTemplate(validator *Validator, node *composer.Node, createOpts packageKey: packageKey{filePath: node.GetRelativeToHead(), name: node.GetOriginalPackageName()}, }) } - fillYamlTemplate(validator, node, createOpts, node.GetOriginalPackageName()) + fillYamlTemplate(validator, node, createOpts, node.GetOriginalPackageName(), node.GetRelativeToHead()) } func fillActiveTemplate(validator *Validator, createOpts types.ZarfCreateOptions) { @@ -122,18 +122,20 @@ func fillActiveTemplate(validator *Validator, createOpts types.ZarfCreateOptions }) } - fillYamlTemplate(validator, &validator.typedZarfPackage, createOpts, validator.typedZarfPackage.Metadata.Name) + fillYamlTemplate(validator, &validator.typedZarfPackage, createOpts, validator.typedZarfPackage.Metadata.Name, "") } -func fillYamlTemplate(validator *Validator, yamlObj any, createOpts types.ZarfCreateOptions, pkgName string) { +func fillYamlTemplate(validator *Validator, yamlObj any, + createOpts types.ZarfCreateOptions, pkgName string, pkgPath string) { templateMap := map[string]string{} + pkgKey := packageKey{name: pkgName, filePath: pkgPath} setVarsAndWarn := func(templatePrefix string, deprecated bool) { yamlTemplates, err := utils.FindYamlTemplates(yamlObj, templatePrefix, "###") if err != nil { validator.addWarning(validatorMessage{ description: fmt.Sprintf("unable to find variables %s", err), - packageKey: packageKey{name: pkgName}, + packageKey: pkgKey, }) } @@ -141,19 +143,17 @@ func fillYamlTemplate(validator *Validator, yamlObj any, createOpts types.ZarfCr if deprecated { validator.addWarning(validatorMessage{ description: fmt.Sprintf(lang.PkgValidateTemplateDeprecation, key, key, key), - packageKey: packageKey{name: pkgName}, + packageKey: pkgKey, }) } _, present := createOpts.SetVariables[key] - if !present && !validator.hasUnSetVarWarning { + if !present && !validator.HasUnsetVarMessageForPkg(pkgKey) { validator.findings = append([]validatorMessage{{ - description: "There are variables that are unset and won't be evaluated during lint", - packageKey: packageKey{name: pkgName}, + description: lang.UnsetVarWarning, + packageKey: pkgKey, }}, validator.findings...) - validator.hasUnSetVarWarning = true } } - for key, value := range createOpts.SetVariables { templateMap[fmt.Sprintf("%s%s###", templatePrefix, key)] = value } diff --git a/src/pkg/packager/lint/validator.go b/src/pkg/packager/lint/validator.go index 7435bffaf5..c27d7a365d 100644 --- a/src/pkg/packager/lint/validator.go +++ b/src/pkg/packager/lint/validator.go @@ -6,8 +6,8 @@ package lint import ( "fmt" - "reflect" + "github.com/defenseunicorns/zarf/src/config/lang" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/types" @@ -61,7 +61,6 @@ type Validator struct { jsonSchema []byte typedZarfPackage types.ZarfPackage untypedZarfPackage interface{} - hasUnSetVarWarning bool } // DisplayFormattedMessage message sent to user based on validator results @@ -82,6 +81,15 @@ func (v Validator) IsSuccess() bool { return true } +func (v Validator) HasUnsetVarMessageForPkg(pk packageKey) bool { + for _, finding := range v.findings { + if finding.description == lang.UnsetVarWarning && finding.packageKey == pk { + return true + } + } + return false +} + func (v Validator) printValidationTable() { if !v.hasFindings() { return @@ -125,15 +133,6 @@ func (v Validator) getUniquePackageKeys() []packageKey { return pks } -func contains(slice []any, item any) bool { - for _, v := range slice { - if reflect.DeepEqual(v, item) { - return true - } - } - return false -} - func (v Validator) getFormattedFindingCount(pk packageKey) string { warningCount := 0 errorCount := 0 From eacd341d4ba8fb8f410ba51ff2038ba5862a4981 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Fri, 8 Dec 2023 21:46:47 +0000 Subject: [PATCH 144/192] refactor to further use packagekey --- src/pkg/packager/lint/lint.go | 53 +++++++++++++++--------------- src/pkg/packager/lint/lint_test.go | 6 ++-- src/pkg/packager/lint/validator.go | 10 +++--- 3 files changed, 34 insertions(+), 35 deletions(-) diff --git a/src/pkg/packager/lint/lint.go b/src/pkg/packager/lint/lint.go index 31592666bc..044a7f0e52 100644 --- a/src/pkg/packager/lint/lint.go +++ b/src/pkg/packager/lint/lint.go @@ -92,24 +92,25 @@ func lintComposableComponenets(validator *Validator, createOpts types.ZarfCreate if path == "" { path = node.GetRelativeToHead() } - checkForVarInComponentImport(validator, node.GetIndex(), node.ZarfComponent, path, node.GetOriginalPackageName()) - fillComponentTemplate(validator, node, createOpts) - lintComponent(validator, node.GetIndex(), node.ZarfComponent, path, node.GetOriginalPackageName()) + pkgKey := packageKey{path: path, name: node.GetOriginalPackageName()} + checkForVarInComponentImport(validator, node.GetIndex(), node.ZarfComponent, pkgKey) + fillComponentTemplate(validator, node, createOpts, pkgKey) + lintComponent(validator, node.GetIndex(), node.ZarfComponent, pkgKey) node = node.Next() } } } -func fillComponentTemplate(validator *Validator, node *composer.Node, createOpts types.ZarfCreateOptions) { +func fillComponentTemplate(validator *Validator, node *composer.Node, createOpts types.ZarfCreateOptions, pkgKey packageKey) { err := packager.ReloadComponentTemplate(&node.ZarfComponent) if err != nil { validator.addWarning(validatorMessage{ description: fmt.Sprintf("unable to find components %s", err), - packageKey: packageKey{filePath: node.GetRelativeToHead(), name: node.GetOriginalPackageName()}, + packageKey: packageKey{path: node.GetRelativeToHead(), name: node.GetOriginalPackageName()}, }) } - fillYamlTemplate(validator, node, createOpts, node.GetOriginalPackageName(), node.GetRelativeToHead()) + fillYamlTemplate(validator, node, createOpts, pkgKey) } func fillActiveTemplate(validator *Validator, createOpts types.ZarfCreateOptions) { @@ -122,13 +123,12 @@ func fillActiveTemplate(validator *Validator, createOpts types.ZarfCreateOptions }) } - fillYamlTemplate(validator, &validator.typedZarfPackage, createOpts, validator.typedZarfPackage.Metadata.Name, "") + fillYamlTemplate(validator, &validator.typedZarfPackage, + createOpts, packageKey{name: validator.typedZarfPackage.Metadata.Name, path: ""}) } -func fillYamlTemplate(validator *Validator, yamlObj any, - createOpts types.ZarfCreateOptions, pkgName string, pkgPath string) { +func fillYamlTemplate(validator *Validator, yamlObj any, createOpts types.ZarfCreateOptions, pkgKey packageKey) { templateMap := map[string]string{} - pkgKey := packageKey{name: pkgName, filePath: pkgPath} setVarsAndWarn := func(templatePrefix string, deprecated bool) { yamlTemplates, err := utils.FindYamlTemplates(yamlObj, templatePrefix, "###") @@ -185,23 +185,23 @@ func isPinnedRepo(repo string) bool { func lintComponents(validator *Validator) { for i, component := range validator.typedZarfPackage.Components { - lintComponent(validator, i, component, "", validator.typedZarfPackage.Metadata.Name) + lintComponent(validator, i, component, packageKey{path: "", name: validator.typedZarfPackage.Metadata.Name}) } } -func lintComponent(validator *Validator, index int, component types.ZarfComponent, path string, packageName string) { - checkForUnpinnedRepos(validator, index, component, path, packageName) - checkForUnpinnedImages(validator, index, component, path, packageName) - checkForUnpinnedFiles(validator, index, component, path, packageName) +func lintComponent(validator *Validator, index int, component types.ZarfComponent, pkgKey packageKey) { + checkForUnpinnedRepos(validator, index, component, pkgKey) + checkForUnpinnedImages(validator, index, component, pkgKey) + checkForUnpinnedFiles(validator, index, component, pkgKey) } -func checkForUnpinnedRepos(validator *Validator, index int, component types.ZarfComponent, path string, pkgName string) { +func checkForUnpinnedRepos(validator *Validator, index int, component types.ZarfComponent, pkgKey packageKey) { for j, repo := range component.Repos { repoYqPath := fmt.Sprintf(".components.[%d].repos.[%d]", index, j) if !isPinnedRepo(repo) { validator.addWarning(validatorMessage{ yqPath: repoYqPath, - packageKey: packageKey{filePath: path, name: pkgName}, + packageKey: pkgKey, description: "Unpinned repository", item: repo, }) @@ -209,14 +209,14 @@ func checkForUnpinnedRepos(validator *Validator, index int, component types.Zarf } } -func checkForUnpinnedImages(validator *Validator, index int, component types.ZarfComponent, path string, pkgName string) { +func checkForUnpinnedImages(validator *Validator, index int, component types.ZarfComponent, pkgKey packageKey) { for j, image := range component.Images { imageYqPath := fmt.Sprintf(".components.[%d].images.[%d]", index, j) pinnedImage, err := isPinnedImage(image) if err != nil { validator.addError(validatorMessage{ yqPath: imageYqPath, - packageKey: packageKey{filePath: path, name: pkgName}, + packageKey: pkgKey, description: "Invalid image format", item: image, }) @@ -225,7 +225,7 @@ func checkForUnpinnedImages(validator *Validator, index int, component types.Zar if !pinnedImage { validator.addWarning(validatorMessage{ yqPath: imageYqPath, - packageKey: packageKey{filePath: path, name: pkgName}, + packageKey: pkgKey, description: "Image not pinned with digest", item: image, }) @@ -233,14 +233,13 @@ func checkForUnpinnedImages(validator *Validator, index int, component types.Zar } } -func checkForUnpinnedFiles(validator *Validator, index int, - component types.ZarfComponent, path string, pkgName string) { +func checkForUnpinnedFiles(validator *Validator, index int, component types.ZarfComponent, pkgKey packageKey) { for j, file := range component.Files { fileYqPath := fmt.Sprintf(".components.[%d].files.[%d]", index, j) if file.Shasum == "" && helpers.IsURL(file.Source) { validator.addWarning(validatorMessage{ yqPath: fileYqPath, - packageKey: packageKey{filePath: path, name: pkgName}, + packageKey: pkgKey, description: "No shasum for remote file", item: file.Source, }) @@ -250,15 +249,15 @@ func checkForUnpinnedFiles(validator *Validator, index int, func lintUnEvaledVariables(validator *Validator) { for i, component := range validator.typedZarfPackage.Components { - checkForVarInComponentImport(validator, i, component, "", validator.typedZarfPackage.Metadata.Name) + checkForVarInComponentImport(validator, i, component, packageKey{name: validator.typedZarfPackage.Metadata.Name, path: ""}) } } -func checkForVarInComponentImport(validator *Validator, index int, component types.ZarfComponent, path string, pkgName string) { +func checkForVarInComponentImport(validator *Validator, index int, component types.ZarfComponent, pkgKey packageKey) { if strings.Contains(component.Import.Path, types.ZarfPackageTemplatePrefix) { validator.addWarning(validatorMessage{ yqPath: fmt.Sprintf(".components.[%d].import.path", index), - packageKey: packageKey{filePath: path, name: pkgName}, + packageKey: pkgKey, description: "Zarf does not evaluate variables at component.x.import.path", item: component.Import.Path, }) @@ -266,7 +265,7 @@ func checkForVarInComponentImport(validator *Validator, index int, component typ if strings.Contains(component.Import.URL, types.ZarfPackageTemplatePrefix) { validator.addWarning(validatorMessage{ yqPath: fmt.Sprintf(".components.[%d].import.url", index), - packageKey: packageKey{filePath: path, name: pkgName}, + packageKey: pkgKey, description: "Zarf does not evaluate variables at component.x.import.url", item: component.Import.URL, }) diff --git a/src/pkg/packager/lint/lint_test.go b/src/pkg/packager/lint/lint_test.go index 4ba03f2955..f91355f2ca 100644 --- a/src/pkg/packager/lint/lint_test.go +++ b/src/pkg/packager/lint/lint_test.go @@ -103,7 +103,7 @@ func TestValidateSchema(t *testing.T) { component := types.ZarfComponent{Repos: []string{ unpinnedRepo, "https://dev.azure.com/defenseunicorns/zarf-public-test/_git/zarf-public-test@v0.0.1"}} - checkForUnpinnedRepos(&validator, 0, component, "", "") + checkForUnpinnedRepos(&validator, 0, component, packageKey{}) require.Equal(t, unpinnedRepo, validator.findings[0].item) require.Equal(t, len(validator.findings), 1) }) @@ -116,7 +116,7 @@ func TestValidateSchema(t *testing.T) { unpinnedImage, "busybox:latest@sha256:3fbc632167424a6d997e74f52b878d7cc478225cffac6bc977eedfe51c7f4e79", badImage}} - checkForUnpinnedImages(&validator, 0, component, "", "") + checkForUnpinnedImages(&validator, 0, component, packageKey{}) require.Equal(t, unpinnedImage, validator.findings[0].item) require.Equal(t, badImage, validator.findings[1].item) require.Equal(t, 2, len(validator.findings)) @@ -132,7 +132,7 @@ func TestValidateSchema(t *testing.T) { }, } component := types.ZarfComponent{Files: zarfFiles} - checkForUnpinnedFiles(&validator, 0, component, "", "") + checkForUnpinnedFiles(&validator, 0, component, packageKey{}) require.Equal(t, filename, validator.findings[0].item) require.Equal(t, 1, len(validator.findings)) }) diff --git a/src/pkg/packager/lint/validator.go b/src/pkg/packager/lint/validator.go index c27d7a365d..1ed20e4dc8 100644 --- a/src/pkg/packager/lint/validator.go +++ b/src/pkg/packager/lint/validator.go @@ -23,8 +23,8 @@ const ( ) type packageKey struct { - filePath string - name string + path string + name string } type validatorMessage struct { @@ -49,7 +49,7 @@ func (v validatorMessage) String() string { if v.item != "" { v.item = fmt.Sprintf(" - %s", v.item) } - if v.packageKey.filePath == "" && v.yqPath == "" && v.item == "" { + if v.packageKey.path == "" && v.yqPath == "" && v.item == "" { return v.description } return fmt.Sprintf("%s%s", v.description, v.item) @@ -108,8 +108,8 @@ func (v Validator) printValidationTable() { } //We should probably move this println into info pterm.Println() - if packageKey.filePath != "" { - message.Infof("Linting package %q at %s", packageKey.name, packageKey.filePath) + if packageKey.path != "" { + message.Infof("Linting package %q at %s", packageKey.name, packageKey.path) } else { message.Infof("Linting package %q", packageKey.name) } From 1862ad5009647bf790c4db594d62569267e0b0cd Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Mon, 11 Dec 2023 14:55:39 +0000 Subject: [PATCH 145/192] additional paths on broken imports, refactors --- Makefile | 2 +- src/config/lang/english.go | 2 +- src/pkg/packager/composer/list.go | 30 +++++++++++++++--------------- src/pkg/packager/lint/lint.go | 20 ++++++++++++++++---- src/pkg/packager/lint/validator.go | 2 +- src/test/e2e/12_lint_test.go | 2 +- 6 files changed, 35 insertions(+), 23 deletions(-) diff --git a/Makefile b/Makefile index b970c1cbf8..29f795ee69 100644 --- a/Makefile +++ b/Makefile @@ -76,7 +76,7 @@ docs-and-schema: ## Generate the Zarf Documentation and Schema hack/gen-cli-docs.sh ZARF_CONFIG=hack/empty-config.toml hack/create-zarf-schema.sh -lint-all-packages: ## Recurisvely lint all zarf.yaml files +lint-packages-and-examples: ## Recurisvely lint all zarf.yaml files in packages or examples hack/lint_all_zarf_packages.sh # INTERNAL: a shim used to build the agent image only if needed on Windows using the `test` command diff --git a/src/config/lang/english.go b/src/config/lang/english.go index 957a2188e2..e24a06716f 100644 --- a/src/config/lang/english.go +++ b/src/config/lang/english.go @@ -33,7 +33,7 @@ const ( // Lint messages const ( - UnsetVarWarning = "There are variables that are unset and won't be evaluated during lint" + UnsetVarLintWarning = "There are variables that are unset and won't be evaluated during lint" ) // Zarf CLI commands. diff --git a/src/pkg/packager/composer/list.go b/src/pkg/packager/composer/list.go index 9b01131178..5ca8cda751 100644 --- a/src/pkg/packager/composer/list.go +++ b/src/pkg/packager/composer/list.go @@ -39,7 +39,7 @@ func (n *Node) GetIndex() int { return n.index } -// OriginalPackageName returns the .metadata.name of the zarf package the component originated from +// GetOriginalPackageName returns the .metadata.name of the zarf package the component originated from func (n *Node) GetOriginalPackageName() string { return n.originalPackageName } @@ -113,13 +113,13 @@ func (ic *ImportChain) append(c types.ZarfComponent, index int, originalPackageN } // NewImportChain creates a new import chain from a component +// Returning the chain on error so we can have additional information to use during lint func NewImportChain(head types.ZarfComponent, index int, originalPackageName, arch, flavor string) (*ImportChain, error) { + ic := &ImportChain{} if arch == "" { - return nil, fmt.Errorf("cannot build import chain: architecture must be provided") + return ic, fmt.Errorf("cannot build import chain: architecture must be provided") } - ic := &ImportChain{} - ic.append(head, index, originalPackageName, ".", nil, nil) history := []string{} @@ -137,16 +137,16 @@ func NewImportChain(head types.ZarfComponent, index int, originalPackageName, ar // TODO: stuff like this should also happen in linting if err := validate.ImportDefinition(&node.ZarfComponent); err != nil { - return nil, err + return ic, err } // ensure that remote components are not importing other remote components if node.prev != nil && node.prev.Import.URL != "" && isRemote { - return nil, fmt.Errorf("detected malformed import chain, cannot import remote components from remote components") + return ic, fmt.Errorf("detected malformed import chain, cannot import remote components from remote components") } // ensure that remote components are not importing local components if node.prev != nil && node.prev.Import.URL != "" && isLocal { - return nil, fmt.Errorf("detected malformed import chain, cannot import local components from remote components") + return ic, fmt.Errorf("detected malformed import chain, cannot import local components from remote components") } var pkg types.ZarfPackage @@ -160,26 +160,26 @@ func NewImportChain(head types.ZarfComponent, index int, originalPackageName, ar prev := node for prev != nil { if prev.relativeToHead == relativeToHead { - return nil, fmt.Errorf("detected circular import chain: %s", strings.Join(history, " -> ")) + return ic, fmt.Errorf("detected circular import chain: %s", strings.Join(history, " -> ")) } prev = prev.prev } // this assumes the composed package is following the zarf layout if err := utils.ReadYaml(filepath.Join(relativeToHead, layout.ZarfYAML), &pkg); err != nil { - return nil, err + return ic, err } } else if isRemote { remote, err := ic.getRemote(node.Import.URL) if err != nil { - return nil, err + return ic, err } // When it's a bad file, the error gets tracked here // Maybe we add something like couldn't fetch zarf yaml to this // So it's clearer to the user the pull failed pkg, err = remote.FetchZarfYAML() if err != nil { - return nil, err + return ic, err } } @@ -196,15 +196,15 @@ func NewImportChain(head types.ZarfComponent, index int, originalPackageName, ar if len(found) == 0 { if isLocal { - return nil, fmt.Errorf("component %q not found in %q", name, filepath.Join(history...)) + return ic, fmt.Errorf("component %q not found in %q", name, filepath.Join(history...)) } else if isRemote { - return nil, fmt.Errorf("component %q not found in %q", name, node.Import.URL) + return ic, fmt.Errorf("component %q not found in %q", name, node.Import.URL) } } else if len(found) > 1 { if isLocal { - return nil, fmt.Errorf("multiple components named %q found in %q satisfying %q", name, filepath.Join(history...), arch) + return ic, fmt.Errorf("multiple components named %q found in %q satisfying %q", name, filepath.Join(history...), arch) } else if isRemote { - return nil, fmt.Errorf("multiple components named %q found in %q satisfying %q", name, node.Import.URL, arch) + return ic, fmt.Errorf("multiple components named %q found in %q satisfying %q", name, node.Import.URL, arch) } } diff --git a/src/pkg/packager/lint/lint.go b/src/pkg/packager/lint/lint.go index 044a7f0e52..b435ca08d2 100644 --- a/src/pkg/packager/lint/lint.go +++ b/src/pkg/packager/lint/lint.go @@ -77,17 +77,28 @@ func lintComposableComponenets(validator *Validator, createOpts types.ZarfCreate } chain, err := composer.NewImportChain(component, i, validator.typedZarfPackage.Metadata.Name, arch, createOpts.Flavor) + baseComponent := chain.Head() + var yqPath string + if baseComponent != nil { + if baseComponent.Import.URL != "" { + yqPath = fmt.Sprintf(".components.[%d].import.url", i) + } + if baseComponent.Import.Path != "" { + yqPath = fmt.Sprintf(".components.[%d].import.path", i) + } + } if err != nil { validator.addError(validatorMessage{ description: err.Error(), packageKey: packageKey{name: validator.typedZarfPackage.Metadata.Name}, + yqPath: yqPath, }) continue } - path := chain.Head().Import.URL + path := baseComponent.Import.URL // Skipping initial component since it will be linted the usual way - node := chain.Head().Next() + node := baseComponent.Next() for node != nil { if path == "" { path = node.GetRelativeToHead() @@ -149,8 +160,9 @@ func fillYamlTemplate(validator *Validator, yamlObj any, createOpts types.ZarfCr _, present := createOpts.SetVariables[key] if !present && !validator.HasUnsetVarMessageForPkg(pkgKey) { validator.findings = append([]validatorMessage{{ - description: lang.UnsetVarWarning, - packageKey: pkgKey, + description: lang.UnsetVarLintWarning, + packageKey: pkgKey, + validationType: validationWarning, }}, validator.findings...) } } diff --git a/src/pkg/packager/lint/validator.go b/src/pkg/packager/lint/validator.go index 1ed20e4dc8..33c1f24139 100644 --- a/src/pkg/packager/lint/validator.go +++ b/src/pkg/packager/lint/validator.go @@ -83,7 +83,7 @@ func (v Validator) IsSuccess() bool { func (v Validator) HasUnsetVarMessageForPkg(pk packageKey) bool { for _, finding := range v.findings { - if finding.description == lang.UnsetVarWarning && finding.packageKey == pk { + if finding.description == lang.UnsetVarLintWarning && finding.packageKey == pk { return true } } diff --git a/src/test/e2e/12_lint_test.go b/src/test/e2e/12_lint_test.go index 161d540bf7..1dce1ff6df 100644 --- a/src/test/e2e/12_lint_test.go +++ b/src/test/e2e/12_lint_test.go @@ -33,7 +33,7 @@ func TestLint(t *testing.T) { strippedStderr := e2e.StripMessageFormatting(stderr) // It's a bit weird to have a period here and not in the other warnings key := "WHATEVER_IMAGE" - require.Contains(t, strippedStderr, "There are variables that are unset and won't be evaluated during lint") + require.Contains(t, strippedStderr, lang.UnsetVarLintWarning) // Multiple spaces in lang require.Contains(t, strippedStderr, fmt.Sprintf(lang.PkgValidateTemplateDeprecation, key, key, key)) require.Contains(t, strippedStderr, ".components.[2].repos.[0] | Unpinned repository") From 9d750cb92ff04128e1ea0524eef6a2f263f5f112 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Mon, 11 Dec 2023 16:31:17 +0000 Subject: [PATCH 146/192] lint, removing duplicate findings, and fixing package path --- src/pkg/message/message.go | 12 +++-- src/pkg/packager/composer/list.go | 3 -- src/pkg/packager/lint/lint.go | 20 ++++---- src/pkg/packager/lint/validator.go | 49 +++++++++---------- .../packages/12-lint/linted-import/zarf.yaml | 5 ++ src/test/packages/12-lint/zarf.yaml | 4 ++ 6 files changed, 51 insertions(+), 42 deletions(-) diff --git a/src/pkg/message/message.go b/src/pkg/message/message.go index ef1d2e235f..cc00cb8b81 100644 --- a/src/pkg/message/message.go +++ b/src/pkg/message/message.go @@ -322,15 +322,19 @@ func Truncate(text string, length int, invert bool) string { func Table(header []string, data [][]string) { pterm.Println() - if len(header) > 0 { - header[0] = fmt.Sprintf(" %s", header[0]) + headerCopy := make([]string, len(header)) + copy(headerCopy, header) + dataCopy := make([][]string, len(data)) + copy(dataCopy, data) + if len(headerCopy) > 0 { + headerCopy[0] = fmt.Sprintf(" %s", headerCopy[0]) } table := pterm.TableData{ - header, + headerCopy, } - for _, row := range data { + for _, row := range dataCopy { if len(row) > 0 { row[0] = fmt.Sprintf(" %s", row[0]) } diff --git a/src/pkg/packager/composer/list.go b/src/pkg/packager/composer/list.go index 5ca8cda751..844a7aa8aa 100644 --- a/src/pkg/packager/composer/list.go +++ b/src/pkg/packager/composer/list.go @@ -174,9 +174,6 @@ func NewImportChain(head types.ZarfComponent, index int, originalPackageName, ar if err != nil { return ic, err } - // When it's a bad file, the error gets tracked here - // Maybe we add something like couldn't fetch zarf yaml to this - // So it's clearer to the user the pull failed pkg, err = remote.FetchZarfYAML() if err != nil { return ic, err diff --git a/src/pkg/packager/lint/lint.go b/src/pkg/packager/lint/lint.go index b435ca08d2..9abb4eb839 100644 --- a/src/pkg/packager/lint/lint.go +++ b/src/pkg/packager/lint/lint.go @@ -78,32 +78,34 @@ func lintComposableComponenets(validator *Validator, createOpts types.ZarfCreate chain, err := composer.NewImportChain(component, i, validator.typedZarfPackage.Metadata.Name, arch, createOpts.Flavor) baseComponent := chain.Head() - var yqPath string + var badImportYqPath string if baseComponent != nil { if baseComponent.Import.URL != "" { - yqPath = fmt.Sprintf(".components.[%d].import.url", i) + badImportYqPath = fmt.Sprintf(".components.[%d].import.url", i) } if baseComponent.Import.Path != "" { - yqPath = fmt.Sprintf(".components.[%d].import.path", i) + badImportYqPath = fmt.Sprintf(".components.[%d].import.path", i) } } if err != nil { validator.addError(validatorMessage{ description: err.Error(), packageKey: packageKey{name: validator.typedZarfPackage.Metadata.Name}, - yqPath: yqPath, + yqPath: badImportYqPath, }) continue } - path := baseComponent.Import.URL // Skipping initial component since it will be linted the usual way node := baseComponent.Next() for node != nil { - if path == "" { - path = node.GetRelativeToHead() + var fileOrOciPath string + if node.Prev().Import.URL != "" { + fileOrOciPath = node.Prev().Import.URL + } else if node.GetRelativeToHead() != "" { + fileOrOciPath = node.GetRelativeToHead() } - pkgKey := packageKey{path: path, name: node.GetOriginalPackageName()} + pkgKey := packageKey{path: fileOrOciPath, name: node.GetOriginalPackageName()} checkForVarInComponentImport(validator, node.GetIndex(), node.ZarfComponent, pkgKey) fillComponentTemplate(validator, node, createOpts, pkgKey) lintComponent(validator, node.GetIndex(), node.ZarfComponent, pkgKey) @@ -158,7 +160,7 @@ func fillYamlTemplate(validator *Validator, yamlObj any, createOpts types.ZarfCr }) } _, present := createOpts.SetVariables[key] - if !present && !validator.HasUnsetVarMessageForPkg(pkgKey) { + if !present && !validator.hasUnsetVarMessageForPkg(pkgKey) { validator.findings = append([]validatorMessage{{ description: lang.UnsetVarLintWarning, packageKey: pkgKey, diff --git a/src/pkg/packager/lint/validator.go b/src/pkg/packager/lint/validator.go index 33c1f24139..17ca700eb6 100644 --- a/src/pkg/packager/lint/validator.go +++ b/src/pkg/packager/lint/validator.go @@ -10,16 +10,17 @@ import ( "github.com/defenseunicorns/zarf/src/config/lang" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/utils" + "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "github.com/defenseunicorns/zarf/src/types" "github.com/fatih/color" "github.com/pterm/pterm" ) -type ValidationType int +type validationType int const ( - validationError ValidationType = 1 - validationWarning ValidationType = 2 + validationError validationType = 1 + validationWarning validationType = 2 ) type packageKey struct { @@ -32,27 +33,23 @@ type validatorMessage struct { description string item string packageKey packageKey - validationType ValidationType + validationType validationType } -func (vt ValidationType) String() string { +func (vt validationType) String() string { if vt == validationError { return utils.ColorWrap("Error", color.FgRed) } else if vt == validationWarning { return utils.ColorWrap("Warning", color.FgYellow) - } else { - return "" } + return "" } -func (v validatorMessage) String() string { - if v.item != "" { - v.item = fmt.Sprintf(" - %s", v.item) - } - if v.packageKey.path == "" && v.yqPath == "" && v.item == "" { - return v.description +func (vm validatorMessage) String() string { + if vm.item != "" { + vm.item = fmt.Sprintf(" - %s", vm.item) } - return fmt.Sprintf("%s%s", v.description, v.item) + return fmt.Sprintf("%s%s", vm.description, vm.item) } // Validator holds the warnings/errors and messaging that we get from validation @@ -81,7 +78,7 @@ func (v Validator) IsSuccess() bool { return true } -func (v Validator) HasUnsetVarMessageForPkg(pk packageKey) bool { +func (v Validator) hasUnsetVarMessageForPkg(pk packageKey) bool { for _, finding := range v.findings { if finding.description == lang.UnsetVarLintWarning && finding.packageKey == pk { return true @@ -95,17 +92,17 @@ func (v Validator) printValidationTable() { return } - packageKeys := v.getUniquePackageKeys() + packageKeys := helpers.Unique(v.getUniquePackageKeys()) connectData := make(map[packageKey][][]string) - for _, packageKey := range packageKeys { - header := []string{"Type", "Path", "Message"} - for _, finding := range v.findings { - if finding.packageKey == packageKey { - connectData[packageKey] = append(connectData[packageKey], - []string{finding.validationType.String(), finding.getPath(), finding.String()}) - } - } + findings := helpers.Unique(v.findings) + for _, finding := range findings { + connectData[finding.packageKey] = append(connectData[finding.packageKey], + []string{finding.validationType.String(), finding.getPath(), finding.String()}) + } + + header := []string{"Type", "Path", "Message"} + for _, packageKey := range packageKeys { //We should probably move this println into info pterm.Println() if packageKey.path != "" { @@ -141,10 +138,10 @@ func (v Validator) getFormattedFindingCount(pk packageKey) string { continue } if finding.validationType == validationWarning { - warningCount += 1 + warningCount++ } if finding.validationType == validationError { - errorCount += 1 + errorCount++ } } wordWarning := "warnings" diff --git a/src/test/packages/12-lint/linted-import/zarf.yaml b/src/test/packages/12-lint/linted-import/zarf.yaml index db2d92415d..a2e33b0841 100644 --- a/src/test/packages/12-lint/linted-import/zarf.yaml +++ b/src/test/packages/12-lint/linted-import/zarf.yaml @@ -16,3 +16,8 @@ components: - busybox@sha256:3fbc632167424a6d997e74f52b878d7cc478225cffac6bc977eedfe51c7f4e79 - busybox:###ZARF_PKG_TMPL_BUSYBOX_IMAGE### - busybox:###ZARF_PKG_TMPL_UNSET### + + - name: oci-games-url + import: + url: oci://🦄/dos-games:1.0.0-skeleton + name: baseline diff --git a/src/test/packages/12-lint/zarf.yaml b/src/test/packages/12-lint/zarf.yaml index 36f762ba0d..78e22cbaf2 100644 --- a/src/test/packages/12-lint/zarf.yaml +++ b/src/test/packages/12-lint/zarf.yaml @@ -43,3 +43,7 @@ components: import: url: oci://🦄/dos-games:1.0.0-skeleton name: baseline + + - name: oci-games-url + import: + path: linted-import From 208ed8e148dd501f11e190bcef778302ca4c446a Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Mon, 11 Dec 2023 16:42:26 +0000 Subject: [PATCH 147/192] fix lint script --- hack/lint_all_zarf_packages.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/hack/lint_all_zarf_packages.sh b/hack/lint_all_zarf_packages.sh index 94d0df4f9f..681d71ced2 100755 --- a/hack/lint_all_zarf_packages.sh +++ b/hack/lint_all_zarf_packages.sh @@ -1,8 +1,10 @@ #!/bin/bash use_build=$1 -BASEDIR=$(dirname "$0") -cd ../$BASEDIR +SCRIPT=$(realpath "$0") +SCRIPTPATH=$(dirname "$SCRIPT") +cd $SCRIPTPATH +cd .. find "." -type f -name 'zarf.yaml' | while read -r yaml_file; do dir=$(dirname "$yaml_file") if [[ "$dir" == *src/test/* ]] && [ "$use_build" != true ]; then From 47c2a4b59f8431c733b6fb65a9088a68ff2c8127 Mon Sep 17 00:00:00 2001 From: Austin Abro <37223396+AustinAbro321@users.noreply.github.com> Date: Mon, 11 Dec 2023 14:36:35 -0500 Subject: [PATCH 148/192] Apply suggestions from code review Co-authored-by: Wayne Starr --- Makefile | 2 +- hack/lint_all_zarf_packages.sh | 6 +----- src/pkg/message/message.go | 1 + src/pkg/packager/lint/lint.go | 6 +++--- 4 files changed, 6 insertions(+), 9 deletions(-) diff --git a/Makefile b/Makefile index 29f795ee69..f5fdb0c02e 100644 --- a/Makefile +++ b/Makefile @@ -76,7 +76,7 @@ docs-and-schema: ## Generate the Zarf Documentation and Schema hack/gen-cli-docs.sh ZARF_CONFIG=hack/empty-config.toml hack/create-zarf-schema.sh -lint-packages-and-examples: ## Recurisvely lint all zarf.yaml files in packages or examples +lint-packages-and-examples: ## Recurisvely lint all zarf.yaml files in the repo except for those dedicated to tests hack/lint_all_zarf_packages.sh # INTERNAL: a shim used to build the agent image only if needed on Windows using the `test` command diff --git a/hack/lint_all_zarf_packages.sh b/hack/lint_all_zarf_packages.sh index 681d71ced2..ebdca11a11 100755 --- a/hack/lint_all_zarf_packages.sh +++ b/hack/lint_all_zarf_packages.sh @@ -11,10 +11,6 @@ find "." -type f -name 'zarf.yaml' | while read -r yaml_file; do continue fi echo "Running 'zarf prepare lint' in directory: $dir" - if [ "$use_build" = true ]; then - ./build/zarf prepare lint $dir - else - zarf prepare lint $dir - fi + go run main.go prepare lint $dir echo "---" done diff --git a/src/pkg/message/message.go b/src/pkg/message/message.go index cc00cb8b81..a04cdaf89e 100644 --- a/src/pkg/message/message.go +++ b/src/pkg/message/message.go @@ -322,6 +322,7 @@ func Truncate(text string, length int, invert bool) string { func Table(header []string, data [][]string) { pterm.Println() + // To avoid side effects make copies of the header and data before adding padding headerCopy := make([]string, len(header)) copy(headerCopy, header) dataCopy := make([][]string, len(data)) diff --git a/src/pkg/packager/lint/lint.go b/src/pkg/packager/lint/lint.go index 9abb4eb839..1dd096452f 100644 --- a/src/pkg/packager/lint/lint.go +++ b/src/pkg/packager/lint/lint.go @@ -49,7 +49,7 @@ func ValidateZarfSchema(createOpts types.ZarfCreateOptions) (*Validator, error) return nil, fmt.Errorf("unable to access directory '%s': %w", createOpts.BaseDir, err) } - lintComposableComponenets(&validator, createOpts) + lintComposableComponents(&validator, createOpts) lintUnEvaledVariables(&validator) @@ -126,9 +126,9 @@ func fillComponentTemplate(validator *Validator, node *composer.Node, createOpts fillYamlTemplate(validator, node, createOpts, pkgKey) } -func fillActiveTemplate(validator *Validator, createOpts types.ZarfCreateOptions) { +func fillPackageTemplate(validator *Validator, createOpts types.ZarfCreateOptions) { - err := packager.FindComponentTemplatesAndReload(&validator.typedZarfPackage) + err := packager.ReloadComponentTemplatesInPackage(&validator.typedZarfPackage) if err != nil { validator.addWarning(validatorMessage{ description: fmt.Sprintf("unable to find components %s", err), From 60bf5047c45ac3f33d4ca05923e1f7a3075d1256 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Mon, 11 Dec 2023 19:50:35 +0000 Subject: [PATCH 149/192] suggestions --- hack/lint_all_zarf_packages.sh | 4 ++-- src/pkg/packager/lint/lint.go | 4 ++-- src/pkg/packager/variables.go | 6 +++--- src/test/packages/12-lint/zarf.yaml | 4 ++++ 4 files changed, 11 insertions(+), 7 deletions(-) diff --git a/hack/lint_all_zarf_packages.sh b/hack/lint_all_zarf_packages.sh index ebdca11a11..c66a96435f 100755 --- a/hack/lint_all_zarf_packages.sh +++ b/hack/lint_all_zarf_packages.sh @@ -1,13 +1,13 @@ #!/bin/bash -use_build=$1 +lint_src_test=$1 SCRIPT=$(realpath "$0") SCRIPTPATH=$(dirname "$SCRIPT") cd $SCRIPTPATH cd .. find "." -type f -name 'zarf.yaml' | while read -r yaml_file; do dir=$(dirname "$yaml_file") - if [[ "$dir" == *src/test/* ]] && [ "$use_build" != true ]; then + if [[ "$dir" == *src/test/* ]] && [ "$lint_src_test" != true ]; then continue fi echo "Running 'zarf prepare lint' in directory: $dir" diff --git a/src/pkg/packager/lint/lint.go b/src/pkg/packager/lint/lint.go index 1dd096452f..3c9ac82d48 100644 --- a/src/pkg/packager/lint/lint.go +++ b/src/pkg/packager/lint/lint.go @@ -53,7 +53,7 @@ func ValidateZarfSchema(createOpts types.ZarfCreateOptions) (*Validator, error) lintUnEvaledVariables(&validator) - fillActiveTemplate(&validator, createOpts) + fillPackageTemplate(&validator, createOpts) lintComponents(&validator) @@ -68,7 +68,7 @@ func ValidateZarfSchema(createOpts types.ZarfCreateOptions) (*Validator, error) return &validator, nil } -func lintComposableComponenets(validator *Validator, createOpts types.ZarfCreateOptions) { +func lintComposableComponents(validator *Validator, createOpts types.ZarfCreateOptions) { for i, component := range validator.typedZarfPackage.Components { arch := config.GetArch(validator.typedZarfPackage.Metadata.Architecture) diff --git a/src/pkg/packager/variables.go b/src/pkg/packager/variables.go index cb223c68bb..fb3e20f6d3 100644 --- a/src/pkg/packager/variables.go +++ b/src/pkg/packager/variables.go @@ -27,8 +27,8 @@ func ReloadComponentTemplate(component *types.ZarfComponent) error { return nil } -// FindComponentTemplatesAndReload appends ###ZARF_COMPONENT_NAME### for each component, assigns value, and reloads -func FindComponentTemplatesAndReload(zarfPackage *types.ZarfPackage) error { +// ReloadComponentTemplatesInPackage appends ###ZARF_COMPONENT_NAME### for each component, assigns value, and reloads +func ReloadComponentTemplatesInPackage(zarfPackage *types.ZarfPackage) error { // iterate through components to and find all ###ZARF_COMPONENT_NAME, assign to component Name and value for i := range zarfPackage.Components { if err := ReloadComponentTemplate(&zarfPackage.Components[i]); err != nil { @@ -78,7 +78,7 @@ func (p *Packager) fillActiveTemplate() error { } // update the component templates on the package - err := FindComponentTemplatesAndReload(&p.cfg.Pkg) + err := ReloadComponentTemplatesInPackage(&p.cfg.Pkg) if err != nil { return err } diff --git a/src/test/packages/12-lint/zarf.yaml b/src/test/packages/12-lint/zarf.yaml index 78e22cbaf2..66b7187a10 100644 --- a/src/test/packages/12-lint/zarf.yaml +++ b/src/test/packages/12-lint/zarf.yaml @@ -47,3 +47,7 @@ components: - name: oci-games-url import: path: linted-import + + - name: import-big-bang + import: + url: oci://localhost:555/big-bang-min:2.10.0-skeleton From 0b7fd718b12f38f4e1a85fe0d32c1355a589cc1c Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Mon, 11 Dec 2023 20:36:25 +0000 Subject: [PATCH 150/192] refactor --- src/cmd/package.go | 18 +----------------- src/cmd/prepare.go | 4 ++-- src/pkg/packager/lint/validator.go | 7 +++---- src/test/e2e/12_lint_test.go | 2 ++ src/test/packages/12-lint/zarf.yaml | 19 ++++++++++--------- 5 files changed, 18 insertions(+), 32 deletions(-) diff --git a/src/cmd/package.go b/src/cmd/package.go index 9cc0464b56..7c1e28ab70 100644 --- a/src/cmd/package.go +++ b/src/cmd/package.go @@ -6,7 +6,6 @@ package cmd import ( "fmt" - "os" "path/filepath" "regexp" "strings" @@ -16,7 +15,6 @@ import ( "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/packager/sources" "github.com/defenseunicorns/zarf/src/pkg/utils" - "github.com/defenseunicorns/zarf/src/types" "oras.land/oras-go/v2/registry" @@ -35,18 +33,6 @@ var packageCmd = &cobra.Command{ Short: lang.CmdPackageShort, } -func setBaseDirectory(args []string, pkgConfig *types.PackagerConfig) { - if len(args) > 0 { - pkgConfig.CreateOpts.BaseDir = args[0] - } else { - var err error - pkgConfig.CreateOpts.BaseDir, err = os.Getwd() - if err != nil { - message.Fatalf(err, lang.CmdPackageCreateErr, err.Error()) - } - } -} - var packageCreateCmd = &cobra.Command{ Use: "create [ DIRECTORY ]", Aliases: []string{"c"}, @@ -54,9 +40,7 @@ var packageCreateCmd = &cobra.Command{ Short: lang.CmdPackageCreateShort, Long: lang.CmdPackageCreateLong, Run: func(cmd *cobra.Command, args []string) { - - // If a directory was provided, use that as the base directory - setBaseDirectory(args, &pkgConfig) + common.SetBaseDirectory(args, &pkgConfig) var isCleanPathRegex = regexp.MustCompile(`^[a-zA-Z0-9\_\-\/\.\~\\:]+$`) if !isCleanPathRegex.MatchString(config.CommonOptions.CachePath) { diff --git a/src/cmd/prepare.go b/src/cmd/prepare.go index 20edca7d8e..50fd443b9f 100644 --- a/src/cmd/prepare.go +++ b/src/cmd/prepare.go @@ -213,7 +213,7 @@ var lintCmd = &cobra.Command{ Short: lang.CmdPrepareLintShort, Long: lang.CmdPrepareLintLong, Run: func(cmd *cobra.Command, args []string) { - setBaseDirectory(args, &pkgConfig) + common.SetBaseDirectory(args, &pkgConfig) v := common.GetViper() pkgConfig.CreateOpts.SetVariables = helpers.TransformAndMergeMap( v.GetStringMapString(common.VPkgCreateSet), pkgConfig.CreateOpts.SetVariables, strings.ToUpper) @@ -247,6 +247,6 @@ func init() { prepareFindImages.Flags().StringVar(&pkgConfig.FindImagesOpts.KubeVersionOverride, "kube-version", "", lang.CmdPrepareFlagKubeVersion) lintCmd.Flags().StringToStringVar(&pkgConfig.CreateOpts.SetVariables, "set", v.GetStringMapString(common.VPkgCreateSet), lang.CmdPackageCreateFlagSet) - + lintCmd.Flags().StringVarP(&pkgConfig.CreateOpts.Flavor, "flavor", "f", v.GetString(common.VPkgCreateFlavor), lang.CmdPackageCreateFlagFlavor) prepareTransformGitLinks.Flags().StringVar(&pkgConfig.InitOpts.GitServer.PushUsername, "git-account", config.ZarfGitPushUser, lang.CmdPrepareFlagGitAccount) } diff --git a/src/pkg/packager/lint/validator.go b/src/pkg/packager/lint/validator.go index 17ca700eb6..ffff9248f0 100644 --- a/src/pkg/packager/lint/validator.go +++ b/src/pkg/packager/lint/validator.go @@ -95,8 +95,7 @@ func (v Validator) printValidationTable() { packageKeys := helpers.Unique(v.getUniquePackageKeys()) connectData := make(map[packageKey][][]string) - findings := helpers.Unique(v.findings) - for _, finding := range findings { + for _, finding := range v.findings { connectData[finding.packageKey] = append(connectData[finding.packageKey], []string{finding.validationType.String(), finding.getPath(), finding.String()}) } @@ -169,10 +168,10 @@ func (v Validator) hasFindings() bool { func (v *Validator) addWarning(vmessage validatorMessage) { vmessage.validationType = validationWarning - v.findings = append(v.findings, vmessage) + v.findings = helpers.Unique(append(v.findings, vmessage)) } func (v *Validator) addError(vMessage validatorMessage) { vMessage.validationType = validationError - v.findings = append(v.findings, vMessage) + v.findings = helpers.Unique(append(v.findings, vMessage)) } diff --git a/src/test/e2e/12_lint_test.go b/src/test/e2e/12_lint_test.go index 1dce1ff6df..de13e7b75e 100644 --- a/src/test/e2e/12_lint_test.go +++ b/src/test/e2e/12_lint_test.go @@ -44,6 +44,8 @@ func TestLint(t *testing.T) { require.Contains(t, strippedStderr, ".components.[1].images.[2] | Image not pinned with digest") require.Contains(t, strippedStderr, ".components.[3].import.path | Zarf does not evaluate variables at component.x.import.path - ###ZARF_PKG_TMPL_PATH###") require.Contains(t, strippedStderr, ".components.[0].images.[0] | Image not pinned with digest - defenseunicorns/zarf-game:multi-tile-dark") + // I need to see if we only want to not include flavors when they're downstream or not include them in general + // require.NotContains(t, strippedStderr, "image-in-flavor-component:unpinned") }) } diff --git a/src/test/packages/12-lint/zarf.yaml b/src/test/packages/12-lint/zarf.yaml index 66b7187a10..75e52da8d5 100644 --- a/src/test/packages/12-lint/zarf.yaml +++ b/src/test/packages/12-lint/zarf.yaml @@ -1,13 +1,8 @@ -kind: ZarfInitConfig +kind: ZarfPackageConfig metadata: - name: init + name: lint description1: Testing bad yaml - -variables: - - name: BUSYBOX_IMAGE - description: "whatever" - components: - name: first-test-component import: @@ -48,6 +43,12 @@ components: import: path: linted-import + # - name: import-big-bang + # import: + # url: oci://localhost:555/big-bang-min:2.10.0-skeleton + - name: import-big-bang - import: - url: oci://localhost:555/big-bang-min:2.10.0-skeleton + only: + flavor: bad-flavor-that-were-not-picking-but-want-to-make-sure-doesnt-show-up-in-lint + images: + - image-in-flavor-component:unpinned From 4cc3bf20d8d14ac8552e137ee513898bf7f8a807 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Mon, 11 Dec 2023 20:36:42 +0000 Subject: [PATCH 151/192] refactor --- src/cmd/common/common.go | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 src/cmd/common/common.go diff --git a/src/cmd/common/common.go b/src/cmd/common/common.go new file mode 100644 index 0000000000..41cb208a7c --- /dev/null +++ b/src/cmd/common/common.go @@ -0,0 +1,26 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package common handles command configuration across all commands +package common + +import ( + "os" + + "github.com/defenseunicorns/zarf/src/config/lang" + "github.com/defenseunicorns/zarf/src/pkg/message" + "github.com/defenseunicorns/zarf/src/types" +) + +// SetBaseDirectory sets base directory on package config when given in args +func SetBaseDirectory(args []string, pkgConfig *types.PackagerConfig) { + if len(args) > 0 { + pkgConfig.CreateOpts.BaseDir = args[0] + } else { + var err error + pkgConfig.CreateOpts.BaseDir, err = os.Getwd() + if err != nil { + message.Fatalf(err, lang.CmdPackageCreateErr, err.Error()) + } + } +} From 516b1a4f7f6c9a2afae3e8dfbe198097c1858a44 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Mon, 11 Dec 2023 21:26:32 +0000 Subject: [PATCH 152/192] no longer linting base and downstream components separtely --- src/pkg/packager/lint/lint.go | 80 +++++++++--------------------- src/pkg/packager/lint/lint_test.go | 4 +- src/pkg/packager/lint/validator.go | 10 ---- src/test/e2e/12_lint_test.go | 4 +- 4 files changed, 28 insertions(+), 70 deletions(-) diff --git a/src/pkg/packager/lint/lint.go b/src/pkg/packager/lint/lint.go index 3c9ac82d48..bdde34ac4d 100644 --- a/src/pkg/packager/lint/lint.go +++ b/src/pkg/packager/lint/lint.go @@ -51,17 +51,11 @@ func ValidateZarfSchema(createOpts types.ZarfCreateOptions) (*Validator, error) lintComposableComponents(&validator, createOpts) - lintUnEvaledVariables(&validator) - - fillPackageTemplate(&validator, createOpts) - - lintComponents(&validator) - if validator.jsonSchema, err = getSchemaFile(); err != nil { return nil, err } - - if err = validateSchema(&validator); err != nil { + pkgKey := packageKey{name: validator.typedZarfPackage.Metadata.Name, path: createOpts.BaseDir} + if err = validateSchema(&validator, pkgKey); err != nil { return nil, err } @@ -96,14 +90,17 @@ func lintComposableComponents(validator *Validator, createOpts types.ZarfCreateO continue } - // Skipping initial component since it will be linted the usual way - node := baseComponent.Next() + node := baseComponent for node != nil { var fileOrOciPath string - if node.Prev().Import.URL != "" { - fileOrOciPath = node.Prev().Import.URL - } else if node.GetRelativeToHead() != "" { - fileOrOciPath = node.GetRelativeToHead() + if node.Prev() != nil { + if node.Prev().Import.URL != "" { + fileOrOciPath = node.Prev().Import.URL + } else if node.GetRelativeToHead() != "" { + fileOrOciPath = filepath.Join(createOpts.BaseDir, node.GetRelativeToHead()) + } + } else { + fileOrOciPath = filepath.Join(createOpts.BaseDir, node.GetRelativeToHead()) } pkgKey := packageKey{path: fileOrOciPath, name: node.GetOriginalPackageName()} checkForVarInComponentImport(validator, node.GetIndex(), node.ZarfComponent, pkgKey) @@ -119,35 +116,17 @@ func fillComponentTemplate(validator *Validator, node *composer.Node, createOpts err := packager.ReloadComponentTemplate(&node.ZarfComponent) if err != nil { validator.addWarning(validatorMessage{ - description: fmt.Sprintf("unable to find components %s", err), - packageKey: packageKey{path: node.GetRelativeToHead(), name: node.GetOriginalPackageName()}, - }) - } - fillYamlTemplate(validator, node, createOpts, pkgKey) -} - -func fillPackageTemplate(validator *Validator, createOpts types.ZarfCreateOptions) { - - err := packager.ReloadComponentTemplatesInPackage(&validator.typedZarfPackage) - if err != nil { - validator.addWarning(validatorMessage{ - description: fmt.Sprintf("unable to find components %s", err), - packageKey: packageKey{name: validator.typedZarfPackage.Metadata.Name}, + description: err.Error(), + packageKey: pkgKey, }) } - - fillYamlTemplate(validator, &validator.typedZarfPackage, - createOpts, packageKey{name: validator.typedZarfPackage.Metadata.Name, path: ""}) -} - -func fillYamlTemplate(validator *Validator, yamlObj any, createOpts types.ZarfCreateOptions, pkgKey packageKey) { templateMap := map[string]string{} setVarsAndWarn := func(templatePrefix string, deprecated bool) { - yamlTemplates, err := utils.FindYamlTemplates(yamlObj, templatePrefix, "###") + yamlTemplates, err := utils.FindYamlTemplates(node, templatePrefix, "###") if err != nil { validator.addWarning(validatorMessage{ - description: fmt.Sprintf("unable to find variables %s", err), + description: err.Error(), packageKey: pkgKey, }) } @@ -160,12 +139,11 @@ func fillYamlTemplate(validator *Validator, yamlObj any, createOpts types.ZarfCr }) } _, present := createOpts.SetVariables[key] - if !present && !validator.hasUnsetVarMessageForPkg(pkgKey) { - validator.findings = append([]validatorMessage{{ - description: lang.UnsetVarLintWarning, - packageKey: pkgKey, - validationType: validationWarning, - }}, validator.findings...) + if !present { + validator.addWarning(validatorMessage{ + description: lang.UnsetVarLintWarning, + packageKey: pkgKey, + }) } } for key, value := range createOpts.SetVariables { @@ -178,7 +156,7 @@ func fillYamlTemplate(validator *Validator, yamlObj any, createOpts types.ZarfCr // [DEPRECATION] Set the Package Variable syntax as well for backward compatibility setVarsAndWarn(types.ZarfPackageVariablePrefix, true) - utils.ReloadYamlTemplate(yamlObj, templateMap) + utils.ReloadYamlTemplate(node, templateMap) } func isPinnedImage(image string) (bool, error) { @@ -197,12 +175,6 @@ func isPinnedRepo(repo string) bool { return (strings.Contains(repo, "@")) } -func lintComponents(validator *Validator) { - for i, component := range validator.typedZarfPackage.Components { - lintComponent(validator, i, component, packageKey{path: "", name: validator.typedZarfPackage.Metadata.Name}) - } -} - func lintComponent(validator *Validator, index int, component types.ZarfComponent, pkgKey packageKey) { checkForUnpinnedRepos(validator, index, component, pkgKey) checkForUnpinnedImages(validator, index, component, pkgKey) @@ -261,12 +233,6 @@ func checkForUnpinnedFiles(validator *Validator, index int, component types.Zarf } } -func lintUnEvaledVariables(validator *Validator) { - for i, component := range validator.typedZarfPackage.Components { - checkForVarInComponentImport(validator, i, component, packageKey{name: validator.typedZarfPackage.Metadata.Name, path: ""}) - } -} - func checkForVarInComponentImport(validator *Validator, index int, component types.ZarfComponent, pkgKey packageKey) { if strings.Contains(component.Import.Path, types.ZarfPackageTemplatePrefix) { validator.addWarning(validatorMessage{ @@ -299,7 +265,7 @@ func makeFieldPathYqCompat(field string) string { return fmt.Sprintf(".%s", wrappedField) } -func validateSchema(validator *Validator) error { +func validateSchema(validator *Validator, pkgKey packageKey) error { schemaLoader := gojsonschema.NewBytesLoader(validator.jsonSchema) documentLoader := gojsonschema.NewGoLoader(validator.untypedZarfPackage) @@ -313,7 +279,7 @@ func validateSchema(validator *Validator) error { validator.addError(validatorMessage{ yqPath: makeFieldPathYqCompat(desc.Field()), description: desc.Description(), - packageKey: packageKey{name: validator.typedZarfPackage.Metadata.Name}, + packageKey: pkgKey, }) } } diff --git a/src/pkg/packager/lint/lint_test.go b/src/pkg/packager/lint/lint_test.go index f91355f2ca..255ce47798 100644 --- a/src/pkg/packager/lint/lint_test.go +++ b/src/pkg/packager/lint/lint_test.go @@ -82,7 +82,9 @@ func TestValidateSchema(t *testing.T) { t.Run("Template in component import success", func(t *testing.T) { unmarshalledYaml := readAndUnmarshalYaml[types.ZarfPackage](t, goodZarfPackage) validator := Validator{typedZarfPackage: unmarshalledYaml} - lintComponents(&validator) + for i, component := range validator.typedZarfPackage.Components { + lintComponent(&validator, i, component, packageKey{path: "", name: validator.typedZarfPackage.Metadata.Name}) + } require.Empty(t, validator.findings) }) diff --git a/src/pkg/packager/lint/validator.go b/src/pkg/packager/lint/validator.go index ffff9248f0..3d86f0c96f 100644 --- a/src/pkg/packager/lint/validator.go +++ b/src/pkg/packager/lint/validator.go @@ -7,7 +7,6 @@ package lint import ( "fmt" - "github.com/defenseunicorns/zarf/src/config/lang" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" @@ -78,15 +77,6 @@ func (v Validator) IsSuccess() bool { return true } -func (v Validator) hasUnsetVarMessageForPkg(pk packageKey) bool { - for _, finding := range v.findings { - if finding.description == lang.UnsetVarLintWarning && finding.packageKey == pk { - return true - } - } - return false -} - func (v Validator) printValidationTable() { if !v.hasFindings() { return diff --git a/src/test/e2e/12_lint_test.go b/src/test/e2e/12_lint_test.go index de13e7b75e..99b6d3c2f5 100644 --- a/src/test/e2e/12_lint_test.go +++ b/src/test/e2e/12_lint_test.go @@ -44,8 +44,8 @@ func TestLint(t *testing.T) { require.Contains(t, strippedStderr, ".components.[1].images.[2] | Image not pinned with digest") require.Contains(t, strippedStderr, ".components.[3].import.path | Zarf does not evaluate variables at component.x.import.path - ###ZARF_PKG_TMPL_PATH###") require.Contains(t, strippedStderr, ".components.[0].images.[0] | Image not pinned with digest - defenseunicorns/zarf-game:multi-tile-dark") - // I need to see if we only want to not include flavors when they're downstream or not include them in general - // require.NotContains(t, strippedStderr, "image-in-flavor-component:unpinned") + // Check to ensure non selected flavor is included + require.NotContains(t, strippedStderr, "image-in-flavor-component:unpinned") }) } From fea98bb7f1d52890591daf485ada4f99d7daf8d0 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Tue, 12 Dec 2023 14:56:02 +0000 Subject: [PATCH 153/192] add tests for lint composable components --- src/pkg/packager/lint/lint.go | 9 ++++--- src/pkg/packager/lint/lint_test.go | 38 +++++++++++++++++++++++------- 2 files changed, 33 insertions(+), 14 deletions(-) diff --git a/src/pkg/packager/lint/lint.go b/src/pkg/packager/lint/lint.go index bdde34ac4d..f9f78970b5 100644 --- a/src/pkg/packager/lint/lint.go +++ b/src/pkg/packager/lint/lint.go @@ -49,7 +49,7 @@ func ValidateZarfSchema(createOpts types.ZarfCreateOptions) (*Validator, error) return nil, fmt.Errorf("unable to access directory '%s': %w", createOpts.BaseDir, err) } - lintComposableComponents(&validator, createOpts) + lintComposableComponents(&validator, &createOpts) if validator.jsonSchema, err = getSchemaFile(); err != nil { return nil, err @@ -62,7 +62,7 @@ func ValidateZarfSchema(createOpts types.ZarfCreateOptions) (*Validator, error) return &validator, nil } -func lintComposableComponents(validator *Validator, createOpts types.ZarfCreateOptions) { +func lintComposableComponents(validator *Validator, createOpts *types.ZarfCreateOptions) { for i, component := range validator.typedZarfPackage.Components { arch := config.GetArch(validator.typedZarfPackage.Metadata.Architecture) @@ -87,7 +87,6 @@ func lintComposableComponents(validator *Validator, createOpts types.ZarfCreateO packageKey: packageKey{name: validator.typedZarfPackage.Metadata.Name}, yqPath: badImportYqPath, }) - continue } node := baseComponent @@ -100,7 +99,7 @@ func lintComposableComponents(validator *Validator, createOpts types.ZarfCreateO fileOrOciPath = filepath.Join(createOpts.BaseDir, node.GetRelativeToHead()) } } else { - fileOrOciPath = filepath.Join(createOpts.BaseDir, node.GetRelativeToHead()) + fileOrOciPath = filepath.Join(createOpts.BaseDir, ".") } pkgKey := packageKey{path: fileOrOciPath, name: node.GetOriginalPackageName()} checkForVarInComponentImport(validator, node.GetIndex(), node.ZarfComponent, pkgKey) @@ -111,7 +110,7 @@ func lintComposableComponents(validator *Validator, createOpts types.ZarfCreateO } } -func fillComponentTemplate(validator *Validator, node *composer.Node, createOpts types.ZarfCreateOptions, pkgKey packageKey) { +func fillComponentTemplate(validator *Validator, node *composer.Node, createOpts *types.ZarfCreateOptions, pkgKey packageKey) { err := packager.ReloadComponentTemplate(&node.ZarfComponent) if err != nil { diff --git a/src/pkg/packager/lint/lint_test.go b/src/pkg/packager/lint/lint_test.go index 255ce47798..8eb2861fb7 100644 --- a/src/pkg/packager/lint/lint_test.go +++ b/src/pkg/packager/lint/lint_test.go @@ -6,7 +6,6 @@ package lint import ( "errors" - "fmt" "os" "testing" @@ -64,7 +63,7 @@ func TestValidateSchema(t *testing.T) { t.Run("validate schema success", func(t *testing.T) { unmarshalledYaml := readAndUnmarshalYaml[interface{}](t, goodZarfPackage) validator := Validator{untypedZarfPackage: unmarshalledYaml, jsonSchema: getZarfSchema(t)} - err := validateSchema(&validator) + err := validateSchema(&validator, packageKey{}) require.NoError(t, err) require.Empty(t, validator.findings) }) @@ -72,7 +71,7 @@ func TestValidateSchema(t *testing.T) { t.Run("validate schema fail", func(t *testing.T) { unmarshalledYaml := readAndUnmarshalYaml[interface{}](t, badZarfPackage) validator := Validator{untypedZarfPackage: unmarshalledYaml, jsonSchema: getZarfSchema(t)} - err := validateSchema(&validator) + err := validateSchema(&validator, packageKey{}) require.NoError(t, err) config.NoColor = true require.Equal(t, "Additional property not-path is not allowed", validator.findings[0].String()) @@ -88,15 +87,20 @@ func TestValidateSchema(t *testing.T) { require.Empty(t, validator.findings) }) - t.Run("Template in component import failure", func(t *testing.T) { + t.Run("Path template in component import failure", func(t *testing.T) { pathVar := "###ZARF_PKG_TMPL_PATH###" - ociPathVar := fmt.Sprintf("oci://%s", pathVar) pathComponent := types.ZarfComponent{Import: types.ZarfComponentImport{Path: pathVar}} - URLComponent := types.ZarfComponent{Import: types.ZarfComponentImport{URL: ociPathVar}} - validator := Validator{typedZarfPackage: types.ZarfPackage{Components: []types.ZarfComponent{pathComponent, URLComponent}}} - lintUnEvaledVariables(&validator) + validator := Validator{typedZarfPackage: types.ZarfPackage{Components: []types.ZarfComponent{pathComponent}}} + checkForVarInComponentImport(&validator, 0, validator.typedZarfPackage.Components[0], packageKey{}) require.Equal(t, pathVar, validator.findings[0].item) - require.Equal(t, ociPathVar, validator.findings[1].item) + }) + + t.Run("OCI template in component import failure", func(t *testing.T) { + ociPathVar := "oci://###ZARF_PKG_TMPL_PATH###" + URLComponent := types.ZarfComponent{Import: types.ZarfComponentImport{URL: ociPathVar}} + validator := Validator{typedZarfPackage: types.ZarfPackage{Components: []types.ZarfComponent{URLComponent}}} + checkForVarInComponentImport(&validator, 0, validator.typedZarfPackage.Components[0], packageKey{}) + require.Equal(t, ociPathVar, validator.findings[0].item) }) t.Run("Unpinnned repo warning", func(t *testing.T) { @@ -152,6 +156,22 @@ func TestValidateSchema(t *testing.T) { require.Equal(t, input, acutal) }) + t.Run("Test comoposable components", func(t *testing.T) { + pathVar := "fake-path" + unpinnedImage := "unpinned:latest" + pathComponent := types.ZarfComponent{ + Import: types.ZarfComponentImport{Path: pathVar}, + Images: []string{unpinnedImage}} + validator := Validator{ + typedZarfPackage: types.ZarfPackage{Components: []types.ZarfComponent{pathComponent}, + Metadata: types.ZarfMetadata{Name: "test-zarf-package"}}} + + createOpts := types.ZarfCreateOptions{Flavor: ""} + lintComposableComponents(&validator, &createOpts) + require.Equal(t, "open fake-path/zarf.yaml: no such file or directory", validator.findings[0].description) + require.Equal(t, unpinnedImage, validator.findings[1].item) + }) + t.Run("isImagePinned", func(t *testing.T) { t.Parallel() tests := []struct { From 9a595514888fdfb347a4eacd11df8fccf399ce60 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Tue, 12 Dec 2023 14:57:10 +0000 Subject: [PATCH 154/192] add additional test --- src/pkg/packager/lint/lint_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/src/pkg/packager/lint/lint_test.go b/src/pkg/packager/lint/lint_test.go index 8eb2861fb7..b40b2601b4 100644 --- a/src/pkg/packager/lint/lint_test.go +++ b/src/pkg/packager/lint/lint_test.go @@ -169,6 +169,7 @@ func TestValidateSchema(t *testing.T) { createOpts := types.ZarfCreateOptions{Flavor: ""} lintComposableComponents(&validator, &createOpts) require.Equal(t, "open fake-path/zarf.yaml: no such file or directory", validator.findings[0].description) + require.Equal(t, ".components.[0].import.path", validator.findings[0].yqPath) require.Equal(t, unpinnedImage, validator.findings[1].item) }) From 22a1b9c17caf8305a11a753ccb701a805390a1c1 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Tue, 12 Dec 2023 15:28:19 +0000 Subject: [PATCH 155/192] adding test to files --- src/pkg/packager/lint/lint.go | 4 +--- src/pkg/packager/lint/lint_test.go | 14 +++++++++++--- src/test/e2e/12_lint_test.go | 7 ++++--- src/test/packages/12-lint/zarf.yaml | 12 +++++++++--- 4 files changed, 25 insertions(+), 12 deletions(-) diff --git a/src/pkg/packager/lint/lint.go b/src/pkg/packager/lint/lint.go index f9f78970b5..c32c0227bd 100644 --- a/src/pkg/packager/lint/lint.go +++ b/src/pkg/packager/lint/lint.go @@ -91,15 +91,13 @@ func lintComposableComponents(validator *Validator, createOpts *types.ZarfCreate node := baseComponent for node != nil { - var fileOrOciPath string + fileOrOciPath := createOpts.BaseDir if node.Prev() != nil { if node.Prev().Import.URL != "" { fileOrOciPath = node.Prev().Import.URL } else if node.GetRelativeToHead() != "" { fileOrOciPath = filepath.Join(createOpts.BaseDir, node.GetRelativeToHead()) } - } else { - fileOrOciPath = filepath.Join(createOpts.BaseDir, ".") } pkgKey := packageKey{path: fileOrOciPath, name: node.GetOriginalPackageName()} checkForVarInComponentImport(validator, node.GetIndex(), node.ZarfComponent, pkgKey) diff --git a/src/pkg/packager/lint/lint_test.go b/src/pkg/packager/lint/lint_test.go index b40b2601b4..61f1cff8c4 100644 --- a/src/pkg/packager/lint/lint_test.go +++ b/src/pkg/packager/lint/lint_test.go @@ -131,15 +131,23 @@ func TestValidateSchema(t *testing.T) { t.Run("Unpinnned file warning", func(t *testing.T) { validator := Validator{} - filename := "http://example.com/file.zip" + fileUrl := "http://example.com/file.zip" + localFile := "local.txt" zarfFiles := []types.ZarfFile{ { - Source: filename, + Source: fileUrl, + }, + { + Source: localFile, + }, + { + Source: fileUrl, + Shasum: "fake-shasum", }, } component := types.ZarfComponent{Files: zarfFiles} checkForUnpinnedFiles(&validator, 0, component, packageKey{}) - require.Equal(t, filename, validator.findings[0].item) + require.Equal(t, fileUrl, validator.findings[0].item) require.Equal(t, 1, len(validator.findings)) }) diff --git a/src/test/e2e/12_lint_test.go b/src/test/e2e/12_lint_test.go index 99b6d3c2f5..3a783fc5be 100644 --- a/src/test/e2e/12_lint_test.go +++ b/src/test/e2e/12_lint_test.go @@ -28,7 +28,7 @@ func TestLint(t *testing.T) { configPath := filepath.Join(path, "zarf-config.toml") os.Setenv("ZARF_CONFIG", configPath) // In this case I'm guessing we should also remove color from the table? - _, stderr, err := e2e.Zarf("prepare", "lint", path) + _, stderr, err := e2e.Zarf("prepare", "lint", path, "-f", "good-flavor") require.Error(t, err, "Require an exit code since there was warnings / errors") strippedStderr := e2e.StripMessageFormatting(stderr) // It's a bit weird to have a period here and not in the other warnings @@ -44,8 +44,9 @@ func TestLint(t *testing.T) { require.Contains(t, strippedStderr, ".components.[1].images.[2] | Image not pinned with digest") require.Contains(t, strippedStderr, ".components.[3].import.path | Zarf does not evaluate variables at component.x.import.path - ###ZARF_PKG_TMPL_PATH###") require.Contains(t, strippedStderr, ".components.[0].images.[0] | Image not pinned with digest - defenseunicorns/zarf-game:multi-tile-dark") - // Check to ensure non selected flavor is included - require.NotContains(t, strippedStderr, "image-in-flavor-component:unpinned") + // Check flavors + require.NotContains(t, strippedStderr, "image-in-bad-flavor-component:unpinned") + require.Contains(t, strippedStderr, "image-in-good-flavor-component:unpinned") }) } diff --git a/src/test/packages/12-lint/zarf.yaml b/src/test/packages/12-lint/zarf.yaml index 75e52da8d5..b404e76932 100644 --- a/src/test/packages/12-lint/zarf.yaml +++ b/src/test/packages/12-lint/zarf.yaml @@ -47,8 +47,14 @@ components: # import: # url: oci://localhost:555/big-bang-min:2.10.0-skeleton - - name: import-big-bang + - name: import-bad-flavor only: - flavor: bad-flavor-that-were-not-picking-but-want-to-make-sure-doesnt-show-up-in-lint + flavor: bad-flavor images: - - image-in-flavor-component:unpinned + - image-in-bad-flavor-component:unpinned + + - name: import-good-flavor + only: + flavor: good-flavor + images: + - image-in-good-flavor-component:unpinned From 08f207640a1c12e698a09a1795361a02a9bab70b Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Tue, 12 Dec 2023 15:56:31 +0000 Subject: [PATCH 156/192] adding test to transform parse image --- src/pkg/packager/lint/lint.go | 1 + src/pkg/transform/image_test.go | 16 +++++++++++++--- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/src/pkg/packager/lint/lint.go b/src/pkg/packager/lint/lint.go index c32c0227bd..bab97d5633 100644 --- a/src/pkg/packager/lint/lint.go +++ b/src/pkg/packager/lint/lint.go @@ -91,6 +91,7 @@ func lintComposableComponents(validator *Validator, createOpts *types.ZarfCreate node := baseComponent for node != nil { + // If it's the first node it will equal the baseDir fileOrOciPath := createOpts.BaseDir if node.Prev() != nil { if node.Prev().Import.URL != "" { diff --git a/src/pkg/transform/image_test.go b/src/pkg/transform/image_test.go index 819f9f672e..2edc2e87ec 100644 --- a/src/pkg/transform/image_test.go +++ b/src/pkg/transform/image_test.go @@ -15,6 +15,7 @@ var imageRefs = []string{ "nginx:1.23.3", "defenseunicorns/zarf-agent:v0.22.1", "defenseunicorns/zarf-agent@sha256:84605f731c6a18194794c51e70021c671ab064654b751aa57e905bce55be13de", + "busybox:latest@sha256:3fbc632167424a6d997e74f52b878d7cc478225cffac6bc977eedfe51c7f4e79", "ghcr.io/stefanprodan/podinfo:6.3.3", "registry1.dso.mil/ironbank/opensource/defenseunicorns/zarf/zarf-agent:v0.25.0", "gitlab.com/project/gitea/gitea:1.19.3-rootless-zarf-3431384023", @@ -33,6 +34,7 @@ func TestImageTransformHost(t *testing.T) { "gitlab.com/project/library/nginx:1.23.3-zarf-3793515731", "gitlab.com/project/defenseunicorns/zarf-agent:v0.22.1-zarf-4283503412", "gitlab.com/project/defenseunicorns/zarf-agent@sha256:84605f731c6a18194794c51e70021c671ab064654b751aa57e905bce55be13de", + "gitlab.com/project/library/busybox@sha256:3fbc632167424a6d997e74f52b878d7cc478225cffac6bc977eedfe51c7f4e79", "gitlab.com/project/stefanprodan/podinfo:6.3.3-zarf-2985051089", "gitlab.com/project/ironbank/opensource/defenseunicorns/zarf/zarf-agent:v0.25.0-zarf-2003217571", "gitlab.com/project/gitea/gitea:1.19.3-rootless-zarf-3431384023", @@ -56,6 +58,7 @@ func TestImageTransformHostWithoutChecksum(t *testing.T) { "gitlab.com/project/library/nginx:1.23.3", "gitlab.com/project/defenseunicorns/zarf-agent:v0.22.1", "gitlab.com/project/defenseunicorns/zarf-agent@sha256:84605f731c6a18194794c51e70021c671ab064654b751aa57e905bce55be13de", + "gitlab.com/project/library/busybox@sha256:3fbc632167424a6d997e74f52b878d7cc478225cffac6bc977eedfe51c7f4e79", "gitlab.com/project/stefanprodan/podinfo:6.3.3", "gitlab.com/project/ironbank/opensource/defenseunicorns/zarf/zarf-agent:v0.25.0", "gitlab.com/project/gitea/gitea:1.19.3-rootless-zarf-3431384023", @@ -79,6 +82,7 @@ func TestParseImageRef(t *testing.T) { {"docker.io/", "library/nginx", "1.23.3", ""}, {"docker.io/", "defenseunicorns/zarf-agent", "v0.22.1", ""}, {"docker.io/", "defenseunicorns/zarf-agent", "", "sha256:84605f731c6a18194794c51e70021c671ab064654b751aa57e905bce55be13de"}, + {"docker.io/", "library/busybox", "latest", "sha256:3fbc632167424a6d997e74f52b878d7cc478225cffac6bc977eedfe51c7f4e79"}, {"ghcr.io/", "stefanprodan/podinfo", "6.3.3", ""}, {"registry1.dso.mil/", "ironbank/opensource/defenseunicorns/zarf/zarf-agent", "v0.25.0", ""}, {"gitlab.com/", "project/gitea/gitea", "1.19.3-rootless-zarf-3431384023", ""}, @@ -89,13 +93,19 @@ func TestParseImageRef(t *testing.T) { require.NoError(t, err) tag := expectedResult[idx][2] digest := expectedResult[idx][3] - tagOrDigest := ":" + tag - if tag == "" { + var tagOrDigest string + var tagAndDigest string + if tag != "" { + tagOrDigest = ":" + tag + tagAndDigest = ":" + tag + } + if digest != "" { tagOrDigest = "@" + digest + tagAndDigest += "@" + digest } path := expectedResult[idx][1] name := expectedResult[idx][0] + path - reference := name + tagOrDigest + reference := name + tagAndDigest require.Equal(t, reference, img.Reference) require.Equal(t, name, img.Name) From 6d88659051fce06ba16cef933bbcec3fa8c9fbfa Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Tue, 12 Dec 2023 16:05:16 +0000 Subject: [PATCH 157/192] fix weird helper code --- src/config/lang/english.go | 2 +- src/pkg/packager/lint/validator.go | 17 +++++------------ 2 files changed, 6 insertions(+), 13 deletions(-) diff --git a/src/config/lang/english.go b/src/config/lang/english.go index e24a06716f..46027d83ba 100644 --- a/src/config/lang/english.go +++ b/src/config/lang/english.go @@ -33,7 +33,7 @@ const ( // Lint messages const ( - UnsetVarLintWarning = "There are variables that are unset and won't be evaluated during lint" + UnsetVarLintWarning = "There are templates that are unset and won't be evaluated during lint" ) // Zarf CLI commands. diff --git a/src/pkg/packager/lint/validator.go b/src/pkg/packager/lint/validator.go index 3d86f0c96f..db05851f9e 100644 --- a/src/pkg/packager/lint/validator.go +++ b/src/pkg/packager/lint/validator.go @@ -12,7 +12,6 @@ import ( "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "github.com/defenseunicorns/zarf/src/types" "github.com/fatih/color" - "github.com/pterm/pterm" ) type validationType int @@ -82,7 +81,7 @@ func (v Validator) printValidationTable() { return } - packageKeys := helpers.Unique(v.getUniquePackageKeys()) + packageKeys := helpers.Unique(v.getAllPackages()) connectData := make(map[packageKey][][]string) for _, finding := range v.findings { @@ -92,12 +91,10 @@ func (v Validator) printValidationTable() { header := []string{"Type", "Path", "Message"} for _, packageKey := range packageKeys { - //We should probably move this println into info - pterm.Println() if packageKey.path != "" { - message.Infof("Linting package %q at %s", packageKey.name, packageKey.path) + message.Notef("Linting package %q at %s", packageKey.name, packageKey.path) } else { - message.Infof("Linting package %q", packageKey.name) + message.Notef("Linting package %q", packageKey.name) } message.Table(header, connectData[packageKey]) @@ -105,15 +102,11 @@ func (v Validator) printValidationTable() { } } -func (v Validator) getUniquePackageKeys() []packageKey { - uniqueKeys := make(map[packageKey]bool) +func (v Validator) getAllPackages() []packageKey { var pks []packageKey for _, finding := range v.findings { - if _, exists := uniqueKeys[finding.packageKey]; !exists { - uniqueKeys[finding.packageKey] = true - pks = append(pks, finding.packageKey) - } + pks = append(pks, finding.packageKey) } return pks From 49bcd5bced33edd628fc87f336c1a97c08fac648 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Tue, 12 Dec 2023 16:46:01 +0000 Subject: [PATCH 158/192] Keeping relative paths for lint --- src/cmd/common/common.go | 10 +--------- src/pkg/packager/lint/lint.go | 14 +++++++++----- src/pkg/packager/lint/lint_test.go | 4 +++- src/pkg/packager/lint/validator.go | 7 +------ 4 files changed, 14 insertions(+), 21 deletions(-) diff --git a/src/cmd/common/common.go b/src/cmd/common/common.go index 41cb208a7c..01a6b104d6 100644 --- a/src/cmd/common/common.go +++ b/src/cmd/common/common.go @@ -5,10 +5,6 @@ package common import ( - "os" - - "github.com/defenseunicorns/zarf/src/config/lang" - "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/types" ) @@ -17,10 +13,6 @@ func SetBaseDirectory(args []string, pkgConfig *types.PackagerConfig) { if len(args) > 0 { pkgConfig.CreateOpts.BaseDir = args[0] } else { - var err error - pkgConfig.CreateOpts.BaseDir, err = os.Getwd() - if err != nil { - message.Fatalf(err, lang.CmdPackageCreateErr, err.Error()) - } + pkgConfig.CreateOpts.BaseDir = "." } } diff --git a/src/pkg/packager/lint/lint.go b/src/pkg/packager/lint/lint.go index bab97d5633..5d5db652a5 100644 --- a/src/pkg/packager/lint/lint.go +++ b/src/pkg/packager/lint/lint.go @@ -54,7 +54,8 @@ func ValidateZarfSchema(createOpts types.ZarfCreateOptions) (*Validator, error) if validator.jsonSchema, err = getSchemaFile(); err != nil { return nil, err } - pkgKey := packageKey{name: validator.typedZarfPackage.Metadata.Name, path: createOpts.BaseDir} + + pkgKey := packageKey{name: validator.typedZarfPackage.Metadata.Name, path: filepath.Join(createOpts.BaseDir, ".")} if err = validateSchema(&validator, pkgKey); err != nil { return nil, err } @@ -72,8 +73,11 @@ func lintComposableComponents(validator *Validator, createOpts *types.ZarfCreate chain, err := composer.NewImportChain(component, i, validator.typedZarfPackage.Metadata.Name, arch, createOpts.Flavor) baseComponent := chain.Head() + var badImportYqPath string + var baseNodeFilePath string if baseComponent != nil { + baseNodeFilePath = filepath.Join(createOpts.BaseDir, baseComponent.GetRelativeToHead()) if baseComponent.Import.URL != "" { badImportYqPath = fmt.Sprintf(".components.[%d].import.url", i) } @@ -84,15 +88,15 @@ func lintComposableComponents(validator *Validator, createOpts *types.ZarfCreate if err != nil { validator.addError(validatorMessage{ description: err.Error(), - packageKey: packageKey{name: validator.typedZarfPackage.Metadata.Name}, - yqPath: badImportYqPath, + packageKey: packageKey{name: validator.typedZarfPackage.Metadata.Name, + path: baseNodeFilePath}, + yqPath: badImportYqPath, }) } node := baseComponent for node != nil { - // If it's the first node it will equal the baseDir - fileOrOciPath := createOpts.BaseDir + fileOrOciPath := baseNodeFilePath if node.Prev() != nil { if node.Prev().Import.URL != "" { fileOrOciPath = node.Prev().Import.URL diff --git a/src/pkg/packager/lint/lint_test.go b/src/pkg/packager/lint/lint_test.go index 61f1cff8c4..97e9dbe933 100644 --- a/src/pkg/packager/lint/lint_test.go +++ b/src/pkg/packager/lint/lint_test.go @@ -174,11 +174,13 @@ func TestValidateSchema(t *testing.T) { typedZarfPackage: types.ZarfPackage{Components: []types.ZarfComponent{pathComponent}, Metadata: types.ZarfMetadata{Name: "test-zarf-package"}}} - createOpts := types.ZarfCreateOptions{Flavor: ""} + createOpts := types.ZarfCreateOptions{Flavor: "", BaseDir: "."} lintComposableComponents(&validator, &createOpts) require.Equal(t, "open fake-path/zarf.yaml: no such file or directory", validator.findings[0].description) require.Equal(t, ".components.[0].import.path", validator.findings[0].yqPath) + require.Equal(t, ".", validator.findings[0].packageKey.path) require.Equal(t, unpinnedImage, validator.findings[1].item) + require.Equal(t, ".", validator.findings[1].packageKey.path) }) t.Run("isImagePinned", func(t *testing.T) { diff --git a/src/pkg/packager/lint/validator.go b/src/pkg/packager/lint/validator.go index db05851f9e..e42b3f5b49 100644 --- a/src/pkg/packager/lint/validator.go +++ b/src/pkg/packager/lint/validator.go @@ -91,12 +91,7 @@ func (v Validator) printValidationTable() { header := []string{"Type", "Path", "Message"} for _, packageKey := range packageKeys { - if packageKey.path != "" { - message.Notef("Linting package %q at %s", packageKey.name, packageKey.path) - } else { - message.Notef("Linting package %q", packageKey.name) - } - + message.Notef("Linting package %q at %s", packageKey.name, packageKey.path) message.Table(header, connectData[packageKey]) message.Info(v.getFormattedFindingCount(packageKey)) } From 11d9d414d86d943b4d28670fa321cf065b9c3e5b Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Tue, 12 Dec 2023 16:48:52 +0000 Subject: [PATCH 159/192] moving color wrap to message --- src/pkg/cluster/state.go | 2 +- src/pkg/message/message.go | 14 +++++++++++++- src/pkg/packager/lint/validator.go | 7 +++---- src/pkg/utils/random.go | 13 ------------- 4 files changed, 17 insertions(+), 19 deletions(-) diff --git a/src/pkg/cluster/state.go b/src/pkg/cluster/state.go index b9025b7b47..aca5f01f82 100644 --- a/src/pkg/cluster/state.go +++ b/src/pkg/cluster/state.go @@ -158,7 +158,7 @@ func (c *Cluster) LoadZarfState() (state *types.ZarfState, err error) { // Set up the API connection secret, err := c.GetSecret(ZarfNamespaceName, ZarfStateSecretName) if err != nil { - return nil, fmt.Errorf("%w. %s", err, utils.ColorWrap("Did you remember to zarf init?", color.Bold)) + return nil, fmt.Errorf("%w. %s", err, message.ColorWrap("Did you remember to zarf init?", color.Bold)) } err = json.Unmarshal(secret.Data[ZarfStateDataKey], &state) diff --git a/src/pkg/message/message.go b/src/pkg/message/message.go index a04cdaf89e..d17baca298 100644 --- a/src/pkg/message/message.go +++ b/src/pkg/message/message.go @@ -14,6 +14,8 @@ import ( "strings" "time" + "github.com/defenseunicorns/zarf/src/config" + "github.com/fatih/color" "github.com/pterm/pterm" "github.com/sergi/go-diff/diffmatchpatch" ) @@ -322,7 +324,7 @@ func Truncate(text string, length int, invert bool) string { func Table(header []string, data [][]string) { pterm.Println() - // To avoid side effects make copies of the header and data before adding padding + // To avoid side effects make copies of the header and data before adding padding headerCopy := make([]string, len(header)) copy(headerCopy, header) dataCopy := make([][]string, len(data)) @@ -345,6 +347,16 @@ func Table(header []string, data [][]string) { pterm.DefaultTable.WithHasHeader().WithData(table).Render() } +// ColorWrap changes a string to an ansi color code and appends the default color to the end +// preventing future characters from taking on the given color +// returns string as normal if color is disabled +func ColorWrap(str string, attr color.Attribute) string { + if config.NoColor { + return str + } + return fmt.Sprintf("\x1b[%dm%s\x1b[0m", attr, str) +} + func debugPrinter(offset int, a ...any) { printer := pterm.Debug.WithShowLineNumber(logLevel > 2).WithLineNumberOffset(offset) now := time.Now().Format(time.RFC3339) diff --git a/src/pkg/packager/lint/validator.go b/src/pkg/packager/lint/validator.go index e42b3f5b49..ca71ce5565 100644 --- a/src/pkg/packager/lint/validator.go +++ b/src/pkg/packager/lint/validator.go @@ -8,7 +8,6 @@ import ( "fmt" "github.com/defenseunicorns/zarf/src/pkg/message" - "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "github.com/defenseunicorns/zarf/src/types" "github.com/fatih/color" @@ -36,9 +35,9 @@ type validatorMessage struct { func (vt validationType) String() string { if vt == validationError { - return utils.ColorWrap("Error", color.FgRed) + return message.ColorWrap("Error", color.FgRed) } else if vt == validationWarning { - return utils.ColorWrap("Warning", color.FgYellow) + return message.ColorWrap("Warning", color.FgYellow) } return "" } @@ -137,7 +136,7 @@ func (vm validatorMessage) getPath() string { if vm.yqPath == "" { return "" } - return utils.ColorWrap(vm.yqPath, color.FgCyan) + return message.ColorWrap(vm.yqPath, color.FgCyan) } func (v Validator) hasFindings() bool { diff --git a/src/pkg/utils/random.go b/src/pkg/utils/random.go index 0cc7b77bad..0c074580d2 100644 --- a/src/pkg/utils/random.go +++ b/src/pkg/utils/random.go @@ -6,11 +6,8 @@ package utils import ( "crypto/rand" - "fmt" - "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/pkg/message" - "github.com/fatih/color" ) // Very limited special chars for git / basic auth @@ -40,13 +37,3 @@ func First30last30(s string) string { return s } - -// ColorWrap changes a string to an ansi color code and appends the default color to the end -// preventing future characters from taking on the given color -// returns string as normal if color is disabled -func ColorWrap(str string, attr color.Attribute) string { - if config.NoColor { - return str - } - return fmt.Sprintf("\x1b[%dm%s\x1b[0m", attr, str) -} From 802662194c144d8d31add21468470f5b4a846f1a Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Tue, 12 Dec 2023 17:35:51 +0000 Subject: [PATCH 160/192] moving random to helpers --- src/config/lang/english.go | 29 +++++++++-------- src/pkg/cluster/state.go | 46 ++++++++++++++++++++------- src/pkg/message/message.go | 9 ++++++ src/pkg/oci/push.go | 3 +- src/pkg/oci/utils.go | 3 +- src/pkg/utils/{ => helpers}/random.go | 22 ++++--------- 6 files changed, 66 insertions(+), 46 deletions(-) rename src/pkg/utils/{ => helpers}/random.go (57%) diff --git a/src/config/lang/english.go b/src/config/lang/english.go index 46027d83ba..e475180414 100644 --- a/src/config/lang/english.go +++ b/src/config/lang/english.go @@ -15,20 +15,21 @@ import "errors" // Debug messages will not be a part of the language strings since they are not intended to be user facing // Include sprintf formatting directives in the string if needed. const ( - ErrLoadState = "Failed to load the Zarf State from the cluster." - ErrSaveState = "Failed to save the Zarf State to the cluster." - ErrLoadPackageSecret = "Failed to load %s's secret from the cluster" - ErrNoClusterConnection = "Failed to connect to the cluster." - ErrTunnelFailed = "Failed to create a tunnel to the cluster." - ErrUnmarshal = "failed to unmarshal file: %w" - ErrWritingFile = "failed to write file %s: %s" - ErrDownloading = "failed to download %s: %s" - ErrCreatingDir = "failed to create directory %s: %s" - ErrRemoveFile = "failed to remove file %s: %s" - ErrUnarchive = "failed to unarchive %s: %s" - ErrConfirmCancel = "confirm selection canceled: %s" - ErrFileExtract = "failed to extract filename %s from archive %s: %s" - ErrFileNameExtract = "failed to extract filename from URL %s: %s" + ErrLoadState = "Failed to load the Zarf State from the cluster." + ErrSaveState = "Failed to save the Zarf State to the cluster." + ErrLoadPackageSecret = "Failed to load %s's secret from the cluster" + ErrNoClusterConnection = "Failed to connect to the cluster." + ErrTunnelFailed = "Failed to create a tunnel to the cluster." + ErrUnmarshal = "failed to unmarshal file: %w" + ErrWritingFile = "failed to write file %s: %s" + ErrDownloading = "failed to download %s: %s" + ErrCreatingDir = "failed to create directory %s: %s" + ErrRemoveFile = "failed to remove file %s: %s" + ErrUnarchive = "failed to unarchive %s: %s" + ErrConfirmCancel = "confirm selection canceled: %s" + ErrFileExtract = "failed to extract filename %s from archive %s: %s" + ErrFileNameExtract = "failed to extract filename from URL %s: %s" + ErrUnableToGenerateRandomSecret = "unable to generate a random secret" ) // Lint messages diff --git a/src/pkg/cluster/state.go b/src/pkg/cluster/state.go index aca5f01f82..1d4127ef27 100644 --- a/src/pkg/cluster/state.go +++ b/src/pkg/cluster/state.go @@ -12,13 +12,13 @@ import ( "slices" "github.com/defenseunicorns/zarf/src/config" + "github.com/defenseunicorns/zarf/src/config/lang" "github.com/defenseunicorns/zarf/src/types" "github.com/fatih/color" "github.com/defenseunicorns/zarf/src/pkg/k8s" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/pki" - "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -72,7 +72,9 @@ func (c *Cluster) InitZarfState(initOptions types.ZarfInitOptions) error { // Defaults state.Distro = distro - state.LoggingSecret = utils.RandomString(config.ZarfGeneratedPasswordLen) + if state.LoggingSecret, err = helpers.RandomString(config.ZarfGeneratedPasswordLen); err != nil { + message.Fatal(err, lang.ErrUnableToGenerateRandomSecret) + } // Setup zarf agent PKI state.AgentTLS = pki.GeneratePKI(config.ZarfAgentHost) @@ -247,7 +249,7 @@ func (c *Cluster) SaveZarfState(state *types.ZarfState) error { // MergeZarfState merges init options for provided services into the provided state to create a new state struct func (c *Cluster) MergeZarfState(oldState *types.ZarfState, initOptions types.ZarfInitOptions, services []string) *types.ZarfState { newState := *oldState - + var err error if slices.Contains(services, message.RegistryKey) { newState.RegistryInfo = helpers.MergeNonZero(newState.RegistryInfo, initOptions.RegistryInfo) // Set the state of the internal registry if it has changed @@ -259,10 +261,14 @@ func (c *Cluster) MergeZarfState(oldState *types.ZarfState, initOptions types.Za // Set the new passwords if they should be autogenerated if newState.RegistryInfo.PushPassword == oldState.RegistryInfo.PushPassword && oldState.RegistryInfo.InternalRegistry { - newState.RegistryInfo.PushPassword = utils.RandomString(config.ZarfGeneratedPasswordLen) + if newState.RegistryInfo.PushPassword, err = helpers.RandomString(config.ZarfGeneratedPasswordLen); err != nil { + message.Fatal(err, lang.ErrUnableToGenerateRandomSecret) + } } if newState.RegistryInfo.PullPassword == oldState.RegistryInfo.PullPassword && oldState.RegistryInfo.InternalRegistry { - newState.RegistryInfo.PullPassword = utils.RandomString(config.ZarfGeneratedPasswordLen) + if newState.RegistryInfo.PullPassword, err = helpers.RandomString(config.ZarfGeneratedPasswordLen); err != nil { + message.Fatal(err, lang.ErrUnableToGenerateRandomSecret) + } } } if slices.Contains(services, message.GitKey) { @@ -277,10 +283,14 @@ func (c *Cluster) MergeZarfState(oldState *types.ZarfState, initOptions types.Za // Set the new passwords if they should be autogenerated if newState.GitServer.PushPassword == oldState.GitServer.PushPassword && oldState.GitServer.InternalServer { - newState.GitServer.PushPassword = utils.RandomString(config.ZarfGeneratedPasswordLen) + if newState.GitServer.PushPassword, err = helpers.RandomString(config.ZarfGeneratedPasswordLen); err != nil { + message.Fatal(err, lang.ErrUnableToGenerateRandomSecret) + } } if newState.GitServer.PullPassword == oldState.GitServer.PullPassword && oldState.GitServer.InternalServer { - newState.GitServer.PullPassword = utils.RandomString(config.ZarfGeneratedPasswordLen) + if newState.GitServer.PullPassword, err = helpers.RandomString(config.ZarfGeneratedPasswordLen); err != nil { + message.Fatal(err, lang.ErrUnableToGenerateRandomSecret) + } } } if slices.Contains(services, message.ArtifactKey) { @@ -306,6 +316,7 @@ func (c *Cluster) MergeZarfState(oldState *types.ZarfState, initOptions types.Za } func (c *Cluster) fillInEmptyContainerRegistryValues(containerRegistry types.RegistryInfo) types.RegistryInfo { + var err error // Set default NodePort if none was provided if containerRegistry.NodePort == 0 { containerRegistry.NodePort = config.ZarfInClusterContainerRegistryNodePort @@ -319,7 +330,9 @@ func (c *Cluster) fillInEmptyContainerRegistryValues(containerRegistry types.Reg // Generate a push-user password if not provided by init flag if containerRegistry.PushPassword == "" { - containerRegistry.PushPassword = utils.RandomString(config.ZarfGeneratedPasswordLen) + if containerRegistry.PushPassword, err = helpers.RandomString(config.ZarfGeneratedPasswordLen); err != nil { + message.Fatal(err, lang.ErrUnableToGenerateRandomSecret) + } } // Set pull-username if not provided by init flag @@ -333,7 +346,9 @@ func (c *Cluster) fillInEmptyContainerRegistryValues(containerRegistry types.Reg } if containerRegistry.PullPassword == "" { if containerRegistry.InternalRegistry { - containerRegistry.PullPassword = utils.RandomString(config.ZarfGeneratedPasswordLen) + if containerRegistry.PullPassword, err = helpers.RandomString(config.ZarfGeneratedPasswordLen); err != nil { + message.Fatal(err, lang.ErrUnableToGenerateRandomSecret) + } } else { // If this is an external registry and a pull-user wasn't provided, use the same credentials as the push user containerRegistry.PullPassword = containerRegistry.PushPassword @@ -341,7 +356,9 @@ func (c *Cluster) fillInEmptyContainerRegistryValues(containerRegistry types.Reg } if containerRegistry.Secret == "" { - containerRegistry.Secret = utils.RandomString(config.ZarfGeneratedSecretLen) + if containerRegistry.Secret, err = helpers.RandomString(config.ZarfGeneratedPasswordLen); err != nil { + message.Fatal(err, lang.ErrUnableToGenerateRandomSecret) + } } return containerRegistry @@ -349,6 +366,7 @@ func (c *Cluster) fillInEmptyContainerRegistryValues(containerRegistry types.Reg // Fill in empty GitServerInfo values with the defaults. func (c *Cluster) fillInEmptyGitServerValues(gitServer types.GitServerInfo) types.GitServerInfo { + var err error // Set default svc url if an external repository was not provided if gitServer.Address == "" { gitServer.Address = config.ZarfInClusterGitServiceURL @@ -357,7 +375,9 @@ func (c *Cluster) fillInEmptyGitServerValues(gitServer types.GitServerInfo) type // Generate a push-user password if not provided by init flag if gitServer.PushPassword == "" { - gitServer.PushPassword = utils.RandomString(config.ZarfGeneratedPasswordLen) + if gitServer.PushPassword, err = helpers.RandomString(config.ZarfGeneratedPasswordLen); err != nil { + message.Fatal(err, lang.ErrUnableToGenerateRandomSecret) + } } // Set read-user information if using an internal repository, otherwise copy from the push-user @@ -370,7 +390,9 @@ func (c *Cluster) fillInEmptyGitServerValues(gitServer types.GitServerInfo) type } if gitServer.PullPassword == "" { if gitServer.InternalServer { - gitServer.PullPassword = utils.RandomString(config.ZarfGeneratedPasswordLen) + if gitServer.PullPassword, err = helpers.RandomString(config.ZarfGeneratedPasswordLen); err != nil { + message.Fatal(err, lang.ErrUnableToGenerateRandomSecret) + } } else { gitServer.PullPassword = gitServer.PushPassword } diff --git a/src/pkg/message/message.go b/src/pkg/message/message.go index d17baca298..b4b6834a0f 100644 --- a/src/pkg/message/message.go +++ b/src/pkg/message/message.go @@ -357,6 +357,15 @@ func ColorWrap(str string, attr color.Attribute) string { return fmt.Sprintf("\x1b[%dm%s\x1b[0m", attr, str) } +// First30last30 returns the source string that has been trimmed to 30 characters at the beginning and end. +func First30last30(s string) string { + if len(s) > 60 { + return s[0:27] + "..." + s[len(s)-26:] + } + + return s +} + func debugPrinter(offset int, a ...any) { printer := pterm.Debug.WithShowLineNumber(logLevel > 2).WithLineNumberOffset(offset) now := time.Now().Format(time.RFC3339) diff --git a/src/pkg/oci/push.go b/src/pkg/oci/push.go index 2d057003f1..7c678ab9b5 100644 --- a/src/pkg/oci/push.go +++ b/src/pkg/oci/push.go @@ -11,7 +11,6 @@ import ( "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/message" - "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/types" ocispec "github.com/opencontainers/image-spec/specs-go/v1" "oras.land/oras-go/v2" @@ -116,7 +115,7 @@ func (o *OrasRemote) PublishPackage(pkg *types.ZarfPackage, paths *layout.Packag // Get all of the layers in the package var descs []ocispec.Descriptor for name, path := range paths.Files() { - spinner.Updatef("Preparing layer %s", utils.First30last30(name)) + spinner.Updatef("Preparing layer %s", message.First30last30(name)) mediaType := ZarfLayerMediaTypeBlob diff --git a/src/pkg/oci/utils.go b/src/pkg/oci/utils.go index b15b14be19..1134e97132 100644 --- a/src/pkg/oci/utils.go +++ b/src/pkg/oci/utils.go @@ -11,7 +11,6 @@ import ( "strings" "github.com/defenseunicorns/zarf/src/pkg/message" - "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "github.com/defenseunicorns/zarf/src/types" ocispec "github.com/opencontainers/image-spec/specs-go/v1" @@ -63,7 +62,7 @@ func (o *OrasRemote) printLayer(desc ocispec.Descriptor, suffix string) error { title := desc.Annotations[ocispec.AnnotationTitle] var layerInfo string if title != "" { - layerInfo = fmt.Sprintf("%s %s", desc.Digest.Encoded()[:12], utils.First30last30(title)) + layerInfo = fmt.Sprintf("%s %s", desc.Digest.Encoded()[:12], message.First30last30(title)) } else { layerInfo = fmt.Sprintf("%s [%s]", desc.Digest.Encoded()[:12], desc.MediaType) } diff --git a/src/pkg/utils/random.go b/src/pkg/utils/helpers/random.go similarity index 57% rename from src/pkg/utils/random.go rename to src/pkg/utils/helpers/random.go index 0c074580d2..63e7bfa89c 100644 --- a/src/pkg/utils/random.go +++ b/src/pkg/utils/helpers/random.go @@ -1,13 +1,11 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: 2021-Present The Zarf Authors -// Package utils provides generic utility functions. -package utils +// Package helpers provides generic helper functions with no external imports +package helpers import ( "crypto/rand" - - "github.com/defenseunicorns/zarf/src/pkg/message" ) // Very limited special chars for git / basic auth @@ -15,25 +13,17 @@ import ( const randomStringChars = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ!~-" // RandomString generates a secure random string of the specified length. -func RandomString(length int) string { +func RandomString(length int) (string, error) { bytes := make([]byte, length) if _, err := rand.Read(bytes); err != nil { - message.Fatal(err, "unable to generate a random secret") + //message.Fatal(err, "unable to generate a random secret") + return "", err } for i, b := range bytes { bytes[i] = randomStringChars[b%byte(len(randomStringChars))] } - return string(bytes) -} - -// First30last30 returns the source string that has been trimmed to 30 characters at the beginning and end. -func First30last30(s string) string { - if len(s) > 60 { - return s[0:27] + "..." + s[len(s)-26:] - } - - return s + return string(bytes), nil } From 5c96525baa7e1958cda4d8b9a9b3bc0bdcc8df8d Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Tue, 12 Dec 2023 17:42:14 +0000 Subject: [PATCH 161/192] update e2e --- src/test/e2e/12_lint_test.go | 4 +++- .../12-lint/###ZARF_PKG_TMPL_PATH###/zarf.yaml | 10 ---------- src/test/packages/12-lint/linted-import/zarf.yaml | 4 ++-- 3 files changed, 5 insertions(+), 13 deletions(-) delete mode 100644 src/test/packages/12-lint/###ZARF_PKG_TMPL_PATH###/zarf.yaml diff --git a/src/test/e2e/12_lint_test.go b/src/test/e2e/12_lint_test.go index 3a783fc5be..fdc6729852 100644 --- a/src/test/e2e/12_lint_test.go +++ b/src/test/e2e/12_lint_test.go @@ -31,7 +31,7 @@ func TestLint(t *testing.T) { _, stderr, err := e2e.Zarf("prepare", "lint", path, "-f", "good-flavor") require.Error(t, err, "Require an exit code since there was warnings / errors") strippedStderr := e2e.StripMessageFormatting(stderr) - // It's a bit weird to have a period here and not in the other warnings + key := "WHATEVER_IMAGE" require.Contains(t, strippedStderr, lang.UnsetVarLintWarning) // Multiple spaces in lang @@ -44,6 +44,8 @@ func TestLint(t *testing.T) { require.Contains(t, strippedStderr, ".components.[1].images.[2] | Image not pinned with digest") require.Contains(t, strippedStderr, ".components.[3].import.path | Zarf does not evaluate variables at component.x.import.path - ###ZARF_PKG_TMPL_PATH###") require.Contains(t, strippedStderr, ".components.[0].images.[0] | Image not pinned with digest - defenseunicorns/zarf-game:multi-tile-dark") + require.Contains(t, strippedStderr, ".components.[3].import.path | open ###ZARF_PKG_TMPL_PATH###/zarf.yaml: no such file or directory") + // Check flavors require.NotContains(t, strippedStderr, "image-in-bad-flavor-component:unpinned") require.Contains(t, strippedStderr, "image-in-good-flavor-component:unpinned") diff --git a/src/test/packages/12-lint/###ZARF_PKG_TMPL_PATH###/zarf.yaml b/src/test/packages/12-lint/###ZARF_PKG_TMPL_PATH###/zarf.yaml deleted file mode 100644 index 9c95401914..0000000000 --- a/src/test/packages/12-lint/###ZARF_PKG_TMPL_PATH###/zarf.yaml +++ /dev/null @@ -1,10 +0,0 @@ -kind: ZarfInitConfig -metadata: - name: init - description: Testing bad yaml imported - -components: - - name: import - files: - - source: https://github.com/k3s-io/k3s/releases/download/v1.28.2+k3s1/k3s - target: src/ diff --git a/src/test/packages/12-lint/linted-import/zarf.yaml b/src/test/packages/12-lint/linted-import/zarf.yaml index a2e33b0841..f5f21981f6 100644 --- a/src/test/packages/12-lint/linted-import/zarf.yaml +++ b/src/test/packages/12-lint/linted-import/zarf.yaml @@ -1,6 +1,6 @@ -kind: ZarfInitConfig +kind: ZarfPackageConfig metadata: - name: init + name: linted-import description: Testing bad yaml imported variables: From 8a1741313d1747a267cd0701484c4886ae16482b Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Tue, 12 Dec 2023 18:21:14 +0000 Subject: [PATCH 162/192] refactor tests --- src/config/lang/english.go | 2 +- src/test/e2e/12_lint_test.go | 10 ++++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/config/lang/english.go b/src/config/lang/english.go index e475180414..da0261c597 100644 --- a/src/config/lang/english.go +++ b/src/config/lang/english.go @@ -34,7 +34,7 @@ const ( // Lint messages const ( - UnsetVarLintWarning = "There are templates that are unset and won't be evaluated during lint" + UnsetVarLintWarning = "There are templates that are not set and won't be evaluated during lint" ) // Zarf CLI commands. diff --git a/src/test/e2e/12_lint_test.go b/src/test/e2e/12_lint_test.go index fdc6729852..90a8390c8a 100644 --- a/src/test/e2e/12_lint_test.go +++ b/src/test/e2e/12_lint_test.go @@ -34,16 +34,18 @@ func TestLint(t *testing.T) { key := "WHATEVER_IMAGE" require.Contains(t, strippedStderr, lang.UnsetVarLintWarning) - // Multiple spaces in lang require.Contains(t, strippedStderr, fmt.Sprintf(lang.PkgValidateTemplateDeprecation, key, key, key)) require.Contains(t, strippedStderr, ".components.[2].repos.[0] | Unpinned repository") require.Contains(t, strippedStderr, ".metadata | Additional property description1 is not allowed") require.Contains(t, strippedStderr, ".components.[0].import | Additional property not-path is not allowed") - require.Contains(t, strippedStderr, ".components.[2].images.[3] | Image not pinned with digest") - require.Contains(t, strippedStderr, ".components.[1].images.[0] | Image not pinned with digest") - require.Contains(t, strippedStderr, ".components.[1].images.[2] | Image not pinned with digest") + // This is testing the import / compose on lint is working + require.Contains(t, strippedStderr, ".components.[1].images.[0] | Image not pinned with digest - registry.com:9001/whatever/image:latest") + // This is testing import / compose + variables are working + require.Contains(t, strippedStderr, ".components.[2].images.[3] | Image not pinned with digest - busybox:latest") require.Contains(t, strippedStderr, ".components.[3].import.path | Zarf does not evaluate variables at component.x.import.path - ###ZARF_PKG_TMPL_PATH###") + // testing OCI imports get linted require.Contains(t, strippedStderr, ".components.[0].images.[0] | Image not pinned with digest - defenseunicorns/zarf-game:multi-tile-dark") + // This is require.Contains(t, strippedStderr, ".components.[3].import.path | open ###ZARF_PKG_TMPL_PATH###/zarf.yaml: no such file or directory") // Check flavors From 363ef42f6d1974eaa60af3408d25490922884505 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Tue, 12 Dec 2023 20:03:30 +0000 Subject: [PATCH 163/192] changing lint all script to use one binary --- Makefile | 2 +- hack/lint_all_zarf_packages.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index f5fdb0c02e..57dd5e3e3d 100644 --- a/Makefile +++ b/Makefile @@ -76,7 +76,7 @@ docs-and-schema: ## Generate the Zarf Documentation and Schema hack/gen-cli-docs.sh ZARF_CONFIG=hack/empty-config.toml hack/create-zarf-schema.sh -lint-packages-and-examples: ## Recurisvely lint all zarf.yaml files in the repo except for those dedicated to tests +lint-packages-and-examples: build-cli ## Recurisvely lint all zarf.yaml files in the repo except for those dedicated to tests hack/lint_all_zarf_packages.sh # INTERNAL: a shim used to build the agent image only if needed on Windows using the `test` command diff --git a/hack/lint_all_zarf_packages.sh b/hack/lint_all_zarf_packages.sh index c66a96435f..b79ef6ed08 100755 --- a/hack/lint_all_zarf_packages.sh +++ b/hack/lint_all_zarf_packages.sh @@ -11,6 +11,6 @@ find "." -type f -name 'zarf.yaml' | while read -r yaml_file; do continue fi echo "Running 'zarf prepare lint' in directory: $dir" - go run main.go prepare lint $dir + ./build/zarf prepare lint $dir echo "---" done From 396a69e0d94615da32416303e5c47536987f186e Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Tue, 12 Dec 2023 21:42:55 +0000 Subject: [PATCH 164/192] refactoring to use node and not use packagekey --- src/pkg/packager/composer/list.go | 52 +++++----- src/pkg/packager/lint/lint.go | 147 +++++++++++++++-------------- src/pkg/packager/lint/lint_test.go | 23 ++--- src/pkg/packager/lint/validator.go | 51 +++++----- src/test/e2e/12_lint_test.go | 7 +- 5 files changed, 141 insertions(+), 139 deletions(-) diff --git a/src/pkg/packager/composer/list.go b/src/pkg/packager/composer/list.go index 844a7aa8aa..b464c133ef 100644 --- a/src/pkg/packager/composer/list.go +++ b/src/pkg/packager/composer/list.go @@ -59,8 +59,21 @@ func (n *Node) Prev() *Node { return n.prev } +func NewNode(c types.ZarfComponent, index int, originalPackageName string, + relativeToHead string, vars []types.ZarfPackageVariable, consts []types.ZarfPackageConstant) Node { + return Node{ + ZarfComponent: c, + index: index, + originalPackageName: originalPackageName, + relativeToHead: relativeToHead, + vars: vars, + consts: consts, + prev: nil, + next: nil, + } +} + // ImportName returns the name of the component to import -// // If the component import has a ComponentName defined, that will be used // otherwise the name of the component will be used func (n *Node) ImportName() string { @@ -91,24 +104,15 @@ func (ic *ImportChain) Tail() *Node { func (ic *ImportChain) append(c types.ZarfComponent, index int, originalPackageName string, relativeToHead string, vars []types.ZarfPackageVariable, consts []types.ZarfPackageConstant) { - node := &Node{ - ZarfComponent: c, - index: index, - originalPackageName: originalPackageName, - relativeToHead: relativeToHead, - vars: vars, - consts: consts, - prev: nil, - next: nil, - } + node := NewNode(c, index, originalPackageName, relativeToHead, vars, consts) if ic.head == nil { - ic.head = node - ic.tail = node + ic.head = &node + ic.tail = &node } else { p := ic.tail node.prev = p - p.next = node - ic.tail = node + p.next = &node + ic.tail = &node } } @@ -151,9 +155,10 @@ func NewImportChain(head types.ZarfComponent, index int, originalPackageName, ar var pkg types.ZarfPackage + var relativeToHead string if isLocal { history = append(history, node.Import.Path) - relativeToHead := filepath.Join(history...) + relativeToHead = filepath.Join(history...) // prevent circular imports (including self-imports) // this is O(n^2) but the import chain should be small @@ -170,6 +175,7 @@ func NewImportChain(head types.ZarfComponent, index int, originalPackageName, ar return ic, err } } else if isRemote { + relativeToHead = node.Import.URL remote, err := ic.getRemote(node.Import.URL) if err != nil { return ic, err @@ -192,20 +198,12 @@ func NewImportChain(head types.ZarfComponent, index int, originalPackageName, ar } if len(found) == 0 { - if isLocal { - return ic, fmt.Errorf("component %q not found in %q", name, filepath.Join(history...)) - } else if isRemote { - return ic, fmt.Errorf("component %q not found in %q", name, node.Import.URL) - } + return ic, fmt.Errorf("component %q not found in %q", name, relativeToHead) } else if len(found) > 1 { - if isLocal { - return ic, fmt.Errorf("multiple components named %q found in %q satisfying %q", name, filepath.Join(history...), arch) - } else if isRemote { - return ic, fmt.Errorf("multiple components named %q found in %q satisfying %q", name, node.Import.URL, arch) - } + return ic, fmt.Errorf("multiple components named %q found in %q satisfying %q", name, relativeToHead, arch) } - ic.append(found[0], index[0], pkg.Metadata.Name, filepath.Join(history...), pkg.Variables, pkg.Constants) + ic.append(found[0], index[0], pkg.Metadata.Name, relativeToHead, pkg.Variables, pkg.Constants) node = node.next } return ic, nil diff --git a/src/pkg/packager/lint/lint.go b/src/pkg/packager/lint/lint.go index 5d5db652a5..8ed830fe9c 100644 --- a/src/pkg/packager/lint/lint.go +++ b/src/pkg/packager/lint/lint.go @@ -49,14 +49,15 @@ func ValidateZarfSchema(createOpts types.ZarfCreateOptions) (*Validator, error) return nil, fmt.Errorf("unable to access directory '%s': %w", createOpts.BaseDir, err) } + validator.baseDir = createOpts.BaseDir + lintComposableComponents(&validator, &createOpts) if validator.jsonSchema, err = getSchemaFile(); err != nil { return nil, err } - pkgKey := packageKey{name: validator.typedZarfPackage.Metadata.Name, path: filepath.Join(createOpts.BaseDir, ".")} - if err = validateSchema(&validator, pkgKey); err != nil { + if err = validateSchema(&validator); err != nil { return nil, err } @@ -75,9 +76,7 @@ func lintComposableComponents(validator *Validator, createOpts *types.ZarfCreate baseComponent := chain.Head() var badImportYqPath string - var baseNodeFilePath string if baseComponent != nil { - baseNodeFilePath = filepath.Join(createOpts.BaseDir, baseComponent.GetRelativeToHead()) if baseComponent.Import.URL != "" { badImportYqPath = fmt.Sprintf(".components.[%d].import.url", i) } @@ -87,39 +86,31 @@ func lintComposableComponents(validator *Validator, createOpts *types.ZarfCreate } if err != nil { validator.addError(validatorMessage{ - description: err.Error(), - packageKey: packageKey{name: validator.typedZarfPackage.Metadata.Name, - path: baseNodeFilePath}, - yqPath: badImportYqPath, + description: err.Error(), + packageRelPath: ".", + packageName: validator.typedZarfPackage.Metadata.Name, + yqPath: badImportYqPath, }) } node := baseComponent for node != nil { - fileOrOciPath := baseNodeFilePath - if node.Prev() != nil { - if node.Prev().Import.URL != "" { - fileOrOciPath = node.Prev().Import.URL - } else if node.GetRelativeToHead() != "" { - fileOrOciPath = filepath.Join(createOpts.BaseDir, node.GetRelativeToHead()) - } - } - pkgKey := packageKey{path: fileOrOciPath, name: node.GetOriginalPackageName()} - checkForVarInComponentImport(validator, node.GetIndex(), node.ZarfComponent, pkgKey) - fillComponentTemplate(validator, node, createOpts, pkgKey) - lintComponent(validator, node.GetIndex(), node.ZarfComponent, pkgKey) + checkForVarInComponentImport(validator, node) + fillComponentTemplate(validator, node, createOpts) + lintComponent(validator, node) node = node.Next() } } } -func fillComponentTemplate(validator *Validator, node *composer.Node, createOpts *types.ZarfCreateOptions, pkgKey packageKey) { +func fillComponentTemplate(validator *Validator, node *composer.Node, createOpts *types.ZarfCreateOptions) { err := packager.ReloadComponentTemplate(&node.ZarfComponent) if err != nil { validator.addWarning(validatorMessage{ - description: err.Error(), - packageKey: pkgKey, + description: err.Error(), + packageRelPath: node.GetRelativeToHead(), + packageName: node.GetOriginalPackageName(), }) } templateMap := map[string]string{} @@ -128,23 +119,26 @@ func fillComponentTemplate(validator *Validator, node *composer.Node, createOpts yamlTemplates, err := utils.FindYamlTemplates(node, templatePrefix, "###") if err != nil { validator.addWarning(validatorMessage{ - description: err.Error(), - packageKey: pkgKey, + description: err.Error(), + packageRelPath: node.GetRelativeToHead(), + packageName: node.GetOriginalPackageName(), }) } for key := range yamlTemplates { if deprecated { validator.addWarning(validatorMessage{ - description: fmt.Sprintf(lang.PkgValidateTemplateDeprecation, key, key, key), - packageKey: pkgKey, + description: fmt.Sprintf(lang.PkgValidateTemplateDeprecation, key, key, key), + packageRelPath: node.GetRelativeToHead(), + packageName: node.GetOriginalPackageName(), }) } _, present := createOpts.SetVariables[key] if !present { validator.addWarning(validatorMessage{ - description: lang.UnsetVarLintWarning, - packageKey: pkgKey, + description: lang.UnsetVarLintWarning, + packageRelPath: node.GetRelativeToHead(), + packageName: node.GetOriginalPackageName(), }) } } @@ -177,79 +171,85 @@ func isPinnedRepo(repo string) bool { return (strings.Contains(repo, "@")) } -func lintComponent(validator *Validator, index int, component types.ZarfComponent, pkgKey packageKey) { - checkForUnpinnedRepos(validator, index, component, pkgKey) - checkForUnpinnedImages(validator, index, component, pkgKey) - checkForUnpinnedFiles(validator, index, component, pkgKey) +func lintComponent(validator *Validator, node *composer.Node) { + checkForUnpinnedRepos(validator, node) + checkForUnpinnedImages(validator, node) + checkForUnpinnedFiles(validator, node) } -func checkForUnpinnedRepos(validator *Validator, index int, component types.ZarfComponent, pkgKey packageKey) { - for j, repo := range component.Repos { - repoYqPath := fmt.Sprintf(".components.[%d].repos.[%d]", index, j) +func checkForUnpinnedRepos(validator *Validator, node *composer.Node) { + for j, repo := range node.Repos { + repoYqPath := fmt.Sprintf(".components.[%d].repos.[%d]", node.GetIndex(), j) if !isPinnedRepo(repo) { validator.addWarning(validatorMessage{ - yqPath: repoYqPath, - packageKey: pkgKey, - description: "Unpinned repository", - item: repo, + yqPath: repoYqPath, + packageRelPath: node.GetRelativeToHead(), + packageName: node.GetOriginalPackageName(), + description: "Unpinned repository", + item: repo, }) } } } -func checkForUnpinnedImages(validator *Validator, index int, component types.ZarfComponent, pkgKey packageKey) { - for j, image := range component.Images { - imageYqPath := fmt.Sprintf(".components.[%d].images.[%d]", index, j) +func checkForUnpinnedImages(validator *Validator, node *composer.Node) { + for j, image := range node.Images { + imageYqPath := fmt.Sprintf(".components.[%d].images.[%d]", node.GetIndex(), j) pinnedImage, err := isPinnedImage(image) if err != nil { validator.addError(validatorMessage{ - yqPath: imageYqPath, - packageKey: pkgKey, - description: "Invalid image format", - item: image, + yqPath: imageYqPath, + packageRelPath: node.GetRelativeToHead(), + packageName: node.GetOriginalPackageName(), + description: "Invalid image format", + item: image, }) continue } if !pinnedImage { validator.addWarning(validatorMessage{ - yqPath: imageYqPath, - packageKey: pkgKey, - description: "Image not pinned with digest", - item: image, + yqPath: imageYqPath, + packageRelPath: node.GetRelativeToHead(), + packageName: node.GetOriginalPackageName(), + description: "Image not pinned with digest", + item: image, }) } } } -func checkForUnpinnedFiles(validator *Validator, index int, component types.ZarfComponent, pkgKey packageKey) { - for j, file := range component.Files { - fileYqPath := fmt.Sprintf(".components.[%d].files.[%d]", index, j) +func checkForUnpinnedFiles(validator *Validator, node *composer.Node) { + for j, file := range node.Files { + fileYqPath := fmt.Sprintf(".components.[%d].files.[%d]", node.GetIndex(), j) if file.Shasum == "" && helpers.IsURL(file.Source) { validator.addWarning(validatorMessage{ - yqPath: fileYqPath, - packageKey: pkgKey, - description: "No shasum for remote file", - item: file.Source, + yqPath: fileYqPath, + packageRelPath: node.GetRelativeToHead(), + packageName: node.GetOriginalPackageName(), + description: "No shasum for remote file", + item: file.Source, }) } } } -func checkForVarInComponentImport(validator *Validator, index int, component types.ZarfComponent, pkgKey packageKey) { - if strings.Contains(component.Import.Path, types.ZarfPackageTemplatePrefix) { +func checkForVarInComponentImport(validator *Validator, node *composer.Node) { + if strings.Contains(node.Import.Path, types.ZarfPackageTemplatePrefix) { validator.addWarning(validatorMessage{ - yqPath: fmt.Sprintf(".components.[%d].import.path", index), - packageKey: pkgKey, - description: "Zarf does not evaluate variables at component.x.import.path", - item: component.Import.Path, + yqPath: fmt.Sprintf(".components.[%d].import.path", node.GetIndex()), + packageRelPath: node.GetRelativeToHead(), + packageName: node.GetOriginalPackageName(), + description: "Zarf does not evaluate variables at component.x.import.path", + item: node.Import.Path, }) } - if strings.Contains(component.Import.URL, types.ZarfPackageTemplatePrefix) { + if strings.Contains(node.Import.URL, types.ZarfPackageTemplatePrefix) { validator.addWarning(validatorMessage{ - yqPath: fmt.Sprintf(".components.[%d].import.url", index), - packageKey: pkgKey, - description: "Zarf does not evaluate variables at component.x.import.url", - item: component.Import.URL, + yqPath: fmt.Sprintf(".components.[%d].import.url", node.GetIndex()), + packageRelPath: node.GetRelativeToHead(), + packageName: node.GetOriginalPackageName(), + description: "Zarf does not evaluate variables at component.x.import.url", + item: node.Import.URL, }) } } @@ -267,7 +267,7 @@ func makeFieldPathYqCompat(field string) string { return fmt.Sprintf(".%s", wrappedField) } -func validateSchema(validator *Validator, pkgKey packageKey) error { +func validateSchema(validator *Validator) error { schemaLoader := gojsonschema.NewBytesLoader(validator.jsonSchema) documentLoader := gojsonschema.NewGoLoader(validator.untypedZarfPackage) @@ -279,9 +279,10 @@ func validateSchema(validator *Validator, pkgKey packageKey) error { if !result.Valid() { for _, desc := range result.Errors() { validator.addError(validatorMessage{ - yqPath: makeFieldPathYqCompat(desc.Field()), - description: desc.Description(), - packageKey: pkgKey, + yqPath: makeFieldPathYqCompat(desc.Field()), + description: desc.Description(), + packageRelPath: ".", + packageName: validator.typedZarfPackage.Metadata.Name, }) } } diff --git a/src/pkg/packager/lint/lint_test.go b/src/pkg/packager/lint/lint_test.go index 97e9dbe933..132cf0596c 100644 --- a/src/pkg/packager/lint/lint_test.go +++ b/src/pkg/packager/lint/lint_test.go @@ -10,6 +10,7 @@ import ( "testing" "github.com/defenseunicorns/zarf/src/config" + "github.com/defenseunicorns/zarf/src/pkg/packager/composer" "github.com/defenseunicorns/zarf/src/types" goyaml "github.com/goccy/go-yaml" "github.com/stretchr/testify/require" @@ -63,7 +64,7 @@ func TestValidateSchema(t *testing.T) { t.Run("validate schema success", func(t *testing.T) { unmarshalledYaml := readAndUnmarshalYaml[interface{}](t, goodZarfPackage) validator := Validator{untypedZarfPackage: unmarshalledYaml, jsonSchema: getZarfSchema(t)} - err := validateSchema(&validator, packageKey{}) + err := validateSchema(&validator) require.NoError(t, err) require.Empty(t, validator.findings) }) @@ -71,7 +72,7 @@ func TestValidateSchema(t *testing.T) { t.Run("validate schema fail", func(t *testing.T) { unmarshalledYaml := readAndUnmarshalYaml[interface{}](t, badZarfPackage) validator := Validator{untypedZarfPackage: unmarshalledYaml, jsonSchema: getZarfSchema(t)} - err := validateSchema(&validator, packageKey{}) + err := validateSchema(&validator) require.NoError(t, err) config.NoColor = true require.Equal(t, "Additional property not-path is not allowed", validator.findings[0].String()) @@ -81,8 +82,8 @@ func TestValidateSchema(t *testing.T) { t.Run("Template in component import success", func(t *testing.T) { unmarshalledYaml := readAndUnmarshalYaml[types.ZarfPackage](t, goodZarfPackage) validator := Validator{typedZarfPackage: unmarshalledYaml} - for i, component := range validator.typedZarfPackage.Components { - lintComponent(&validator, i, component, packageKey{path: "", name: validator.typedZarfPackage.Metadata.Name}) + for _, component := range validator.typedZarfPackage.Components { + lintComponent(&validator, &composer.Node{ZarfComponent: component}) } require.Empty(t, validator.findings) }) @@ -91,7 +92,7 @@ func TestValidateSchema(t *testing.T) { pathVar := "###ZARF_PKG_TMPL_PATH###" pathComponent := types.ZarfComponent{Import: types.ZarfComponentImport{Path: pathVar}} validator := Validator{typedZarfPackage: types.ZarfPackage{Components: []types.ZarfComponent{pathComponent}}} - checkForVarInComponentImport(&validator, 0, validator.typedZarfPackage.Components[0], packageKey{}) + checkForVarInComponentImport(&validator, &composer.Node{ZarfComponent: pathComponent}) require.Equal(t, pathVar, validator.findings[0].item) }) @@ -99,7 +100,7 @@ func TestValidateSchema(t *testing.T) { ociPathVar := "oci://###ZARF_PKG_TMPL_PATH###" URLComponent := types.ZarfComponent{Import: types.ZarfComponentImport{URL: ociPathVar}} validator := Validator{typedZarfPackage: types.ZarfPackage{Components: []types.ZarfComponent{URLComponent}}} - checkForVarInComponentImport(&validator, 0, validator.typedZarfPackage.Components[0], packageKey{}) + checkForVarInComponentImport(&validator, &composer.Node{ZarfComponent: URLComponent}) require.Equal(t, ociPathVar, validator.findings[0].item) }) @@ -109,7 +110,7 @@ func TestValidateSchema(t *testing.T) { component := types.ZarfComponent{Repos: []string{ unpinnedRepo, "https://dev.azure.com/defenseunicorns/zarf-public-test/_git/zarf-public-test@v0.0.1"}} - checkForUnpinnedRepos(&validator, 0, component, packageKey{}) + checkForUnpinnedRepos(&validator, &composer.Node{ZarfComponent: component}) require.Equal(t, unpinnedRepo, validator.findings[0].item) require.Equal(t, len(validator.findings), 1) }) @@ -122,7 +123,7 @@ func TestValidateSchema(t *testing.T) { unpinnedImage, "busybox:latest@sha256:3fbc632167424a6d997e74f52b878d7cc478225cffac6bc977eedfe51c7f4e79", badImage}} - checkForUnpinnedImages(&validator, 0, component, packageKey{}) + checkForUnpinnedImages(&validator, &composer.Node{ZarfComponent: component}) require.Equal(t, unpinnedImage, validator.findings[0].item) require.Equal(t, badImage, validator.findings[1].item) require.Equal(t, 2, len(validator.findings)) @@ -146,7 +147,7 @@ func TestValidateSchema(t *testing.T) { }, } component := types.ZarfComponent{Files: zarfFiles} - checkForUnpinnedFiles(&validator, 0, component, packageKey{}) + checkForUnpinnedFiles(&validator, &composer.Node{ZarfComponent: component}) require.Equal(t, fileUrl, validator.findings[0].item) require.Equal(t, 1, len(validator.findings)) }) @@ -178,9 +179,9 @@ func TestValidateSchema(t *testing.T) { lintComposableComponents(&validator, &createOpts) require.Equal(t, "open fake-path/zarf.yaml: no such file or directory", validator.findings[0].description) require.Equal(t, ".components.[0].import.path", validator.findings[0].yqPath) - require.Equal(t, ".", validator.findings[0].packageKey.path) + require.Equal(t, ".", validator.findings[0].packageRelPath) require.Equal(t, unpinnedImage, validator.findings[1].item) - require.Equal(t, ".", validator.findings[1].packageKey.path) + require.Equal(t, ".", validator.findings[1].packageRelPath) }) t.Run("isImagePinned", func(t *testing.T) { diff --git a/src/pkg/packager/lint/validator.go b/src/pkg/packager/lint/validator.go index ca71ce5565..6b687549c6 100644 --- a/src/pkg/packager/lint/validator.go +++ b/src/pkg/packager/lint/validator.go @@ -6,6 +6,7 @@ package lint import ( "fmt" + "path/filepath" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" @@ -20,16 +21,12 @@ const ( validationWarning validationType = 2 ) -type packageKey struct { - path string - name string -} - type validatorMessage struct { yqPath string description string item string - packageKey packageKey + packageRelPath string + packageName string validationType validationType } @@ -55,6 +52,7 @@ type Validator struct { jsonSchema []byte typedZarfPackage types.ZarfPackage untypedZarfPackage interface{} + baseDir string } // DisplayFormattedMessage message sent to user based on validator results @@ -75,42 +73,41 @@ func (v Validator) IsSuccess() bool { return true } +func (v Validator) packageRelPathToUser(vm validatorMessage) string { + if helpers.IsOCIURL(vm.packageRelPath) { + return vm.packageRelPath + } + return filepath.Join(v.baseDir, vm.packageRelPath) +} + func (v Validator) printValidationTable() { if !v.hasFindings() { return } - packageKeys := helpers.Unique(v.getAllPackages()) - connectData := make(map[packageKey][][]string) - + mapOfFindingsByPath := make(map[string][]validatorMessage) for _, finding := range v.findings { - connectData[finding.packageKey] = append(connectData[finding.packageKey], - []string{finding.validationType.String(), finding.getPath(), finding.String()}) + mapOfFindingsByPath[finding.packageRelPath] = append(mapOfFindingsByPath[finding.packageRelPath], finding) } header := []string{"Type", "Path", "Message"} - for _, packageKey := range packageKeys { - message.Notef("Linting package %q at %s", packageKey.name, packageKey.path) - message.Table(header, connectData[packageKey]) - message.Info(v.getFormattedFindingCount(packageKey)) - } -} - -func (v Validator) getAllPackages() []packageKey { - var pks []packageKey - for _, finding := range v.findings { - pks = append(pks, finding.packageKey) + for packageRelPath, findings := range mapOfFindingsByPath { + lintData := [][]string{} + for _, finding := range findings { + lintData = append(lintData, []string{finding.validationType.String(), finding.getPath(), finding.String()}) + } + message.Notef("Linting package %q at %s", findings[0].packageName, v.packageRelPathToUser(findings[0])) + message.Table(header, lintData) + message.Info(v.getFormattedFindingCount(packageRelPath, findings[0].packageName)) } - - return pks } -func (v Validator) getFormattedFindingCount(pk packageKey) string { +func (v Validator) getFormattedFindingCount(relPath string, packageName string) string { warningCount := 0 errorCount := 0 for _, finding := range v.findings { - if finding.packageKey != pk { + if finding.packageRelPath != relPath { continue } if finding.validationType == validationWarning { @@ -129,7 +126,7 @@ func (v Validator) getFormattedFindingCount(pk packageKey) string { wordError = "error" } return fmt.Sprintf("%d %s and %d %s in %q", - warningCount, wordWarning, errorCount, wordError, pk.name) + warningCount, wordWarning, errorCount, wordError, packageName) } func (vm validatorMessage) getPath() string { diff --git a/src/test/e2e/12_lint_test.go b/src/test/e2e/12_lint_test.go index 90a8390c8a..1be8a62299 100644 --- a/src/test/e2e/12_lint_test.go +++ b/src/test/e2e/12_lint_test.go @@ -41,7 +41,7 @@ func TestLint(t *testing.T) { // This is testing the import / compose on lint is working require.Contains(t, strippedStderr, ".components.[1].images.[0] | Image not pinned with digest - registry.com:9001/whatever/image:latest") // This is testing import / compose + variables are working - require.Contains(t, strippedStderr, ".components.[2].images.[3] | Image not pinned with digest - busybox:latest") + require.Contains(t, strippedStderr, ".components.[2].images.[3] | Image not pinned with digest - busybox:latest") require.Contains(t, strippedStderr, ".components.[3].import.path | Zarf does not evaluate variables at component.x.import.path - ###ZARF_PKG_TMPL_PATH###") // testing OCI imports get linted require.Contains(t, strippedStderr, ".components.[0].images.[0] | Image not pinned with digest - defenseunicorns/zarf-game:multi-tile-dark") @@ -51,6 +51,11 @@ func TestLint(t *testing.T) { // Check flavors require.NotContains(t, strippedStderr, "image-in-bad-flavor-component:unpinned") require.Contains(t, strippedStderr, "image-in-good-flavor-component:unpinned") + + // Check reported filepaths + require.Contains(t, strippedStderr, "Linting package \"dos-games\" at oci://🦄/dos-games:1.0.0-skeleton") + require.Contains(t, strippedStderr, "Linting package \"lint\" at src/test/packages/12-lint") + }) } From 10bdff2507857e4404491528a7c0438ab634b1b0 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Tue, 12 Dec 2023 21:46:01 +0000 Subject: [PATCH 165/192] removing new node --- src/pkg/packager/composer/list.go | 33 +++++++++++++------------------ 1 file changed, 14 insertions(+), 19 deletions(-) diff --git a/src/pkg/packager/composer/list.go b/src/pkg/packager/composer/list.go index b464c133ef..f761fbebd0 100644 --- a/src/pkg/packager/composer/list.go +++ b/src/pkg/packager/composer/list.go @@ -59,20 +59,6 @@ func (n *Node) Prev() *Node { return n.prev } -func NewNode(c types.ZarfComponent, index int, originalPackageName string, - relativeToHead string, vars []types.ZarfPackageVariable, consts []types.ZarfPackageConstant) Node { - return Node{ - ZarfComponent: c, - index: index, - originalPackageName: originalPackageName, - relativeToHead: relativeToHead, - vars: vars, - consts: consts, - prev: nil, - next: nil, - } -} - // ImportName returns the name of the component to import // If the component import has a ComponentName defined, that will be used // otherwise the name of the component will be used @@ -104,15 +90,24 @@ func (ic *ImportChain) Tail() *Node { func (ic *ImportChain) append(c types.ZarfComponent, index int, originalPackageName string, relativeToHead string, vars []types.ZarfPackageVariable, consts []types.ZarfPackageConstant) { - node := NewNode(c, index, originalPackageName, relativeToHead, vars, consts) + node := &Node{ + ZarfComponent: c, + index: index, + originalPackageName: originalPackageName, + relativeToHead: relativeToHead, + vars: vars, + consts: consts, + prev: nil, + next: nil, + } if ic.head == nil { - ic.head = &node - ic.tail = &node + ic.head = node + ic.tail = node } else { p := ic.tail node.prev = p - p.next = &node - ic.tail = &node + p.next = node + ic.tail = node } } From e6eefad56c4d1b414d1a909fce44050c2338fdca Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Tue, 12 Dec 2023 21:52:13 +0000 Subject: [PATCH 166/192] method renaem --- src/cmd/prepare.go | 2 +- src/pkg/packager/lint/lint.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/cmd/prepare.go b/src/cmd/prepare.go index 50fd443b9f..0342dc2939 100644 --- a/src/cmd/prepare.go +++ b/src/cmd/prepare.go @@ -217,7 +217,7 @@ var lintCmd = &cobra.Command{ v := common.GetViper() pkgConfig.CreateOpts.SetVariables = helpers.TransformAndMergeMap( v.GetStringMapString(common.VPkgCreateSet), pkgConfig.CreateOpts.SetVariables, strings.ToUpper) - validator, err := lint.ValidateZarfSchema(pkgConfig.CreateOpts) + validator, err := lint.ValidateZarf(pkgConfig.CreateOpts) if err != nil { message.Fatal(err, err.Error()) } diff --git a/src/pkg/packager/lint/lint.go b/src/pkg/packager/lint/lint.go index 8ed830fe9c..0870d73a15 100644 --- a/src/pkg/packager/lint/lint.go +++ b/src/pkg/packager/lint/lint.go @@ -31,9 +31,9 @@ func getSchemaFile() ([]byte, error) { return ZarfSchema.ReadFile("zarf.schema.json") } -// ValidateZarfSchema validates a zarf file against the zarf schema, returns *validator with warnings or errors if they exist +// ValidateZarf validates a zarf file against the zarf schema, returns *validator with warnings or errors if they exist // along with an error if the validation itself failed -func ValidateZarfSchema(createOpts types.ZarfCreateOptions) (*Validator, error) { +func ValidateZarf(createOpts types.ZarfCreateOptions) (*Validator, error) { validator := Validator{} var err error From 7165595fe2fb367fa83d28923ee05343e58b03c6 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Tue, 12 Dec 2023 22:00:32 +0000 Subject: [PATCH 167/192] make docs and schema --- docs/2-the-zarf-cli/100-cli-commands/zarf_prepare_lint.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/2-the-zarf-cli/100-cli-commands/zarf_prepare_lint.md b/docs/2-the-zarf-cli/100-cli-commands/zarf_prepare_lint.md index 648956db33..ef96568acc 100644 --- a/docs/2-the-zarf-cli/100-cli-commands/zarf_prepare_lint.md +++ b/docs/2-the-zarf-cli/100-cli-commands/zarf_prepare_lint.md @@ -14,6 +14,7 @@ zarf prepare lint [ DIRECTORY ] [flags] ## Options ``` + -f, --flavor string The flavor of components to include in the resulting package (i.e. have a matching or empty "only.flavor" key) -h, --help help for lint --set stringToString Specify package variables to set on the command line (KEY=value) (default []) ``` From 4f7fc77192b7193bb87aefe8b8c031c62b65fc81 Mon Sep 17 00:00:00 2001 From: Austin Abro <37223396+AustinAbro321@users.noreply.github.com> Date: Wed, 13 Dec 2023 09:16:18 -0500 Subject: [PATCH 168/192] Apply suggestions from code review Co-authored-by: razzle Co-authored-by: Wayne Starr --- src/pkg/packager/composer/list.go | 2 +- src/pkg/packager/lint/lint.go | 6 +++--- src/pkg/packager/lint/lint_test.go | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/pkg/packager/composer/list.go b/src/pkg/packager/composer/list.go index f761fbebd0..7c4e071fba 100644 --- a/src/pkg/packager/composer/list.go +++ b/src/pkg/packager/composer/list.go @@ -78,7 +78,7 @@ type ImportChain struct { remote *oci.OrasRemote } -// Head eturns the first node in the import chain +// Head returns the first node in the import chain func (ic *ImportChain) Head() *Node { return ic.head } diff --git a/src/pkg/packager/lint/lint.go b/src/pkg/packager/lint/lint.go index 0870d73a15..df337bd70f 100644 --- a/src/pkg/packager/lint/lint.go +++ b/src/pkg/packager/lint/lint.go @@ -33,7 +33,7 @@ func getSchemaFile() ([]byte, error) { // ValidateZarf validates a zarf file against the zarf schema, returns *validator with warnings or errors if they exist // along with an error if the validation itself failed -func ValidateZarf(createOpts types.ZarfCreateOptions) (*Validator, error) { +func Validate(createOpts types.ZarfCreateOptions) (*Validator, error) { validator := Validator{} var err error @@ -51,7 +51,7 @@ func ValidateZarf(createOpts types.ZarfCreateOptions) (*Validator, error) { validator.baseDir = createOpts.BaseDir - lintComposableComponents(&validator, &createOpts) + lintComponents(&validator, &createOpts) if validator.jsonSchema, err = getSchemaFile(); err != nil { return nil, err @@ -201,7 +201,7 @@ func checkForUnpinnedImages(validator *Validator, node *composer.Node) { yqPath: imageYqPath, packageRelPath: node.GetRelativeToHead(), packageName: node.GetOriginalPackageName(), - description: "Invalid image format", + description: "Invalid image reference", item: image, }) continue diff --git a/src/pkg/packager/lint/lint_test.go b/src/pkg/packager/lint/lint_test.go index 132cf0596c..725eca2fa5 100644 --- a/src/pkg/packager/lint/lint_test.go +++ b/src/pkg/packager/lint/lint_test.go @@ -165,7 +165,7 @@ func TestValidateSchema(t *testing.T) { require.Equal(t, input, acutal) }) - t.Run("Test comoposable components", func(t *testing.T) { + t.Run("Test composable components", func(t *testing.T) { pathVar := "fake-path" unpinnedImage := "unpinned:latest" pathComponent := types.ZarfComponent{ From b1e8ebe1de24159fc42f1d4b4670027bfe85fe56 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Wed, 13 Dec 2023 14:19:30 +0000 Subject: [PATCH 169/192] clean up --- src/cmd/prepare.go | 2 +- src/pkg/packager/lint/lint.go | 2 +- src/pkg/packager/lint/lint_test.go | 2 +- src/test/e2e/12_lint_test.go | 9 ++++----- src/test/packages/12-lint/zarf.yaml | 4 ---- 5 files changed, 7 insertions(+), 12 deletions(-) diff --git a/src/cmd/prepare.go b/src/cmd/prepare.go index 0342dc2939..9d19b73a73 100644 --- a/src/cmd/prepare.go +++ b/src/cmd/prepare.go @@ -217,7 +217,7 @@ var lintCmd = &cobra.Command{ v := common.GetViper() pkgConfig.CreateOpts.SetVariables = helpers.TransformAndMergeMap( v.GetStringMapString(common.VPkgCreateSet), pkgConfig.CreateOpts.SetVariables, strings.ToUpper) - validator, err := lint.ValidateZarf(pkgConfig.CreateOpts) + validator, err := lint.Validate(pkgConfig.CreateOpts) if err != nil { message.Fatal(err, err.Error()) } diff --git a/src/pkg/packager/lint/lint.go b/src/pkg/packager/lint/lint.go index df337bd70f..ab2fa10e70 100644 --- a/src/pkg/packager/lint/lint.go +++ b/src/pkg/packager/lint/lint.go @@ -64,7 +64,7 @@ func Validate(createOpts types.ZarfCreateOptions) (*Validator, error) { return &validator, nil } -func lintComposableComponents(validator *Validator, createOpts *types.ZarfCreateOptions) { +func lintComponents(validator *Validator, createOpts *types.ZarfCreateOptions) { for i, component := range validator.typedZarfPackage.Components { arch := config.GetArch(validator.typedZarfPackage.Metadata.Architecture) diff --git a/src/pkg/packager/lint/lint_test.go b/src/pkg/packager/lint/lint_test.go index 725eca2fa5..7c1ca28a06 100644 --- a/src/pkg/packager/lint/lint_test.go +++ b/src/pkg/packager/lint/lint_test.go @@ -176,7 +176,7 @@ func TestValidateSchema(t *testing.T) { Metadata: types.ZarfMetadata{Name: "test-zarf-package"}}} createOpts := types.ZarfCreateOptions{Flavor: "", BaseDir: "."} - lintComposableComponents(&validator, &createOpts) + lintComponents(&validator, &createOpts) require.Equal(t, "open fake-path/zarf.yaml: no such file or directory", validator.findings[0].description) require.Equal(t, ".components.[0].import.path", validator.findings[0].yqPath) require.Equal(t, ".", validator.findings[0].packageRelPath) diff --git a/src/test/e2e/12_lint_test.go b/src/test/e2e/12_lint_test.go index 1be8a62299..1f6ffa2e2a 100644 --- a/src/test/e2e/12_lint_test.go +++ b/src/test/e2e/12_lint_test.go @@ -27,7 +27,6 @@ func TestLint(t *testing.T) { path := filepath.Join("src", "test", "packages", "12-lint") configPath := filepath.Join(path, "zarf-config.toml") os.Setenv("ZARF_CONFIG", configPath) - // In this case I'm guessing we should also remove color from the table? _, stderr, err := e2e.Zarf("prepare", "lint", path, "-f", "good-flavor") require.Error(t, err, "Require an exit code since there was warnings / errors") strippedStderr := e2e.StripMessageFormatting(stderr) @@ -38,14 +37,14 @@ func TestLint(t *testing.T) { require.Contains(t, strippedStderr, ".components.[2].repos.[0] | Unpinned repository") require.Contains(t, strippedStderr, ".metadata | Additional property description1 is not allowed") require.Contains(t, strippedStderr, ".components.[0].import | Additional property not-path is not allowed") - // This is testing the import / compose on lint is working + // Testing the import / compose on lint is working require.Contains(t, strippedStderr, ".components.[1].images.[0] | Image not pinned with digest - registry.com:9001/whatever/image:latest") - // This is testing import / compose + variables are working + // Testing import / compose + variables are working require.Contains(t, strippedStderr, ".components.[2].images.[3] | Image not pinned with digest - busybox:latest") require.Contains(t, strippedStderr, ".components.[3].import.path | Zarf does not evaluate variables at component.x.import.path - ###ZARF_PKG_TMPL_PATH###") - // testing OCI imports get linted + // Testing OCI imports get linted require.Contains(t, strippedStderr, ".components.[0].images.[0] | Image not pinned with digest - defenseunicorns/zarf-game:multi-tile-dark") - // This is + // Testing a bad path leads to a finding in lint require.Contains(t, strippedStderr, ".components.[3].import.path | open ###ZARF_PKG_TMPL_PATH###/zarf.yaml: no such file or directory") // Check flavors diff --git a/src/test/packages/12-lint/zarf.yaml b/src/test/packages/12-lint/zarf.yaml index b404e76932..efddf42eea 100644 --- a/src/test/packages/12-lint/zarf.yaml +++ b/src/test/packages/12-lint/zarf.yaml @@ -43,10 +43,6 @@ components: import: path: linted-import - # - name: import-big-bang - # import: - # url: oci://localhost:555/big-bang-min:2.10.0-skeleton - - name: import-bad-flavor only: flavor: bad-flavor From 0884591d8d2e4c9a5db50eb8eaf0343eefa007d2 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Wed, 13 Dec 2023 15:37:27 +0000 Subject: [PATCH 170/192] changing how errors are sent through in zarf state --- src/cmd/tools/zarf.go | 6 +++-- src/config/lang/english.go | 1 + src/pkg/cluster/state.go | 40 ++++++++++++++++-------------- src/pkg/packager/lint/lint.go | 2 +- src/pkg/packager/lint/lint_test.go | 8 +++--- 5 files changed, 32 insertions(+), 25 deletions(-) diff --git a/src/cmd/tools/zarf.go b/src/cmd/tools/zarf.go index bbd8020e07..8f86593268 100644 --- a/src/cmd/tools/zarf.go +++ b/src/cmd/tools/zarf.go @@ -88,8 +88,10 @@ var updateCredsCmd = &cobra.Command{ // If no distro the zarf secret did not load properly message.Fatalf(nil, lang.ErrLoadState) } - - newState := c.MergeZarfState(oldState, updateCredsInitOpts, args) + var newState *types.ZarfState + if newState, err = c.MergeZarfState(oldState, updateCredsInitOpts, args); err != nil { + message.Fatal(err, lang.CmdToolsUpdateCredsUnableUpdateCreds) + } message.PrintCredentialUpdates(oldState, newState, args) diff --git a/src/config/lang/english.go b/src/config/lang/english.go index da0261c597..f9004da591 100644 --- a/src/config/lang/english.go +++ b/src/config/lang/english.go @@ -564,6 +564,7 @@ $ zarf tools update-creds artifact --artifact-push-username={USERNAME} --artifac CmdToolsUpdateCredsUnableUpdateRegistry = "Unable to update Zarf Registry values: %s" CmdToolsUpdateCredsUnableUpdateGit = "Unable to update Zarf Git Server values: %s" CmdToolsUpdateCredsUnableUpdateAgent = "Unable to update Zarf Agent TLS secrets: %s" + CmdToolsUpdateCredsUnableUpdateCreds = "Unable to update Zarf credentials" // zarf version CmdVersionShort = "Shows the version of the running Zarf binary" diff --git a/src/pkg/cluster/state.go b/src/pkg/cluster/state.go index 1d4127ef27..5fe6e4d9dd 100644 --- a/src/pkg/cluster/state.go +++ b/src/pkg/cluster/state.go @@ -73,7 +73,7 @@ func (c *Cluster) InitZarfState(initOptions types.ZarfInitOptions) error { // Defaults state.Distro = distro if state.LoggingSecret, err = helpers.RandomString(config.ZarfGeneratedPasswordLen); err != nil { - message.Fatal(err, lang.ErrUnableToGenerateRandomSecret) + return fmt.Errorf("%s: %w", lang.ErrUnableToGenerateRandomSecret, err) } // Setup zarf agent PKI @@ -112,8 +112,12 @@ func (c *Cluster) InitZarfState(initOptions types.ZarfInitOptions) error { return fmt.Errorf("unable get default Zarf service account: %w", err) } - state.GitServer = c.fillInEmptyGitServerValues(initOptions.GitServer) - state.RegistryInfo = c.fillInEmptyContainerRegistryValues(initOptions.RegistryInfo) + if state.GitServer, err = c.fillInEmptyGitServerValues(initOptions.GitServer); err != nil { + return err + } + if state.RegistryInfo, err = c.fillInEmptyContainerRegistryValues(initOptions.RegistryInfo); err != nil { + return err + } state.ArtifactServer = c.fillInEmptyArtifactServerValues(initOptions.ArtifactServer) } else { if helpers.IsNotZeroAndNotEqual(initOptions.GitServer, state.GitServer) { @@ -247,7 +251,7 @@ func (c *Cluster) SaveZarfState(state *types.ZarfState) error { } // MergeZarfState merges init options for provided services into the provided state to create a new state struct -func (c *Cluster) MergeZarfState(oldState *types.ZarfState, initOptions types.ZarfInitOptions, services []string) *types.ZarfState { +func (c *Cluster) MergeZarfState(oldState *types.ZarfState, initOptions types.ZarfInitOptions, services []string) (*types.ZarfState, error) { newState := *oldState var err error if slices.Contains(services, message.RegistryKey) { @@ -262,12 +266,12 @@ func (c *Cluster) MergeZarfState(oldState *types.ZarfState, initOptions types.Za // Set the new passwords if they should be autogenerated if newState.RegistryInfo.PushPassword == oldState.RegistryInfo.PushPassword && oldState.RegistryInfo.InternalRegistry { if newState.RegistryInfo.PushPassword, err = helpers.RandomString(config.ZarfGeneratedPasswordLen); err != nil { - message.Fatal(err, lang.ErrUnableToGenerateRandomSecret) + return nil, fmt.Errorf("%s: %w", lang.ErrUnableToGenerateRandomSecret, err) } } if newState.RegistryInfo.PullPassword == oldState.RegistryInfo.PullPassword && oldState.RegistryInfo.InternalRegistry { if newState.RegistryInfo.PullPassword, err = helpers.RandomString(config.ZarfGeneratedPasswordLen); err != nil { - message.Fatal(err, lang.ErrUnableToGenerateRandomSecret) + return nil, fmt.Errorf("%s: %w", lang.ErrUnableToGenerateRandomSecret, err) } } } @@ -284,12 +288,12 @@ func (c *Cluster) MergeZarfState(oldState *types.ZarfState, initOptions types.Za // Set the new passwords if they should be autogenerated if newState.GitServer.PushPassword == oldState.GitServer.PushPassword && oldState.GitServer.InternalServer { if newState.GitServer.PushPassword, err = helpers.RandomString(config.ZarfGeneratedPasswordLen); err != nil { - message.Fatal(err, lang.ErrUnableToGenerateRandomSecret) + return nil, fmt.Errorf("%s: %w", lang.ErrUnableToGenerateRandomSecret, err) } } if newState.GitServer.PullPassword == oldState.GitServer.PullPassword && oldState.GitServer.InternalServer { if newState.GitServer.PullPassword, err = helpers.RandomString(config.ZarfGeneratedPasswordLen); err != nil { - message.Fatal(err, lang.ErrUnableToGenerateRandomSecret) + return nil, fmt.Errorf("%s: %w", lang.ErrUnableToGenerateRandomSecret, err) } } } @@ -312,10 +316,10 @@ func (c *Cluster) MergeZarfState(oldState *types.ZarfState, initOptions types.Za newState.AgentTLS = pki.GeneratePKI(config.ZarfAgentHost) } - return &newState + return &newState, nil } -func (c *Cluster) fillInEmptyContainerRegistryValues(containerRegistry types.RegistryInfo) types.RegistryInfo { +func (c *Cluster) fillInEmptyContainerRegistryValues(containerRegistry types.RegistryInfo) (types.RegistryInfo, error) { var err error // Set default NodePort if none was provided if containerRegistry.NodePort == 0 { @@ -331,7 +335,7 @@ func (c *Cluster) fillInEmptyContainerRegistryValues(containerRegistry types.Reg // Generate a push-user password if not provided by init flag if containerRegistry.PushPassword == "" { if containerRegistry.PushPassword, err = helpers.RandomString(config.ZarfGeneratedPasswordLen); err != nil { - message.Fatal(err, lang.ErrUnableToGenerateRandomSecret) + return containerRegistry, fmt.Errorf("%s: %w", lang.ErrUnableToGenerateRandomSecret, err) } } @@ -347,7 +351,7 @@ func (c *Cluster) fillInEmptyContainerRegistryValues(containerRegistry types.Reg if containerRegistry.PullPassword == "" { if containerRegistry.InternalRegistry { if containerRegistry.PullPassword, err = helpers.RandomString(config.ZarfGeneratedPasswordLen); err != nil { - message.Fatal(err, lang.ErrUnableToGenerateRandomSecret) + return containerRegistry, fmt.Errorf("%s: %w", lang.ErrUnableToGenerateRandomSecret, err) } } else { // If this is an external registry and a pull-user wasn't provided, use the same credentials as the push user @@ -357,15 +361,15 @@ func (c *Cluster) fillInEmptyContainerRegistryValues(containerRegistry types.Reg if containerRegistry.Secret == "" { if containerRegistry.Secret, err = helpers.RandomString(config.ZarfGeneratedPasswordLen); err != nil { - message.Fatal(err, lang.ErrUnableToGenerateRandomSecret) + return containerRegistry, fmt.Errorf("%s: %w", lang.ErrUnableToGenerateRandomSecret, err) } } - return containerRegistry + return containerRegistry, nil } // Fill in empty GitServerInfo values with the defaults. -func (c *Cluster) fillInEmptyGitServerValues(gitServer types.GitServerInfo) types.GitServerInfo { +func (c *Cluster) fillInEmptyGitServerValues(gitServer types.GitServerInfo) (types.GitServerInfo, error) { var err error // Set default svc url if an external repository was not provided if gitServer.Address == "" { @@ -376,7 +380,7 @@ func (c *Cluster) fillInEmptyGitServerValues(gitServer types.GitServerInfo) type // Generate a push-user password if not provided by init flag if gitServer.PushPassword == "" { if gitServer.PushPassword, err = helpers.RandomString(config.ZarfGeneratedPasswordLen); err != nil { - message.Fatal(err, lang.ErrUnableToGenerateRandomSecret) + return gitServer, fmt.Errorf("%s: %w", lang.ErrUnableToGenerateRandomSecret, err) } } @@ -391,14 +395,14 @@ func (c *Cluster) fillInEmptyGitServerValues(gitServer types.GitServerInfo) type if gitServer.PullPassword == "" { if gitServer.InternalServer { if gitServer.PullPassword, err = helpers.RandomString(config.ZarfGeneratedPasswordLen); err != nil { - message.Fatal(err, lang.ErrUnableToGenerateRandomSecret) + return gitServer, fmt.Errorf("%s: %w", lang.ErrUnableToGenerateRandomSecret, err) } } else { gitServer.PullPassword = gitServer.PushPassword } } - return gitServer + return gitServer, nil } // Fill in empty ArtifactServerInfo values with the defaults. diff --git a/src/pkg/packager/lint/lint.go b/src/pkg/packager/lint/lint.go index ab2fa10e70..82b45a2b3f 100644 --- a/src/pkg/packager/lint/lint.go +++ b/src/pkg/packager/lint/lint.go @@ -31,7 +31,7 @@ func getSchemaFile() ([]byte, error) { return ZarfSchema.ReadFile("zarf.schema.json") } -// ValidateZarf validates a zarf file against the zarf schema, returns *validator with warnings or errors if they exist +// Validate validates a zarf file against the zarf schema, returns *validator with warnings or errors if they exist // along with an error if the validation itself failed func Validate(createOpts types.ZarfCreateOptions) (*Validator, error) { validator := Validator{} diff --git a/src/pkg/packager/lint/lint_test.go b/src/pkg/packager/lint/lint_test.go index 7c1ca28a06..ee56a2b29d 100644 --- a/src/pkg/packager/lint/lint_test.go +++ b/src/pkg/packager/lint/lint_test.go @@ -132,23 +132,23 @@ func TestValidateSchema(t *testing.T) { t.Run("Unpinnned file warning", func(t *testing.T) { validator := Validator{} - fileUrl := "http://example.com/file.zip" + fileURL := "http://example.com/file.zip" localFile := "local.txt" zarfFiles := []types.ZarfFile{ { - Source: fileUrl, + Source: fileURL, }, { Source: localFile, }, { - Source: fileUrl, + Source: fileURL, Shasum: "fake-shasum", }, } component := types.ZarfComponent{Files: zarfFiles} checkForUnpinnedFiles(&validator, &composer.Node{ZarfComponent: component}) - require.Equal(t, fileUrl, validator.findings[0].item) + require.Equal(t, fileURL, validator.findings[0].item) require.Equal(t, 1, len(validator.findings)) }) From 4a10df8eeb34e79dd1e623716b6bba74d4c15f27 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Wed, 13 Dec 2023 15:46:12 +0000 Subject: [PATCH 171/192] added comment --- src/pkg/packager/composer/list.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/pkg/packager/composer/list.go b/src/pkg/packager/composer/list.go index 7c4e071fba..d1400709bc 100644 --- a/src/pkg/packager/composer/list.go +++ b/src/pkg/packager/composer/list.go @@ -183,6 +183,8 @@ func NewImportChain(head types.ZarfComponent, index int, originalPackageName, ar name := node.ImportName() + // 'found' and 'index' are parallel slices. Each element in found[x] corresponds to pkg[index[x]] + // found[0] and pkg[index[0]] would be the same componenet for example found := []types.ZarfComponent{} index := []int{} for i, component := range pkg.Components { From ec64dbc06ad2e61969585f109c19dd904bed0a33 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Wed, 13 Dec 2023 16:06:52 +0000 Subject: [PATCH 172/192] build correct package in make file --- Makefile | 15 +++++++++++---- hack/lint_all_zarf_packages.sh | 7 ++++--- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index 57dd5e3e3d..7f4cd4e76e 100644 --- a/Makefile +++ b/Makefile @@ -8,8 +8,10 @@ KEY ?= "" # Figure out which Zarf binary we should use based on the operating system we are on ZARF_BIN := ./build/zarf +BUILD_CLI_FOR_SYSTEM := build-cli-linux-amd ifeq ($(OS),Windows_NT) ZARF_BIN := $(addsuffix .exe,$(ZARF_BIN)) + BUILD_CLI_FOR_SYSTEM := build-cli-windows-amd else UNAME_S := $(shell uname -s) UNAME_P := $(shell uname -p) @@ -19,9 +21,11 @@ else endif ifeq ($(UNAME_P),i386) ZARF_BIN := $(addsuffix -intel,$(ZARF_BIN)) + BUILD_CLI_FOR_SYSTEM = build-cli-mac-intel endif ifeq ($(UNAME_P),arm) ZARF_BIN := $(addsuffix -apple,$(ZARF_BIN)) + BUILD_CLI_FOR_SYSTEM = build-cli-mac-apple endif endif endif @@ -35,8 +39,8 @@ BUILD_ARGS := -s -w -X 'github.com/defenseunicorns/zarf/src/config.CLIVersion=$( .PHONY: help help: ## Display this help information @grep -E '^[a-zA-Z0-9_-]+:.*?## .*$$' $(MAKEFILE_LIST) \ - | sort | awk 'BEGIN {FS = ":.*?## "}; \ - {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' + | sort | awk 'BEGIN {FS = ":.*?## "}; \ + {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' clean: ## Clean the build directory rm -rf build @@ -72,12 +76,15 @@ build-cli-linux: build-cli-linux-amd build-cli-linux-arm ## Build the Zarf CLI f build-cli: build-cli-linux-amd build-cli-linux-arm build-cli-mac-intel build-cli-mac-apple build-cli-windows-amd build-cli-windows-arm ## Build the CLI +build-cli-for-system: + $(MAKE) $(BUILD_CLI_FOR_SYSTEM) + docs-and-schema: ## Generate the Zarf Documentation and Schema hack/gen-cli-docs.sh ZARF_CONFIG=hack/empty-config.toml hack/create-zarf-schema.sh -lint-packages-and-examples: build-cli ## Recurisvely lint all zarf.yaml files in the repo except for those dedicated to tests - hack/lint_all_zarf_packages.sh +lint-packages-and-examples: build-cli-for-system ## Recurisvely lint all zarf.yaml files in the repo except for those dedicated to tests + hack/lint_all_zarf_packages.sh $(ZARF_BIN) # INTERNAL: a shim used to build the agent image only if needed on Windows using the `test` command init-package-local-agent: diff --git a/hack/lint_all_zarf_packages.sh b/hack/lint_all_zarf_packages.sh index b79ef6ed08..e065628ac7 100755 --- a/hack/lint_all_zarf_packages.sh +++ b/hack/lint_all_zarf_packages.sh @@ -1,16 +1,17 @@ #!/bin/bash -lint_src_test=$1 +ZARF_BIN=$1 +LINT_SRC_TEST=$2 SCRIPT=$(realpath "$0") SCRIPTPATH=$(dirname "$SCRIPT") cd $SCRIPTPATH cd .. find "." -type f -name 'zarf.yaml' | while read -r yaml_file; do dir=$(dirname "$yaml_file") - if [[ "$dir" == *src/test/* ]] && [ "$lint_src_test" != true ]; then + if [[ "$dir" == *src/test/* ]] && [ "$LINT_SRC_TEST" != true ]; then continue fi echo "Running 'zarf prepare lint' in directory: $dir" - ./build/zarf prepare lint $dir + $ZARF_BIN prepare lint $dir echo "---" done From 27d41d524addee197ad8570c174dfb1e3eaf4fff Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Wed, 13 Dec 2023 16:50:33 +0000 Subject: [PATCH 173/192] fixing windows tests --- src/pkg/packager/lint/lint_test.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/pkg/packager/lint/lint_test.go b/src/pkg/packager/lint/lint_test.go index ee56a2b29d..1d40cdce3e 100644 --- a/src/pkg/packager/lint/lint_test.go +++ b/src/pkg/packager/lint/lint_test.go @@ -6,7 +6,9 @@ package lint import ( "errors" + "fmt" "os" + "path/filepath" "testing" "github.com/defenseunicorns/zarf/src/config" @@ -177,7 +179,7 @@ func TestValidateSchema(t *testing.T) { createOpts := types.ZarfCreateOptions{Flavor: "", BaseDir: "."} lintComponents(&validator, &createOpts) - require.Equal(t, "open fake-path/zarf.yaml: no such file or directory", validator.findings[0].description) + require.Equal(t, fmt.Sprintf("open %s: no such file or directory", filepath.Join("fake-path", "zarf.yaml")), validator.findings[0].description) require.Equal(t, ".components.[0].import.path", validator.findings[0].yqPath) require.Equal(t, ".", validator.findings[0].packageRelPath) require.Equal(t, unpinnedImage, validator.findings[1].item) From a35c83c1221af1bc318e906f91b9f9dd33d729e2 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Wed, 13 Dec 2023 18:05:32 +0000 Subject: [PATCH 174/192] fixing windows tests --- src/pkg/packager/lint/lint_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/pkg/packager/lint/lint_test.go b/src/pkg/packager/lint/lint_test.go index 1d40cdce3e..5f81660d32 100644 --- a/src/pkg/packager/lint/lint_test.go +++ b/src/pkg/packager/lint/lint_test.go @@ -179,7 +179,8 @@ func TestValidateSchema(t *testing.T) { createOpts := types.ZarfCreateOptions{Flavor: "", BaseDir: "."} lintComponents(&validator, &createOpts) - require.Equal(t, fmt.Sprintf("open %s: no such file or directory", filepath.Join("fake-path", "zarf.yaml")), validator.findings[0].description) + // Require.contains rather than equals since the error message changes from linux to windows + require.Contains(t, validator.findings[0].description, fmt.Sprintf("open %s", filepath.Join("fake-path", "zarf.yaml"))) require.Equal(t, ".components.[0].import.path", validator.findings[0].yqPath) require.Equal(t, ".", validator.findings[0].packageRelPath) require.Equal(t, unpinnedImage, validator.findings[1].item) From 4291ecc787c0036011f5f988a0aa97599ef9ac32 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Wed, 13 Dec 2023 18:32:03 +0000 Subject: [PATCH 175/192] fixing windows tests --- src/test/e2e/12_lint_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/e2e/12_lint_test.go b/src/test/e2e/12_lint_test.go index 1f6ffa2e2a..3cb18dceb1 100644 --- a/src/test/e2e/12_lint_test.go +++ b/src/test/e2e/12_lint_test.go @@ -45,7 +45,7 @@ func TestLint(t *testing.T) { // Testing OCI imports get linted require.Contains(t, strippedStderr, ".components.[0].images.[0] | Image not pinned with digest - defenseunicorns/zarf-game:multi-tile-dark") // Testing a bad path leads to a finding in lint - require.Contains(t, strippedStderr, ".components.[3].import.path | open ###ZARF_PKG_TMPL_PATH###/zarf.yaml: no such file or directory") + require.Contains(t, strippedStderr, fmt.Sprintf(".components.[3].import.path | open %s: no such file or directory", filepath.Join("###ZARF_PKG_TMPL_PATH###", "zarf.yaml"))) // Check flavors require.NotContains(t, strippedStderr, "image-in-bad-flavor-component:unpinned") From 8d347f9388bff8838b832206b0d7719f0761ce02 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Wed, 13 Dec 2023 19:06:33 +0000 Subject: [PATCH 176/192] fixing windows tests --- src/test/e2e/12_lint_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/e2e/12_lint_test.go b/src/test/e2e/12_lint_test.go index 3cb18dceb1..19f9e72703 100644 --- a/src/test/e2e/12_lint_test.go +++ b/src/test/e2e/12_lint_test.go @@ -45,7 +45,7 @@ func TestLint(t *testing.T) { // Testing OCI imports get linted require.Contains(t, strippedStderr, ".components.[0].images.[0] | Image not pinned with digest - defenseunicorns/zarf-game:multi-tile-dark") // Testing a bad path leads to a finding in lint - require.Contains(t, strippedStderr, fmt.Sprintf(".components.[3].import.path | open %s: no such file or directory", filepath.Join("###ZARF_PKG_TMPL_PATH###", "zarf.yaml"))) + require.Contains(t, strippedStderr, fmt.Sprintf(".components.[3].import.path | open %s", filepath.Join("###ZARF_PKG_TMPL_PATH###", "zarf.yaml"))) // Check flavors require.NotContains(t, strippedStderr, "image-in-bad-flavor-component:unpinned") From 558980ad73c2ed3a29b77307c3a854181c4c5cb7 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Wed, 13 Dec 2023 19:23:35 +0000 Subject: [PATCH 177/192] refactor and renames --- src/cmd/common/{common.go => utils.go} | 0 src/pkg/packager/lint/validator.go | 26 +++++++++++++------------- 2 files changed, 13 insertions(+), 13 deletions(-) rename src/cmd/common/{common.go => utils.go} (100%) diff --git a/src/cmd/common/common.go b/src/cmd/common/utils.go similarity index 100% rename from src/cmd/common/common.go rename to src/cmd/common/utils.go diff --git a/src/pkg/packager/lint/validator.go b/src/pkg/packager/lint/validator.go index 6b687549c6..830ac21ff2 100644 --- a/src/pkg/packager/lint/validator.go +++ b/src/pkg/packager/lint/validator.go @@ -14,11 +14,11 @@ import ( "github.com/fatih/color" ) -type validationType int +type category int const ( - validationError validationType = 1 - validationWarning validationType = 2 + categoryError category = 1 + categoryWarning category = 2 ) type validatorMessage struct { @@ -27,13 +27,13 @@ type validatorMessage struct { item string packageRelPath string packageName string - validationType validationType + category category } -func (vt validationType) String() string { - if vt == validationError { +func (c category) String() string { + if c == categoryError { return message.ColorWrap("Error", color.FgRed) - } else if vt == validationWarning { + } else if c == categoryWarning { return message.ColorWrap("Warning", color.FgYellow) } return "" @@ -66,7 +66,7 @@ func (v Validator) DisplayFormattedMessage() { // IsSuccess returns true if there are not any errors func (v Validator) IsSuccess() bool { for _, finding := range v.findings { - if finding.validationType == validationError { + if finding.category == categoryError { return false } } @@ -95,7 +95,7 @@ func (v Validator) printValidationTable() { for packageRelPath, findings := range mapOfFindingsByPath { lintData := [][]string{} for _, finding := range findings { - lintData = append(lintData, []string{finding.validationType.String(), finding.getPath(), finding.String()}) + lintData = append(lintData, []string{finding.category.String(), finding.getPath(), finding.String()}) } message.Notef("Linting package %q at %s", findings[0].packageName, v.packageRelPathToUser(findings[0])) message.Table(header, lintData) @@ -110,10 +110,10 @@ func (v Validator) getFormattedFindingCount(relPath string, packageName string) if finding.packageRelPath != relPath { continue } - if finding.validationType == validationWarning { + if finding.category == categoryWarning { warningCount++ } - if finding.validationType == validationError { + if finding.category == categoryError { errorCount++ } } @@ -141,11 +141,11 @@ func (v Validator) hasFindings() bool { } func (v *Validator) addWarning(vmessage validatorMessage) { - vmessage.validationType = validationWarning + vmessage.category = categoryWarning v.findings = helpers.Unique(append(v.findings, vmessage)) } func (v *Validator) addError(vMessage validatorMessage) { - vMessage.validationType = validationError + vMessage.category = categoryError v.findings = helpers.Unique(append(v.findings, vMessage)) } From fa9970045a7b021c39a89bbe1cb193ad31e47de2 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Wed, 13 Dec 2023 20:26:50 +0000 Subject: [PATCH 178/192] fix windows tests --- src/test/e2e/12_lint_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/test/e2e/12_lint_test.go b/src/test/e2e/12_lint_test.go index 19f9e72703..e48741f5c4 100644 --- a/src/test/e2e/12_lint_test.go +++ b/src/test/e2e/12_lint_test.go @@ -24,10 +24,10 @@ func TestLint(t *testing.T) { t.Run("zarf test lint fail", func(t *testing.T) { t.Log("E2E: Test lint on schema fail") - path := filepath.Join("src", "test", "packages", "12-lint") - configPath := filepath.Join(path, "zarf-config.toml") + testPackagePath := filepath.Join("src", "test", "packages", "12-lint") + configPath := filepath.Join(testPackagePath, "zarf-config.toml") os.Setenv("ZARF_CONFIG", configPath) - _, stderr, err := e2e.Zarf("prepare", "lint", path, "-f", "good-flavor") + _, stderr, err := e2e.Zarf("prepare", "lint", testPackagePath, "-f", "good-flavor") require.Error(t, err, "Require an exit code since there was warnings / errors") strippedStderr := e2e.StripMessageFormatting(stderr) @@ -53,7 +53,7 @@ func TestLint(t *testing.T) { // Check reported filepaths require.Contains(t, strippedStderr, "Linting package \"dos-games\" at oci://🦄/dos-games:1.0.0-skeleton") - require.Contains(t, strippedStderr, "Linting package \"lint\" at src/test/packages/12-lint") + require.Contains(t, strippedStderr, fmt.Sprintf("Linting package \"lint\" at %s", testPackagePath)) }) From b41744947e36b360734221031ca610f2f89878d5 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Fri, 15 Dec 2023 16:10:35 +0000 Subject: [PATCH 179/192] make docs and schema --- docs/2-the-zarf-cli/100-cli-commands/zarf_dev.md | 2 +- src/test/e2e/12_lint_test.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/2-the-zarf-cli/100-cli-commands/zarf_dev.md b/docs/2-the-zarf-cli/100-cli-commands/zarf_dev.md index ae76b14632..0e44236b00 100644 --- a/docs/2-the-zarf-cli/100-cli-commands/zarf_dev.md +++ b/docs/2-the-zarf-cli/100-cli-commands/zarf_dev.md @@ -28,7 +28,7 @@ Commands useful for developing packages * [zarf dev deploy](zarf_dev_deploy.md) - [beta] Creates and deploys a Zarf package from a given directory * [zarf dev find-images](zarf_dev_find-images.md) - Evaluates components in a Zarf file to identify images specified in their helm charts and manifests * [zarf dev generate-config](zarf_dev_generate-config.md) - Generates a config file for Zarf -* [zarf dev lint](zarf_dev_lint.md) - Verifies the package schema +* [zarf dev lint](zarf_dev_lint.md) - Lints the given package for valid schema and recommended practices * [zarf dev patch-git](zarf_dev_patch-git.md) - Converts all .git URLs to the specified Zarf HOST and with the Zarf URL pattern in a given FILE. NOTE: This should only be used for manifests that are not mutated by the Zarf Agent Mutating Webhook. * [zarf dev sha256sum](zarf_dev_sha256sum.md) - Generates a SHA256SUM for the given file diff --git a/src/test/e2e/12_lint_test.go b/src/test/e2e/12_lint_test.go index e48741f5c4..a2d6443ed8 100644 --- a/src/test/e2e/12_lint_test.go +++ b/src/test/e2e/12_lint_test.go @@ -17,7 +17,7 @@ func TestLint(t *testing.T) { t.Log("E2E: Test lint on schema success") // This runs lint on the zarf.yaml in the base directory of the repo - _, _, err := e2e.Zarf("prepare", "lint") + _, _, err := e2e.Zarf("dev", "lint") require.NoError(t, err, "Expect no error here because the yaml file is following schema") }) @@ -27,7 +27,7 @@ func TestLint(t *testing.T) { testPackagePath := filepath.Join("src", "test", "packages", "12-lint") configPath := filepath.Join(testPackagePath, "zarf-config.toml") os.Setenv("ZARF_CONFIG", configPath) - _, stderr, err := e2e.Zarf("prepare", "lint", testPackagePath, "-f", "good-flavor") + _, stderr, err := e2e.Zarf("dev", "lint", testPackagePath, "-f", "good-flavor") require.Error(t, err, "Require an exit code since there was warnings / errors") strippedStderr := e2e.StripMessageFormatting(stderr) From cd807d78e05c48a4fdf298a075cce96429d5eeeb Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Fri, 15 Dec 2023 16:28:53 +0000 Subject: [PATCH 180/192] rename var, dry cmds --- src/cmd/dev.go | 20 ++----------------- src/pkg/packager/composer/list.go | 32 +++++++++++++++---------------- src/pkg/packager/composer/oci.go | 4 ++-- src/pkg/packager/lint/lint.go | 20 +++++++++---------- 4 files changed, 30 insertions(+), 46 deletions(-) diff --git a/src/cmd/dev.go b/src/cmd/dev.go index 62d9548f48..1d18da26e7 100644 --- a/src/cmd/dev.go +++ b/src/cmd/dev.go @@ -40,15 +40,7 @@ var devDeployCmd = &cobra.Command{ Short: lang.CmdDevDeployShort, Long: lang.CmdDevDeployLong, Run: func(cmd *cobra.Command, args []string) { - if len(args) > 0 { - pkgConfig.CreateOpts.BaseDir = args[0] - } else { - var err error - pkgConfig.CreateOpts.BaseDir, err = os.Getwd() - if err != nil { - message.Fatalf(err, lang.CmdPackageCreateErr, err.Error()) - } - } + common.SetBaseDirectory(args, &pkgConfig) v := common.GetViper() pkgConfig.CreateOpts.SetVariables = helpers.TransformAndMergeMap( @@ -194,15 +186,7 @@ var devFindImagesCmd = &cobra.Command{ Long: lang.CmdPrepareFindImagesLong, Run: func(cmd *cobra.Command, args []string) { // If a directory was provided, use that as the base directory - if len(args) > 0 { - pkgConfig.CreateOpts.BaseDir = args[0] - } else { - cwd, err := os.Getwd() - if err != nil { - message.Fatalf(err, lang.CmdPrepareFindImagesErr, err.Error()) - } - pkgConfig.CreateOpts.BaseDir = cwd - } + common.SetBaseDirectory(args, &pkgConfig) // Ensure uppercase keys from viper v := common.GetViper() diff --git a/src/pkg/packager/composer/list.go b/src/pkg/packager/composer/list.go index d1400709bc..db2d6452c0 100644 --- a/src/pkg/packager/composer/list.go +++ b/src/pkg/packager/composer/list.go @@ -27,7 +27,7 @@ type Node struct { vars []types.ZarfPackageVariable consts []types.ZarfPackageConstant - relativeToHead string + relativePathOrUrl string originalPackageName string prev *Node @@ -44,9 +44,9 @@ func (n *Node) GetOriginalPackageName() string { return n.originalPackageName } -// GetRelativeToHead gets the path from downstream zarf file to upstream imported zarf file -func (n *Node) GetRelativeToHead() string { - return n.relativeToHead +// GetRelativePathOrUrl gets the path from downstream zarf file to upstream imported zarf file +func (n *Node) GetRelativePathOrUrl() string { + return n.relativePathOrUrl } // Next returns next node in the chain @@ -89,12 +89,12 @@ func (ic *ImportChain) Tail() *Node { } func (ic *ImportChain) append(c types.ZarfComponent, index int, originalPackageName string, - relativeToHead string, vars []types.ZarfPackageVariable, consts []types.ZarfPackageConstant) { + relativePathOrUrl string, vars []types.ZarfPackageVariable, consts []types.ZarfPackageConstant) { node := &Node{ ZarfComponent: c, index: index, originalPackageName: originalPackageName, - relativeToHead: relativeToHead, + relativePathOrUrl: relativePathOrUrl, vars: vars, consts: consts, prev: nil, @@ -150,27 +150,27 @@ func NewImportChain(head types.ZarfComponent, index int, originalPackageName, ar var pkg types.ZarfPackage - var relativeToHead string + var relativePathOrUrl string if isLocal { history = append(history, node.Import.Path) - relativeToHead = filepath.Join(history...) + relativePathOrUrl = filepath.Join(history...) // prevent circular imports (including self-imports) // this is O(n^2) but the import chain should be small prev := node for prev != nil { - if prev.relativeToHead == relativeToHead { + if prev.relativePathOrUrl == relativePathOrUrl { return ic, fmt.Errorf("detected circular import chain: %s", strings.Join(history, " -> ")) } prev = prev.prev } // this assumes the composed package is following the zarf layout - if err := utils.ReadYaml(filepath.Join(relativeToHead, layout.ZarfYAML), &pkg); err != nil { + if err := utils.ReadYaml(filepath.Join(relativePathOrUrl, layout.ZarfYAML), &pkg); err != nil { return ic, err } } else if isRemote { - relativeToHead = node.Import.URL + relativePathOrUrl = node.Import.URL remote, err := ic.getRemote(node.Import.URL) if err != nil { return ic, err @@ -195,12 +195,12 @@ func NewImportChain(head types.ZarfComponent, index int, originalPackageName, ar } if len(found) == 0 { - return ic, fmt.Errorf("component %q not found in %q", name, relativeToHead) + return ic, fmt.Errorf("component %q not found in %q", name, relativePathOrUrl) } else if len(found) > 1 { - return ic, fmt.Errorf("multiple components named %q found in %q satisfying %q", name, relativeToHead, arch) + return ic, fmt.Errorf("multiple components named %q found in %q satisfying %q", name, relativePathOrUrl, arch) } - ic.append(found[0], index[0], pkg.Metadata.Name, relativeToHead, pkg.Variables, pkg.Constants) + ic.append(found[0], index[0], pkg.Metadata.Name, relativePathOrUrl, pkg.Variables, pkg.Constants) node = node.next } return ic, nil @@ -274,7 +274,7 @@ func (ic *ImportChain) Compose() (composed *types.ZarfComponent, err error) { // start overriding with the tail node node := ic.tail for node != nil { - fixPaths(&node.ZarfComponent, node.relativeToHead) + fixPaths(&node.ZarfComponent, node.relativePathOrUrl) // perform overrides here err := overrideMetadata(composed, node.ZarfComponent) @@ -286,7 +286,7 @@ func (ic *ImportChain) Compose() (composed *types.ZarfComponent, err error) { overrideResources(composed, node.ZarfComponent) overrideActions(composed, node.ZarfComponent) - composeExtensions(composed, node.ZarfComponent, node.relativeToHead) + composeExtensions(composed, node.ZarfComponent, node.relativePathOrUrl) node = node.prev } diff --git a/src/pkg/packager/composer/oci.go b/src/pkg/packager/composer/oci.go index 5a7b4637ed..de0cfdd8ae 100644 --- a/src/pkg/packager/composer/oci.go +++ b/src/pkg/packager/composer/oci.go @@ -112,9 +112,9 @@ func (ic *ImportChain) fetchOCISkeleton() error { if err != nil { return err } - // the tail node is the only node whose relativeToHead is based solely upon cwd<->cache + // the tail node is the only node whose relativePathOrUrl is based solely upon cwd<->cache // contrary to the other nodes, which are based upon the previous node - ic.tail.relativeToHead = rel + ic.tail.relativePathOrUrl = rel if oci.IsEmptyDescriptor(componentDesc) { // nothing was fetched, nothing to extract diff --git a/src/pkg/packager/lint/lint.go b/src/pkg/packager/lint/lint.go index 82b45a2b3f..81bdc51533 100644 --- a/src/pkg/packager/lint/lint.go +++ b/src/pkg/packager/lint/lint.go @@ -109,7 +109,7 @@ func fillComponentTemplate(validator *Validator, node *composer.Node, createOpts if err != nil { validator.addWarning(validatorMessage{ description: err.Error(), - packageRelPath: node.GetRelativeToHead(), + packageRelPath: node.GetRelativePathOrUrl(), packageName: node.GetOriginalPackageName(), }) } @@ -120,7 +120,7 @@ func fillComponentTemplate(validator *Validator, node *composer.Node, createOpts if err != nil { validator.addWarning(validatorMessage{ description: err.Error(), - packageRelPath: node.GetRelativeToHead(), + packageRelPath: node.GetRelativePathOrUrl(), packageName: node.GetOriginalPackageName(), }) } @@ -129,7 +129,7 @@ func fillComponentTemplate(validator *Validator, node *composer.Node, createOpts if deprecated { validator.addWarning(validatorMessage{ description: fmt.Sprintf(lang.PkgValidateTemplateDeprecation, key, key, key), - packageRelPath: node.GetRelativeToHead(), + packageRelPath: node.GetRelativePathOrUrl(), packageName: node.GetOriginalPackageName(), }) } @@ -137,7 +137,7 @@ func fillComponentTemplate(validator *Validator, node *composer.Node, createOpts if !present { validator.addWarning(validatorMessage{ description: lang.UnsetVarLintWarning, - packageRelPath: node.GetRelativeToHead(), + packageRelPath: node.GetRelativePathOrUrl(), packageName: node.GetOriginalPackageName(), }) } @@ -183,7 +183,7 @@ func checkForUnpinnedRepos(validator *Validator, node *composer.Node) { if !isPinnedRepo(repo) { validator.addWarning(validatorMessage{ yqPath: repoYqPath, - packageRelPath: node.GetRelativeToHead(), + packageRelPath: node.GetRelativePathOrUrl(), packageName: node.GetOriginalPackageName(), description: "Unpinned repository", item: repo, @@ -199,7 +199,7 @@ func checkForUnpinnedImages(validator *Validator, node *composer.Node) { if err != nil { validator.addError(validatorMessage{ yqPath: imageYqPath, - packageRelPath: node.GetRelativeToHead(), + packageRelPath: node.GetRelativePathOrUrl(), packageName: node.GetOriginalPackageName(), description: "Invalid image reference", item: image, @@ -209,7 +209,7 @@ func checkForUnpinnedImages(validator *Validator, node *composer.Node) { if !pinnedImage { validator.addWarning(validatorMessage{ yqPath: imageYqPath, - packageRelPath: node.GetRelativeToHead(), + packageRelPath: node.GetRelativePathOrUrl(), packageName: node.GetOriginalPackageName(), description: "Image not pinned with digest", item: image, @@ -224,7 +224,7 @@ func checkForUnpinnedFiles(validator *Validator, node *composer.Node) { if file.Shasum == "" && helpers.IsURL(file.Source) { validator.addWarning(validatorMessage{ yqPath: fileYqPath, - packageRelPath: node.GetRelativeToHead(), + packageRelPath: node.GetRelativePathOrUrl(), packageName: node.GetOriginalPackageName(), description: "No shasum for remote file", item: file.Source, @@ -237,7 +237,7 @@ func checkForVarInComponentImport(validator *Validator, node *composer.Node) { if strings.Contains(node.Import.Path, types.ZarfPackageTemplatePrefix) { validator.addWarning(validatorMessage{ yqPath: fmt.Sprintf(".components.[%d].import.path", node.GetIndex()), - packageRelPath: node.GetRelativeToHead(), + packageRelPath: node.GetRelativePathOrUrl(), packageName: node.GetOriginalPackageName(), description: "Zarf does not evaluate variables at component.x.import.path", item: node.Import.Path, @@ -246,7 +246,7 @@ func checkForVarInComponentImport(validator *Validator, node *composer.Node) { if strings.Contains(node.Import.URL, types.ZarfPackageTemplatePrefix) { validator.addWarning(validatorMessage{ yqPath: fmt.Sprintf(".components.[%d].import.url", node.GetIndex()), - packageRelPath: node.GetRelativeToHead(), + packageRelPath: node.GetRelativePathOrUrl(), packageName: node.GetOriginalPackageName(), description: "Zarf does not evaluate variables at component.x.import.url", item: node.Import.URL, From c796ffc6bcb5cf0bea1bd7ee94bfd4d7c869c425 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Fri, 15 Dec 2023 16:40:21 +0000 Subject: [PATCH 181/192] var rename --- src/pkg/packager/composer/list.go | 32 +++++++++++++------------- src/pkg/packager/composer/oci.go | 4 ++-- src/pkg/packager/composer/pathfixer.go | 30 ++++++++++++------------ src/pkg/packager/lint/lint.go | 20 ++++++++-------- 4 files changed, 43 insertions(+), 43 deletions(-) diff --git a/src/pkg/packager/composer/list.go b/src/pkg/packager/composer/list.go index db2d6452c0..ea39628c64 100644 --- a/src/pkg/packager/composer/list.go +++ b/src/pkg/packager/composer/list.go @@ -27,7 +27,7 @@ type Node struct { vars []types.ZarfPackageVariable consts []types.ZarfPackageConstant - relativePathOrUrl string + relativeToHeadOrUrl string originalPackageName string prev *Node @@ -44,9 +44,9 @@ func (n *Node) GetOriginalPackageName() string { return n.originalPackageName } -// GetRelativePathOrUrl gets the path from downstream zarf file to upstream imported zarf file -func (n *Node) GetRelativePathOrUrl() string { - return n.relativePathOrUrl +// GetRelativeToHeadOrUrl gets the path from downstream zarf file to upstream imported zarf file +func (n *Node) GetRelativeToHeadOrUrl() string { + return n.relativeToHeadOrUrl } // Next returns next node in the chain @@ -89,12 +89,12 @@ func (ic *ImportChain) Tail() *Node { } func (ic *ImportChain) append(c types.ZarfComponent, index int, originalPackageName string, - relativePathOrUrl string, vars []types.ZarfPackageVariable, consts []types.ZarfPackageConstant) { + relativeToHeadOrUrl string, vars []types.ZarfPackageVariable, consts []types.ZarfPackageConstant) { node := &Node{ ZarfComponent: c, index: index, originalPackageName: originalPackageName, - relativePathOrUrl: relativePathOrUrl, + relativeToHeadOrUrl: relativeToHeadOrUrl, vars: vars, consts: consts, prev: nil, @@ -150,27 +150,27 @@ func NewImportChain(head types.ZarfComponent, index int, originalPackageName, ar var pkg types.ZarfPackage - var relativePathOrUrl string + var relativeToHeadOrUrl string if isLocal { history = append(history, node.Import.Path) - relativePathOrUrl = filepath.Join(history...) + relativeToHeadOrUrl = filepath.Join(history...) // prevent circular imports (including self-imports) // this is O(n^2) but the import chain should be small prev := node for prev != nil { - if prev.relativePathOrUrl == relativePathOrUrl { + if prev.relativeToHeadOrUrl == relativeToHeadOrUrl { return ic, fmt.Errorf("detected circular import chain: %s", strings.Join(history, " -> ")) } prev = prev.prev } // this assumes the composed package is following the zarf layout - if err := utils.ReadYaml(filepath.Join(relativePathOrUrl, layout.ZarfYAML), &pkg); err != nil { + if err := utils.ReadYaml(filepath.Join(relativeToHeadOrUrl, layout.ZarfYAML), &pkg); err != nil { return ic, err } } else if isRemote { - relativePathOrUrl = node.Import.URL + relativeToHeadOrUrl = node.Import.URL remote, err := ic.getRemote(node.Import.URL) if err != nil { return ic, err @@ -195,12 +195,12 @@ func NewImportChain(head types.ZarfComponent, index int, originalPackageName, ar } if len(found) == 0 { - return ic, fmt.Errorf("component %q not found in %q", name, relativePathOrUrl) + return ic, fmt.Errorf("component %q not found in %q", name, relativeToHeadOrUrl) } else if len(found) > 1 { - return ic, fmt.Errorf("multiple components named %q found in %q satisfying %q", name, relativePathOrUrl, arch) + return ic, fmt.Errorf("multiple components named %q found in %q satisfying %q", name, relativeToHeadOrUrl, arch) } - ic.append(found[0], index[0], pkg.Metadata.Name, relativePathOrUrl, pkg.Variables, pkg.Constants) + ic.append(found[0], index[0], pkg.Metadata.Name, relativeToHeadOrUrl, pkg.Variables, pkg.Constants) node = node.next } return ic, nil @@ -274,7 +274,7 @@ func (ic *ImportChain) Compose() (composed *types.ZarfComponent, err error) { // start overriding with the tail node node := ic.tail for node != nil { - fixPaths(&node.ZarfComponent, node.relativePathOrUrl) + fixPaths(&node.ZarfComponent, node.relativeToHeadOrUrl) // perform overrides here err := overrideMetadata(composed, node.ZarfComponent) @@ -286,7 +286,7 @@ func (ic *ImportChain) Compose() (composed *types.ZarfComponent, err error) { overrideResources(composed, node.ZarfComponent) overrideActions(composed, node.ZarfComponent) - composeExtensions(composed, node.ZarfComponent, node.relativePathOrUrl) + composeExtensions(composed, node.ZarfComponent, node.relativeToHeadOrUrl) node = node.prev } diff --git a/src/pkg/packager/composer/oci.go b/src/pkg/packager/composer/oci.go index de0cfdd8ae..3e3dcce03c 100644 --- a/src/pkg/packager/composer/oci.go +++ b/src/pkg/packager/composer/oci.go @@ -112,9 +112,9 @@ func (ic *ImportChain) fetchOCISkeleton() error { if err != nil { return err } - // the tail node is the only node whose relativePathOrUrl is based solely upon cwd<->cache + // the tail node is the only node whose relativeToHeadOrUrl is based solely upon cwd<->cache // contrary to the other nodes, which are based upon the previous node - ic.tail.relativePathOrUrl = rel + ic.tail.relativeToHeadOrUrl = rel if oci.IsEmptyDescriptor(componentDesc) { // nothing was fetched, nothing to extract diff --git a/src/pkg/packager/composer/pathfixer.go b/src/pkg/packager/composer/pathfixer.go index b52c2c3257..b6856a1b10 100644 --- a/src/pkg/packager/composer/pathfixer.go +++ b/src/pkg/packager/composer/pathfixer.go @@ -20,30 +20,30 @@ func makePathRelativeTo(path, relativeTo string) string { return filepath.Join(relativeTo, path) } -func fixPaths(child *types.ZarfComponent, relativeToHead string) { +func fixPaths(child *types.ZarfComponent, relativeToHeadOrUrl string) { for fileIdx, file := range child.Files { - composed := makePathRelativeTo(file.Source, relativeToHead) + composed := makePathRelativeTo(file.Source, relativeToHeadOrUrl) child.Files[fileIdx].Source = composed } for chartIdx, chart := range child.Charts { for valuesIdx, valuesFile := range chart.ValuesFiles { - composed := makePathRelativeTo(valuesFile, relativeToHead) + composed := makePathRelativeTo(valuesFile, relativeToHeadOrUrl) child.Charts[chartIdx].ValuesFiles[valuesIdx] = composed } if child.Charts[chartIdx].LocalPath != "" { - composed := makePathRelativeTo(chart.LocalPath, relativeToHead) + composed := makePathRelativeTo(chart.LocalPath, relativeToHeadOrUrl) child.Charts[chartIdx].LocalPath = composed } } for manifestIdx, manifest := range child.Manifests { for fileIdx, file := range manifest.Files { - composed := makePathRelativeTo(file, relativeToHead) + composed := makePathRelativeTo(file, relativeToHeadOrUrl) child.Manifests[manifestIdx].Files[fileIdx] = composed } for kustomizeIdx, kustomization := range manifest.Kustomizations { - composed := makePathRelativeTo(kustomization, relativeToHead) + composed := makePathRelativeTo(kustomization, relativeToHeadOrUrl) // kustomizations can use non-standard urls, so we need to check if the composed path exists on the local filesystem abs, _ := filepath.Abs(composed) invalid := utils.InvalidPath(abs) @@ -54,31 +54,31 @@ func fixPaths(child *types.ZarfComponent, relativeToHead string) { } for dataInjectionsIdx, dataInjection := range child.DataInjections { - composed := makePathRelativeTo(dataInjection.Source, relativeToHead) + composed := makePathRelativeTo(dataInjection.Source, relativeToHeadOrUrl) child.DataInjections[dataInjectionsIdx].Source = composed } defaultDir := child.Actions.OnCreate.Defaults.Dir - child.Actions.OnCreate.Before = fixActionPaths(child.Actions.OnCreate.Before, defaultDir, relativeToHead) - child.Actions.OnCreate.After = fixActionPaths(child.Actions.OnCreate.After, defaultDir, relativeToHead) - child.Actions.OnCreate.OnFailure = fixActionPaths(child.Actions.OnCreate.OnFailure, defaultDir, relativeToHead) - child.Actions.OnCreate.OnSuccess = fixActionPaths(child.Actions.OnCreate.OnSuccess, defaultDir, relativeToHead) + child.Actions.OnCreate.Before = fixActionPaths(child.Actions.OnCreate.Before, defaultDir, relativeToHeadOrUrl) + child.Actions.OnCreate.After = fixActionPaths(child.Actions.OnCreate.After, defaultDir, relativeToHeadOrUrl) + child.Actions.OnCreate.OnFailure = fixActionPaths(child.Actions.OnCreate.OnFailure, defaultDir, relativeToHeadOrUrl) + child.Actions.OnCreate.OnSuccess = fixActionPaths(child.Actions.OnCreate.OnSuccess, defaultDir, relativeToHeadOrUrl) // deprecated if child.DeprecatedCosignKeyPath != "" { - composed := makePathRelativeTo(child.DeprecatedCosignKeyPath, relativeToHead) + composed := makePathRelativeTo(child.DeprecatedCosignKeyPath, relativeToHeadOrUrl) child.DeprecatedCosignKeyPath = composed } } // fixActionPaths takes a slice of actions and mutates the Dir to be relative to the head node -func fixActionPaths(actions []types.ZarfComponentAction, defaultDir, relativeToHead string) []types.ZarfComponentAction { +func fixActionPaths(actions []types.ZarfComponentAction, defaultDir, relativeToHeadOrUrl string) []types.ZarfComponentAction { for actionIdx, action := range actions { var composed string if action.Dir != nil { - composed = makePathRelativeTo(*action.Dir, relativeToHead) + composed = makePathRelativeTo(*action.Dir, relativeToHeadOrUrl) } else { - composed = makePathRelativeTo(defaultDir, relativeToHead) + composed = makePathRelativeTo(defaultDir, relativeToHeadOrUrl) } actions[actionIdx].Dir = &composed } diff --git a/src/pkg/packager/lint/lint.go b/src/pkg/packager/lint/lint.go index 81bdc51533..f0c39264b2 100644 --- a/src/pkg/packager/lint/lint.go +++ b/src/pkg/packager/lint/lint.go @@ -109,7 +109,7 @@ func fillComponentTemplate(validator *Validator, node *composer.Node, createOpts if err != nil { validator.addWarning(validatorMessage{ description: err.Error(), - packageRelPath: node.GetRelativePathOrUrl(), + packageRelPath: node.GetRelativeToHeadOrUrl(), packageName: node.GetOriginalPackageName(), }) } @@ -120,7 +120,7 @@ func fillComponentTemplate(validator *Validator, node *composer.Node, createOpts if err != nil { validator.addWarning(validatorMessage{ description: err.Error(), - packageRelPath: node.GetRelativePathOrUrl(), + packageRelPath: node.GetRelativeToHeadOrUrl(), packageName: node.GetOriginalPackageName(), }) } @@ -129,7 +129,7 @@ func fillComponentTemplate(validator *Validator, node *composer.Node, createOpts if deprecated { validator.addWarning(validatorMessage{ description: fmt.Sprintf(lang.PkgValidateTemplateDeprecation, key, key, key), - packageRelPath: node.GetRelativePathOrUrl(), + packageRelPath: node.GetRelativeToHeadOrUrl(), packageName: node.GetOriginalPackageName(), }) } @@ -137,7 +137,7 @@ func fillComponentTemplate(validator *Validator, node *composer.Node, createOpts if !present { validator.addWarning(validatorMessage{ description: lang.UnsetVarLintWarning, - packageRelPath: node.GetRelativePathOrUrl(), + packageRelPath: node.GetRelativeToHeadOrUrl(), packageName: node.GetOriginalPackageName(), }) } @@ -183,7 +183,7 @@ func checkForUnpinnedRepos(validator *Validator, node *composer.Node) { if !isPinnedRepo(repo) { validator.addWarning(validatorMessage{ yqPath: repoYqPath, - packageRelPath: node.GetRelativePathOrUrl(), + packageRelPath: node.GetRelativeToHeadOrUrl(), packageName: node.GetOriginalPackageName(), description: "Unpinned repository", item: repo, @@ -199,7 +199,7 @@ func checkForUnpinnedImages(validator *Validator, node *composer.Node) { if err != nil { validator.addError(validatorMessage{ yqPath: imageYqPath, - packageRelPath: node.GetRelativePathOrUrl(), + packageRelPath: node.GetRelativeToHeadOrUrl(), packageName: node.GetOriginalPackageName(), description: "Invalid image reference", item: image, @@ -209,7 +209,7 @@ func checkForUnpinnedImages(validator *Validator, node *composer.Node) { if !pinnedImage { validator.addWarning(validatorMessage{ yqPath: imageYqPath, - packageRelPath: node.GetRelativePathOrUrl(), + packageRelPath: node.GetRelativeToHeadOrUrl(), packageName: node.GetOriginalPackageName(), description: "Image not pinned with digest", item: image, @@ -224,7 +224,7 @@ func checkForUnpinnedFiles(validator *Validator, node *composer.Node) { if file.Shasum == "" && helpers.IsURL(file.Source) { validator.addWarning(validatorMessage{ yqPath: fileYqPath, - packageRelPath: node.GetRelativePathOrUrl(), + packageRelPath: node.GetRelativeToHeadOrUrl(), packageName: node.GetOriginalPackageName(), description: "No shasum for remote file", item: file.Source, @@ -237,7 +237,7 @@ func checkForVarInComponentImport(validator *Validator, node *composer.Node) { if strings.Contains(node.Import.Path, types.ZarfPackageTemplatePrefix) { validator.addWarning(validatorMessage{ yqPath: fmt.Sprintf(".components.[%d].import.path", node.GetIndex()), - packageRelPath: node.GetRelativePathOrUrl(), + packageRelPath: node.GetRelativeToHeadOrUrl(), packageName: node.GetOriginalPackageName(), description: "Zarf does not evaluate variables at component.x.import.path", item: node.Import.Path, @@ -246,7 +246,7 @@ func checkForVarInComponentImport(validator *Validator, node *composer.Node) { if strings.Contains(node.Import.URL, types.ZarfPackageTemplatePrefix) { validator.addWarning(validatorMessage{ yqPath: fmt.Sprintf(".components.[%d].import.url", node.GetIndex()), - packageRelPath: node.GetRelativePathOrUrl(), + packageRelPath: node.GetRelativeToHeadOrUrl(), packageName: node.GetOriginalPackageName(), description: "Zarf does not evaluate variables at component.x.import.url", item: node.Import.URL, From dafa68d606db8cb2d4ade8f59714b9dbdb0128ce Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Fri, 15 Dec 2023 16:44:13 +0000 Subject: [PATCH 182/192] lint --- Makefile | 4 ++-- src/pkg/packager/composer/list.go | 32 +++++++++++++------------- src/pkg/packager/composer/oci.go | 4 ++-- src/pkg/packager/composer/pathfixer.go | 30 ++++++++++++------------ src/pkg/packager/lint/lint.go | 20 ++++++++-------- 5 files changed, 45 insertions(+), 45 deletions(-) diff --git a/Makefile b/Makefile index 7f4cd4e76e..02133e9de1 100644 --- a/Makefile +++ b/Makefile @@ -39,8 +39,8 @@ BUILD_ARGS := -s -w -X 'github.com/defenseunicorns/zarf/src/config.CLIVersion=$( .PHONY: help help: ## Display this help information @grep -E '^[a-zA-Z0-9_-]+:.*?## .*$$' $(MAKEFILE_LIST) \ - | sort | awk 'BEGIN {FS = ":.*?## "}; \ - {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' + | sort | awk 'BEGIN {FS = ":.*?## "}; \ + {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' clean: ## Clean the build directory rm -rf build diff --git a/src/pkg/packager/composer/list.go b/src/pkg/packager/composer/list.go index ea39628c64..4004b7290d 100644 --- a/src/pkg/packager/composer/list.go +++ b/src/pkg/packager/composer/list.go @@ -27,7 +27,7 @@ type Node struct { vars []types.ZarfPackageVariable consts []types.ZarfPackageConstant - relativeToHeadOrUrl string + relativeToHeadOrURL string originalPackageName string prev *Node @@ -44,9 +44,9 @@ func (n *Node) GetOriginalPackageName() string { return n.originalPackageName } -// GetRelativeToHeadOrUrl gets the path from downstream zarf file to upstream imported zarf file -func (n *Node) GetRelativeToHeadOrUrl() string { - return n.relativeToHeadOrUrl +// GetRelativeToHeadOrURL gets the path from downstream zarf file to upstream imported zarf file +func (n *Node) GetRelativeToHeadOrURL() string { + return n.relativeToHeadOrURL } // Next returns next node in the chain @@ -89,12 +89,12 @@ func (ic *ImportChain) Tail() *Node { } func (ic *ImportChain) append(c types.ZarfComponent, index int, originalPackageName string, - relativeToHeadOrUrl string, vars []types.ZarfPackageVariable, consts []types.ZarfPackageConstant) { + relativeToHeadOrURL string, vars []types.ZarfPackageVariable, consts []types.ZarfPackageConstant) { node := &Node{ ZarfComponent: c, index: index, originalPackageName: originalPackageName, - relativeToHeadOrUrl: relativeToHeadOrUrl, + relativeToHeadOrURL: relativeToHeadOrURL, vars: vars, consts: consts, prev: nil, @@ -150,27 +150,27 @@ func NewImportChain(head types.ZarfComponent, index int, originalPackageName, ar var pkg types.ZarfPackage - var relativeToHeadOrUrl string + var relativeToHeadOrURL string if isLocal { history = append(history, node.Import.Path) - relativeToHeadOrUrl = filepath.Join(history...) + relativeToHeadOrURL = filepath.Join(history...) // prevent circular imports (including self-imports) // this is O(n^2) but the import chain should be small prev := node for prev != nil { - if prev.relativeToHeadOrUrl == relativeToHeadOrUrl { + if prev.relativeToHeadOrURL == relativeToHeadOrURL { return ic, fmt.Errorf("detected circular import chain: %s", strings.Join(history, " -> ")) } prev = prev.prev } // this assumes the composed package is following the zarf layout - if err := utils.ReadYaml(filepath.Join(relativeToHeadOrUrl, layout.ZarfYAML), &pkg); err != nil { + if err := utils.ReadYaml(filepath.Join(relativeToHeadOrURL, layout.ZarfYAML), &pkg); err != nil { return ic, err } } else if isRemote { - relativeToHeadOrUrl = node.Import.URL + relativeToHeadOrURL = node.Import.URL remote, err := ic.getRemote(node.Import.URL) if err != nil { return ic, err @@ -195,12 +195,12 @@ func NewImportChain(head types.ZarfComponent, index int, originalPackageName, ar } if len(found) == 0 { - return ic, fmt.Errorf("component %q not found in %q", name, relativeToHeadOrUrl) + return ic, fmt.Errorf("component %q not found in %q", name, relativeToHeadOrURL) } else if len(found) > 1 { - return ic, fmt.Errorf("multiple components named %q found in %q satisfying %q", name, relativeToHeadOrUrl, arch) + return ic, fmt.Errorf("multiple components named %q found in %q satisfying %q", name, relativeToHeadOrURL, arch) } - ic.append(found[0], index[0], pkg.Metadata.Name, relativeToHeadOrUrl, pkg.Variables, pkg.Constants) + ic.append(found[0], index[0], pkg.Metadata.Name, relativeToHeadOrURL, pkg.Variables, pkg.Constants) node = node.next } return ic, nil @@ -274,7 +274,7 @@ func (ic *ImportChain) Compose() (composed *types.ZarfComponent, err error) { // start overriding with the tail node node := ic.tail for node != nil { - fixPaths(&node.ZarfComponent, node.relativeToHeadOrUrl) + fixPaths(&node.ZarfComponent, node.relativeToHeadOrURL) // perform overrides here err := overrideMetadata(composed, node.ZarfComponent) @@ -286,7 +286,7 @@ func (ic *ImportChain) Compose() (composed *types.ZarfComponent, err error) { overrideResources(composed, node.ZarfComponent) overrideActions(composed, node.ZarfComponent) - composeExtensions(composed, node.ZarfComponent, node.relativeToHeadOrUrl) + composeExtensions(composed, node.ZarfComponent, node.relativeToHeadOrURL) node = node.prev } diff --git a/src/pkg/packager/composer/oci.go b/src/pkg/packager/composer/oci.go index 3e3dcce03c..9c62ec4612 100644 --- a/src/pkg/packager/composer/oci.go +++ b/src/pkg/packager/composer/oci.go @@ -112,9 +112,9 @@ func (ic *ImportChain) fetchOCISkeleton() error { if err != nil { return err } - // the tail node is the only node whose relativeToHeadOrUrl is based solely upon cwd<->cache + // the tail node is the only node whose relativeToHeadOrURL is based solely upon cwd<->cache // contrary to the other nodes, which are based upon the previous node - ic.tail.relativeToHeadOrUrl = rel + ic.tail.relativeToHeadOrURL = rel if oci.IsEmptyDescriptor(componentDesc) { // nothing was fetched, nothing to extract diff --git a/src/pkg/packager/composer/pathfixer.go b/src/pkg/packager/composer/pathfixer.go index b6856a1b10..c06bcb6e46 100644 --- a/src/pkg/packager/composer/pathfixer.go +++ b/src/pkg/packager/composer/pathfixer.go @@ -20,30 +20,30 @@ func makePathRelativeTo(path, relativeTo string) string { return filepath.Join(relativeTo, path) } -func fixPaths(child *types.ZarfComponent, relativeToHeadOrUrl string) { +func fixPaths(child *types.ZarfComponent, relativeToHeadOrURL string) { for fileIdx, file := range child.Files { - composed := makePathRelativeTo(file.Source, relativeToHeadOrUrl) + composed := makePathRelativeTo(file.Source, relativeToHeadOrURL) child.Files[fileIdx].Source = composed } for chartIdx, chart := range child.Charts { for valuesIdx, valuesFile := range chart.ValuesFiles { - composed := makePathRelativeTo(valuesFile, relativeToHeadOrUrl) + composed := makePathRelativeTo(valuesFile, relativeToHeadOrURL) child.Charts[chartIdx].ValuesFiles[valuesIdx] = composed } if child.Charts[chartIdx].LocalPath != "" { - composed := makePathRelativeTo(chart.LocalPath, relativeToHeadOrUrl) + composed := makePathRelativeTo(chart.LocalPath, relativeToHeadOrURL) child.Charts[chartIdx].LocalPath = composed } } for manifestIdx, manifest := range child.Manifests { for fileIdx, file := range manifest.Files { - composed := makePathRelativeTo(file, relativeToHeadOrUrl) + composed := makePathRelativeTo(file, relativeToHeadOrURL) child.Manifests[manifestIdx].Files[fileIdx] = composed } for kustomizeIdx, kustomization := range manifest.Kustomizations { - composed := makePathRelativeTo(kustomization, relativeToHeadOrUrl) + composed := makePathRelativeTo(kustomization, relativeToHeadOrURL) // kustomizations can use non-standard urls, so we need to check if the composed path exists on the local filesystem abs, _ := filepath.Abs(composed) invalid := utils.InvalidPath(abs) @@ -54,31 +54,31 @@ func fixPaths(child *types.ZarfComponent, relativeToHeadOrUrl string) { } for dataInjectionsIdx, dataInjection := range child.DataInjections { - composed := makePathRelativeTo(dataInjection.Source, relativeToHeadOrUrl) + composed := makePathRelativeTo(dataInjection.Source, relativeToHeadOrURL) child.DataInjections[dataInjectionsIdx].Source = composed } defaultDir := child.Actions.OnCreate.Defaults.Dir - child.Actions.OnCreate.Before = fixActionPaths(child.Actions.OnCreate.Before, defaultDir, relativeToHeadOrUrl) - child.Actions.OnCreate.After = fixActionPaths(child.Actions.OnCreate.After, defaultDir, relativeToHeadOrUrl) - child.Actions.OnCreate.OnFailure = fixActionPaths(child.Actions.OnCreate.OnFailure, defaultDir, relativeToHeadOrUrl) - child.Actions.OnCreate.OnSuccess = fixActionPaths(child.Actions.OnCreate.OnSuccess, defaultDir, relativeToHeadOrUrl) + child.Actions.OnCreate.Before = fixActionPaths(child.Actions.OnCreate.Before, defaultDir, relativeToHeadOrURL) + child.Actions.OnCreate.After = fixActionPaths(child.Actions.OnCreate.After, defaultDir, relativeToHeadOrURL) + child.Actions.OnCreate.OnFailure = fixActionPaths(child.Actions.OnCreate.OnFailure, defaultDir, relativeToHeadOrURL) + child.Actions.OnCreate.OnSuccess = fixActionPaths(child.Actions.OnCreate.OnSuccess, defaultDir, relativeToHeadOrURL) // deprecated if child.DeprecatedCosignKeyPath != "" { - composed := makePathRelativeTo(child.DeprecatedCosignKeyPath, relativeToHeadOrUrl) + composed := makePathRelativeTo(child.DeprecatedCosignKeyPath, relativeToHeadOrURL) child.DeprecatedCosignKeyPath = composed } } // fixActionPaths takes a slice of actions and mutates the Dir to be relative to the head node -func fixActionPaths(actions []types.ZarfComponentAction, defaultDir, relativeToHeadOrUrl string) []types.ZarfComponentAction { +func fixActionPaths(actions []types.ZarfComponentAction, defaultDir, relativeToHeadOrURL string) []types.ZarfComponentAction { for actionIdx, action := range actions { var composed string if action.Dir != nil { - composed = makePathRelativeTo(*action.Dir, relativeToHeadOrUrl) + composed = makePathRelativeTo(*action.Dir, relativeToHeadOrURL) } else { - composed = makePathRelativeTo(defaultDir, relativeToHeadOrUrl) + composed = makePathRelativeTo(defaultDir, relativeToHeadOrURL) } actions[actionIdx].Dir = &composed } diff --git a/src/pkg/packager/lint/lint.go b/src/pkg/packager/lint/lint.go index f0c39264b2..4a2fafc640 100644 --- a/src/pkg/packager/lint/lint.go +++ b/src/pkg/packager/lint/lint.go @@ -109,7 +109,7 @@ func fillComponentTemplate(validator *Validator, node *composer.Node, createOpts if err != nil { validator.addWarning(validatorMessage{ description: err.Error(), - packageRelPath: node.GetRelativeToHeadOrUrl(), + packageRelPath: node.GetRelativeToHeadOrURL(), packageName: node.GetOriginalPackageName(), }) } @@ -120,7 +120,7 @@ func fillComponentTemplate(validator *Validator, node *composer.Node, createOpts if err != nil { validator.addWarning(validatorMessage{ description: err.Error(), - packageRelPath: node.GetRelativeToHeadOrUrl(), + packageRelPath: node.GetRelativeToHeadOrURL(), packageName: node.GetOriginalPackageName(), }) } @@ -129,7 +129,7 @@ func fillComponentTemplate(validator *Validator, node *composer.Node, createOpts if deprecated { validator.addWarning(validatorMessage{ description: fmt.Sprintf(lang.PkgValidateTemplateDeprecation, key, key, key), - packageRelPath: node.GetRelativeToHeadOrUrl(), + packageRelPath: node.GetRelativeToHeadOrURL(), packageName: node.GetOriginalPackageName(), }) } @@ -137,7 +137,7 @@ func fillComponentTemplate(validator *Validator, node *composer.Node, createOpts if !present { validator.addWarning(validatorMessage{ description: lang.UnsetVarLintWarning, - packageRelPath: node.GetRelativeToHeadOrUrl(), + packageRelPath: node.GetRelativeToHeadOrURL(), packageName: node.GetOriginalPackageName(), }) } @@ -183,7 +183,7 @@ func checkForUnpinnedRepos(validator *Validator, node *composer.Node) { if !isPinnedRepo(repo) { validator.addWarning(validatorMessage{ yqPath: repoYqPath, - packageRelPath: node.GetRelativeToHeadOrUrl(), + packageRelPath: node.GetRelativeToHeadOrURL(), packageName: node.GetOriginalPackageName(), description: "Unpinned repository", item: repo, @@ -199,7 +199,7 @@ func checkForUnpinnedImages(validator *Validator, node *composer.Node) { if err != nil { validator.addError(validatorMessage{ yqPath: imageYqPath, - packageRelPath: node.GetRelativeToHeadOrUrl(), + packageRelPath: node.GetRelativeToHeadOrURL(), packageName: node.GetOriginalPackageName(), description: "Invalid image reference", item: image, @@ -209,7 +209,7 @@ func checkForUnpinnedImages(validator *Validator, node *composer.Node) { if !pinnedImage { validator.addWarning(validatorMessage{ yqPath: imageYqPath, - packageRelPath: node.GetRelativeToHeadOrUrl(), + packageRelPath: node.GetRelativeToHeadOrURL(), packageName: node.GetOriginalPackageName(), description: "Image not pinned with digest", item: image, @@ -224,7 +224,7 @@ func checkForUnpinnedFiles(validator *Validator, node *composer.Node) { if file.Shasum == "" && helpers.IsURL(file.Source) { validator.addWarning(validatorMessage{ yqPath: fileYqPath, - packageRelPath: node.GetRelativeToHeadOrUrl(), + packageRelPath: node.GetRelativeToHeadOrURL(), packageName: node.GetOriginalPackageName(), description: "No shasum for remote file", item: file.Source, @@ -237,7 +237,7 @@ func checkForVarInComponentImport(validator *Validator, node *composer.Node) { if strings.Contains(node.Import.Path, types.ZarfPackageTemplatePrefix) { validator.addWarning(validatorMessage{ yqPath: fmt.Sprintf(".components.[%d].import.path", node.GetIndex()), - packageRelPath: node.GetRelativeToHeadOrUrl(), + packageRelPath: node.GetRelativeToHeadOrURL(), packageName: node.GetOriginalPackageName(), description: "Zarf does not evaluate variables at component.x.import.path", item: node.Import.Path, @@ -246,7 +246,7 @@ func checkForVarInComponentImport(validator *Validator, node *composer.Node) { if strings.Contains(node.Import.URL, types.ZarfPackageTemplatePrefix) { validator.addWarning(validatorMessage{ yqPath: fmt.Sprintf(".components.[%d].import.url", node.GetIndex()), - packageRelPath: node.GetRelativeToHeadOrUrl(), + packageRelPath: node.GetRelativeToHeadOrURL(), packageName: node.GetOriginalPackageName(), description: "Zarf does not evaluate variables at component.x.import.url", item: node.Import.URL, From 4d77c22f07e56fac0fdb65268fafe2d79bd7c313 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Fri, 15 Dec 2023 17:45:08 +0000 Subject: [PATCH 183/192] changign lang var name, refactoring / separating out var names --- src/cmd/dev.go | 4 +-- src/config/lang/english.go | 6 ++-- src/pkg/packager/composer/list.go | 46 +++++++++++++++++--------- src/pkg/packager/composer/list_test.go | 4 +-- src/pkg/packager/composer/oci.go | 2 +- 5 files changed, 39 insertions(+), 23 deletions(-) diff --git a/src/cmd/dev.go b/src/cmd/dev.go index 1d18da26e7..9f08e53bb3 100644 --- a/src/cmd/dev.go +++ b/src/cmd/dev.go @@ -229,8 +229,8 @@ var devLintCmd = &cobra.Command{ Use: "lint [ DIRECTORY ]", Args: cobra.MaximumNArgs(1), Aliases: []string{"l"}, - Short: lang.CmdPrepareLintShort, - Long: lang.CmdPrepareLintLong, + Short: lang.CmdDevLintShort, + Long: lang.CmdDevLintLong, Run: func(cmd *cobra.Command, args []string) { common.SetBaseDirectory(args, &pkgConfig) v := common.GetViper() diff --git a/src/config/lang/english.go b/src/config/lang/english.go index 989c0a0ab0..957073f956 100644 --- a/src/config/lang/english.go +++ b/src/config/lang/english.go @@ -371,9 +371,9 @@ $ zarf package publish ./path/to/dir oci://my-registry.com/my-namespace CmdPrepareFlagGitAccount = "User or organization name for the git account that the repos are created under." CmdPrepareFlagKubeVersion = "Override the default helm template KubeVersion when performing a package chart template" - CmdPrepareLintShort = "Lints the given package for valid schema and recommended practices" - CmdPrepareLintLong = "Verifies the package schema, checks if any variables won't be evaluated, and checks for unpinned images/repos/files" - CmdPrepareLintErr = "Unable to lint package: %s" + CmdDevLintShort = "Lints the given package for valid schema and recommended practices" + CmdDevLintLong = "Verifies the package schema, checks if any variables won't be evaluated, and checks for unpinned images/repos/files" + CmdDevLintErr = "Unable to lint package: %s" // zarf tools CmdToolsShort = "Collection of additional tools to make airgap easier" diff --git a/src/pkg/packager/composer/list.go b/src/pkg/packager/composer/list.go index 4004b7290d..c9b490fd75 100644 --- a/src/pkg/packager/composer/list.go +++ b/src/pkg/packager/composer/list.go @@ -27,7 +27,8 @@ type Node struct { vars []types.ZarfPackageVariable consts []types.ZarfPackageConstant - relativeToHeadOrURL string + relativeToHead string + importURL string originalPackageName string prev *Node @@ -46,7 +47,10 @@ func (n *Node) GetOriginalPackageName() string { // GetRelativeToHeadOrURL gets the path from downstream zarf file to upstream imported zarf file func (n *Node) GetRelativeToHeadOrURL() string { - return n.relativeToHeadOrURL + if n.importURL != "" { + return n.importURL + } + return n.relativeToHead } // Next returns next node in the chain @@ -89,12 +93,13 @@ func (ic *ImportChain) Tail() *Node { } func (ic *ImportChain) append(c types.ZarfComponent, index int, originalPackageName string, - relativeToHeadOrURL string, vars []types.ZarfPackageVariable, consts []types.ZarfPackageConstant) { + relativeToHead string, importURL string, vars []types.ZarfPackageVariable, consts []types.ZarfPackageConstant) { node := &Node{ ZarfComponent: c, index: index, originalPackageName: originalPackageName, - relativeToHeadOrURL: relativeToHeadOrURL, + relativeToHead: relativeToHead, + importURL: importURL, vars: vars, consts: consts, prev: nil, @@ -119,7 +124,7 @@ func NewImportChain(head types.ZarfComponent, index int, originalPackageName, ar return ic, fmt.Errorf("cannot build import chain: architecture must be provided") } - ic.append(head, index, originalPackageName, ".", nil, nil) + ic.append(head, index, originalPackageName, ".", "", nil, nil) history := []string{} @@ -150,27 +155,28 @@ func NewImportChain(head types.ZarfComponent, index int, originalPackageName, ar var pkg types.ZarfPackage - var relativeToHeadOrURL string + var relativeToHead string + var importURL string if isLocal { history = append(history, node.Import.Path) - relativeToHeadOrURL = filepath.Join(history...) + relativeToHead = filepath.Join(history...) // prevent circular imports (including self-imports) // this is O(n^2) but the import chain should be small prev := node for prev != nil { - if prev.relativeToHeadOrURL == relativeToHeadOrURL { + if prev.relativeToHead == relativeToHead { return ic, fmt.Errorf("detected circular import chain: %s", strings.Join(history, " -> ")) } prev = prev.prev } // this assumes the composed package is following the zarf layout - if err := utils.ReadYaml(filepath.Join(relativeToHeadOrURL, layout.ZarfYAML), &pkg); err != nil { + if err := utils.ReadYaml(filepath.Join(relativeToHead, layout.ZarfYAML), &pkg); err != nil { return ic, err } } else if isRemote { - relativeToHeadOrURL = node.Import.URL + importURL = node.Import.URL remote, err := ic.getRemote(node.Import.URL) if err != nil { return ic, err @@ -195,12 +201,22 @@ func NewImportChain(head types.ZarfComponent, index int, originalPackageName, ar } if len(found) == 0 { - return ic, fmt.Errorf("component %q not found in %q", name, relativeToHeadOrURL) + componentNotFound := "component %q not found in %q" + if isLocal { + return ic, fmt.Errorf(componentNotFound, name, relativeToHead) + } else if isRemote { + return ic, fmt.Errorf(componentNotFound, name, importURL) + } } else if len(found) > 1 { - return ic, fmt.Errorf("multiple components named %q found in %q satisfying %q", name, relativeToHeadOrURL, arch) + multipleComponentsFound := "multiple components named %q found in %q satisfying %q" + if isLocal { + return ic, fmt.Errorf(multipleComponentsFound, name, relativeToHead, arch) + } else if isRemote { + return ic, fmt.Errorf(multipleComponentsFound, name, importURL, arch) + } } - ic.append(found[0], index[0], pkg.Metadata.Name, relativeToHeadOrURL, pkg.Variables, pkg.Constants) + ic.append(found[0], index[0], pkg.Metadata.Name, relativeToHead, importURL, pkg.Variables, pkg.Constants) node = node.next } return ic, nil @@ -274,7 +290,7 @@ func (ic *ImportChain) Compose() (composed *types.ZarfComponent, err error) { // start overriding with the tail node node := ic.tail for node != nil { - fixPaths(&node.ZarfComponent, node.relativeToHeadOrURL) + fixPaths(&node.ZarfComponent, node.relativeToHead) // perform overrides here err := overrideMetadata(composed, node.ZarfComponent) @@ -286,7 +302,7 @@ func (ic *ImportChain) Compose() (composed *types.ZarfComponent, err error) { overrideResources(composed, node.ZarfComponent) overrideActions(composed, node.ZarfComponent) - composeExtensions(composed, node.ZarfComponent, node.relativeToHeadOrURL) + composeExtensions(composed, node.ZarfComponent, node.relativeToHead) node = node.prev } diff --git a/src/pkg/packager/composer/list_test.go b/src/pkg/packager/composer/list_test.go index d80f3633c3..f904820bd3 100644 --- a/src/pkg/packager/composer/list_test.go +++ b/src/pkg/packager/composer/list_test.go @@ -447,12 +447,12 @@ func createChainFromSlice(components []types.ZarfComponent) (ic *ImportChain) { return ic } - ic.append(components[0], 0, testPackageName, ".", nil, nil) + ic.append(components[0], 0, testPackageName, ".", "", nil, nil) history := []string{} for idx := 1; idx < len(components); idx++ { history = append(history, components[idx-1].Import.Path) - ic.append(components[idx], idx, testPackageName, filepath.Join(history...), nil, nil) + ic.append(components[idx], idx, testPackageName, filepath.Join(history...), "", nil, nil) } return ic diff --git a/src/pkg/packager/composer/oci.go b/src/pkg/packager/composer/oci.go index 9c62ec4612..b0170c3f9d 100644 --- a/src/pkg/packager/composer/oci.go +++ b/src/pkg/packager/composer/oci.go @@ -114,7 +114,7 @@ func (ic *ImportChain) fetchOCISkeleton() error { } // the tail node is the only node whose relativeToHeadOrURL is based solely upon cwd<->cache // contrary to the other nodes, which are based upon the previous node - ic.tail.relativeToHeadOrURL = rel + ic.tail.relativeToHead = rel if oci.IsEmptyDescriptor(componentDesc) { // nothing was fetched, nothing to extract From 12e31fab48f5d4df4112b72c85da0b3a00807f23 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Fri, 15 Dec 2023 17:50:29 +0000 Subject: [PATCH 184/192] renames --- src/pkg/packager/composer/list.go | 2 +- src/pkg/packager/composer/oci.go | 2 +- src/pkg/packager/composer/pathfixer.go | 30 +++++++++++++------------- 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/src/pkg/packager/composer/list.go b/src/pkg/packager/composer/list.go index c9b490fd75..64acbec041 100644 --- a/src/pkg/packager/composer/list.go +++ b/src/pkg/packager/composer/list.go @@ -45,7 +45,7 @@ func (n *Node) GetOriginalPackageName() string { return n.originalPackageName } -// GetRelativeToHeadOrURL gets the path from downstream zarf file to upstream imported zarf file +// GetRelativeToHeadOrURL gets the path from the base zarf file to the imported zarf file func (n *Node) GetRelativeToHeadOrURL() string { if n.importURL != "" { return n.importURL diff --git a/src/pkg/packager/composer/oci.go b/src/pkg/packager/composer/oci.go index b0170c3f9d..5a7b4637ed 100644 --- a/src/pkg/packager/composer/oci.go +++ b/src/pkg/packager/composer/oci.go @@ -112,7 +112,7 @@ func (ic *ImportChain) fetchOCISkeleton() error { if err != nil { return err } - // the tail node is the only node whose relativeToHeadOrURL is based solely upon cwd<->cache + // the tail node is the only node whose relativeToHead is based solely upon cwd<->cache // contrary to the other nodes, which are based upon the previous node ic.tail.relativeToHead = rel diff --git a/src/pkg/packager/composer/pathfixer.go b/src/pkg/packager/composer/pathfixer.go index c06bcb6e46..b52c2c3257 100644 --- a/src/pkg/packager/composer/pathfixer.go +++ b/src/pkg/packager/composer/pathfixer.go @@ -20,30 +20,30 @@ func makePathRelativeTo(path, relativeTo string) string { return filepath.Join(relativeTo, path) } -func fixPaths(child *types.ZarfComponent, relativeToHeadOrURL string) { +func fixPaths(child *types.ZarfComponent, relativeToHead string) { for fileIdx, file := range child.Files { - composed := makePathRelativeTo(file.Source, relativeToHeadOrURL) + composed := makePathRelativeTo(file.Source, relativeToHead) child.Files[fileIdx].Source = composed } for chartIdx, chart := range child.Charts { for valuesIdx, valuesFile := range chart.ValuesFiles { - composed := makePathRelativeTo(valuesFile, relativeToHeadOrURL) + composed := makePathRelativeTo(valuesFile, relativeToHead) child.Charts[chartIdx].ValuesFiles[valuesIdx] = composed } if child.Charts[chartIdx].LocalPath != "" { - composed := makePathRelativeTo(chart.LocalPath, relativeToHeadOrURL) + composed := makePathRelativeTo(chart.LocalPath, relativeToHead) child.Charts[chartIdx].LocalPath = composed } } for manifestIdx, manifest := range child.Manifests { for fileIdx, file := range manifest.Files { - composed := makePathRelativeTo(file, relativeToHeadOrURL) + composed := makePathRelativeTo(file, relativeToHead) child.Manifests[manifestIdx].Files[fileIdx] = composed } for kustomizeIdx, kustomization := range manifest.Kustomizations { - composed := makePathRelativeTo(kustomization, relativeToHeadOrURL) + composed := makePathRelativeTo(kustomization, relativeToHead) // kustomizations can use non-standard urls, so we need to check if the composed path exists on the local filesystem abs, _ := filepath.Abs(composed) invalid := utils.InvalidPath(abs) @@ -54,31 +54,31 @@ func fixPaths(child *types.ZarfComponent, relativeToHeadOrURL string) { } for dataInjectionsIdx, dataInjection := range child.DataInjections { - composed := makePathRelativeTo(dataInjection.Source, relativeToHeadOrURL) + composed := makePathRelativeTo(dataInjection.Source, relativeToHead) child.DataInjections[dataInjectionsIdx].Source = composed } defaultDir := child.Actions.OnCreate.Defaults.Dir - child.Actions.OnCreate.Before = fixActionPaths(child.Actions.OnCreate.Before, defaultDir, relativeToHeadOrURL) - child.Actions.OnCreate.After = fixActionPaths(child.Actions.OnCreate.After, defaultDir, relativeToHeadOrURL) - child.Actions.OnCreate.OnFailure = fixActionPaths(child.Actions.OnCreate.OnFailure, defaultDir, relativeToHeadOrURL) - child.Actions.OnCreate.OnSuccess = fixActionPaths(child.Actions.OnCreate.OnSuccess, defaultDir, relativeToHeadOrURL) + child.Actions.OnCreate.Before = fixActionPaths(child.Actions.OnCreate.Before, defaultDir, relativeToHead) + child.Actions.OnCreate.After = fixActionPaths(child.Actions.OnCreate.After, defaultDir, relativeToHead) + child.Actions.OnCreate.OnFailure = fixActionPaths(child.Actions.OnCreate.OnFailure, defaultDir, relativeToHead) + child.Actions.OnCreate.OnSuccess = fixActionPaths(child.Actions.OnCreate.OnSuccess, defaultDir, relativeToHead) // deprecated if child.DeprecatedCosignKeyPath != "" { - composed := makePathRelativeTo(child.DeprecatedCosignKeyPath, relativeToHeadOrURL) + composed := makePathRelativeTo(child.DeprecatedCosignKeyPath, relativeToHead) child.DeprecatedCosignKeyPath = composed } } // fixActionPaths takes a slice of actions and mutates the Dir to be relative to the head node -func fixActionPaths(actions []types.ZarfComponentAction, defaultDir, relativeToHeadOrURL string) []types.ZarfComponentAction { +func fixActionPaths(actions []types.ZarfComponentAction, defaultDir, relativeToHead string) []types.ZarfComponentAction { for actionIdx, action := range actions { var composed string if action.Dir != nil { - composed = makePathRelativeTo(*action.Dir, relativeToHeadOrURL) + composed = makePathRelativeTo(*action.Dir, relativeToHead) } else { - composed = makePathRelativeTo(defaultDir, relativeToHeadOrURL) + composed = makePathRelativeTo(defaultDir, relativeToHead) } actions[actionIdx].Dir = &composed } From 1479cc835f4225ba62ffd5b9ff60d5a5701ebc44 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Mon, 18 Dec 2023 17:15:48 +0000 Subject: [PATCH 185/192] moving cmds to dev over lint --- src/cmd/dev.go | 50 +++++++++++++++++++------------------- src/config/lang/english.go | 38 ++++++++++++++--------------- 2 files changed, 44 insertions(+), 44 deletions(-) diff --git a/src/cmd/dev.go b/src/cmd/dev.go index 9f08e53bb3..63426692ea 100644 --- a/src/cmd/dev.go +++ b/src/cmd/dev.go @@ -63,7 +63,7 @@ var devDeployCmd = &cobra.Command{ var devTransformGitLinksCmd = &cobra.Command{ Use: "patch-git HOST FILE", Aliases: []string{"p"}, - Short: lang.CmdPreparePatchGitShort, + Short: lang.CmdDevPatchGitShort, Args: cobra.ExactArgs(2), Run: func(cmd *cobra.Command, args []string) { host, fileName := args[0], args[1] @@ -71,7 +71,7 @@ var devTransformGitLinksCmd = &cobra.Command{ // Read the contents of the given file content, err := os.ReadFile(fileName) if err != nil { - message.Fatalf(err, lang.CmdPreparePatchGitFileReadErr, fileName) + message.Fatalf(err, lang.CmdDevPatchGitFileReadErr, fileName) } pkgConfig.InitOpts.GitServer.Address = host @@ -86,17 +86,17 @@ var devTransformGitLinksCmd = &cobra.Command{ // Ask the user before this destructive action confirm := false prompt := &survey.Confirm{ - Message: fmt.Sprintf(lang.CmdPreparePatchGitOverwritePrompt, fileName), + Message: fmt.Sprintf(lang.CmdDevPatchGitOverwritePrompt, fileName), } if err := survey.AskOne(prompt, &confirm); err != nil { - message.Fatalf(nil, lang.CmdPreparePatchGitOverwriteErr, err.Error()) + message.Fatalf(nil, lang.CmdDevPatchGitOverwriteErr, err.Error()) } if confirm { // Overwrite the file err = os.WriteFile(fileName, []byte(processedText), 0640) if err != nil { - message.Fatal(err, lang.CmdPreparePatchGitFileWriteErr) + message.Fatal(err, lang.CmdDevPatchGitFileWriteErr) } } @@ -106,7 +106,7 @@ var devTransformGitLinksCmd = &cobra.Command{ var devSha256SumCmd = &cobra.Command{ Use: "sha256sum { FILE | URL }", Aliases: []string{"s"}, - Short: lang.CmdPrepareSha256sumShort, + Short: lang.CmdDevSha256sumShort, Args: cobra.ExactArgs(1), Run: func(cmd *cobra.Command, args []string) { fileName := args[0] @@ -116,11 +116,11 @@ var devSha256SumCmd = &cobra.Command{ var err error if helpers.IsURL(fileName) { - message.Warn(lang.CmdPrepareSha256sumRemoteWarning) + message.Warn(lang.CmdDevSha256sumRemoteWarning) fileBase, err := helpers.ExtractBasePathFromURL(fileName) if err != nil { - message.Fatalf(err, lang.CmdPrepareSha256sumHashErr, err.Error()) + message.Fatalf(err, lang.CmdDevSha256sumHashErr, err.Error()) } if fileBase == "" { @@ -129,13 +129,13 @@ var devSha256SumCmd = &cobra.Command{ tmp, err = utils.MakeTempDir(config.CommonOptions.TempDirectory) if err != nil { - message.Fatalf(err, lang.CmdPrepareSha256sumHashErr, err.Error()) + message.Fatalf(err, lang.CmdDevSha256sumHashErr, err.Error()) } downloadPath := filepath.Join(tmp, fileBase) err = utils.DownloadToFile(fileName, downloadPath, "") if err != nil { - message.Fatalf(err, lang.CmdPrepareSha256sumHashErr, err.Error()) + message.Fatalf(err, lang.CmdDevSha256sumHashErr, err.Error()) } fileName = downloadPath @@ -147,7 +147,7 @@ var devSha256SumCmd = &cobra.Command{ if tmp == "" { tmp, err = utils.MakeTempDir(config.CommonOptions.TempDirectory) if err != nil { - message.Fatalf(err, lang.CmdPrepareSha256sumHashErr, err.Error()) + message.Fatalf(err, lang.CmdDevSha256sumHashErr, err.Error()) } defer os.RemoveAll(tmp) } @@ -156,7 +156,7 @@ var devSha256SumCmd = &cobra.Command{ err = archiver.Extract(fileName, extractPath, tmp) if err != nil { - message.Fatalf(err, lang.CmdPrepareSha256sumHashErr, err.Error()) + message.Fatalf(err, lang.CmdDevSha256sumHashErr, err.Error()) } fileName = extractedFile @@ -164,14 +164,14 @@ var devSha256SumCmd = &cobra.Command{ data, err = os.Open(fileName) if err != nil { - message.Fatalf(err, lang.CmdPrepareSha256sumHashErr, err.Error()) + message.Fatalf(err, lang.CmdDevSha256sumHashErr, err.Error()) } defer data.Close() var hash string hash, err = helpers.GetSHA256Hash(data) if err != nil { - message.Fatalf(err, lang.CmdPrepareSha256sumHashErr, err.Error()) + message.Fatalf(err, lang.CmdDevSha256sumHashErr, err.Error()) } else { fmt.Println(hash) } @@ -182,8 +182,8 @@ var devFindImagesCmd = &cobra.Command{ Use: "find-images [ PACKAGE ]", Aliases: []string{"f"}, Args: cobra.MaximumNArgs(1), - Short: lang.CmdPrepareFindImagesShort, - Long: lang.CmdPrepareFindImagesLong, + Short: lang.CmdDevFindImagesShort, + Long: lang.CmdDevFindImagesLong, Run: func(cmd *cobra.Command, args []string) { // If a directory was provided, use that as the base directory common.SetBaseDirectory(args, &pkgConfig) @@ -199,7 +199,7 @@ var devFindImagesCmd = &cobra.Command{ // Find all the images the package might need if _, err := pkgClient.FindImages(); err != nil { - message.Fatalf(err, lang.CmdPrepareFindImagesErr, err.Error()) + message.Fatalf(err, lang.CmdDevFindImagesErr, err.Error()) } }, } @@ -208,8 +208,8 @@ var devGenConfigFileCmd = &cobra.Command{ Use: "generate-config [ FILENAME ]", Aliases: []string{"gc"}, Args: cobra.MaximumNArgs(1), - Short: lang.CmdPrepareGenerateConfigShort, - Long: lang.CmdPrepareGenerateConfigLong, + Short: lang.CmdDevGenerateConfigShort, + Long: lang.CmdDevGenerateConfigLong, Run: func(cmd *cobra.Command, args []string) { fileName := "zarf-config.toml" @@ -220,7 +220,7 @@ var devGenConfigFileCmd = &cobra.Command{ v := common.GetViper() if err := v.SafeWriteConfigAs(fileName); err != nil { - message.Fatalf(err, lang.CmdPrepareGenerateConfigErr, fileName) + message.Fatalf(err, lang.CmdDevGenerateConfigErr, fileName) } }, } @@ -260,17 +260,17 @@ func init() { bindDevDeployFlags(v) - devSha256SumCmd.Flags().StringVarP(&extractPath, "extract-path", "e", "", lang.CmdPrepareFlagExtractPath) + devSha256SumCmd.Flags().StringVarP(&extractPath, "extract-path", "e", "", lang.CmdDevFlagExtractPath) - devFindImagesCmd.Flags().StringVarP(&pkgConfig.FindImagesOpts.RepoHelmChartPath, "repo-chart-path", "p", "", lang.CmdPrepareFlagRepoChartPath) + devFindImagesCmd.Flags().StringVarP(&pkgConfig.FindImagesOpts.RepoHelmChartPath, "repo-chart-path", "p", "", lang.CmdDevFlagRepoChartPath) // use the package create config for this and reset it here to avoid overwriting the config.CreateOptions.SetVariables - devFindImagesCmd.Flags().StringToStringVar(&pkgConfig.CreateOpts.SetVariables, "set", v.GetStringMapString(common.VPkgCreateSet), lang.CmdPrepareFlagSet) + devFindImagesCmd.Flags().StringToStringVar(&pkgConfig.CreateOpts.SetVariables, "set", v.GetStringMapString(common.VPkgCreateSet), lang.CmdDevFlagSet) // allow for the override of the default helm KubeVersion - devFindImagesCmd.Flags().StringVar(&pkgConfig.FindImagesOpts.KubeVersionOverride, "kube-version", "", lang.CmdPrepareFlagKubeVersion) + devFindImagesCmd.Flags().StringVar(&pkgConfig.FindImagesOpts.KubeVersionOverride, "kube-version", "", lang.CmdDevFlagKubeVersion) devLintCmd.Flags().StringToStringVar(&pkgConfig.CreateOpts.SetVariables, "set", v.GetStringMapString(common.VPkgCreateSet), lang.CmdPackageCreateFlagSet) devLintCmd.Flags().StringVarP(&pkgConfig.CreateOpts.Flavor, "flavor", "f", v.GetString(common.VPkgCreateFlavor), lang.CmdPackageCreateFlagFlavor) - devTransformGitLinksCmd.Flags().StringVar(&pkgConfig.InitOpts.GitServer.PushUsername, "git-account", config.ZarfGitPushUser, lang.CmdPrepareFlagGitAccount) + devTransformGitLinksCmd.Flags().StringVar(&pkgConfig.InitOpts.GitServer.PushUsername, "git-account", config.ZarfGitPushUser, lang.CmdDevFlagGitAccount) } func bindDevDeployFlags(v *viper.Viper) { diff --git a/src/config/lang/english.go b/src/config/lang/english.go index 957073f956..60f4d7617a 100644 --- a/src/config/lang/english.go +++ b/src/config/lang/english.go @@ -342,34 +342,34 @@ $ zarf package publish ./path/to/dir oci://my-registry.com/my-namespace CmdDevDeployFlagNoYolo = "Disable the YOLO mode default override and create / deploy the package as-defined" CmdDevDeployErr = "Failed to dev deploy: %s" - CmdPreparePatchGitShort = "Converts all .git URLs to the specified Zarf HOST and with the Zarf URL pattern in a given FILE. NOTE:\n" + + CmdDevPatchGitShort = "Converts all .git URLs to the specified Zarf HOST and with the Zarf URL pattern in a given FILE. NOTE:\n" + "This should only be used for manifests that are not mutated by the Zarf Agent Mutating Webhook." - CmdPreparePatchGitOverwritePrompt = "Overwrite the file %s with these changes?" - CmdPreparePatchGitOverwriteErr = "Confirm overwrite canceled: %s" - CmdPreparePatchGitFileReadErr = "Unable to read the file %s" - CmdPreparePatchGitFileWriteErr = "Unable to write the changes back to the file" + CmdDevPatchGitOverwritePrompt = "Overwrite the file %s with these changes?" + CmdDevPatchGitOverwriteErr = "Confirm overwrite canceled: %s" + CmdDevPatchGitFileReadErr = "Unable to read the file %s" + CmdDevPatchGitFileWriteErr = "Unable to write the changes back to the file" - CmdPrepareSha256sumShort = "Generates a SHA256SUM for the given file" - CmdPrepareSha256sumRemoteWarning = "This is a remote source. If a published checksum is available you should use that rather than calculating it directly from the remote link." - CmdPrepareSha256sumHashErr = "Unable to compute the SHA256SUM hash: %s" + CmdDevSha256sumShort = "Generates a SHA256SUM for the given file" + CmdDevSha256sumRemoteWarning = "This is a remote source. If a published checksum is available you should use that rather than calculating it directly from the remote link." + CmdDevSha256sumHashErr = "Unable to compute the SHA256SUM hash: %s" - CmdPrepareFindImagesShort = "Evaluates components in a Zarf file to identify images specified in their helm charts and manifests" - CmdPrepareFindImagesLong = "Evaluates components in a Zarf file to identify images specified in their helm charts and manifests.\n\n" + + CmdDevFindImagesShort = "Evaluates components in a Zarf file to identify images specified in their helm charts and manifests" + CmdDevFindImagesLong = "Evaluates components in a Zarf file to identify images specified in their helm charts and manifests.\n\n" + "Components that have repos that host helm charts can be processed by providing the --repo-chart-path." - CmdPrepareFindImagesErr = "Unable to find images: %s" + CmdDevFindImagesErr = "Unable to find images: %s" - CmdPrepareGenerateConfigShort = "Generates a config file for Zarf" - CmdPrepareGenerateConfigLong = "Generates a Zarf config file for controlling how the Zarf CLI operates. Optionally accepts a filename to write the config to.\n\n" + + CmdDevGenerateConfigShort = "Generates a config file for Zarf" + CmdDevGenerateConfigLong = "Generates a Zarf config file for controlling how the Zarf CLI operates. Optionally accepts a filename to write the config to.\n\n" + "The extension will determine the format of the config file, e.g. env-1.yaml, env-2.json, env-3.toml etc.\n" + "Accepted extensions are json, toml, yaml.\n\n" + "NOTE: This file must not already exist. If no filename is provided, the config will be written to the current working directory as zarf-config.toml." - CmdPrepareGenerateConfigErr = "Unable to write the config file %s, make sure the file doesn't already exist" + CmdDevGenerateConfigErr = "Unable to write the config file %s, make sure the file doesn't already exist" - CmdPrepareFlagExtractPath = `The path inside of an archive to use to calculate the sha256sum (i.e. for use with "files.extractPath")` - CmdPrepareFlagSet = "Specify package variables to set on the command line (KEY=value). Note, if using a config file, this will be set by [package.create.set]." - CmdPrepareFlagRepoChartPath = `If git repos hold helm charts, often found with gitops tools, specify the chart path, e.g. "/" or "/chart"` - CmdPrepareFlagGitAccount = "User or organization name for the git account that the repos are created under." - CmdPrepareFlagKubeVersion = "Override the default helm template KubeVersion when performing a package chart template" + CmdDevFlagExtractPath = `The path inside of an archive to use to calculate the sha256sum (i.e. for use with "files.extractPath")` + CmdDevFlagSet = "Specify package variables to set on the command line (KEY=value). Note, if using a config file, this will be set by [package.create.set]." + CmdDevFlagRepoChartPath = `If git repos hold helm charts, often found with gitops tools, specify the chart path, e.g. "/" or "/chart"` + CmdDevFlagGitAccount = "User or organization name for the git account that the repos are created under." + CmdDevFlagKubeVersion = "Override the default helm template KubeVersion when performing a package chart template" CmdDevLintShort = "Lints the given package for valid schema and recommended practices" CmdDevLintLong = "Verifies the package schema, checks if any variables won't be evaluated, and checks for unpinned images/repos/files" From deb0fa3e0881a0a38228b697ab272cbbe8ac1af5 Mon Sep 17 00:00:00 2001 From: Austin Abro <37223396+AustinAbro321@users.noreply.github.com> Date: Mon, 18 Dec 2023 12:18:08 -0500 Subject: [PATCH 186/192] Update Makefile Co-authored-by: Lucas Rodriguez --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index f62031004f..27f0c8ed28 100644 --- a/Makefile +++ b/Makefile @@ -96,7 +96,7 @@ docs-and-schema: ## Generate the Zarf Documentation and Schema hack/gen-cli-docs.sh ZARF_CONFIG=hack/empty-config.toml hack/create-zarf-schema.sh -lint-packages-and-examples: build-cli-for-system ## Recurisvely lint all zarf.yaml files in the repo except for those dedicated to tests +lint-packages-and-examples: build-cli-for-system ## Recursively lint all zarf.yaml files in the repo except for those dedicated to tests hack/lint_all_zarf_packages.sh $(ZARF_BIN) # INTERNAL: a shim used to build the agent image only if needed on Windows using the `test` command From 9473c25386060f35d10c5289e3440cbd6a5bb62e Mon Sep 17 00:00:00 2001 From: Austin Abro <37223396+AustinAbro321@users.noreply.github.com> Date: Mon, 18 Dec 2023 12:18:19 -0500 Subject: [PATCH 187/192] Update src/pkg/packager/composer/list.go Co-authored-by: Wayne Starr --- src/pkg/packager/composer/list.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pkg/packager/composer/list.go b/src/pkg/packager/composer/list.go index 64acbec041..0331004d60 100644 --- a/src/pkg/packager/composer/list.go +++ b/src/pkg/packager/composer/list.go @@ -46,7 +46,7 @@ func (n *Node) GetOriginalPackageName() string { } // GetRelativeToHeadOrURL gets the path from the base zarf file to the imported zarf file -func (n *Node) GetRelativeToHeadOrURL() string { +func (n *Node) ImportLocation() string { if n.importURL != "" { return n.importURL } From be35e72b9282b9ee5fe31029808d3bc7adf41fa1 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Mon, 18 Dec 2023 18:47:51 +0000 Subject: [PATCH 188/192] refactor name --- src/pkg/packager/composer/list.go | 2 +- src/pkg/packager/lint/lint.go | 20 ++++++++++---------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/pkg/packager/composer/list.go b/src/pkg/packager/composer/list.go index 0331004d60..55e2c3b548 100644 --- a/src/pkg/packager/composer/list.go +++ b/src/pkg/packager/composer/list.go @@ -45,7 +45,7 @@ func (n *Node) GetOriginalPackageName() string { return n.originalPackageName } -// GetRelativeToHeadOrURL gets the path from the base zarf file to the imported zarf file +// ImportLocation gets the path from the base zarf file to the imported zarf file func (n *Node) ImportLocation() string { if n.importURL != "" { return n.importURL diff --git a/src/pkg/packager/lint/lint.go b/src/pkg/packager/lint/lint.go index 4a2fafc640..bc8b5df336 100644 --- a/src/pkg/packager/lint/lint.go +++ b/src/pkg/packager/lint/lint.go @@ -109,7 +109,7 @@ func fillComponentTemplate(validator *Validator, node *composer.Node, createOpts if err != nil { validator.addWarning(validatorMessage{ description: err.Error(), - packageRelPath: node.GetRelativeToHeadOrURL(), + packageRelPath: node.ImportLocation(), packageName: node.GetOriginalPackageName(), }) } @@ -120,7 +120,7 @@ func fillComponentTemplate(validator *Validator, node *composer.Node, createOpts if err != nil { validator.addWarning(validatorMessage{ description: err.Error(), - packageRelPath: node.GetRelativeToHeadOrURL(), + packageRelPath: node.ImportLocation(), packageName: node.GetOriginalPackageName(), }) } @@ -129,7 +129,7 @@ func fillComponentTemplate(validator *Validator, node *composer.Node, createOpts if deprecated { validator.addWarning(validatorMessage{ description: fmt.Sprintf(lang.PkgValidateTemplateDeprecation, key, key, key), - packageRelPath: node.GetRelativeToHeadOrURL(), + packageRelPath: node.ImportLocation(), packageName: node.GetOriginalPackageName(), }) } @@ -137,7 +137,7 @@ func fillComponentTemplate(validator *Validator, node *composer.Node, createOpts if !present { validator.addWarning(validatorMessage{ description: lang.UnsetVarLintWarning, - packageRelPath: node.GetRelativeToHeadOrURL(), + packageRelPath: node.ImportLocation(), packageName: node.GetOriginalPackageName(), }) } @@ -183,7 +183,7 @@ func checkForUnpinnedRepos(validator *Validator, node *composer.Node) { if !isPinnedRepo(repo) { validator.addWarning(validatorMessage{ yqPath: repoYqPath, - packageRelPath: node.GetRelativeToHeadOrURL(), + packageRelPath: node.ImportLocation(), packageName: node.GetOriginalPackageName(), description: "Unpinned repository", item: repo, @@ -199,7 +199,7 @@ func checkForUnpinnedImages(validator *Validator, node *composer.Node) { if err != nil { validator.addError(validatorMessage{ yqPath: imageYqPath, - packageRelPath: node.GetRelativeToHeadOrURL(), + packageRelPath: node.ImportLocation(), packageName: node.GetOriginalPackageName(), description: "Invalid image reference", item: image, @@ -209,7 +209,7 @@ func checkForUnpinnedImages(validator *Validator, node *composer.Node) { if !pinnedImage { validator.addWarning(validatorMessage{ yqPath: imageYqPath, - packageRelPath: node.GetRelativeToHeadOrURL(), + packageRelPath: node.ImportLocation(), packageName: node.GetOriginalPackageName(), description: "Image not pinned with digest", item: image, @@ -224,7 +224,7 @@ func checkForUnpinnedFiles(validator *Validator, node *composer.Node) { if file.Shasum == "" && helpers.IsURL(file.Source) { validator.addWarning(validatorMessage{ yqPath: fileYqPath, - packageRelPath: node.GetRelativeToHeadOrURL(), + packageRelPath: node.ImportLocation(), packageName: node.GetOriginalPackageName(), description: "No shasum for remote file", item: file.Source, @@ -237,7 +237,7 @@ func checkForVarInComponentImport(validator *Validator, node *composer.Node) { if strings.Contains(node.Import.Path, types.ZarfPackageTemplatePrefix) { validator.addWarning(validatorMessage{ yqPath: fmt.Sprintf(".components.[%d].import.path", node.GetIndex()), - packageRelPath: node.GetRelativeToHeadOrURL(), + packageRelPath: node.ImportLocation(), packageName: node.GetOriginalPackageName(), description: "Zarf does not evaluate variables at component.x.import.path", item: node.Import.Path, @@ -246,7 +246,7 @@ func checkForVarInComponentImport(validator *Validator, node *composer.Node) { if strings.Contains(node.Import.URL, types.ZarfPackageTemplatePrefix) { validator.addWarning(validatorMessage{ yqPath: fmt.Sprintf(".components.[%d].import.url", node.GetIndex()), - packageRelPath: node.GetRelativeToHeadOrURL(), + packageRelPath: node.ImportLocation(), packageName: node.GetOriginalPackageName(), description: "Zarf does not evaluate variables at component.x.import.url", item: node.Import.URL, From 8702613f67cdde5fbd425504ecc943dfe19a1280 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Mon, 18 Dec 2023 19:59:55 +0000 Subject: [PATCH 189/192] making remotes deterministic --- src/pkg/packager/composer/list.go | 14 +++++++------- src/pkg/packager/composer/list_test.go | 4 ++-- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/pkg/packager/composer/list.go b/src/pkg/packager/composer/list.go index 55e2c3b548..f46235f212 100644 --- a/src/pkg/packager/composer/list.go +++ b/src/pkg/packager/composer/list.go @@ -28,7 +28,6 @@ type Node struct { consts []types.ZarfPackageConstant relativeToHead string - importURL string originalPackageName string prev *Node @@ -47,8 +46,10 @@ func (n *Node) GetOriginalPackageName() string { // ImportLocation gets the path from the base zarf file to the imported zarf file func (n *Node) ImportLocation() string { - if n.importURL != "" { - return n.importURL + if n.prev != nil { + if n.prev.ZarfComponent.Import.URL != "" { + return n.prev.ZarfComponent.Import.URL + } } return n.relativeToHead } @@ -93,13 +94,12 @@ func (ic *ImportChain) Tail() *Node { } func (ic *ImportChain) append(c types.ZarfComponent, index int, originalPackageName string, - relativeToHead string, importURL string, vars []types.ZarfPackageVariable, consts []types.ZarfPackageConstant) { + relativeToHead string, vars []types.ZarfPackageVariable, consts []types.ZarfPackageConstant) { node := &Node{ ZarfComponent: c, index: index, originalPackageName: originalPackageName, relativeToHead: relativeToHead, - importURL: importURL, vars: vars, consts: consts, prev: nil, @@ -124,7 +124,7 @@ func NewImportChain(head types.ZarfComponent, index int, originalPackageName, ar return ic, fmt.Errorf("cannot build import chain: architecture must be provided") } - ic.append(head, index, originalPackageName, ".", "", nil, nil) + ic.append(head, index, originalPackageName, ".", nil, nil) history := []string{} @@ -216,7 +216,7 @@ func NewImportChain(head types.ZarfComponent, index int, originalPackageName, ar } } - ic.append(found[0], index[0], pkg.Metadata.Name, relativeToHead, importURL, pkg.Variables, pkg.Constants) + ic.append(found[0], index[0], pkg.Metadata.Name, relativeToHead, pkg.Variables, pkg.Constants) node = node.next } return ic, nil diff --git a/src/pkg/packager/composer/list_test.go b/src/pkg/packager/composer/list_test.go index f904820bd3..d80f3633c3 100644 --- a/src/pkg/packager/composer/list_test.go +++ b/src/pkg/packager/composer/list_test.go @@ -447,12 +447,12 @@ func createChainFromSlice(components []types.ZarfComponent) (ic *ImportChain) { return ic } - ic.append(components[0], 0, testPackageName, ".", "", nil, nil) + ic.append(components[0], 0, testPackageName, ".", nil, nil) history := []string{} for idx := 1; idx < len(components); idx++ { history = append(history, components[idx-1].Import.Path) - ic.append(components[idx], idx, testPackageName, filepath.Join(history...), "", nil, nil) + ic.append(components[idx], idx, testPackageName, filepath.Join(history...), nil, nil) } return ic From 4c08da84cc8ed7126865c21b3d94d561ff47d16d Mon Sep 17 00:00:00 2001 From: Austin Abro <37223396+AustinAbro321@users.noreply.github.com> Date: Mon, 18 Dec 2023 15:00:07 -0500 Subject: [PATCH 190/192] Update hack/lint_all_zarf_packages.sh Co-authored-by: razzle --- hack/lint_all_zarf_packages.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hack/lint_all_zarf_packages.sh b/hack/lint_all_zarf_packages.sh index e065628ac7..358a921702 100755 --- a/hack/lint_all_zarf_packages.sh +++ b/hack/lint_all_zarf_packages.sh @@ -12,6 +12,6 @@ find "." -type f -name 'zarf.yaml' | while read -r yaml_file; do continue fi echo "Running 'zarf prepare lint' in directory: $dir" - $ZARF_BIN prepare lint $dir + $ZARF_BIN prepare lint "$dir" echo "---" done From b69842171507b6ceb4f04d6890a694f165f6bcad Mon Sep 17 00:00:00 2001 From: Austin Abro <37223396+AustinAbro321@users.noreply.github.com> Date: Mon, 18 Dec 2023 15:00:18 -0500 Subject: [PATCH 191/192] Update hack/lint_all_zarf_packages.sh Co-authored-by: razzle --- hack/lint_all_zarf_packages.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hack/lint_all_zarf_packages.sh b/hack/lint_all_zarf_packages.sh index 358a921702..5f41d3d887 100755 --- a/hack/lint_all_zarf_packages.sh +++ b/hack/lint_all_zarf_packages.sh @@ -4,7 +4,7 @@ ZARF_BIN=$1 LINT_SRC_TEST=$2 SCRIPT=$(realpath "$0") SCRIPTPATH=$(dirname "$SCRIPT") -cd $SCRIPTPATH +cd "$SCRIPTPATH" || exit cd .. find "." -type f -name 'zarf.yaml' | while read -r yaml_file; do dir=$(dirname "$yaml_file") From 3a9f65d02848b698b8aba796f2ded068d5b9b08b Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Mon, 18 Dec 2023 20:16:49 +0000 Subject: [PATCH 192/192] defaulting to build in make file --- Makefile | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 27f0c8ed28..5ab1738a69 100644 --- a/Makefile +++ b/Makefile @@ -29,7 +29,6 @@ else endif endif endif -.DEFAULT_GOAL := help CLI_VERSION ?= $(if $(shell git describe --tags),$(shell git describe --tags),"UnknownVersion") BUILD_ARGS := -s -w -X github.com/defenseunicorns/zarf/src/config.CLIVersion=$(CLI_VERSION) @@ -49,6 +48,8 @@ BUILD_DATE := $(shell date -u +'%Y-%m-%dT%H:%M:%SZ') BUILD_ARGS += -X k8s.io/component-base/version.gitCommit=$(GIT_SHA) BUILD_ARGS += -X k8s.io/component-base/version.buildDate=$(BUILD_DATE) +.DEFAULT_GOAL := build + .PHONY: help help: ## Display this help information @grep -E '^[a-zA-Z0-9_-]+:.*?## .*$$' $(MAKEFILE_LIST) \ @@ -66,6 +67,9 @@ delete-packages: ## Delete all Zarf package tarballs in the project recursively find . -type f -name 'zarf-package-*' -delete # Note: the path to the main.go file is not used due to https://github.com/golang/go/issues/51831#issuecomment-1074188363 +.PHONY: build +build: ## Build the Zarf CLI for the machines OS and architecture + $(MAKE) $(BUILD_CLI_FOR_SYSTEM) build-cli-linux-amd: ## Build the Zarf CLI for Linux on AMD64 CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags="$(BUILD_ARGS)" -o build/zarf . @@ -89,9 +93,6 @@ build-cli-linux: build-cli-linux-amd build-cli-linux-arm ## Build the Zarf CLI f build-cli: build-cli-linux-amd build-cli-linux-arm build-cli-mac-intel build-cli-mac-apple build-cli-windows-amd build-cli-windows-arm ## Build the CLI -build-cli-for-system: - $(MAKE) $(BUILD_CLI_FOR_SYSTEM) - docs-and-schema: ## Generate the Zarf Documentation and Schema hack/gen-cli-docs.sh ZARF_CONFIG=hack/empty-config.toml hack/create-zarf-schema.sh