From 682c05617df72a7580eedc44a87d09af3374e72c Mon Sep 17 00:00:00 2001 From: razzle Date: Mon, 25 Dec 2023 18:31:03 -0500 Subject: [PATCH 1/4] cleanup Signed-off-by: razzle --- src/pkg/packager/common.go | 28 +++------------------------- src/pkg/packager/create_stages.go | 22 ++++++++++++++++++++-- src/pkg/packager/sources/split.go | 10 +++++----- src/pkg/packager/sources/tarball.go | 1 - src/pkg/packager/sources/utils.go | 4 ++++ 5 files changed, 32 insertions(+), 33 deletions(-) diff --git a/src/pkg/packager/common.go b/src/pkg/packager/common.go index 3f127062ca..6543451305 100644 --- a/src/pkg/packager/common.go +++ b/src/pkg/packager/common.go @@ -171,28 +171,6 @@ func GetInitPackageName(arch string) string { return fmt.Sprintf("zarf-init-%s-%s.tar.zst", arch, config.CLIVersion) } -// GetPackageName returns the formatted name of the package. -func (p *Packager) GetPackageName() string { - if p.isInitConfig() { - return GetInitPackageName(p.arch) - } - - packageName := p.cfg.Pkg.Metadata.Name - suffix := "tar.zst" - if p.cfg.Pkg.Metadata.Uncompressed { - suffix = "tar" - } - - packageFileName := fmt.Sprintf("%s%s-%s", config.ZarfPackagePrefix, packageName, p.arch) - if p.cfg.Pkg.Build.Differential { - packageFileName = fmt.Sprintf("%s-%s-differential-%s", packageFileName, p.cfg.CreateOpts.DifferentialData.DifferentialPackageVersion, p.cfg.Pkg.Metadata.Version) - } else if p.cfg.Pkg.Metadata.Version != "" { - packageFileName = fmt.Sprintf("%s-%s", packageFileName, p.cfg.Pkg.Metadata.Version) - } - - return fmt.Sprintf("%s.%s", packageFileName, suffix) -} - // ClearTempPaths removes the temp directory and any files within it. func (p *Packager) ClearTempPaths() { // Remove the temp directory, but don't throw an error if it fails @@ -332,7 +310,7 @@ func (p *Packager) archivePackage(destinationTarball string) error { } spinner.Updatef("Wrote %s to %s", p.layout.Base, destinationTarball) - f, err := os.Stat(destinationTarball) + info, err := os.Stat(destinationTarball) if err != nil { return fmt.Errorf("unable to read the package archive: %w", err) } @@ -341,7 +319,7 @@ func (p *Packager) archivePackage(destinationTarball string) error { chunkSize := p.cfg.CreateOpts.MaxPackageSizeMB * 1000 * 1000 // If a chunk size was specified and the package is larger than the chunk size, split it into chunks. - if p.cfg.CreateOpts.MaxPackageSizeMB > 0 && f.Size() > int64(chunkSize) { + if p.cfg.CreateOpts.MaxPackageSizeMB > 0 && info.Size() > int64(chunkSize) { spinner.Updatef("Package is larger than %dMB, splitting into multiple files", p.cfg.CreateOpts.MaxPackageSizeMB) chunks, sha256sum, err := utils.SplitFile(destinationTarball, chunkSize) if err != nil { @@ -359,7 +337,7 @@ func (p *Packager) archivePackage(destinationTarball string) error { // Marshal the data into a json file. jsonData, err := json.Marshal(types.ZarfSplitPackageData{ Count: len(chunks), - Bytes: f.Size(), + Bytes: info.Size(), Sha256Sum: sha256sum, }) if err != nil { diff --git a/src/pkg/packager/create_stages.go b/src/pkg/packager/create_stages.go index 482cac8bd4..1ae3260011 100644 --- a/src/pkg/packager/create_stages.go +++ b/src/pkg/packager/create_stages.go @@ -268,8 +268,26 @@ func (p *Packager) output() error { message.ZarfCommand("package deploy %s %s", helpers.OCIURLPrefix+remote.Repo().Reference.String(), flags) message.ZarfCommand("package pull %s %s", helpers.OCIURLPrefix+remote.Repo().Reference.String(), flags) } else { - // Use the output path if the user specified it. - packageName := filepath.Join(p.cfg.CreateOpts.Output, p.GetPackageName()) + var packageName string + + if p.isInitConfig() { + packageName = GetInitPackageName(p.arch) + } else { + packageName = p.cfg.Pkg.Metadata.Name + suffix := "tar.zst" + if p.cfg.Pkg.Metadata.Uncompressed { + suffix = "tar" + } + packageFileName := fmt.Sprintf("%s%s-%s", config.ZarfPackagePrefix, packageName, p.arch) + if p.cfg.Pkg.Build.Differential { + packageFileName = fmt.Sprintf("%s-%s-differential-%s", packageFileName, p.cfg.CreateOpts.DifferentialData.DifferentialPackageVersion, p.cfg.Pkg.Metadata.Version) + } else if p.cfg.Pkg.Metadata.Version != "" { + packageFileName = fmt.Sprintf("%s-%s", packageFileName, p.cfg.Pkg.Metadata.Version) + } + + packageName = fmt.Sprintf("%s.%s", packageFileName, suffix) + packageName = filepath.Join(p.cfg.CreateOpts.Output, packageName) + } // Try to remove the package if it already exists. _ = os.Remove(packageName) diff --git a/src/pkg/packager/sources/split.go b/src/pkg/packager/sources/split.go index 296b26deb2..1aade2eee5 100644 --- a/src/pkg/packager/sources/split.go +++ b/src/pkg/packager/sources/split.go @@ -40,9 +40,9 @@ func (s *SplitTarballSource) Collect(dir string) (string, error) { // Ensure the files are in order so they are appended in the correct order sort.Strings(fileList) - reassmbled := filepath.Join(dir, filepath.Base(strings.Replace(s.PackageSource, ".part000", "", 1))) + reassembled := filepath.Join(dir, filepath.Base(strings.Replace(s.PackageSource, ".part000", "", 1))) // Create the new package - pkgFile, err := os.Create(reassmbled) + pkgFile, err := os.Create(reassembled) if err != nil { return "", fmt.Errorf("unable to create new package file: %s", err) } @@ -87,7 +87,7 @@ func (s *SplitTarballSource) Collect(dir string) (string, error) { } } - if err := utils.SHAsMatch(reassmbled, pkgData.Sha256Sum); err != nil { + if err := utils.SHAsMatch(reassembled, pkgData.Sha256Sum); err != nil { return "", fmt.Errorf("package integrity check failed: %w", err) } @@ -97,9 +97,9 @@ func (s *SplitTarballSource) Collect(dir string) (string, error) { } // communicate to the user that the package was reassembled - message.Infof("Reassembled package to: %q", reassmbled) + message.Infof("Reassembled package to: %q", reassembled) - return reassmbled, nil + return reassembled, nil } // LoadPackage loads a package from a split tarball. diff --git a/src/pkg/packager/sources/tarball.go b/src/pkg/packager/sources/tarball.go index e5b13a2749..e65d0b962a 100644 --- a/src/pkg/packager/sources/tarball.go +++ b/src/pkg/packager/sources/tarball.go @@ -45,7 +45,6 @@ func (s *TarballSource) LoadPackage(dst *layout.PackagePaths, unarchiveAll bool) pathsExtracted := []string{} - // Walk the package so that was can dynamically load a .tar or a .tar.zst without caring about filenames. err = archiver.Walk(s.PackageSource, func(f archiver.File) error { if f.IsDir() { return nil diff --git a/src/pkg/packager/sources/utils.go b/src/pkg/packager/sources/utils.go index 97337f4944..383cc262cd 100644 --- a/src/pkg/packager/sources/utils.go +++ b/src/pkg/packager/sources/utils.go @@ -92,6 +92,10 @@ func RenameFromMetadata(path string) (string, error) { return "", err } + if pkg.Metadata.Name == "" { + return "", fmt.Errorf("%q does not contain a zarf.yaml", path) + } + name := NameFromMetadata(&pkg, false) name = fmt.Sprintf("%s%s", name, ext) From b1f143b56ee27a9e08c88d69dbd0ec7bf3184001 Mon Sep 17 00:00:00 2001 From: razzle Date: Mon, 25 Dec 2023 18:37:16 -0500 Subject: [PATCH 2/4] fix make target Signed-off-by: razzle --- Makefile | 2 +- examples/composable-packages/zarf.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 5ab1738a69..6a6b216a01 100644 --- a/Makefile +++ b/Makefile @@ -97,7 +97,7 @@ docs-and-schema: ## Generate the Zarf Documentation and Schema hack/gen-cli-docs.sh ZARF_CONFIG=hack/empty-config.toml hack/create-zarf-schema.sh -lint-packages-and-examples: build-cli-for-system ## Recursively lint all zarf.yaml files in the repo except for those dedicated to tests +lint-packages-and-examples: build ## Recursively lint all zarf.yaml files in the repo except for those dedicated to tests hack/lint_all_zarf_packages.sh $(ZARF_BIN) # INTERNAL: a shim used to build the agent image only if needed on Windows using the `test` command diff --git a/examples/composable-packages/zarf.yaml b/examples/composable-packages/zarf.yaml index 34f499d670..efd16ce6d9 100644 --- a/examples/composable-packages/zarf.yaml +++ b/examples/composable-packages/zarf.yaml @@ -31,7 +31,7 @@ components: # default: false # the initial value overrides the child component import: # The URL to the skeleton package containing this component's package definition - url: oci://🦄/dos-games:1.0.0-skeleton + url: oci://🦄/dos-games:1.0.0 # Example optional custom name to point to in the imported package (default is to use this component's name) name: baseline # Un'name'd Zarf primitives will be appended to the end of the primitive's list for that component. From e754068d48267e0abdcbcb57c543c0b0742b7ace Mon Sep 17 00:00:00 2001 From: razzle Date: Tue, 2 Jan 2024 17:52:47 -0500 Subject: [PATCH 3/4] scope creep Signed-off-by: razzle --- src/pkg/packager/common.go | 28 +++++++++++++++++++++++++--- src/pkg/packager/create_stages.go | 22 ++-------------------- 2 files changed, 27 insertions(+), 23 deletions(-) diff --git a/src/pkg/packager/common.go b/src/pkg/packager/common.go index 6543451305..e1cbfb0a20 100644 --- a/src/pkg/packager/common.go +++ b/src/pkg/packager/common.go @@ -171,6 +171,28 @@ func GetInitPackageName(arch string) string { return fmt.Sprintf("zarf-init-%s-%s.tar.zst", arch, config.CLIVersion) } +// GetPackageName returns the formatted name of the package. +func (p *Packager) GetPackageName() string { + if p.isInitConfig() { + return GetInitPackageName(p.arch) + } + + packageName := p.cfg.Pkg.Metadata.Name + suffix := "tar.zst" + if p.cfg.Pkg.Metadata.Uncompressed { + suffix = "tar" + } + + packageFileName := fmt.Sprintf("%s%s-%s", config.ZarfPackagePrefix, packageName, p.arch) + if p.cfg.Pkg.Build.Differential { + packageFileName = fmt.Sprintf("%s-%s-differential-%s", packageFileName, p.cfg.CreateOpts.DifferentialData.DifferentialPackageVersion, p.cfg.Pkg.Metadata.Version) + } else if p.cfg.Pkg.Metadata.Version != "" { + packageFileName = fmt.Sprintf("%s-%s", packageFileName, p.cfg.Pkg.Metadata.Version) + } + + return fmt.Sprintf("%s.%s", packageFileName, suffix) +} + // ClearTempPaths removes the temp directory and any files within it. func (p *Packager) ClearTempPaths() { // Remove the temp directory, but don't throw an error if it fails @@ -310,7 +332,7 @@ func (p *Packager) archivePackage(destinationTarball string) error { } spinner.Updatef("Wrote %s to %s", p.layout.Base, destinationTarball) - info, err := os.Stat(destinationTarball) + fi, err := os.Stat(destinationTarball) if err != nil { return fmt.Errorf("unable to read the package archive: %w", err) } @@ -319,7 +341,7 @@ func (p *Packager) archivePackage(destinationTarball string) error { chunkSize := p.cfg.CreateOpts.MaxPackageSizeMB * 1000 * 1000 // If a chunk size was specified and the package is larger than the chunk size, split it into chunks. - if p.cfg.CreateOpts.MaxPackageSizeMB > 0 && info.Size() > int64(chunkSize) { + if p.cfg.CreateOpts.MaxPackageSizeMB > 0 && fi.Size() > int64(chunkSize) { spinner.Updatef("Package is larger than %dMB, splitting into multiple files", p.cfg.CreateOpts.MaxPackageSizeMB) chunks, sha256sum, err := utils.SplitFile(destinationTarball, chunkSize) if err != nil { @@ -337,7 +359,7 @@ func (p *Packager) archivePackage(destinationTarball string) error { // Marshal the data into a json file. jsonData, err := json.Marshal(types.ZarfSplitPackageData{ Count: len(chunks), - Bytes: info.Size(), + Bytes: fi.Size(), Sha256Sum: sha256sum, }) if err != nil { diff --git a/src/pkg/packager/create_stages.go b/src/pkg/packager/create_stages.go index 1ae3260011..482cac8bd4 100644 --- a/src/pkg/packager/create_stages.go +++ b/src/pkg/packager/create_stages.go @@ -268,26 +268,8 @@ func (p *Packager) output() error { message.ZarfCommand("package deploy %s %s", helpers.OCIURLPrefix+remote.Repo().Reference.String(), flags) message.ZarfCommand("package pull %s %s", helpers.OCIURLPrefix+remote.Repo().Reference.String(), flags) } else { - var packageName string - - if p.isInitConfig() { - packageName = GetInitPackageName(p.arch) - } else { - packageName = p.cfg.Pkg.Metadata.Name - suffix := "tar.zst" - if p.cfg.Pkg.Metadata.Uncompressed { - suffix = "tar" - } - packageFileName := fmt.Sprintf("%s%s-%s", config.ZarfPackagePrefix, packageName, p.arch) - if p.cfg.Pkg.Build.Differential { - packageFileName = fmt.Sprintf("%s-%s-differential-%s", packageFileName, p.cfg.CreateOpts.DifferentialData.DifferentialPackageVersion, p.cfg.Pkg.Metadata.Version) - } else if p.cfg.Pkg.Metadata.Version != "" { - packageFileName = fmt.Sprintf("%s-%s", packageFileName, p.cfg.Pkg.Metadata.Version) - } - - packageName = fmt.Sprintf("%s.%s", packageFileName, suffix) - packageName = filepath.Join(p.cfg.CreateOpts.Output, packageName) - } + // Use the output path if the user specified it. + packageName := filepath.Join(p.cfg.CreateOpts.Output, p.GetPackageName()) // Try to remove the package if it already exists. _ = os.Remove(packageName) From 78624ba1b410d81fd357e3bb1a2bcee845da36ba Mon Sep 17 00:00:00 2001 From: razzle Date: Wed, 3 Jan 2024 10:43:34 -0600 Subject: [PATCH 4/4] remove skeleton refs Signed-off-by: razzle --- src/test/e2e/12_lint_test.go | 2 +- src/test/packages/12-lint/linted-import/zarf.yaml | 2 +- src/test/packages/12-lint/zarf.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/test/e2e/12_lint_test.go b/src/test/e2e/12_lint_test.go index a2d6443ed8..cf132c4068 100644 --- a/src/test/e2e/12_lint_test.go +++ b/src/test/e2e/12_lint_test.go @@ -52,7 +52,7 @@ func TestLint(t *testing.T) { require.Contains(t, strippedStderr, "image-in-good-flavor-component:unpinned") // Check reported filepaths - require.Contains(t, strippedStderr, "Linting package \"dos-games\" at oci://🦄/dos-games:1.0.0-skeleton") + require.Contains(t, strippedStderr, "Linting package \"dos-games\" at oci://🦄/dos-games:1.0.0") require.Contains(t, strippedStderr, fmt.Sprintf("Linting package \"lint\" at %s", testPackagePath)) }) diff --git a/src/test/packages/12-lint/linted-import/zarf.yaml b/src/test/packages/12-lint/linted-import/zarf.yaml index f5f21981f6..4a99f73e96 100644 --- a/src/test/packages/12-lint/linted-import/zarf.yaml +++ b/src/test/packages/12-lint/linted-import/zarf.yaml @@ -19,5 +19,5 @@ components: - name: oci-games-url import: - url: oci://🦄/dos-games:1.0.0-skeleton + url: oci://🦄/dos-games:1.0.0 name: baseline diff --git a/src/test/packages/12-lint/zarf.yaml b/src/test/packages/12-lint/zarf.yaml index efddf42eea..980fa78eeb 100644 --- a/src/test/packages/12-lint/zarf.yaml +++ b/src/test/packages/12-lint/zarf.yaml @@ -36,7 +36,7 @@ components: - name: oci-games-url import: - url: oci://🦄/dos-games:1.0.0-skeleton + url: oci://🦄/dos-games:1.0.0 name: baseline - name: oci-games-url