Merge pull request #3009 from infosiftr/architectures

Add initial Architectures support (and the beginnings of a "put-shared" subcommand for bringing all the SharedTags and arch-specific images together with manifest lists/indexes)

bashbrew/go/src/bashbrew/cmd-build.go

@@ -81,7 +81,7 @@ func cmdBuild(c *cli.Context) error {
 					return cli.NewMultiError(fmt.Errorf(`failed fetching git repo for %q (tags %q)`, r.RepoName, entry.TagsString()), err)
 				}
 
-				archive, err := gitArchive(commit, entry.Directory)
+				archive, err := gitArchive(commit, entry.ArchDirectory(arch))
 				if err != nil {
 					return cli.NewMultiError(fmt.Errorf(`failed generating git archive for %q (tags %q)`, r.RepoName, entry.TagsString()), err)
 				}

bashbrew/go/src/bashbrew/cmd-put-shared.go

@@ -0,0 +1,46 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"path"
+	"strings"
+
+	"github.com/codegangsta/cli"
+)
+
+func cmdPutShared(c *cli.Context) error {
+	repos, err := repos(c.Bool("all"), c.Args()...)
+	if err != nil {
+		return cli.NewMultiError(fmt.Errorf(`failed gathering repo list`), err)
+	}
+
+	namespace := c.String("namespace")
+
+	if namespace == "" {
+		return fmt.Errorf(`"--namespace" is a required flag for "put-shared"`)
+	}
+
+	fmt.Fprintf(os.Stderr, "warning: this subcommand is still a big WIP -- it doesn't do anything yet!\n")
+
+	for _, repo := range repos {
+		r, err := fetch(repo)
+		if err != nil {
+			return cli.NewMultiError(fmt.Errorf(`failed fetching repo %q`, repo), err)
+		}
+
+		// TODO handle all multi-architecture tags first (regardless of whether they have SharedTags)
+
+		targetRepo := path.Join(namespace, r.RepoName)
+		for _, group := range r.Manifest.GetSharedTagGroups() {
+			// TODO build up a YAML file
+			entryTags := []string{}
+			for _, entry := range group.Entries {
+				entryTags = append(entryTags, entry.Tags[0])
+			}
+			fmt.Printf("Putting %s (tags %s) <= %s\n", targetRepo, strings.Join(group.SharedTags, ", "), strings.Join(entryTags, ", "))
+		}
+	}
+
+	return nil
+}

bashbrew/go/src/bashbrew/config.go

@@ -18,6 +18,8 @@ type FlagsConfigEntry struct {
 
 	Commands []string `delim:"," strip:"\n\r\t "`
 
+	// TODO arch namespace mappings (for intermediate pushing before put-shared, and for put-shared to pull from to join together in one big happy family)
+
 	Library    string
 	Cache      string
 	Debug      string
@@ -26,6 +28,7 @@ type FlagsConfigEntry struct {
 	BuildOrder string
 	Pull       string
 
+	Arch                 string
 	Constraints          []string `delim:"," strip:"\n\r\t "`
 	ExclusiveConstraints string
 	ApplyConstraints     string
@@ -55,6 +58,9 @@ func (dst *FlagsConfigEntry) Apply(src FlagsConfigEntry) {
 	if src.Pull != "" {
 		dst.Pull = src.Pull
 	}
+	if src.Arch != "" {
+		dst.Arch = src.Arch
+	}
 	if len(src.Constraints) > 0 {
 		dst.Constraints = src.Constraints[:]
 	}
@@ -73,6 +79,7 @@ func (config FlagsConfigEntry) Vars() map[string]map[string]interface{} {
 			"cache":   config.Cache,
 			"debug":   config.Debug,
 
+			"arch":                  config.Arch,
 			"constraint":            config.Constraints,
 			"exclusive-constraints": config.ExclusiveConstraints,
 		},

bashbrew/go/src/bashbrew/docker.go

@@ -24,7 +24,7 @@ func (r Repo) DockerFrom(entry *manifest.Manifest2822Entry) (string, error) {
 		return "", err
 	}
 
-	dockerfileFile := path.Join(entry.Directory, "Dockerfile")
+	dockerfileFile := path.Join(entry.ArchDirectory(arch), "Dockerfile")
 
 	cacheKey := strings.Join([]string{
 		commit,
@@ -130,9 +130,9 @@ func (r Repo) dockerBuildUniqueBits(entry *manifest.Manifest2822Entry) ([]string
 		dockerFromIdCache[from] = fromId
 	}
 	return []string{
-		entry.GitRepo,
-		entry.GitCommit,
-		entry.Directory,
+		entry.ArchGitRepo(arch),
+		entry.ArchGitCommit(arch),
+		entry.ArchDirectory(arch),
 		fromId,
 	}, nil
 }

bashbrew/go/src/bashbrew/git.go

@@ -102,12 +102,12 @@ var gitRepoCache = map[string]string{}
 
 func (r Repo) fetchGitRepo(entry *manifest.Manifest2822Entry) (string, error) {
 	cacheKey := strings.Join([]string{
-		entry.GitRepo,
-		entry.GitFetch,
-		entry.GitCommit,
+		entry.ArchGitRepo(arch),
+		entry.ArchGitFetch(arch),
+		entry.ArchGitCommit(arch),
 	}, "\n")
 	if commit, ok := gitRepoCache[cacheKey]; ok {
-		entry.GitCommit = commit
+		entry.SetGitCommit(arch, commit)
 		return commit, nil
 	}
 
@@ -116,27 +116,27 @@ func (r Repo) fetchGitRepo(entry *manifest.Manifest2822Entry) (string, error) {
 		return "", err
 	}
 
-	if manifest.GitCommitRegex.MatchString(entry.GitCommit) {
-		commit, err := getGitCommit(entry.GitCommit)
+	if manifest.GitCommitRegex.MatchString(entry.ArchGitCommit(arch)) {
+		commit, err := getGitCommit(entry.ArchGitCommit(arch))
 		if err == nil {
 			gitRepoCache[cacheKey] = commit
-			entry.GitCommit = commit
+			entry.SetGitCommit(arch, commit)
 			return commit, nil
 		}
 	}
 
-	fetchString := entry.GitFetch + ":"
-	if entry.GitCommit == "FETCH_HEAD" {
+	fetchString := entry.ArchGitFetch(arch) + ":"
+	if entry.ArchGitCommit(arch) == "FETCH_HEAD" {
 		// fetch remote tag references to a local tag ref so that we can cache them and not re-fetch every time
 		localRef := "refs/tags/" + gitNormalizeForTagUsage(cacheKey)
 		commit, err := getGitCommit(localRef)
 		if err == nil {
 			gitRepoCache[cacheKey] = commit
-			entry.GitCommit = commit
+			entry.SetGitCommit(arch, commit)
 			return commit, nil
 		}
 		fetchString += localRef
-	} else if entry.GitFetch == manifest.DefaultLineBasedFetch {
+	} else if entry.ArchGitFetch(arch) == manifest.DefaultLineBasedFetch {
 		// backwards compat (see manifest/line-based.go in go-dockerlibrary)
 		refBase := "refs/remotes"
 		refBaseDir := filepath.Join(gitCache(), refBase)
@@ -154,17 +154,17 @@ func (r Repo) fetchGitRepo(entry *manifest.Manifest2822Entry) (string, error) {
 		// we create a temporary remote dir so that we can clean it up completely afterwards
 	}
 
-	if strings.HasPrefix(entry.GitRepo, "git://github.com/") {
-		fmt.Fprintf(os.Stderr, "warning: insecure protocol git:// detected: %s\n", entry.GitRepo)
-		entry.GitRepo = strings.Replace(entry.GitRepo, "git://", "https://", 1)
+	if strings.HasPrefix(entry.ArchGitRepo(arch), "git://github.com/") {
+		fmt.Fprintf(os.Stderr, "warning: insecure protocol git:// detected: %s\n", entry.ArchGitRepo(arch))
+		entry.SetGitRepo(arch, strings.Replace(entry.ArchGitRepo(arch), "git://", "https://", 1))
 	}
 
-	_, err = git("fetch", "--quiet", "--no-tags", entry.GitRepo, fetchString)
+	_, err = git("fetch", "--quiet", "--no-tags", entry.ArchGitRepo(arch), fetchString)
 	if err != nil {
 		return "", err
 	}
 
-	commit, err := getGitCommit(entry.GitCommit)
+	commit, err := getGitCommit(entry.ArchGitCommit(arch))
 	if err != nil {
 		return "", err
 	}
@@ -175,6 +175,6 @@ func (r Repo) fetchGitRepo(entry *manifest.Manifest2822Entry) (string, error) {
 	}
 
 	gitRepoCache[cacheKey] = commit
-	entry.GitCommit = commit
+	entry.SetGitCommit(arch, commit)
 	return commit, nil
 }

bashbrew/go/src/bashbrew/main.go

@@ -6,6 +6,8 @@ import (
 	"path/filepath"
 
 	"github.com/codegangsta/cli"
+
+	"github.com/docker-library/go-dockerlibrary/manifest"
 )
 
 // TODO somewhere, ensure that the Docker engine we're talking to is API version 1.22+ (Docker 1.10+)
@@ -18,6 +20,7 @@ var (
 	defaultLibrary string
 	defaultCache   string
 
+	arch                 string
 	constraints          []string
 	exclusiveConstraints bool
 
@@ -27,6 +30,7 @@ var (
 	// separated so that FlagsConfig.ApplyTo can access them
 	flagEnvVars = map[string]string{
 		"debug":   "BASHBREW_DEBUG",
+		"arch":    "BASHBREW_ARCH",
 		"config":  "BASHBREW_CONFIG",
 		"library": "BASHBREW_LIBRARY",
 		"cache":   "BASHBREW_CACHE",
@@ -72,6 +76,12 @@ func main() {
 			Usage: "do not apply any sorting, even via --build-order",
 		},
 
+		cli.StringFlag{
+			Name:   "arch",
+			Value:  manifest.DefaultArchitecture,
+			EnvVar: flagEnvVars["arch"],
+			Usage:  "the current platform architecture",
+		},
 		cli.StringSliceFlag{
 			Name:  "constraint",
 			Usage: "build constraints (see Constraints in Manifest2822Entry)",
@@ -127,6 +137,7 @@ func main() {
 			debugFlag = c.GlobalBool("debug")
 			noSortFlag = c.GlobalBool("no-sort")
 
+			arch = c.GlobalString("arch")
 			constraints = c.GlobalStringSlice("constraint")
 			exclusiveConstraints = c.GlobalBool("exclusive-constraints")
 
@@ -222,7 +233,18 @@ func main() {
 			Before: subcommandBeforeFactory("push"),
 			Action: cmdPush,
 		},
+		{
+			Name:  "put-shared",
+			Usage: `updated shared tags in the registry`,
+			Flags: []cli.Flag{
+				commonFlags["all"],
+				commonFlags["namespace"],
+			},
+			Before: subcommandBeforeFactory("put-shared"),
+			Action: cmdPutShared,
+		},
 
+		// TODO --depth flag for children and parents
 		{
 			Name: "children",
 			Aliases: []string{

bashbrew/go/src/bashbrew/repo.go

@@ -79,6 +79,15 @@ var haveOutputSkippedMessage = map[string]bool{}
 func (r Repo) SkipConstraints(entry manifest.Manifest2822Entry) bool {
 	repoTag := r.RepoName + ":" + entry.Tags[0]
 
+	// TODO decide if "arch" and "constraints" should be handled separately (but probably not)
+	if !entry.HasArchitecture(arch) {
+		if !haveOutputSkippedMessage[repoTag] {
+			fmt.Fprintf(os.Stderr, "skipping %q (due to architecture %q; only %q supported)\n", repoTag, arch, entry.ArchitecturesString())
+			haveOutputSkippedMessage[repoTag] = true
+		}
+		return true
+	}
+
 	if len(entry.Constraints) == 0 {
 		if exclusiveConstraints {
 			if !haveOutputSkippedMessage[repoTag] {

bashbrew/go/vendor/manifest

@@ -10,7 +10,7 @@
 		{
 			"importpath": "github.com/docker-library/go-dockerlibrary",
 			"repository": "https://github.com/docker-library/go-dockerlibrary",
-			"revision": "08ef5a968ebdd83dcc42998a96b6528837b55273",
+			"revision": "663a091da13fc848e27a16048fb39c4e4067056e",
 			"branch": "master"
 		},
 		{

bashbrew/go/vendor/src/github.com/docker-library/go-dockerlibrary/README.md

@@ -0,0 +1,5 @@
+# `import "github.com/docker-library/go-dockerlibrary/manifest"`
+
+[![Travis Build Status](https://travis-ci.org/docker-library/go-dockerlibrary.svg?branch=master)](https://travis-ci.org/docker-library/go-dockerlibrary) [![GoDoc](https://godoc.org/github.com/docker-library/go-dockerlibrary?status.svg)](https://godoc.org/github.com/docker-library/go-dockerlibrary) [![codecov](https://codecov.io/gh/docker-library/go-dockerlibrary/branch/master/graph/badge.svg)](https://codecov.io/gh/docker-library/go-dockerlibrary)
+
+This package contains the core parsing elements of [the `bashbrew` tool used by the Docker Official Images](https://github.com/docker-library/official-images/tree/master/bashbrew).