From 8488b2f8fc38bfecaf7cc5c55f355b2afa70b3f0 Mon Sep 17 00:00:00 2001 From: Tianon Gravi Date: Tue, 24 Jan 2017 11:21:10 -0800 Subject: [PATCH 1/3] Add initial support for an explicit "SharedTags" field Things we know we'll need to be able to do with the new `SharedTags` (and are thus included in this initial file format support): - get list of all shared tags from a manifest (`manifest.GetAllSharedTags()`) - get list of shared tags from an entry (`entry.SharedTags`) - get list of entries given a shared tag (`manifest.GetSharedTag(tag)`) --- manifest/rfc2822.go | 75 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 74 insertions(+), 1 deletion(-) diff --git a/manifest/rfc2822.go b/manifest/rfc2822.go index 1046449..07d1b46 100644 --- a/manifest/rfc2822.go +++ b/manifest/rfc2822.go @@ -27,7 +27,8 @@ type Manifest2822Entry struct { Maintainers []string `delim:"," strip:"\n\r\t "` - Tags []string `delim:"," strip:"\n\r\t "` + Tags []string `delim:"," strip:"\n\r\t "` + SharedTags []string `delim:"," strip:"\n\r\t "` GitRepo string GitFetch string @@ -46,6 +47,7 @@ func (entry Manifest2822Entry) Clone() Manifest2822Entry { // SLICES! grr entry.Maintainers = append([]string{}, entry.Maintainers...) entry.Tags = append([]string{}, entry.Tags...) + entry.SharedTags = append([]string{}, entry.SharedTags...) entry.Constraints = append([]string{}, entry.Constraints...) return entry } @@ -60,6 +62,10 @@ func (entry Manifest2822Entry) TagsString() string { return strings.Join(entry.Tags, StringSeparator2822) } +func (entry Manifest2822Entry) SharedTagsString() string { + return strings.Join(entry.SharedTags, StringSeparator2822) +} + func (entry Manifest2822Entry) ConstraintsString() string { return strings.Join(entry.Constraints, StringSeparator2822) } @@ -77,6 +83,9 @@ func (entry Manifest2822Entry) ClearDefaults(defaults Manifest2822Entry) Manifes if entry.TagsString() == defaults.TagsString() { entry.Tags = nil } + if entry.SharedTagsString() == defaults.SharedTagsString() { + entry.SharedTags = nil + } if entry.GitRepo == defaults.GitRepo { entry.GitRepo = "" } @@ -103,6 +112,9 @@ func (entry Manifest2822Entry) String() string { if str := entry.TagsString(); str != "" { ret = append(ret, "Tags: "+str) } + if str := entry.SharedTagsString(); str != "" { + ret = append(ret, "SharedTags: "+str) + } if str := entry.GitRepo; str != "" { ret = append(ret, "GitRepo: "+str) } @@ -145,6 +157,16 @@ func (entry Manifest2822Entry) HasTag(tag string) bool { return false } +// HasSharedTag returns true if the given tag exists in entry.SharedTags. +func (entry Manifest2822Entry) HasSharedTag(tag string) bool { + for _, existingTag := range entry.SharedTags { + if tag == existingTag { + return true + } + } + return false +} + func (manifest Manifest2822) GetTag(tag string) *Manifest2822Entry { for _, entry := range manifest.Entries { if entry.HasTag(tag) { @@ -154,6 +176,27 @@ func (manifest Manifest2822) GetTag(tag string) *Manifest2822Entry { return nil } +// GetSharedTag returns a list of entries with the given tag in entry.SharedTags (or the empty list if there are no entries with the given tag). +func (manifest Manifest2822) GetSharedTag(tag string) []Manifest2822Entry { + ret := []Manifest2822Entry{} + for _, entry := range manifest.Entries { + if entry.HasSharedTag(tag) { + ret = append(ret, entry) + } + } + return ret +} + +// GetAllSharedTags returns a list of the sum of all SharedTags in all entries of this image manifest (in the order they appear in the file). +func (manifest Manifest2822) GetAllSharedTags() []string { + fakeEntry := Manifest2822Entry{} + for _, entry := range manifest.Entries { + fakeEntry.SharedTags = append(fakeEntry.SharedTags, entry.SharedTags...) + } + fakeEntry.DeduplicateSharedTags() + return fakeEntry.SharedTags +} + func (manifest *Manifest2822) AddEntry(entry Manifest2822Entry) error { if len(entry.Tags) < 1 { return fmt.Errorf("missing Tags") @@ -165,20 +208,36 @@ func (manifest *Manifest2822) AddEntry(entry Manifest2822Entry) error { return fmt.Errorf("Tags %q has invalid Maintainers: %q (expected format %q)", strings.Join(invalidMaintainers, ", "), MaintainersFormat) } + entry.DeduplicateSharedTags() + seenTag := map[string]bool{} for _, tag := range entry.Tags { if otherEntry := manifest.GetTag(tag); otherEntry != nil { return fmt.Errorf("Tags %q includes duplicate tag: %q (duplicated in %q)", entry.TagsString(), tag, otherEntry.TagsString()) } + if otherEntries := manifest.GetSharedTag(tag); len(otherEntries) > 0 { + return fmt.Errorf("Tags %q includes tag conflicting with a shared tag: %q (shared tag in %q)", entry.TagsString(), tag, otherEntries[0].TagsString()) + } if seenTag[tag] { return fmt.Errorf("Tags %q includes duplicate tag: %q", entry.TagsString(), tag) } seenTag[tag] = true } + for _, tag := range entry.SharedTags { + if otherEntry := manifest.GetTag(tag); otherEntry != nil { + return fmt.Errorf("Tags %q includes conflicting shared tag: %q (duplicated in %q)", entry.TagsString(), tag, otherEntry.TagsString()) + } + if seenTag[tag] { + return fmt.Errorf("Tags %q includes duplicate tag: %q (in SharedTags)", entry.TagsString(), tag) + } + seenTag[tag] = true + } for i, existingEntry := range manifest.Entries { if existingEntry.SameBuildArtifacts(entry) { manifest.Entries[i].Tags = append(existingEntry.Tags, entry.Tags...) + manifest.Entries[i].SharedTags = append(existingEntry.SharedTags, entry.SharedTags...) + manifest.Entries[i].DeduplicateSharedTags() return nil } } @@ -210,6 +269,20 @@ func (entry Manifest2822Entry) InvalidMaintainers() []string { return invalid } +// DeduplicateSharedTags will remove duplicate values from entry.SharedTags, preserving order. +func (entry *Manifest2822Entry) DeduplicateSharedTags() { + aggregate := []string{} + seen := map[string]bool{} + for _, tag := range entry.SharedTags { + if seen[tag] { + continue + } + seen[tag] = true + aggregate = append(aggregate, tag) + } + entry.SharedTags = aggregate +} + type decoderWrapper struct { *control.Decoder } From c48fd9a499666bb316c455d56fb7d84417386db6 Mon Sep 17 00:00:00 2001 From: Tianon Gravi Date: Fri, 10 Feb 2017 10:25:54 -0800 Subject: [PATCH 2/3] First pass at implementing GetSharedTagGroups --- manifest/rfc2822.go | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/manifest/rfc2822.go b/manifest/rfc2822.go index 07d1b46..9b926e6 100644 --- a/manifest/rfc2822.go +++ b/manifest/rfc2822.go @@ -197,6 +197,37 @@ func (manifest Manifest2822) GetAllSharedTags() []string { return fakeEntry.SharedTags } +type SharedTagGroup struct { + SharedTags []string + Entries []*Manifest2822Entry +} + +// GetSharedTagGroups returns a map of shared tag groups to the list of entries they share (as described in https://github.com/docker-library/go-dockerlibrary/pull/2#issuecomment-277853597). +func (manifest Manifest2822) GetSharedTagGroups() []SharedTagGroup { + inter := map[string][]string{} + interKeySep := "," + for _, sharedTag := range manifest.GetAllSharedTags() { + interKeyParts := []string{} + for _, entry := range manifest.GetSharedTag(sharedTag) { + interKeyParts = append(interKeyParts, entry.Tags[0]) + } + interKey := strings.Join(interKeyParts, interKeySep) + inter[interKey] = append(inter[interKey], sharedTag) + } + ret := []SharedTagGroup{} + for tags, sharedTags := range inter { + group := SharedTagGroup{ + SharedTags: sharedTags, + Entries: []*Manifest2822Entry{}, + } + for _, tag := range strings.Split(tags, interKeySep) { + group.Entries = append(group.Entries, manifest.GetTag(tag)) + } + ret = append(ret, group) + } + return ret +} + func (manifest *Manifest2822) AddEntry(entry Manifest2822Entry) error { if len(entry.Tags) < 1 { return fmt.Errorf("missing Tags") From ff63f9df03e97ce72509d6fd8111ec77a6ea6950 Mon Sep 17 00:00:00 2001 From: Tianon Gravi Date: Fri, 10 Feb 2017 11:05:35 -0800 Subject: [PATCH 3/3] Fix GetSharedTagGroups ordering and update our hacky example code to have some SharedTags examples --- manifest/example.go | 14 +++++++++++++- manifest/rfc2822.go | 12 ++++++++---- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/manifest/example.go b/manifest/example.go index d4f364f..b0dce0b 100644 --- a/manifest/example.go +++ b/manifest/example.go @@ -20,6 +20,7 @@ Maintainers: InfoSiftr (@infosiftr), Johan Euphrosine (@proppy) GitRepo: https://github.com/docker-library/golang.git GitFetch: refs/heads/master +SharedTags: latest # hi @@ -29,18 +30,20 @@ GitFetch: refs/heads/master # Go 1.6 -Tags: 1.6.1, 1.6, 1, latest +Tags: 1.6.1, 1.6, 1 GitCommit: 0ce80411b9f41e9c3a21fc0a1bffba6ae761825a Directory: 1.6 # Go 1.5 Tags: 1.5.3 +SharedTags: 1.5.3-debian, 1.5-debian GitCommit: d7e2a8d90a9b8f5dfd5bcd428e0c33b68c40cc19 Directory: 1.5 Tags: 1.5 +SharedTags: 1.5-debian GitCommit: d7e2a8d90a9b8f5dfd5bcd428e0c33b68c40cc19 Directory: 1.5 @@ -51,6 +54,15 @@ Directory: 1.5 } fmt.Printf("-------------\n2822:\n%s\n", man) + fmt.Printf("\nShared Tag Groups:\n") + for _, group := range man.GetSharedTagGroups() { + fmt.Printf("\n - %s\n", strings.Join(group.SharedTags, ", ")) + for _, entry := range group.Entries { + fmt.Printf(" - %s\n", entry.TagsString()) + } + } + fmt.Printf("\n") + man, err = manifest.Parse(bufio.NewReader(strings.NewReader(` # first set a: b@c d diff --git a/manifest/rfc2822.go b/manifest/rfc2822.go index 9b926e6..88a8efe 100644 --- a/manifest/rfc2822.go +++ b/manifest/rfc2822.go @@ -199,12 +199,13 @@ func (manifest Manifest2822) GetAllSharedTags() []string { type SharedTagGroup struct { SharedTags []string - Entries []*Manifest2822Entry + Entries []*Manifest2822Entry } // GetSharedTagGroups returns a map of shared tag groups to the list of entries they share (as described in https://github.com/docker-library/go-dockerlibrary/pull/2#issuecomment-277853597). func (manifest Manifest2822) GetSharedTagGroups() []SharedTagGroup { inter := map[string][]string{} + interOrder := []string{} // order matters, and maps randomize order interKeySep := "," for _, sharedTag := range manifest.GetAllSharedTags() { interKeyParts := []string{} @@ -212,13 +213,16 @@ func (manifest Manifest2822) GetSharedTagGroups() []SharedTagGroup { interKeyParts = append(interKeyParts, entry.Tags[0]) } interKey := strings.Join(interKeyParts, interKeySep) + if _, ok := inter[interKey]; !ok { + interOrder = append(interOrder, interKey) + } inter[interKey] = append(inter[interKey], sharedTag) } ret := []SharedTagGroup{} - for tags, sharedTags := range inter { + for _, tags := range interOrder { group := SharedTagGroup{ - SharedTags: sharedTags, - Entries: []*Manifest2822Entry{}, + SharedTags: inter[tags], + Entries: []*Manifest2822Entry{}, } for _, tag := range strings.Split(tags, interKeySep) { group.Entries = append(group.Entries, manifest.GetTag(tag))