diff --git a/cache/blobs.go b/cache/blobs.go index 8d2beefd0654..038edabf0e6f 100644 --- a/cache/blobs.go +++ b/cache/blobs.go @@ -6,6 +6,7 @@ import ( "fmt" "io" "os" + "path" "strconv" "github.com/containerd/containerd/content" @@ -123,7 +124,7 @@ func computeBlobChain(ctx context.Context, sr *immutableRef, createIfNeeded bool lowerRef = sr.layerParent } var lower []mount.Mount - if lowerRef != nil { + if sr.cm.Snapshotter.Name() != "overlaybd" && lowerRef != nil { m, err := lowerRef.Mount(ctx, true, s) if err != nil { return nil, err @@ -146,7 +147,20 @@ func computeBlobChain(ctx context.Context, sr *immutableRef, createIfNeeded bool upperRef = sr } var upper []mount.Mount - if upperRef != nil { + if sr.cm.Snapshotter.Name() == "overlaybd" { + snStat, err := sr.cm.Snapshotter.Stat(ctx, sr.getSnapshotID()) + if err != nil { + return nil, err + } + fsPath := path.Dir(snStat.Labels[LabelLocalOverlayBDPath]) + upper = []mount.Mount{ + { + Source: fsPath, + Type: "bind", + Options: []string{"ro", "rbind"}, + }, + } + } else if upperRef != nil { m, err := upperRef.Mount(ctx, true, s) if err != nil { return nil, err @@ -234,6 +248,10 @@ func computeBlobChain(ctx context.Context, sr *immutableRef, createIfNeeded bool if desc.Annotations == nil { desc.Annotations = map[string]string{} } + if sr.cm.Snapshotter.Name() == "overlaybd" { + desc.Annotations[labelKeyOverlayBDBlobDigest] = string(desc.Digest) + desc.Annotations[labelKeyOverlayBDBlobSize] = fmt.Sprintf("%d", desc.Size) + } if finalize != nil { a, err := finalize(ctx, sr.cm.ContentStore) if err != nil { diff --git a/cache/manager.go b/cache/manager.go index 58e28b47434b..dd18965d78d0 100644 --- a/cache/manager.go +++ b/cache/manager.go @@ -14,6 +14,7 @@ import ( "github.com/containerd/containerd/filters" "github.com/containerd/containerd/gc" "github.com/containerd/containerd/leases" + "github.com/containerd/containerd/snapshots" "github.com/docker/docker/pkg/idtools" "github.com/moby/buildkit/cache/metadata" "github.com/moby/buildkit/client" @@ -611,6 +612,11 @@ func (cm *cacheManager) New(ctx context.Context, s ImmutableRef, sess session.Gr }); rerr != nil { return nil, rerr } + } else if cm.Snapshotter.Name() == "overlaybd" && parent != nil { + rwLabels := make(map[string]string) + rwLabels["containerd.io/snapshot/overlaybd.writable"] = "dev" + logrus.Infof("Prepare writable dev") + err = cm.Snapshotter.Prepare(ctx, snapshotID, parentSnapshotID, snapshots.WithLabels(rwLabels)) } else { err = cm.Snapshotter.Prepare(ctx, snapshotID, parentSnapshotID) } diff --git a/cache/refs.go b/cache/refs.go index bbc38c481245..f860d9a56fee 100644 --- a/cache/refs.go +++ b/cache/refs.go @@ -36,6 +36,15 @@ import ( "golang.org/x/sync/errgroup" ) +const ( + LabelLocalOverlayBDPath = "containerd.io/snapshot/overlaybd.localcommitpath" + labelKeyOverlayBDBlobDigest = "containerd.io/snapshot/overlaybd/blob-digest" + labelKeyOverlayBDBlobSize = "containerd.io/snapshot/overlaybd/blob-size" + labelKeyOverlayBDBlobFsType = "containerd.io/snapshot/overlaybd/blob-fs-type" +) + +var overlaybdAnnotations = []string{labelKeyOverlayBDBlobFsType, labelKeyOverlayBDBlobSize, labelKeyOverlayBDBlobDigest} + // Ref is a reference to cacheable objects. type Ref interface { Mountable @@ -881,7 +890,10 @@ func filterAnnotationsForSave(a map[string]string) (b map[string]string) { if a == nil { return nil } - for _, k := range append(eStargzAnnotations, containerdUncompressed) { + annotations := eStargzAnnotations + annotations = append(annotations, containerdUncompressed) + annotations = append(annotations, overlaybdAnnotations...) + for _, k := range annotations { v, ok := a[k] if !ok { continue @@ -957,6 +969,10 @@ func (sr *immutableRef) Extract(ctx context.Context, s session.Group) (rerr erro return err } return rerr + } else if sr.cm.Snapshotter.Name() == "overlaybd" { + if rerr = sr.prepareRemoteSnapshotsOverlaybdMode(ctx, s); rerr == nil { + return nil + } } return sr.unlazy(ctx, sr.descHandlers, sr.progress, s, true) @@ -1073,6 +1089,60 @@ func (sr *immutableRef) prepareRemoteSnapshotsStargzMode(ctx context.Context, s return err } +func (sr *immutableRef) prepareRemoteSnapshotsOverlaybdMode(ctx context.Context, s session.Group) error { + _, err := sr.sizeG.Do(ctx, sr.ID()+"-prepare-remote-snapshot", func(ctx context.Context) (_ interface{}, rerr error) { + dhs := sr.descHandlers + for _, r := range sr.layerChain() { + r := r + snapshotID := r.getSnapshotID() + if _, err := r.cm.Snapshotter.Stat(ctx, snapshotID); err == nil { + continue + } + + dh := dhs[digest.Digest(r.getBlob())] + if dh == nil { + // We cannot prepare remote snapshots without descHandler. + return nil, nil + } + + defaultLabels := snapshots.FilterInheritedLabels(dh.SnapshotLabels) + if defaultLabels == nil { + defaultLabels = make(map[string]string) + } + defaultLabels["containerd.io/snapshot.ref"] = snapshotID + + // Prepare remote snapshots + var ( + key = fmt.Sprintf("tmp-%s %s", identity.NewID(), r.getChainID()) + opts = []snapshots.Opt{ + snapshots.WithLabels(defaultLabels), + } + ) + parentID := "" + if r.layerParent != nil { + parentID = r.layerParent.getSnapshotID() + } + if err := r.cm.Snapshotter.Prepare(ctx, key, parentID, opts...); err != nil { + if errdefs.IsAlreadyExists(err) { + // Check if the targeting snapshot ID has been prepared as + // a remote snapshot in the snapshotter. + _, err := r.cm.Snapshotter.Stat(ctx, snapshotID) + if err == nil { // usable as remote snapshot without unlazying. + // Try the next layer as well. + continue + } + } + } + + // This layer and all upper layers cannot be prepared without unlazying. + break + } + + return nil, nil + }) + return err +} + func makeTmpLabelsStargzMode(labels map[string]string, s session.Group) (fields []string, res map[string]string) { res = make(map[string]string) // Append unique ID to labels for avoiding collision of labels among calls diff --git a/test/Dockerfile b/test/Dockerfile new file mode 100644 index 000000000000..e33537699031 --- /dev/null +++ b/test/Dockerfile @@ -0,0 +1,2 @@ +FROM docker.io/overlaybd/redis:6.2.6_obd +RUN dd if=/dev/urandom of=/opt/x bs=1M count=1 diff --git a/util/estargz/labels.go b/util/estargz/labels.go index 50f9bf53bb10..ba93f603c4c1 100644 --- a/util/estargz/labels.go +++ b/util/estargz/labels.go @@ -19,6 +19,7 @@ func SnapshotLabels(ref string, descs []ocispecs.Descriptor, targetIndex int) ma labels[k] = desc.Annotations[k] } labels["containerd.io/snapshot/remote/stargz.reference"] = ref + labels["containerd.io/snapshot/image-ref"] = ref labels["containerd.io/snapshot/remote/stargz.digest"] = desc.Digest.String() var ( layersKey = "containerd.io/snapshot/remote/stargz.layers"