Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Proposal PR: Userspace Convertor: Add Deduplication Version Support #271

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
+291 −89
Open

Proposal PR: Userspace Convertor: Add Deduplication Version Support #271

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
41 changes: 37 additions & 4 deletions cmd/convertor/builder/overlaybd_builder_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import (
"testing"

testingresources "github.com/containerd/accelerated-container-image/cmd/convertor/testingresources"
"github.com/containerd/accelerated-container-image/pkg/version"
"github.com/containerd/containerd/errdefs"
"github.com/containerd/containerd/images"
_ "github.com/containerd/containerd/pkg/testutil" // Handle custom root flag
Expand All @@ -31,7 +32,10 @@ import (

func Test_overlaybd_builder_CheckForConvertedLayer(t *testing.T) {
ctx := context.Background()
db := testingresources.NewLocalDB()
versionDB := testingresources.Localdb{
Version: version.GetUserSpaceConsistencyVersion(),
}
db := &versionDB // Reset DB
resolver := testingresources.GetTestResolver(t, ctx)
fetcher := testingresources.GetTestFetcherFromResolver(t, ctx, resolver, testingresources.DockerV2_Manifest_Simple_Ref)
base := &builderEngineBase{
Expand Down Expand Up @@ -83,6 +87,13 @@ func Test_overlaybd_builder_CheckForConvertedLayer(t *testing.T) {
testingresources.Assert(t, desc.Digest == targetDesc.Digest, "CheckForConvertedLayer() returned incorrect digest")
})

versionDB.Version.LayerVersion = "A" // Change version to something else
t.Run("Entry in DB but wrong version (should return not found)", func(t *testing.T) {
_, err := e.CheckForConvertedLayer(ctx, 0)
testingresources.Assert(t, errdefs.IsNotFound(err), fmt.Sprintf("CheckForConvertedLayer() returned an unexpected Error: %v", err))
})
versionDB.Version = version.GetUserSpaceConsistencyVersion() // Reset version

// cross repo mount (change target repo)
base.repository = "hello-world2"
newImageRef := "sample.localstore.io/hello-world2:amd64"
Expand All @@ -107,7 +118,13 @@ func Test_overlaybd_builder_CheckForConvertedLayer(t *testing.T) {
rc.Close()
})

base.db = testingresources.NewLocalDB() // Reset DB
versionDB.Version.LayerVersion = "A" // Change version to something else
t.Run("Cross Repo Entry in DB but wrong version (should return not found)", func(t *testing.T) {
_, err := e.CheckForConvertedLayer(ctx, 0)
testingresources.Assert(t, errdefs.IsNotFound(err), fmt.Sprintf("CheckForConvertedLayer() returned an unexpected Error: %v", err))
})

base.db = testingresources.NewLocalDB(version.GetUserSpaceConsistencyVersion()) // Reset DB
digestNotInRegistry := digest.FromString("Not in reg")
err = base.db.CreateLayerEntry(ctx, e.host, e.repository, digestNotInRegistry, fakeChainId, 10)
if err != nil {
Expand All @@ -124,7 +141,10 @@ func Test_overlaybd_builder_CheckForConvertedLayer(t *testing.T) {

func Test_overlaybd_builder_CheckForConvertedManifest(t *testing.T) {
ctx := context.Background()
db := testingresources.NewLocalDB()
versionDB := testingresources.Localdb{
Version: version.GetUserSpaceConsistencyVersion(),
}
db := &versionDB
resolver := testingresources.GetTestResolver(t, ctx)
fetcher := testingresources.GetTestFetcherFromResolver(t, ctx, resolver, testingresources.DockerV2_Manifest_Simple_Ref)

Expand Down Expand Up @@ -178,6 +198,13 @@ func Test_overlaybd_builder_CheckForConvertedManifest(t *testing.T) {
testingresources.Assert(t, desc.Digest == outputDesc.Digest, "CheckForConvertedManifest() returned incorrect digest")
})

versionDB.Version.ManifestVersion = "A" // Change version to something else
t.Run("Entry in DB but wrong version (should return not found)", func(t *testing.T) {
_, err := e.CheckForConvertedManifest(ctx)
testingresources.Assert(t, errdefs.IsNotFound(err), fmt.Sprintf("CheckForConvertedManifest() returned an unexpected Error: %v", err))
})
versionDB.Version = version.GetUserSpaceConsistencyVersion() // Reset version

// cross repo mount (change target repo)
base.repository = "hello-world2"
newImageRef := "sample.localstore.io/hello-world2:amd64"
Expand Down Expand Up @@ -207,7 +234,13 @@ func Test_overlaybd_builder_CheckForConvertedManifest(t *testing.T) {
rc.Close()
})

base.db = testingresources.NewLocalDB() // Reset DB
versionDB.Version.ManifestVersion = "A" // Change version to something else
t.Run("Cross Repo Entry in DB but wrong version (should return not found)", func(t *testing.T) {
_, err := e.CheckForConvertedManifest(ctx)
testingresources.Assert(t, errdefs.IsNotFound(err), fmt.Sprintf("CheckForConvertedManifest() returned an unexpected Error: %v", err))
})

base.db = testingresources.NewLocalDB(version.GetUserSpaceConsistencyVersion())
digestNotInRegistry := digest.FromString("Not in reg")
err = base.db.CreateManifestEntry(ctx, e.host, e.repository, outputDesc.MediaType, inputDesc.Digest, digestNotInRegistry, outputDesc.Size)
if err != nil {
Expand Down
2 changes: 2 additions & 0 deletions cmd/convertor/database/database.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ type LayerEntry struct {
Repository string
ChainID string
Host string
Version string
}

type ManifestEntry struct {
Expand All @@ -51,4 +52,5 @@ type ManifestEntry struct {
Repository string
Host string
MediaType string
Version string
}
33 changes: 18 additions & 15 deletions cmd/convertor/database/mysql.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,36 +21,39 @@ import (
"database/sql"
"fmt"

"github.com/containerd/accelerated-container-image/pkg/version"
"github.com/containerd/containerd/log"
"github.com/opencontainers/go-digest"
)

type sqldb struct {
db *sql.DB
db *sql.DB
version version.UserspaceVersion
}

func NewSqlDB(db *sql.DB) ConversionDatabase {
func NewSqlDB(db *sql.DB, version version.UserspaceVersion) ConversionDatabase {
return &sqldb{
db: db,
db: db,
version: version,
}
}

func (m *sqldb) CreateLayerEntry(ctx context.Context, host, repository string, convertedDigest digest.Digest, chainID string, size int64) error {
_, err := m.db.ExecContext(ctx, "insert into overlaybd_layers(host, repo, chain_id, data_digest, data_size) values(?, ?, ?, ?, ?)", host, repository, chainID, convertedDigest, size)
_, err := m.db.ExecContext(ctx, "insert into overlaybd_layers(host, repo, chain_id, data_digest, data_size, version) values(?, ?, ?, ?, ?, ?)", host, repository, chainID, convertedDigest, size, m.version.LayerVersion)
return err
}

func (m *sqldb) GetLayerEntryForRepo(ctx context.Context, host, repository, chainID string) *LayerEntry {
var entry LayerEntry
row := m.db.QueryRowContext(ctx, "select host, repo, chain_id, data_digest, data_size from overlaybd_layers where host=? and repo=? and chain_id=?", host, repository, chainID)
if err := row.Scan(&entry.Host, &entry.Repository, &entry.ChainID, &entry.ConvertedDigest, &entry.DataSize); err != nil {
row := m.db.QueryRowContext(ctx, "select host, repo, chain_id, data_digest, data_size, version from overlaybd_layers where host=? and repo=? and chain_id=? and version=?", host, repository, chainID, m.version.LayerVersion)
if err := row.Scan(&entry.Host, &entry.Repository, &entry.ChainID, &entry.ConvertedDigest, &entry.DataSize, &entry.Version); err != nil {
return nil
}
return &entry
}

func (m *sqldb) GetCrossRepoLayerEntries(ctx context.Context, host, chainID string) []*LayerEntry {
rows, err := m.db.QueryContext(ctx, "select host, repo, chain_id, data_digest, data_size from overlaybd_layers where host=? and chain_id=?", host, chainID)
rows, err := m.db.QueryContext(ctx, "select host, repo, chain_id, data_digest, data_size, version from overlaybd_layers where host=? and chain_id=? and version=?", host, chainID, m.version.LayerVersion)
if err != nil {
if err == sql.ErrNoRows {
return nil
Expand All @@ -61,7 +64,7 @@ func (m *sqldb) GetCrossRepoLayerEntries(ctx context.Context, host, chainID stri
var entries []*LayerEntry
for rows.Next() {
var entry LayerEntry
err = rows.Scan(&entry.Host, &entry.Repository, &entry.ChainID, &entry.ConvertedDigest, &entry.DataSize)
err = rows.Scan(&entry.Host, &entry.Repository, &entry.ChainID, &entry.ConvertedDigest, &entry.DataSize, &entry.Version)
if err != nil {
continue
}
Expand All @@ -72,29 +75,29 @@ func (m *sqldb) GetCrossRepoLayerEntries(ctx context.Context, host, chainID stri
}

func (m *sqldb) DeleteLayerEntry(ctx context.Context, host, repository string, chainID string) error {
_, err := m.db.Exec("delete from overlaybd_layers where host=? and repo=? and chain_id=?", host, repository, chainID)
_, err := m.db.Exec("delete from overlaybd_layers where host=? and repo=? and chain_id=? and version=?", host, repository, chainID, m.version.LayerVersion)
if err != nil {
return fmt.Errorf("failed to remove invalid record in db: %w", err)
}
return nil
}

func (m *sqldb) CreateManifestEntry(ctx context.Context, host, repository, mediaType string, original, convertedDigest digest.Digest, size int64) error {
_, err := m.db.ExecContext(ctx, "insert into overlaybd_manifests(host, repo, src_digest, out_digest, data_size, mediatype) values(?, ?, ?, ?, ?, ?)", host, repository, original, convertedDigest, size, mediaType)
_, err := m.db.ExecContext(ctx, "insert into overlaybd_manifests(host, repo, src_digest, out_digest, data_size, mediatype, version) values(?, ?, ?, ?, ?, ?, ?)", host, repository, original, convertedDigest, size, mediaType, m.version.ManifestVersion)
return err
}

func (m *sqldb) GetManifestEntryForRepo(ctx context.Context, host, repository, mediaType string, original digest.Digest) *ManifestEntry {
var entry ManifestEntry
row := m.db.QueryRowContext(ctx, "select host, repo, src_digest, out_digest, data_size, mediatype from overlaybd_manifests where host=? and repo=? and src_digest=? and mediatype=?", host, repository, original, mediaType)
if err := row.Scan(&entry.Host, &entry.Repository, &entry.OriginalDigest, &entry.ConvertedDigest, &entry.DataSize, &entry.MediaType); err != nil {
row := m.db.QueryRowContext(ctx, "select host, repo, src_digest, out_digest, data_size, mediatype, version from overlaybd_manifests where host=? and repo=? and src_digest=? and mediatype=? and version=?", host, repository, original, mediaType, m.version.ManifestVersion)
if err := row.Scan(&entry.Host, &entry.Repository, &entry.OriginalDigest, &entry.ConvertedDigest, &entry.DataSize, &entry.MediaType, &entry.Version); err != nil {
return nil
}
return &entry
}

func (m *sqldb) GetCrossRepoManifestEntries(ctx context.Context, host, mediaType string, original digest.Digest) []*ManifestEntry {
rows, err := m.db.QueryContext(ctx, "select host, repo, src_digest, out_digest, data_size, mediatype from overlaybd_manifests where host=? and src_digest=? and mediatype=?", host, original, mediaType)
rows, err := m.db.QueryContext(ctx, "select host, repo, src_digest, out_digest, data_size, mediatype, version from overlaybd_manifests where host=? and src_digest=? and mediatype=? and version=?", host, original, mediaType, m.version.ManifestVersion)
if err != nil {
if err == sql.ErrNoRows {
return nil
Expand All @@ -105,7 +108,7 @@ func (m *sqldb) GetCrossRepoManifestEntries(ctx context.Context, host, mediaType
var entries []*ManifestEntry
for rows.Next() {
var entry ManifestEntry
err = rows.Scan(&entry.Host, &entry.Repository, &entry.OriginalDigest, &entry.ConvertedDigest, &entry.DataSize, &entry.MediaType)
err = rows.Scan(&entry.Host, &entry.Repository, &entry.OriginalDigest, &entry.ConvertedDigest, &entry.DataSize, &entry.MediaType, &entry.Version)
if err != nil {
continue
}
Expand All @@ -116,7 +119,7 @@ func (m *sqldb) GetCrossRepoManifestEntries(ctx context.Context, host, mediaType
}

func (m *sqldb) DeleteManifestEntry(ctx context.Context, host, repository, mediaType string, original digest.Digest) error {
_, err := m.db.Exec("delete from overlaybd_manifests where host=? and repo=? and src_digest=? and mediatype=?", host, repository, original, mediaType)
_, err := m.db.Exec("delete from overlaybd_manifests where host=? and repo=? and src_digest=? and mediatype=? and version=?", host, repository, original, mediaType, m.version.ManifestVersion)
if err != nil {
return fmt.Errorf("failed to remove invalid record in db: %w", err)
}
Expand Down
11 changes: 10 additions & 1 deletion cmd/convertor/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ import (

"github.com/containerd/accelerated-container-image/cmd/convertor/builder"
"github.com/containerd/accelerated-container-image/cmd/convertor/database"
"github.com/containerd/accelerated-container-image/pkg/version"
_ "github.com/go-sql-driver/mysql"
"github.com/sirupsen/logrus"

Expand All @@ -46,6 +47,8 @@ var (
overlaybd string
dbstr string
dbType string
dbLayerVersion string
dbMnfstVersion string
concurrencyLimit int

// certification
Expand Down Expand Up @@ -116,7 +119,11 @@ var (
logrus.Errorf("failed to open the provided mysql db: %v", err)
os.Exit(1)
}
opt.DB = database.NewSqlDB(db)
userspaceVersion := version.UserspaceVersion{
LayerVersion: dbLayerVersion,
ManifestVersion: dbMnfstVersion,
}
opt.DB = database.NewSqlDB(db, userspaceVersion)
case "":
default:
logrus.Warnf("db-type %s was provided but is not one of known db types. Available: mysql", dbType)
Expand Down Expand Up @@ -172,6 +179,8 @@ func init() {
rootCmd.Flags().BoolVar(&reserve, "reserve", false, "reserve tmp data")
rootCmd.Flags().BoolVar(&noUpload, "no-upload", false, "don't upload layer and manifest")
rootCmd.Flags().BoolVar(&dumpManifest, "dump-manifest", false, "dump manifest")
rootCmd.Flags().StringVar(&dbLayerVersion, "db-layer-version", version.GetUserSpaceConsistencyVersion().LayerVersion, "version of db to use for conversion deduplication. Default 1")
rootCmd.Flags().StringVar(&dbMnfstVersion, "db-manifest-version", version.GetUserSpaceConsistencyVersion().ManifestVersion, "version of db to use for conversion deduplication. Default 1-1")

rootCmd.MarkFlagRequired("repository")
rootCmd.MarkFlagRequired("input-tag")
Expand Down
Loading
Loading