Add DB v6 support to grype-db-manager

.github/actions/bootstrap/action.yaml

@@ -13,10 +13,6 @@ inputs:
     description: "Go version to install"
     required: true
     default: "1.23.x"
-  python-version:
-    description: "Python version to install"
-    required: true
-    default: "3.11"
   poetry-version:
     description: "Poetry version to install"
     required: true
@@ -52,10 +48,10 @@ runs:
       with:
         go-version: ${{ inputs.go-version }}
 
-    - uses: actions/setup-python@v4
+    - uses: actions/setup-python@v5
       if: inputs.python == 'true'
       with:
-        python-version: ${{ inputs.python-version }}
+        python-version-file: "pyproject.toml"
 
     - name: Install poetry
       if: inputs.python == 'true'
@@ -69,9 +65,9 @@ runs:
       id: cache
       with:
         path: ~/.virtualenvs
-        key: ${{ inputs.cache-key-prefix }}-python-${{ inputs.python-version }}-poetry-${{ inputs.poetry-version }}-${{ hashFiles('poetry.lock') }}
+        key: ${{ inputs.cache-key-prefix }}-${{ hashFiles('poetry.lock') }}
         restore-keys: |
-          ${{ inputs.cache-key-prefix }}-python-${{ inputs.python-version }}-poetry-${{ inputs.poetry-version }}
+          ${{ inputs.cache-key-prefix }}-
 
     - name: Setup Poetry config
       if: inputs.python == 'true'

config/grype-db-manager/include.d/validate.yaml

@@ -1,9 +1,23 @@
 # validate:
 
 listing:
-  image: "centos:8.2.2004"
-  minimum-packages: 85
-  minimum-vulnerabilities: 400
+  image: "alpine:3.9.2"
+  minimum-packages: 10
+  minimum-vulnerabilities: 90
+
+expected-providers:
+  - alpine
+  - amazon
+  - chainguard
+  - debian
+  - github
+  - mariner
+  - nvd
+  - oracle
+  - rhel
+  - sles
+  - ubuntu
+  - wolfi
 
 default-max-year: 2021
 gates:

go.mod

@@ -8,7 +8,7 @@ require (
 	github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d
 	github.com/adrg/xdg v0.5.3
 	github.com/anchore/go-logger v0.0.0-20230725134548-c21dafa1ec5a
-	github.com/anchore/grype v0.86.2-0.20241223182831-3baa3d2ca99a
+	github.com/anchore/grype v0.86.2-0.20250108224532-ed98490e966c
 	github.com/anchore/syft v1.18.2-0.20241216153735-397eb9c10acd
 	github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de
 	github.com/dave/jennifer v1.7.1
@@ -114,7 +114,7 @@ require (
 	github.com/felixge/fgprof v0.9.3 // indirect
 	github.com/felixge/httpsnoop v1.0.4 // indirect
 	github.com/fsnotify/fsnotify v1.7.0 // indirect
-	github.com/gabriel-vasile/mimetype v1.4.7 // indirect
+	github.com/gabriel-vasile/mimetype v1.4.8 // indirect
 	github.com/github/go-spdx/v2 v2.3.2 // indirect
 	github.com/glebarez/go-sqlite v1.21.2 // indirect
 	github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
@@ -223,7 +223,7 @@ require (
 	golang.org/x/crypto v0.31.0 // indirect
 	golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect
 	golang.org/x/mod v0.22.0 // indirect
-	golang.org/x/net v0.32.0 // indirect
+	golang.org/x/net v0.33.0 // indirect
 	golang.org/x/oauth2 v0.19.0 // indirect
 	golang.org/x/sys v0.28.0 // indirect
 	golang.org/x/term v0.27.0 // indirect

go.sum

@@ -252,8 +252,8 @@ github.com/anchore/go-testutils v0.0.0-20200925183923-d5f45b0d3c04 h1:VzprUTpc0v
 github.com/anchore/go-testutils v0.0.0-20200925183923-d5f45b0d3c04/go.mod h1:6dK64g27Qi1qGQZ67gFmBFvEHScy0/C8qhQhNe5B5pQ=
 github.com/anchore/go-version v1.2.2-0.20210903204242-51efa5b487c4 h1:rmZG77uXgE+o2gozGEBoUMpX27lsku+xrMwlmBZJtbg=
 github.com/anchore/go-version v1.2.2-0.20210903204242-51efa5b487c4/go.mod h1:Bkc+JYWjMCF8OyZ340IMSIi2Ebf3uwByOk6ho4wne1E=
-github.com/anchore/grype v0.86.2-0.20241223182831-3baa3d2ca99a h1:7vWYvOsRaJmFUOk4YKrRdaFGi/SIo32IY8+tcUZiyQk=
-github.com/anchore/grype v0.86.2-0.20241223182831-3baa3d2ca99a/go.mod h1:rqbEt5hrFdlJ8nrk8VFv0A77XD4gzhkWk8LU9iU77AA=
+github.com/anchore/grype v0.86.2-0.20250108224532-ed98490e966c h1:7qALL8DBiu63wyqKp8WCjt76eSWqIR9E2AKjkwswbOo=
+github.com/anchore/grype v0.86.2-0.20250108224532-ed98490e966c/go.mod h1:tzS7xESazfn00HpttqVG2XdWFTHDSU+yKBEPrpSPtHU=
 github.com/anchore/packageurl-go v0.1.1-0.20241018175412-5c22e6360c4f h1:dAQPIrQ3a5PBqZeZ+B9NGZsGmodk4NO9OjDIsQmQyQM=
 github.com/anchore/packageurl-go v0.1.1-0.20241018175412-5c22e6360c4f/go.mod h1:KoYIv7tdP5+CC9VGkeZV4/vGCKsY55VvoG+5dadg4YI=
 github.com/anchore/stereoscope v0.0.11 h1:d+dePyWyQzoQehnWOnx/aISW5HW1zLAQKzvaFIpydsU=
@@ -435,8 +435,8 @@ github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7z
 github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU=
 github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA=
 github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM=
-github.com/gabriel-vasile/mimetype v1.4.7 h1:SKFKl7kD0RiPdbht0s7hFtjl489WcQ1VyPW8ZzUMYCA=
-github.com/gabriel-vasile/mimetype v1.4.7/go.mod h1:GDlAgAyIRT27BhFl53XNAFtfjzOkLaF35JdEG0P7LtU=
+github.com/gabriel-vasile/mimetype v1.4.8 h1:FfZ3gj38NjllZIeJAmMhr+qKL8Wu+nOoI3GqacKw1NM=
+github.com/gabriel-vasile/mimetype v1.4.8/go.mod h1:ByKUIKGjh1ODkGM1asKUbQZOLGrPjydw3hYPU2YU9t8=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/github/go-spdx/v2 v2.3.2 h1:IfdyNHTqzs4zAJjXdVQfRnxt1XMfycXoHBE2Vsm1bjs=
 github.com/github/go-spdx/v2 v2.3.2/go.mod h1:2ZxKsOhvBp+OYBDlsGnUMcchLeo2mrpEBn2L1C+U3IQ=
@@ -858,8 +858,8 @@ github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
 github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
 github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
-github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8=
-github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4=
+github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII=
+github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
@@ -1142,8 +1142,8 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug
 golang.org/x/net v0.0.0-20220909164309-bea034e7d591/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
 golang.org/x/net v0.0.0-20221014081412-f15817d10f9b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
 golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
-golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI=
-golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs=
+golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I=
+golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=

manager/src/grype_db_manager/cli/config.py

@@ -81,6 +81,22 @@ class Validate:
     default_max_year: int = 2021
     gates: list[ValidateDB] = field(default_factory=list)
     listing: ValidateListing = field(default_factory=ValidateListing)
+    expected_providers: list[str] = field(
+        default_factory=lambda: [
+            "alpine",
+            "amazon",
+            "chainguard",
+            "debian",
+            "github",
+            "mariner",
+            "nvd",
+            "oracle",
+            "rhel",
+            "sles",
+            "ubuntu",
+            "wolfi",
+        ],
+    )
 
 
 @dataclass()

manager/src/grype_db_manager/cli/db.py

@@ -8,7 +8,7 @@
 from yardstick.cli import config as ycfg
 from yardstick.cli.validate import validate as yardstick_validate
 
-from grype_db_manager import db, s3utils
+from grype_db_manager import db, grypedb, s3utils
 from grype_db_manager.cli import config, error
 from grype_db_manager.db.format import Format
 from grype_db_manager.grypedb import DB_DIR, DBManager, GrypeDB
@@ -54,6 +54,13 @@ def clear_dbs(cfg: config.Application) -> None:
         click.echo("no databases to clear")
 
 
+def remove_db(cfg: config.Application, db_uuid: str) -> None:
+    db_manager = DBManager(root_dir=cfg.data.root)
+    if db_manager.remove_db(db_uuid=db_uuid):
+        click.echo(f"database {db_uuid!r} removed")
+    click.echo(f"no database found with session id {db_uuid}")
+
+
 @group.command(name="build", help="build and validate a grype database")
 @click.option("--schema-version", "-s", required=True, help="the DB schema version to build")
 @click.pass_obj
@@ -96,7 +103,7 @@ def show_db(cfg: config.Application, db_uuid: str) -> None:
     "--skip-namespace-check",
     "skip_namespace_check",
     is_flag=True,
-    help="do not ensure the minimum expected namespaces are present",
+    help="do not ensure the minimum expected namespaces are present (for v6+ this is a providers-based check)",
 )
 @click.argument("db-uuid")
 @click.pass_obj
@@ -120,13 +127,52 @@ def validate_db(
         return
 
     if not skip_namespace_check:
-        # ensure the minimum number of namespaces are present
-        db_manager.validate_namespaces(db_uuid=db_uuid)
+        if db_info.schema_version < 6:
+            # ensure the minimum number of namespaces are present
+            db_manager.validate_namespaces(db_uuid=db_uuid)
+        else:
+            # ensure the minimum number of namespaces are present
+            db_manager.validate_providers(db_uuid=db_uuid, expected=cfg.validate.expected_providers)
+
+    _validate_db(ctx, cfg, db_info, images, db_uuid, verbosity, recapture)
+
+    logging.info(f"validating latest.json {db_uuid}")
+
+    if db_info.schema_version >= 6:
+        _validate_latest(cfg, db_info.latest_path, db_info.archive_path)
+
+    click.echo(f"{Format.BOLD}{Format.OKGREEN}Validation passed{Format.RESET}")
 
+
+def _validate_db(
+    ctx: click.Context,
+    cfg: config.Application,
+    db_info: grypedb.DBInfo,
+    images: list[str],
+    db_uuid: str,
+    verbosity: int,
+    recapture: bool,
+) -> None:
     # resolve tool versions and install them
     yardstick.store.config.set_values(store_root=cfg.data.yardstick_root)
 
     grype_version = db.schema.grype_version(db_info.schema_version)
+    basis_grype_version = grype_version
+
+    yardstick_profile_data = ycfg.Profiles()
+    profile_name = None
+    if db_info.schema_version >= 6:
+        # TODO: we don't have any published v6 grype databases yet
+        basis_grype_version = db.schema.grype_version(5)
+        # TODO: we can remove this once v6 lands on grype@main without a feature flag
+        profile_name = "v6"
+        yardstick_profile_data = ycfg.Profiles(
+            data={
+                "grype[custom-db]": {
+                    profile_name: {"config_path": "./.grype-v6.yaml"},
+                },
+            },
+        )
 
     result_sets = {}
     for idx, rs in enumerate(cfg.validate.gates):
@@ -153,19 +199,18 @@ def validate_db(
                         label="custom-db",
                         name="grype",
                         version=grype_version + f"+import-db={db_info.archive_path}",
+                        profile=profile_name,
                     ),
                     ycfg.Tool(
                         name="grype",
-                        version=grype_version,
+                        version=basis_grype_version,
                     ),
                 ],
             ),
         )
 
     yardstick_cfg = ycfg.Application(
-        profiles=ycfg.Profiles(
-            data={},
-        ),
+        profiles=yardstick_profile_data,
         store_root=cfg.data.yardstick_root,
         default_max_year=cfg.validate.default_max_year,
         result_sets=result_sets,
@@ -193,6 +238,31 @@ def validate_db(
     )
 
 
+def _validate_latest(cfg: config.Application, latest_file: str, archive_path: str) -> None:
+    with open(latest_file) as f:
+        latest_obj = db.Latest.from_json(f.read())
+
+    if not cfg.validate.listing.image:
+        msg = "no image specified to validate against"
+        raise ValueError(msg)
+
+    if not cfg.validate.listing.minimum_packages:
+        msg = "minimum packages must be specified"
+        raise ValueError(msg)
+
+    if not cfg.validate.listing.minimum_vulnerabilities:
+        msg = "minimum vulnerabilities must be specified"
+        raise ValueError(msg)
+
+    db.latest.smoke_test(
+        latest_obj,
+        archive_path,
+        image=cfg.validate.listing.image,
+        minimum_packages=cfg.validate.listing.minimum_packages,
+        minimum_vulnerabilities=cfg.validate.listing.minimum_vulnerabilities,
+    )
+
+
 @group.command(name="upload", help="upload a grype database")
 @click.option("--ttl-seconds", "-t", default=DEFAULT_TTL_SECONDS, help="the TTL for the uploaded DB (should be relatively high)")
 @click.argument("db-uuid")
@@ -208,22 +278,35 @@ def upload_db(cfg: config.Application, db_uuid: str, ttl_seconds: int) -> None:
     db_manager = DBManager(root_dir=cfg.data.root)
     db_info = db_manager.get_db_info(db_uuid=db_uuid)
 
-    key = f"{s3_path}/{os.path.basename(db_info.archive_path)}"
+    if db_info.schema_version >= 6:
+        if not os.path.exists(db_info.archive_path):
+            msg = f"latest.json file not found for DB {db_uuid!r}"
+            raise ValueError(msg)
+
+        # /databases -> /databases/v6 , and is dynamic based on the schema version
+        s3_path = f"{s3_path}/v{db_info.schema_version}"
 
-    # TODO: we have folks that require legacy behavior, where the content type was application/x-tar
-    kwargs = {}
-    if db_info.archive_path.endswith(".tar.gz"):
-        kwargs["ContentType"] = "application/x-tar"
+    db_key = f"{s3_path}/{os.path.basename(db_info.archive_path)}"
+    latest_key = f"{s3_path}/latest.json"
 
     s3utils.upload_file(
         bucket=s3_bucket,
-        key=key,
+        key=db_key,
         path=db_info.archive_path,
         CacheControl=f"public,max-age={ttl_seconds}",
-        **kwargs,
     )
 
-    click.echo(f"DB {db_uuid!r} uploaded to s3://{s3_bucket}/{s3_path}")
+    click.echo(f"DB archive {db_uuid!r} uploaded to s3://{s3_bucket}/{s3_path}")
+
+    if db_info.schema_version >= 6:
+        s3utils.upload_file(
+            bucket=s3_bucket,
+            key=latest_key,
+            path=db_info.latest_path,
+            CacheControl="public,max-age=300",  # 5 minutes
+        )
+
+        click.echo(f"DB latest.json {db_uuid!r} uploaded to s3://{s3_bucket}/{s3_path}")
 
 
 @group.command(name="build-and-upload", help="upload a grype database")
@@ -234,7 +317,7 @@ def upload_db(cfg: config.Application, db_uuid: str, ttl_seconds: int) -> None:
     "--skip-namespace-check",
     "skip_namespace_check",
     is_flag=True,
-    help="do not ensure the minimum expected namespaces are present",
+    help="do not ensure the minimum expected namespaces are present (for v6+ this is a providers-based check)",
 )
 @click.option("--verbose", "-v", "verbosity", count=True, help="show details of all comparisons")
 @click.pass_obj

manager/src/grype_db_manager/cli/listing.py

@@ -9,7 +9,7 @@
 from grype_db_manager.db.format import Format
 
 
-@click.group(name="listing", help="manage the grype-db listing file")
+@click.group(name="listing", help="manage the grype-db listing file (only schemas v1-v5)")
 @click.pass_obj
 def group(_: config.Application) -> None:
     pass
@@ -110,7 +110,7 @@ def validate_listing(cfg: config.Application, listing_file: str) -> None:
         raise ValueError(msg)
 
     if cfg.validate.listing.override_grype_version and not cfg.validate.listing.override_db_schema_version:
-        msg = "ovrerride db schema version must be specified if override grype version is specified"
+        msg = "override db schema version must be specified if override grype version is specified"
         raise ValueError(msg)
 
     override_schema_release = None

manager/src/grype_db_manager/data/schema-info.json

@@ -24,6 +24,11 @@
       "schema": "5",
       "grype-version": "main",
       "supported": true
+    },
+    {
+      "schema": "6",
+      "grype-version": "feat/v6-query-api",
+      "supported": false
     }
   ]
 }

manager/src/grype_db_manager/db/__init__.py

@@ -1,11 +1,14 @@
 from . import listing, metadata, schema
+from .latest import Latest
 from .listing import Listing
 from .metadata import Metadata
 from .validation import capture_results
 
 __all__ = [
+    "Latest",
     "Listing",
     "Metadata",
+    "latest",
     "listing",
     "metadata",
     "schema",