add depsdev scanner implementation (guacsec#2385)

Signed-off-by: Brandon Lum <[email protected]>

pkg/ingestor/parser/common/scanner/scanner.go

@@ -19,8 +19,11 @@ import (
 	"context"
 	"fmt"
 	"net/http"
+	"time"
 
+	deps_dev_client "github.com/guacsec/guac/internal/client/depsdevclient"
 	"github.com/guacsec/guac/pkg/assembler"
+	"github.com/guacsec/guac/pkg/assembler/clients/generated"
 	cd_certifier "github.com/guacsec/guac/pkg/certifier/clearlydefined"
 	eol_certifier "github.com/guacsec/guac/pkg/certifier/eol"
 	osv_certifier "github.com/guacsec/guac/pkg/certifier/osv"
@@ -114,8 +117,68 @@ func purlsLicenseScanWithClient(
 	return certLegalIngest, hasSourceAtIngest, nil
 }
 
+// PurlDepsDevScan scans the purls and returns for metadata linked to the repository
+// and returns the list of scorecards and sources associations it finds from Deps.dev
+// This generally takes about 300ms - 600ms. With tests including 1-30 PURLs.
 func PurlsDepsDevScan(ctx context.Context, purls []string) ([]assembler.CertifyScorecardIngest, []assembler.HasSourceAtIngest, error) {
-	return nil, nil, fmt.Errorf("Unimplemented")
+	certifyScorecards := []assembler.CertifyScorecardIngest{}
+	hasSourceAts := []assembler.HasSourceAtIngest{}
+
+	ddc, err := deps_dev_client.NewDepsClient(ctx)
+	if err != nil {
+		return nil, nil, fmt.Errorf("unable to initialize deps.dev client: %w", err)
+
+	}
+
+	ddc.RetrieveVersionsAndProjects(ctx, purls)
+	components, err := ddc.GetMetadata(ctx, purls)
+	if err != nil {
+		return nil, nil, fmt.Errorf("unable to get metadata for purls in deps.dev: %w", err)
+	}
+
+	for _, c := range components {
+		hasSourceAt := createDepsDevHasSourceAtIngest(c.CurrentPackage, c.Source, c.UpdateTime.UTC())
+		scorecard := createDepsDevScorecardIngest(c.Source, c.Scorecard)
+		if hasSourceAt != nil {
+			hasSourceAts = append(hasSourceAts, *hasSourceAt)
+		}
+		if scorecard != nil {
+			certifyScorecards = append(certifyScorecards, *scorecard)
+		}
+	}
+	return certifyScorecards, hasSourceAts, nil
+}
+
+func createDepsDevHasSourceAtIngest(pkg *generated.PkgInputSpec, src *generated.SourceInputSpec, knownSince time.Time) *assembler.HasSourceAtIngest {
+	if pkg != nil && src != nil {
+		return &assembler.HasSourceAtIngest{
+			Pkg: pkg,
+			PkgMatchFlag: generated.MatchFlags{
+				Pkg: generated.PkgMatchTypeAllVersions,
+			},
+			Src: src,
+			HasSourceAt: &generated.HasSourceAtInputSpec{
+				KnownSince:    knownSince,
+				Justification: "collected via deps.dev",
+				Collector:     "ingest_depsdev_scanner",
+				Origin:        "deps.dev",
+			},
+		}
+	}
+	return nil
+}
+
+func createDepsDevScorecardIngest(src *generated.SourceInputSpec, scorecard *generated.ScorecardInputSpec) *assembler.CertifyScorecardIngest {
+	if src != nil && scorecard != nil {
+		scorecard.Origin = "deps.dev"
+		scorecard.Collector = "ingest_depsdev_scanner"
+
+		return &assembler.CertifyScorecardIngest{
+			Source:    src,
+			Scorecard: scorecard,
+		}
+	}
+	return nil
 }
 
 // runQueryOnBatchedPurls runs EvaluateClearlyDefinedDefinition from the clearly defined

pkg/ingestor/parser/common/scanner/scanner_test.go

@@ -700,3 +700,315 @@ func parseEOLValue(value string) map[string]string {
 	}
 	return result
 }
+
+func TestPurlsDepsDevScan(t *testing.T) {
+	ctx := logging.WithLogger(context.Background())
+	tm, _ := time.Parse(time.RFC3339, "2022-11-21T17:45:50.52Z")
+	tests := []struct {
+		name     string
+		purls    []string
+		wantCSs  []assembler.CertifyScorecardIngest
+		wantHSAs []assembler.HasSourceAtIngest
+		wantErr  bool
+	}{
+		{
+			name:     "no purl",
+			purls:    []string{""},
+			wantCSs:  []assembler.CertifyScorecardIngest{},
+			wantHSAs: []assembler.HasSourceAtIngest{},
+			wantErr:  false,
+		},
+		{
+			name:  "valid maven/org.webjars.npm/a",
+			purls: []string{"pkg:maven/org.webjars.npm/[email protected]"},
+			wantCSs: []assembler.CertifyScorecardIngest{
+				{
+					Source: &generated.SourceInputSpec{
+						Type:      "git",
+						Namespace: "github.com/alfateam",
+						Name:      "a",
+					},
+					Scorecard: &generated.ScorecardInputSpec{
+						// Not including all checks since they are not stable for testing
+						// and are not used anyway. As long as its a non-empty list.
+						Checks: []generated.ScorecardCheckInputSpec{
+							{
+								Check: "Pinned-Dependencies",
+								Score: -1,
+							},
+							{
+								Check: "Token-Permissions",
+								Score: -1,
+							},
+							{
+								Check: "Maintained",
+								Score: 0,
+							},
+							{
+								Check: "Binary-Artifacts",
+								Score: 10,
+							},
+						},
+						AggregateScore:   3.0999999046325684,
+						TimeScanned:      tm,
+						ScorecardVersion: "v5.0.0-94-g51f31c98",
+						ScorecardCommit:  "51f31c9882b6e5998e0df571096147a99842092b",
+						Origin:           "deps.dev",
+						Collector:        "ingest_depsdev_scanner",
+					},
+				},
+			},
+			wantHSAs: []assembler.HasSourceAtIngest{
+				{
+					Pkg: &generated.PkgInputSpec{
+						Type:      "maven",
+						Namespace: ptrfrom.String("org.webjars.npm"),
+						Name:      "a",
+						Version:   ptrfrom.String("2.1.2"),
+						Subpath:   ptrfrom.String(""),
+					},
+					PkgMatchFlag: generated.MatchFlags{Pkg: "ALL_VERSIONS"},
+					Src: &generated.SourceInputSpec{
+						Type:      "git",
+						Namespace: "github.com/alfateam",
+						Name:      "a",
+					},
+					HasSourceAt: &generated.HasSourceAtInputSpec{
+						KnownSince:    tm,
+						Justification: "collected via deps.dev",
+						Origin:        "deps.dev",
+						Collector:     "ingest_depsdev_scanner",
+					},
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name:     "non-existent purl",
+			purls:    []string{"pkg:notexist/ns/[email protected]"},
+			wantCSs:  []assembler.CertifyScorecardIngest{},
+			wantHSAs: []assembler.HasSourceAtIngest{},
+			wantErr:  false,
+		},
+		{
+			name:     "malformed purl",
+			purls:    []string{"not-a-purl"},
+			wantCSs:  []assembler.CertifyScorecardIngest{},
+			wantHSAs: []assembler.HasSourceAtIngest{},
+			wantErr:  false,
+		},
+		{
+			name:  "valid pypi/wheel-axle-runtime",
+			purls: []string{"pkg:pypi/[email protected]"},
+			wantCSs: []assembler.CertifyScorecardIngest{
+				{
+					Source: &generated.SourceInputSpec{
+						Type:      "git",
+						Namespace: "github.com/karellen",
+						Name:      "wheel-axle-runtime",
+					},
+					Scorecard: &generated.ScorecardInputSpec{
+						// Not including all checks since they are not stable for testing
+						// and are not used anyway. As long as its a non-empty list.
+						Checks: []generated.ScorecardCheckInputSpec{
+							{
+								Check: "Pinned-Dependencies",
+								Score: 0,
+							},
+							{
+								Check: "Token-Permissions",
+								Score: 8,
+							},
+							{
+								Check: "Maintained",
+								Score: 1,
+							},
+							{
+								Check: "Binary-Artifacts",
+								Score: 8,
+							},
+						},
+						AggregateScore:   3.700000047683716,
+						TimeScanned:      tm,
+						ScorecardVersion: "v5.0.0-94-g51f31c98",
+						ScorecardCommit:  "51f31c9882b6e5998e0df571096147a99842092b",
+						Origin:           "deps.dev",
+						Collector:        "ingest_depsdev_scanner",
+					},
+				},
+			},
+			wantHSAs: []assembler.HasSourceAtIngest{
+				{
+					Pkg: &generated.PkgInputSpec{
+						Type:      "pypi",
+						Namespace: ptrfrom.String(""),
+						Name:      "wheel-axle-runtime",
+						Version:   ptrfrom.String("0.0.4"),
+						Subpath:   ptrfrom.String(""),
+					},
+					PkgMatchFlag: generated.MatchFlags{Pkg: "ALL_VERSIONS"},
+					Src: &generated.SourceInputSpec{
+						Type:      "git",
+						Namespace: "github.com/karellen",
+						Name:      "wheel-axle-runtime",
+					},
+					HasSourceAt: &generated.HasSourceAtInputSpec{
+						KnownSince:    tm,
+						Justification: "collected via deps.dev",
+						Origin:        "deps.dev",
+						Collector:     "ingest_depsdev_scanner",
+					},
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "multiple valid PURLs",
+			purls: []string{
+				"pkg:maven/org.webjars.npm/[email protected]",
+				"pkg:pypi/[email protected]",
+			},
+			wantCSs: []assembler.CertifyScorecardIngest{
+				{
+					Source: &generated.SourceInputSpec{
+						Type:      "git",
+						Namespace: "github.com/alfateam",
+						Name:      "a",
+					},
+					Scorecard: &generated.ScorecardInputSpec{
+						// Not including all checks since they are not stable for testing
+						// and are not used anyway. As long as its a non-empty list.
+						Checks: []generated.ScorecardCheckInputSpec{
+							{
+								Check: "Pinned-Dependencies",
+								Score: -1,
+							},
+							{
+								Check: "Token-Permissions",
+								Score: -1,
+							},
+							{
+								Check: "Maintained",
+								Score: 0,
+							},
+							{
+								Check: "Binary-Artifacts",
+								Score: 10,
+							},
+						},
+						AggregateScore:   3.0999999046325684,
+						TimeScanned:      tm,
+						ScorecardVersion: "v5.0.0-94-g51f31c98",
+						ScorecardCommit:  "51f31c9882b6e5998e0df571096147a99842092b",
+						Origin:           "deps.dev",
+						Collector:        "ingest_depsdev_scanner",
+					},
+				},
+				{
+					Source: &generated.SourceInputSpec{
+						Type:      "git",
+						Namespace: "github.com/karellen",
+						Name:      "wheel-axle-runtime",
+					},
+					Scorecard: &generated.ScorecardInputSpec{
+						// Not including all checks since they are not stable for testing
+						// and are not used anyway. As long as its a non-empty list.
+						Checks: []generated.ScorecardCheckInputSpec{
+							{
+								Check: "Pinned-Dependencies",
+								Score: 0,
+							},
+							{
+								Check: "Token-Permissions",
+								Score: 8,
+							},
+							{
+								Check: "Maintained",
+								Score: 1,
+							},
+							{
+								Check: "Binary-Artifacts",
+								Score: 8,
+							},
+						},
+						AggregateScore:   3.700000047683716,
+						TimeScanned:      tm,
+						ScorecardVersion: "v5.0.0-94-g51f31c98",
+						ScorecardCommit:  "51f31c9882b6e5998e0df571096147a99842092b",
+						Origin:           "deps.dev",
+						Collector:        "ingest_depsdev_scanner",
+					},
+				},
+			},
+			wantHSAs: []assembler.HasSourceAtIngest{
+				{
+					Pkg: &generated.PkgInputSpec{
+						Type:      "maven",
+						Namespace: ptrfrom.String("org.webjars.npm"),
+						Name:      "a",
+						Version:   ptrfrom.String("2.1.2"),
+						Subpath:   ptrfrom.String(""),
+					},
+					PkgMatchFlag: generated.MatchFlags{Pkg: "ALL_VERSIONS"},
+					Src: &generated.SourceInputSpec{
+						Type:      "git",
+						Namespace: "github.com/alfateam",
+						Name:      "a",
+					},
+					HasSourceAt: &generated.HasSourceAtInputSpec{
+						KnownSince:    tm,
+						Justification: "collected via deps.dev",
+						Origin:        "deps.dev",
+						Collector:     "ingest_depsdev_scanner",
+					},
+				},
+				{
+					Pkg: &generated.PkgInputSpec{
+						Type:      "pypi",
+						Namespace: ptrfrom.String(""),
+						Name:      "wheel-axle-runtime",
+						Version:   ptrfrom.String("0.0.4"),
+						Subpath:   ptrfrom.String(""),
+					},
+					PkgMatchFlag: generated.MatchFlags{Pkg: "ALL_VERSIONS"},
+					Src: &generated.SourceInputSpec{
+						Type:      "git",
+						Namespace: "github.com/karellen",
+						Name:      "wheel-axle-runtime",
+					},
+					HasSourceAt: &generated.HasSourceAtInputSpec{
+						KnownSince:    tm,
+						Justification: "collected via deps.dev",
+						Origin:        "deps.dev",
+						Collector:     "ingest_depsdev_scanner",
+					},
+				},
+			},
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			gotCSs, gotHSAs, err := PurlsDepsDevScan(ctx, tt.purls)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("PurlsToScan() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if diff := cmp.Diff(
+				tt.wantCSs,
+				gotCSs,
+				cmpopts.IgnoreFields(generated.ScorecardInputSpec{},
+					"AggregateScore", "TimeScanned",
+					"ScorecardVersion", "ScorecardCommit", "DocumentRef"),
+				cmpopts.IgnoreTypes(generated.ScorecardCheckInputSpec{}),
+			); diff != "" {
+				t.Errorf("Unexpected results. (-want +got):\n%s", diff)
+			}
+			if diff := cmp.Diff(tt.wantHSAs, gotHSAs,
+				cmpopts.IgnoreFields(generated.HasSourceAtInputSpec{}, "KnownSince", "DocumentRef"),
+			); diff != "" {
+				t.Errorf("Unexpected results. (-want +got):\n%s", diff)
+			}
+		})
+	}
+}