ci: validate contents of ndc-hub

.github/workflows/validate.yaml

@@ -0,0 +1,25 @@
+name: Validate
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+jobs:
+  contents:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      - name: Setup Go
+        uses: actions/setup-go@v4
+        with:
+          go-version: 1.21.x
+
+      - name: Run Validations
+        run: |
+          export NDC_HUB_GIT_REPO_FILE_PATH=$(pwd)
+          cd registry-automation
+          go run main.go validate

registry-automation/cmd/validate.go

@@ -0,0 +1,89 @@
+package cmd
+
+import (
+	"fmt"
+	"io/fs"
+	"os"
+	"path/filepath"
+
+	"github.com/hasura/ndc-hub/registry-automation/pkg/ndchub"
+	"github.com/hasura/ndc-hub/registry-automation/pkg/validate"
+	"github.com/spf13/cobra"
+)
+
+var validateCmd = &cobra.Command{
+	Use:   "validate",
+	Short: "Validate the contents of ndc-hub",
+	Run:   executeValidateCmd,
+}
+
+func init() {
+	rootCmd.AddCommand(validateCmd)
+}
+
+func executeValidateCmd(cmd *cobra.Command, args []string) {
+	ndcHubGitRepoFilePath := os.Getenv("NDC_HUB_GIT_REPO_FILE_PATH")
+	if ndcHubGitRepoFilePath == "" {
+		fmt.Println("please set a value for NDC_HUB_GIT_REPO_FILE_PATH env var")
+		os.Exit(1)
+		return
+	}
+
+	registryFolder := filepath.Join(ndcHubGitRepoFilePath, "registry")
+	_, err := os.Stat(registryFolder)
+	if err != nil {
+		fmt.Println("error while finding the registry folder", err)
+		os.Exit(1)
+		return
+	}
+	if os.IsNotExist(err) {
+		fmt.Println("registry folder does not exist")
+		os.Exit(1)
+		return
+	}
+
+	type connectorPackaging struct {
+		filePath         string
+		connectorPackage *ndchub.ConnectorPackaging
+	}
+	var connectorPkgs []connectorPackaging
+	err = filepath.WalkDir(registryFolder, func(path string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+
+		if filepath.Base(path) == ndchub.ConnectorPackagingJSON {
+			cp, err := ndchub.GetConnectorPackaging(path)
+			if err != nil {
+				return err
+			}
+			if cp != nil {
+				connectorPkgs = append(connectorPkgs, connectorPackaging{filePath: path, connectorPackage: cp})
+			}
+		}
+
+		return nil
+	})
+	if err != nil {
+		fmt.Println("error while walking the registry folder", err)
+		os.Exit(1)
+		return
+	}
+
+	hasError := false
+
+	fmt.Println("Validating `connector-packaging.json` contents")
+	for _, cp := range connectorPkgs {
+		err := validate.ConnectorPackaging(cp.connectorPackage)
+		if err != nil {
+			fmt.Println("error validating connector packaging", cp.filePath, err)
+			hasError = true
+		}
+	}
+	fmt.Println("Completed validating `connector-packaging.json` contents")
+
+	if hasError {
+		fmt.Println("Exiting with a non-zero error code due to the error(s) in validation")
+		os.Exit(1)
+	}
+}

registry-automation/go.mod

@@ -1,6 +1,8 @@
 module github.com/hasura/ndc-hub/registry-automation
 
-go 1.21.4
+go 1.22.0
+
+toolchain go1.23.2
 
 require (
 	github.com/cloudinary/cloudinary-go/v2 v2.8.0
@@ -9,12 +11,14 @@ require (
 )
 
 require (
+	github.com/Masterminds/semver/v3 v3.3.1 // indirect
 	github.com/creasty/defaults v1.7.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/gorilla/schema v1.4.1 // indirect
 	github.com/matryer/is v1.4.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/stretchr/objx v0.5.2 // indirect
+	golang.org/x/mod v0.22.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )
 

registry-automation/go.sum

@@ -14,6 +14,8 @@ cloud.google.com/go/longrunning v0.5.9/go.mod h1:HD+0l9/OOW0za6UWdKJtXoFAX/BGg/3
 cloud.google.com/go/storage v1.43.0 h1:CcxnSohZwizt4LCzQHWvBf1/kvtHUn7gk9QERXPyXFs=
 cloud.google.com/go/storage v1.43.0/go.mod h1:ajvxEa7WmZS1PxvKRq4bq0tFT3vMd502JwstCcYv0Q0=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/Masterminds/semver/v3 v3.3.1 h1:QtNSWtVZ3nBfk8mAOu/B6v7FMJ+NHTIgUPi7rj+4nv4=
+github.com/Masterminds/semver/v3 v3.3.1/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/cloudinary/cloudinary-go/v2 v2.8.0 h1:6o2mL5Obm92Q0TuX6yXfdpXSImbsYVYlOPOnpwjfobo=
@@ -121,6 +123,8 @@ golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
 golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/mod v0.22.0 h1:D4nJWe9zXqHOmWqj4VMOJhvzj7bEZg4wEYa759z1pH4=
+golang.org/x/mod v0.22.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=

registry-automation/pkg/ndchub/ndchub.go

@@ -0,0 +1,61 @@
+package ndchub
+
+import (
+	"encoding/json"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+const (
+	MetadataJSON           = "metadata.json"
+	ConnectorPackagingJSON = "connector-packaging.json"
+)
+
+type Checksum struct {
+	Type  string `json:"type"`
+	Value string `json:"value"`
+}
+
+type Source struct {
+	Hash string `json:"hash"`
+}
+
+type ConnectorPackaging struct {
+	Namespace string `json:"-"`
+	Name      string `json:"-"`
+
+	Version  string   `json:"version"`
+	URI      string   `json:"uri"`
+	Checksum Checksum `json:"checksum"`
+	Source   Source   `json:"source"`
+}
+
+func GetConnectorPackaging(path string) (*ConnectorPackaging, error) {
+	if strings.Contains(path, "aliased_connectors") {
+		// It should be safe to ignore aliased_connectors
+		// as their slug is not used in the connector init process
+		return nil, nil
+	}
+
+	// path looks like this: /some/folder/ndc-hub/registry/hasura/turso/releases/v0.1.0/connector-packaging.json
+	versionFolder := filepath.Dir(path)
+	releasesFolder := filepath.Dir(versionFolder)
+	connectorFolder := filepath.Dir(releasesFolder)
+	namespaceFolder := filepath.Dir(connectorFolder)
+
+	connectorPackagingContent, err := os.ReadFile(path)
+	if err != nil {
+		return nil, err
+	}
+
+	var connectorPackaging ConnectorPackaging
+	err = json.Unmarshal(connectorPackagingContent, &connectorPackaging)
+	if err != nil {
+		return nil, err
+	}
+	connectorPackaging.Namespace = filepath.Base(namespaceFolder)
+	connectorPackaging.Name = filepath.Base(connectorFolder)
+
+	return &connectorPackaging, nil
+}

registry-automation/pkg/validate/connector_packaging.go

@@ -0,0 +1,21 @@
+package validate
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/hasura/ndc-hub/registry-automation/pkg/ndchub"
+	"golang.org/x/mod/semver"
+)
+
+func ConnectorPackaging(cp *ndchub.ConnectorPackaging) error {
+	// validate version field
+	if !strings.HasPrefix(cp.Version, "v") {
+		return fmt.Errorf("version must start with 'v': but got %s", cp.Version)
+	}
+	if !semver.IsValid(cp.Version) {
+		return fmt.Errorf("invalid semantic version: %s", cp.Version)
+	}
+
+	return nil
+}

registry-automation/pkg/validate/connector_packaging_test.go

@@ -0,0 +1,39 @@
+package validate
+
+import (
+	"testing"
+
+	"github.com/hasura/ndc-hub/registry-automation/pkg/ndchub"
+)
+
+func TestConnectorPackaging(t *testing.T) {
+	testCases := []struct {
+		name    string
+		version string
+		wantErr bool
+	}{
+		{"Valid version", "v1.0.0", false},
+		{"Valid version with pre-release", "v1.0.0-alpha.1", false},
+		{"Valid version with build metadata", "v1.0.0+build.1", false},
+		{"Missing v prefix", "1.0.0", true},
+		{"Empty version", "", true},
+		{"Invalid characters", "vabc.1.0", true},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			cp := &ndchub.ConnectorPackaging{
+				Version: tc.version,
+			}
+
+			err := ConnectorPackaging(cp)
+
+			if tc.wantErr && err == nil {
+				t.Errorf("ConnectorPackaging() error = nil, wantErr %v", tc.wantErr)
+			}
+			if !tc.wantErr && err != nil {
+				t.Errorf("ConnectorPackaging() error = %v, wantErr %v", err, tc.wantErr)
+			}
+		})
+	}
+}

registry/hasura/cassandra/releases/v1.0.7/connector-packaging.json

@@ -1,5 +1,5 @@
 {
-    "version": "1.0.7",
+    "version": "v1.0.7",
     "uri": "https://github.com/hasura/ndc-cassandra/releases/download/v1.0.7/connector-definition.tgz",
     "checksum": {
         "type": "sha256",

registry/hasura/clickhouse/releases/v1.0.5/connector-packaging.json

@@ -1,5 +1,5 @@
 {
-  "version": "1.0.5",
+  "version": "v1.0.5",
   "uri": "https://github.com/hasura/ndc-clickhouse/releases/download/v1.0.5/connector-definition.tgz",
   "checksum": {
     "type": "sha256",

registry/hasura/duckdb/releases/v0.1.3/connector-packaging.json

@@ -1,5 +1,5 @@
 {
-  "version": "0.1.3",
+  "version": "v0.1.3",
   "uri": "https://github.com/hasura/ndc-duckdb/releases/download/v0.1.3/connector-definition.tgz",
   "checksum": {
     "type": "sha256",

registry/hasura/duckdb/releases/v0.1.4/connector-packaging.json

@@ -1,5 +1,5 @@
 {
-  "version": "0.1.4",
+  "version": "v0.1.4",
   "uri": "https://github.com/hasura/ndc-duckdb/releases/download/v0.1.4/connector-definition.tgz",
   "checksum": {
     "type": "sha256",

registry/hasura/postgres/releases/v1.2.0/connector-packaging.json

@@ -1,5 +1,5 @@
 {
-  "version": "1.2.0",
+  "version": "v1.2.0",
   "uri": "https://github.com/hasura/ndc-postgres/releases/download/v1.2.0/package.tar.gz",
   "checksum": {
     "type": "sha256",