diff --git a/.github/workflows/validate.yaml b/.github/workflows/validate.yaml new file mode 100644 index 00000000..18fdf4f8 --- /dev/null +++ b/.github/workflows/validate.yaml @@ -0,0 +1,25 @@ +name: Validate + +on: + push: + branches: [main] + pull_request: + branches: [main] + +jobs: + contents: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: Setup Go + uses: actions/setup-go@v4 + with: + go-version: 1.21.x + + - name: Run Validations + run: | + export NDC_HUB_GIT_REPO_FILE_PATH=$(pwd) + cd registry-automation + go run main.go validate diff --git a/registry-automation/cmd/validate.go b/registry-automation/cmd/validate.go new file mode 100644 index 00000000..be7ec68c --- /dev/null +++ b/registry-automation/cmd/validate.go @@ -0,0 +1,89 @@ +package cmd + +import ( + "fmt" + "io/fs" + "os" + "path/filepath" + + "github.com/hasura/ndc-hub/registry-automation/pkg/ndchub" + "github.com/hasura/ndc-hub/registry-automation/pkg/validate" + "github.com/spf13/cobra" +) + +var validateCmd = &cobra.Command{ + Use: "validate", + Short: "Validate the contents of ndc-hub", + Run: executeValidateCmd, +} + +func init() { + rootCmd.AddCommand(validateCmd) +} + +func executeValidateCmd(cmd *cobra.Command, args []string) { + ndcHubGitRepoFilePath := os.Getenv("NDC_HUB_GIT_REPO_FILE_PATH") + if ndcHubGitRepoFilePath == "" { + fmt.Println("please set a value for NDC_HUB_GIT_REPO_FILE_PATH env var") + os.Exit(1) + return + } + + registryFolder := filepath.Join(ndcHubGitRepoFilePath, "registry") + _, err := os.Stat(registryFolder) + if err != nil { + fmt.Println("error while finding the registry folder", err) + os.Exit(1) + return + } + if os.IsNotExist(err) { + fmt.Println("registry folder does not exist") + os.Exit(1) + return + } + + type connectorPackaging struct { + filePath string + connectorPackage *ndchub.ConnectorPackaging + } + var connectorPkgs []connectorPackaging + err = filepath.WalkDir(registryFolder, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + + if filepath.Base(path) == ndchub.ConnectorPackagingJSON { + cp, err := ndchub.GetConnectorPackaging(path) + if err != nil { + return err + } + if cp != nil { + connectorPkgs = append(connectorPkgs, connectorPackaging{filePath: path, connectorPackage: cp}) + } + } + + return nil + }) + if err != nil { + fmt.Println("error while walking the registry folder", err) + os.Exit(1) + return + } + + hasError := false + + fmt.Println("Validating `connector-packaging.json` contents") + for _, cp := range connectorPkgs { + err := validate.ConnectorPackaging(cp.connectorPackage) + if err != nil { + fmt.Println("error validating connector packaging", cp.filePath, err) + hasError = true + } + } + fmt.Println("Completed validating `connector-packaging.json` contents") + + if hasError { + fmt.Println("Exiting with a non-zero error code due to the error(s) in validation") + os.Exit(1) + } +} diff --git a/registry-automation/go.mod b/registry-automation/go.mod index 2c03147d..e45613fd 100644 --- a/registry-automation/go.mod +++ b/registry-automation/go.mod @@ -1,6 +1,8 @@ module github.com/hasura/ndc-hub/registry-automation -go 1.21.4 +go 1.22.0 + +toolchain go1.23.2 require ( github.com/cloudinary/cloudinary-go/v2 v2.8.0 @@ -9,12 +11,14 @@ require ( ) require ( + github.com/Masterminds/semver/v3 v3.3.1 // indirect github.com/creasty/defaults v1.7.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/gorilla/schema v1.4.1 // indirect github.com/matryer/is v1.4.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/stretchr/objx v0.5.2 // indirect + golang.org/x/mod v0.22.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/registry-automation/go.sum b/registry-automation/go.sum index f44c7142..ccfd4422 100644 --- a/registry-automation/go.sum +++ b/registry-automation/go.sum @@ -14,6 +14,8 @@ cloud.google.com/go/longrunning v0.5.9/go.mod h1:HD+0l9/OOW0za6UWdKJtXoFAX/BGg/3 cloud.google.com/go/storage v1.43.0 h1:CcxnSohZwizt4LCzQHWvBf1/kvtHUn7gk9QERXPyXFs= cloud.google.com/go/storage v1.43.0/go.mod h1:ajvxEa7WmZS1PxvKRq4bq0tFT3vMd502JwstCcYv0Q0= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/Masterminds/semver/v3 v3.3.1 h1:QtNSWtVZ3nBfk8mAOu/B6v7FMJ+NHTIgUPi7rj+4nv4= +github.com/Masterminds/semver/v3 v3.3.1/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudinary/cloudinary-go/v2 v2.8.0 h1:6o2mL5Obm92Q0TuX6yXfdpXSImbsYVYlOPOnpwjfobo= @@ -121,6 +123,8 @@ golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mod v0.22.0 h1:D4nJWe9zXqHOmWqj4VMOJhvzj7bEZg4wEYa759z1pH4= +golang.org/x/mod v0.22.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= diff --git a/registry-automation/pkg/ndchub/ndchub.go b/registry-automation/pkg/ndchub/ndchub.go new file mode 100644 index 00000000..94e8fd90 --- /dev/null +++ b/registry-automation/pkg/ndchub/ndchub.go @@ -0,0 +1,61 @@ +package ndchub + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" +) + +const ( + MetadataJSON = "metadata.json" + ConnectorPackagingJSON = "connector-packaging.json" +) + +type Checksum struct { + Type string `json:"type"` + Value string `json:"value"` +} + +type Source struct { + Hash string `json:"hash"` +} + +type ConnectorPackaging struct { + Namespace string `json:"-"` + Name string `json:"-"` + + Version string `json:"version"` + URI string `json:"uri"` + Checksum Checksum `json:"checksum"` + Source Source `json:"source"` +} + +func GetConnectorPackaging(path string) (*ConnectorPackaging, error) { + if strings.Contains(path, "aliased_connectors") { + // It should be safe to ignore aliased_connectors + // as their slug is not used in the connector init process + return nil, nil + } + + // path looks like this: /some/folder/ndc-hub/registry/hasura/turso/releases/v0.1.0/connector-packaging.json + versionFolder := filepath.Dir(path) + releasesFolder := filepath.Dir(versionFolder) + connectorFolder := filepath.Dir(releasesFolder) + namespaceFolder := filepath.Dir(connectorFolder) + + connectorPackagingContent, err := os.ReadFile(path) + if err != nil { + return nil, err + } + + var connectorPackaging ConnectorPackaging + err = json.Unmarshal(connectorPackagingContent, &connectorPackaging) + if err != nil { + return nil, err + } + connectorPackaging.Namespace = filepath.Base(namespaceFolder) + connectorPackaging.Name = filepath.Base(connectorFolder) + + return &connectorPackaging, nil +} diff --git a/registry-automation/pkg/validate/connector_packaging.go b/registry-automation/pkg/validate/connector_packaging.go new file mode 100644 index 00000000..988d1335 --- /dev/null +++ b/registry-automation/pkg/validate/connector_packaging.go @@ -0,0 +1,21 @@ +package validate + +import ( + "fmt" + "strings" + + "github.com/hasura/ndc-hub/registry-automation/pkg/ndchub" + "golang.org/x/mod/semver" +) + +func ConnectorPackaging(cp *ndchub.ConnectorPackaging) error { + // validate version field + if !strings.HasPrefix(cp.Version, "v") { + return fmt.Errorf("version must start with 'v': but got %s", cp.Version) + } + if !semver.IsValid(cp.Version) { + return fmt.Errorf("invalid semantic version: %s", cp.Version) + } + + return nil +} diff --git a/registry-automation/pkg/validate/connector_packaging_test.go b/registry-automation/pkg/validate/connector_packaging_test.go new file mode 100644 index 00000000..143a8b05 --- /dev/null +++ b/registry-automation/pkg/validate/connector_packaging_test.go @@ -0,0 +1,39 @@ +package validate + +import ( + "testing" + + "github.com/hasura/ndc-hub/registry-automation/pkg/ndchub" +) + +func TestConnectorPackaging(t *testing.T) { + testCases := []struct { + name string + version string + wantErr bool + }{ + {"Valid version", "v1.0.0", false}, + {"Valid version with pre-release", "v1.0.0-alpha.1", false}, + {"Valid version with build metadata", "v1.0.0+build.1", false}, + {"Missing v prefix", "1.0.0", true}, + {"Empty version", "", true}, + {"Invalid characters", "vabc.1.0", true}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + cp := &ndchub.ConnectorPackaging{ + Version: tc.version, + } + + err := ConnectorPackaging(cp) + + if tc.wantErr && err == nil { + t.Errorf("ConnectorPackaging() error = nil, wantErr %v", tc.wantErr) + } + if !tc.wantErr && err != nil { + t.Errorf("ConnectorPackaging() error = %v, wantErr %v", err, tc.wantErr) + } + }) + } +} diff --git a/registry/hasura/cassandra/releases/v1.0.7/connector-packaging.json b/registry/hasura/cassandra/releases/v1.0.7/connector-packaging.json index 4d10f659..ebf8fcc1 100644 --- a/registry/hasura/cassandra/releases/v1.0.7/connector-packaging.json +++ b/registry/hasura/cassandra/releases/v1.0.7/connector-packaging.json @@ -1,5 +1,5 @@ { - "version": "1.0.7", + "version": "v1.0.7", "uri": "https://github.com/hasura/ndc-cassandra/releases/download/v1.0.7/connector-definition.tgz", "checksum": { "type": "sha256", diff --git a/registry/hasura/clickhouse/releases/v1.0.5/connector-packaging.json b/registry/hasura/clickhouse/releases/v1.0.5/connector-packaging.json index 5fd0351d..e73ebcfb 100644 --- a/registry/hasura/clickhouse/releases/v1.0.5/connector-packaging.json +++ b/registry/hasura/clickhouse/releases/v1.0.5/connector-packaging.json @@ -1,5 +1,5 @@ { - "version": "1.0.5", + "version": "v1.0.5", "uri": "https://github.com/hasura/ndc-clickhouse/releases/download/v1.0.5/connector-definition.tgz", "checksum": { "type": "sha256", diff --git a/registry/hasura/duckdb/releases/v0.1.3/connector-packaging.json b/registry/hasura/duckdb/releases/v0.1.3/connector-packaging.json index 101a99dd..c5d519a6 100644 --- a/registry/hasura/duckdb/releases/v0.1.3/connector-packaging.json +++ b/registry/hasura/duckdb/releases/v0.1.3/connector-packaging.json @@ -1,5 +1,5 @@ { - "version": "0.1.3", + "version": "v0.1.3", "uri": "https://github.com/hasura/ndc-duckdb/releases/download/v0.1.3/connector-definition.tgz", "checksum": { "type": "sha256", diff --git a/registry/hasura/duckdb/releases/v0.1.4/connector-packaging.json b/registry/hasura/duckdb/releases/v0.1.4/connector-packaging.json index 3ca821e5..3f433eed 100644 --- a/registry/hasura/duckdb/releases/v0.1.4/connector-packaging.json +++ b/registry/hasura/duckdb/releases/v0.1.4/connector-packaging.json @@ -1,5 +1,5 @@ { - "version": "0.1.4", + "version": "v0.1.4", "uri": "https://github.com/hasura/ndc-duckdb/releases/download/v0.1.4/connector-definition.tgz", "checksum": { "type": "sha256", diff --git a/registry/hasura/postgres/releases/v1.2.0/connector-packaging.json b/registry/hasura/postgres/releases/v1.2.0/connector-packaging.json index 33b8bb6f..59e6b858 100644 --- a/registry/hasura/postgres/releases/v1.2.0/connector-packaging.json +++ b/registry/hasura/postgres/releases/v1.2.0/connector-packaging.json @@ -1,5 +1,5 @@ { - "version": "1.2.0", + "version": "v1.2.0", "uri": "https://github.com/hasura/ndc-postgres/releases/download/v1.2.0/package.tar.gz", "checksum": { "type": "sha256",