lib add github token optional to avoid being rate limited

cli/go.mod

@@ -3,7 +3,7 @@ module qovery-ai-migration
 go 1.22.5
 
 require (
-	github.com/Qovery/qovery-migration-ai-agent v0.0.0-20240914163651-758fbd47e2b5
+	github.com/Qovery/qovery-migration-ai-agent v0.0.0-20240915120017-84ea9a36ec8c
 	github.com/joho/godotenv v1.5.1
 	github.com/schollz/progressbar/v3 v3.14.6
 	github.com/spf13/cobra v1.8.1

cli/go.sum

@@ -12,6 +12,8 @@ github.com/Qovery/qovery-migration-ai-agent v0.0.0-20240914154405-cef2f94d96e5 h
 github.com/Qovery/qovery-migration-ai-agent v0.0.0-20240914154405-cef2f94d96e5/go.mod h1:BK4dodrSYDAJySssVnQDgzCjaR1yvonA3hNYsDigYLQ=
 github.com/Qovery/qovery-migration-ai-agent v0.0.0-20240914163651-758fbd47e2b5 h1:4saOX9GD/Es+LckVManmU8fVW8qAJRY6hT9QCV1RLrg=
 github.com/Qovery/qovery-migration-ai-agent v0.0.0-20240914163651-758fbd47e2b5/go.mod h1:BK4dodrSYDAJySssVnQDgzCjaR1yvonA3hNYsDigYLQ=
+github.com/Qovery/qovery-migration-ai-agent v0.0.0-20240915120017-84ea9a36ec8c h1:S7LeYYwn2ZfqVJQXfgBSZJdjpGOLRybvInddSayR5Y8=
+github.com/Qovery/qovery-migration-ai-agent v0.0.0-20240915120017-84ea9a36ec8c/go.mod h1:jdwL5TgbMsrvvowpDiJsXZUg1sYR67WbKRtqINMUgGQ=
 github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=

go.mod

@@ -5,13 +5,17 @@ go 1.22.5
 require (
 	github.com/google/go-github/v39 v39.2.0
 	github.com/stretchr/testify v1.9.0
+	golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be
 )
 
 require (
 	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/golang/protobuf v1.3.2 // indirect
 	github.com/google/go-querystring v1.1.0 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/stretchr/objx v0.5.2 // indirect
 	golang.org/x/crypto v0.27.0 // indirect
+	golang.org/x/net v0.21.0 // indirect
+	google.golang.org/appengine v1.6.7 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

go.sum

@@ -1,6 +1,7 @@
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs=
 github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ=
@@ -16,12 +17,14 @@ github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/
 github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
 github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 h1:HWj/xjIHfjYU5nVXpTM0s39J9CbLn7Cc5a7IC5rwsMQ=
 golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A=
 golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70=
 golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4=
+golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be h1:vEDujvNQGv4jgYKudGeI/+DAX4Jffq6hpD55MmoEvKs=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -32,6 +35,7 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c=
 google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

pkg/migration/migration.go

@@ -8,6 +8,7 @@ import (
 	"github.com/Qovery/qovery-migration-ai-agent/pkg/heroku"
 	"github.com/Qovery/qovery-migration-ai-agent/pkg/qovery"
 	"github.com/google/go-github/v39/github"
+	"golang.org/x/oauth2"
 	"io/ioutil"
 	"os"
 	"os/exec"
@@ -35,7 +36,7 @@ type ProgressUpdate struct {
 }
 
 // GenerateMigrationAssets generates all necessary assets for migration and reports progress
-func GenerateMigrationAssets(source, herokuAPIKey, claudeAPIKey, qoveryAPIKey, destination string, progressChan chan<- ProgressUpdate) (*Assets, error) {
+func GenerateMigrationAssets(source, herokuAPIKey, claudeAPIKey, qoveryAPIKey, githubToken, destination string, progressChan chan<- ProgressUpdate) (*Assets, error) {
 	claudeClient := claude.NewClaudeClient(claudeAPIKey)
 	qoveryProvider := qovery.NewQoveryProvider(qoveryAPIKey)
 
@@ -83,7 +84,7 @@ func GenerateMigrationAssets(source, herokuAPIKey, claudeAPIKey, qoveryAPIKey, d
 
 	progressChan <- ProgressUpdate{Stage: "Generating Terraform configs", Progress: 0.7}
 
-	terraformMain, terraformVariables, err := generateTerraform(qoveryConfigs, destination, claudeClient)
+	terraformMain, terraformVariables, err := generateTerraform(qoveryConfigs, destination, claudeClient, githubToken)
 	if err != nil {
 		return nil, fmt.Errorf("error generating Terraform configs: %w", err)
 	}
@@ -125,24 +126,24 @@ type TerraformExample struct {
 }
 
 // generateTerraform generates Terraform configurations for Qovery
-func generateTerraform(qoveryConfigs map[string]interface{}, destination string, claudeClient *claude.ClaudeClient) (string, string, error) {
+func generateTerraform(qoveryConfigs map[string]interface{}, destination string, claudeClient *claude.ClaudeClient, githubToken string) (string, string, error) {
 	configJSON, err := json.Marshal(qoveryConfigs)
 	if err != nil {
 		return "", "", fmt.Errorf("error marshaling Qovery configs: %w", err)
 	}
 
-	officialExamples, err := loadTerraformExamples("Qovery", "terraform-examples", "examples")
+	officialExamples, err := loadTerraformExamples("Qovery", "terraform-examples", "examples", githubToken)
 	if err != nil {
 		return "", "", fmt.Errorf("error loading Terraform examples: %w", err)
 	}
 
-	airbyteExample, err := loadTerraformExamples("evoxmusic", "qovery-airbyte", ".")
+	airbyteExample, err := loadTerraformExamples("evoxmusic", "qovery-airbyte", ".", githubToken)
 	if err != nil {
 		return "", "", fmt.Errorf("error loading Airbyte Terraform example: %w", err)
 	}
 
 	// https://github.com/Qovery/terraform-provider-qovery/tree/main/docs
-	qoveryTerraformDocMarkdown, err := loadMarkdownFiles("Qovery", "terraform-provider-qovery", "main")
+	qoveryTerraformDocMarkdown, err := loadMarkdownFiles("Qovery", "terraform-provider-qovery", "main", githubToken)
 	if err != nil {
 		return "", "", fmt.Errorf("error loading Qovery Terraform Provide markdown documentation: %w", err)
 	}
@@ -201,8 +202,21 @@ Instructions:
 	return finalMainTf, variablesTf, nil
 }
 
-func loadTerraformExamples(owner string, repo string, exampleDir string) ([]TerraformExample, error) {
-	client := github.NewClient(nil)
+// NewGitHubClient creates a new GitHub client with optional authentication
+func NewGitHubClient(token string) *github.Client {
+	ctx := context.Background()
+	if token != "" {
+		ts := oauth2.StaticTokenSource(
+			&oauth2.Token{AccessToken: token},
+		)
+		tc := oauth2.NewClient(ctx, ts)
+		return github.NewClient(tc)
+	}
+	return github.NewClient(nil)
+}
+
+func loadTerraformExamples(owner, repo, exampleDir, token string) ([]TerraformExample, error) {
+	client := NewGitHubClient(token)
 
 	_, dirContent, _, err := client.Repositories.GetContents(context.Background(), owner, repo, exampleDir, nil)
 	if err != nil {
@@ -234,8 +248,8 @@ func loadTerraformExamples(owner string, repo string, exampleDir string) ([]Terr
 	return examples, nil
 }
 
-func loadMarkdownFiles(owner, repo, branch string) (map[string]string, error) {
-	client := github.NewClient(nil)
+func loadMarkdownFiles(owner, repo, branch, token string) (map[string]string, error) {
+	client := NewGitHubClient(token)
 	ctx := context.Background()
 
 	result := make(map[string]string)

web/backend/go.mod

@@ -3,7 +3,7 @@ module backend
 go 1.22.5
 
 require (
-	github.com/Qovery/qovery-migration-ai-agent v0.0.0-20240913112237-53423fc0d9da
+	github.com/Qovery/qovery-migration-ai-agent v0.0.0-20240915120017-84ea9a36ec8c
 	github.com/gin-contrib/cors v1.7.2
 	github.com/gin-gonic/gin v1.10.0
 )
@@ -32,10 +32,10 @@ require (
 	github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
 	github.com/ugorji/go/codec v1.2.12 // indirect
 	golang.org/x/arch v0.8.0 // indirect
-	golang.org/x/crypto v0.23.0 // indirect
+	golang.org/x/crypto v0.27.0 // indirect
 	golang.org/x/net v0.25.0 // indirect
-	golang.org/x/sys v0.20.0 // indirect
-	golang.org/x/text v0.15.0 // indirect
+	golang.org/x/sys v0.25.0 // indirect
+	golang.org/x/text v0.18.0 // indirect
 	google.golang.org/protobuf v1.34.1 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

web/backend/go.sum

@@ -1,5 +1,7 @@
 github.com/Qovery/qovery-migration-ai-agent v0.0.0-20240913112237-53423fc0d9da h1:sPQ8Wmg9QvNANjDTO+w1eDijWVShptH2a1kVB1x9XjE=
 github.com/Qovery/qovery-migration-ai-agent v0.0.0-20240913112237-53423fc0d9da/go.mod h1:BK4dodrSYDAJySssVnQDgzCjaR1yvonA3hNYsDigYLQ=
+github.com/Qovery/qovery-migration-ai-agent v0.0.0-20240915120017-84ea9a36ec8c h1:S7LeYYwn2ZfqVJQXfgBSZJdjpGOLRybvInddSayR5Y8=
+github.com/Qovery/qovery-migration-ai-agent v0.0.0-20240915120017-84ea9a36ec8c/go.mod h1:jdwL5TgbMsrvvowpDiJsXZUg1sYR67WbKRtqINMUgGQ=
 github.com/bytedance/sonic v1.11.6 h1:oUp34TzMlL+OY1OUWxHqsdkgC/Zfc85zGqw9siXjrc0=
 github.com/bytedance/sonic v1.11.6/go.mod h1:LysEHSvpvDySVdC2f87zGWf6CIKJcAvqab1ZaiQtds4=
 github.com/bytedance/sonic/loader v0.1.1 h1:c+e5Pt1k/cy5wMveRDyk2X4B9hF4g7an8N3zCYjJFNM=
@@ -89,6 +91,8 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk
 golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI=
 golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
+golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A=
+golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70=
 golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac=
@@ -101,12 +105,15 @@ golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y=
 golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk=
 golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224=
+golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

web/frontend/src/pages/ReviewPage.jsx

@@ -53,6 +53,8 @@ function ReviewPage({migrationData}) {
         return (<div className="container">
                 <div className="migration-form">
                     <p className="subtitle">Generating migration files...</p>
+                    <p className="info-message">This process may take a few minutes depending on the number of services
+                        to migrate. Please do not reload the page.</p>
                     <QoveryLoader/>
                 </div>
             </div>);