diff --git a/pkgs/defang/cli.nix b/pkgs/defang/cli.nix index b4e4a2b83..91046827d 100644 --- a/pkgs/defang/cli.nix +++ b/pkgs/defang/cli.nix @@ -6,7 +6,7 @@ buildGoModule { pname = "defang-cli"; version = "git"; src = ../../src; - vendorHash = "sha256-P0CsjnnUmkFgljNotsFboEoHU6UoEIOgA5lBX2FXAxY="; + vendorHash = "sha256-HIkjHfMrBsApMjYFffiX7HMF34s8doO0x7BYnQqty6Q="; subPackages = [ "cmd/cli" ]; @@ -15,7 +15,6 @@ buildGoModule { ]; CGO_ENABLED = 0; - GOFLAGS = [ "-trimpath" ]; ldflags = [ "-s" "-w" ]; doCheck = false; # some unit tests need internet access diff --git a/src/go.mod b/src/go.mod index 6647059ff..306bca80e 100644 --- a/src/go.mod +++ b/src/go.mod @@ -25,7 +25,7 @@ require ( github.com/awslabs/goformation/v7 v7.13.1 github.com/bufbuild/connect-go v1.10.0 github.com/compose-spec/compose-go/v2 v2.4.3 - github.com/digitalocean/godo v1.118.0 + github.com/digitalocean/godo v1.131.1 github.com/docker/docker v25.0.6+incompatible github.com/google/uuid v1.6.0 github.com/googleapis/gax-go/v2 v2.13.0 @@ -42,7 +42,7 @@ require ( github.com/spf13/cobra v1.8.0 github.com/spf13/pflag v1.0.5 golang.org/x/mod v0.17.0 - golang.org/x/oauth2 v0.23.0 + golang.org/x/oauth2 v0.24.0 golang.org/x/sys v0.28.0 golang.org/x/term v0.27.0 google.golang.org/api v0.203.0 @@ -143,6 +143,6 @@ require ( golang.org/x/exp v0.0.0-20240112132812-db7319d0e0e3 // indirect; compose-go is using the older slices.sortFunc API golang.org/x/sync v0.10.0 // indirect golang.org/x/text v0.21.0 // indirect - golang.org/x/time v0.7.0 // indirect + golang.org/x/time v0.8.0 // indirect golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect ) diff --git a/src/go.sum b/src/go.sum index cb09519ab..5bf29d688 100644 --- a/src/go.sum +++ b/src/go.sum @@ -124,8 +124,8 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/digitalocean/godo v1.118.0 h1:lkzGFQmACrVCp7UqH1sAi4JK/PWwlc5aaxubgorKmC4= -github.com/digitalocean/godo v1.118.0/go.mod h1:Vk0vpCot2HOAJwc5WE8wljZGtJ3ZtWIc8MQ8rF38sdo= +github.com/digitalocean/godo v1.131.1 h1:2QsRwjNukKgOQbflMxOsTDoC05o5UKBpqQMFKXegYKE= +github.com/digitalocean/godo v1.131.1/go.mod h1:PU8JB6I1XYkQIdHFop8lLAY9ojp6M0XcU0TWaQSxbrc= github.com/distribution/reference v0.5.0 h1:/FUIFXtfc/x2gpa5/VGfiGLuOIdYa1t65IKK2OFGvA0= github.com/distribution/reference v0.5.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= github.com/docker/docker v25.0.6+incompatible h1:5cPwbwriIcsua2REJe8HqQV+6WlWc1byg2QSXzBxBGg= @@ -360,8 +360,8 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4= golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.23.0 h1:PbgcYx2W7i4LvjJWEbf0ngHV6qJYr86PkAV3bXdLEbs= -golang.org/x/oauth2 v0.23.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/oauth2 v0.24.0 h1:KTBBxWqUa0ykRPLtV69rRto9TLXcqYkeswu48x/gvNE= +golang.org/x/oauth2 v0.24.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -395,8 +395,8 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= -golang.org/x/time v0.7.0 h1:ntUhktv3OPE6TgYxXWv9vKvUSJyIFJlyohwbkEwPrKQ= -golang.org/x/time v0.7.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/time v0.8.0 h1:9i3RxcPv3PZnitoVGMPDKZSq1xW1gK1Xy3ArNOGZfEg= +golang.org/x/time v0.8.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= diff --git a/src/pkg/cli/client/byoc/aws/byoc.go b/src/pkg/cli/client/byoc/aws/byoc.go index e88ea8962..4cb007a5f 100644 --- a/src/pkg/cli/client/byoc/aws/byoc.go +++ b/src/pkg/cli/client/byoc/aws/byoc.go @@ -932,13 +932,13 @@ func (b *ByocAws) DeleteConfig(ctx context.Context, secrets *defangv1.Secrets) e return nil } -type awsObj struct{ obj s3types.Object } +type s3Obj struct{ obj s3types.Object } -func (a awsObj) Name() string { +func (a s3Obj) Name() string { return *a.obj.Key } -func (a awsObj) Size() int64 { +func (a s3Obj) Size() int64 { return *a.obj.Size } @@ -957,6 +957,10 @@ func (b *ByocAws) BootstrapList(ctx context.Context) ([]string, error) { } s3client := s3.NewFromConfig(cfg) + return ListPulumiStacks(ctx, s3client, bucketName) +} + +func ListPulumiStacks(ctx context.Context, s3client *s3.Client, bucketName string) ([]string, error) { prefix := `.pulumi/stacks/` // TODO: should we filter on `projectName`? term.Debug("Listing stacks in bucket:", bucketName) @@ -972,7 +976,7 @@ func (b *ByocAws) BootstrapList(ctx context.Context) ([]string, error) { if obj.Key == nil || obj.Size == nil { continue } - stack, err := b.ParsePulumiStackObject(ctx, awsObj{obj}, bucketName, prefix, func(ctx context.Context, bucket, path string) ([]byte, error) { + stack, err := byoc.ParsePulumiStackObject(ctx, s3Obj{obj}, bucketName, prefix, func(ctx context.Context, bucket, path string) ([]byte, error) { getObjectOutput, err := s3client.GetObject(ctx, &s3.GetObjectInput{ Bucket: &bucket, Key: &path, diff --git a/src/pkg/cli/client/byoc/aws/byoc_test.go b/src/pkg/cli/client/byoc/aws/byoc_test.go index 80ecaf6be..429b4597a 100644 --- a/src/pkg/cli/client/byoc/aws/byoc_test.go +++ b/src/pkg/cli/client/byoc/aws/byoc_test.go @@ -88,15 +88,15 @@ func (f FakeLoader) LoadProjectName(ctx context.Context) (string, error) { return f.ProjectName, nil } -//go:embed test_ecs_events/*.json +//go:embed testdata/*.json var testDir embed.FS -//go:embed test_ecs_events/*.events +//go:embed testdata/*.events var expectedDir embed.FS func TestSubscribe(t *testing.T) { t.Skip("Pending test") - tests, err := testDir.ReadDir("test_ecs_events") + tests, err := testDir.ReadDir("testdata") if err != nil { t.Fatalf("failed to load ecs events test files: %v", err) } @@ -125,7 +125,7 @@ func TestSubscribe(t *testing.T) { go func() { defer wg.Done() - filename := path.Join("test_ecs_events", name+".events") + filename := path.Join("testdata", name+".events") ef, _ := expectedDir.ReadFile(filename) dec := json.NewDecoder(bytes.NewReader(ef)) @@ -148,7 +148,7 @@ func TestSubscribe(t *testing.T) { } }() - data, err := testDir.ReadFile(path.Join("test_ecs_events", tt.Name())) + data, err := testDir.ReadFile(path.Join("testdata", tt.Name())) if err != nil { t.Fatalf("failed to read test file: %v", err) } diff --git a/src/pkg/cli/client/byoc/aws/test_ecs_events/build-failure-o4epidtq3j3b.json b/src/pkg/cli/client/byoc/aws/testdata/build-failure-o4epidtq3j3b.json similarity index 100% rename from src/pkg/cli/client/byoc/aws/test_ecs_events/build-failure-o4epidtq3j3b.json rename to src/pkg/cli/client/byoc/aws/testdata/build-failure-o4epidtq3j3b.json diff --git a/src/pkg/cli/client/byoc/aws/test_ecs_events/build-failure.events b/src/pkg/cli/client/byoc/aws/testdata/build-failure.events similarity index 100% rename from src/pkg/cli/client/byoc/aws/test_ecs_events/build-failure.events rename to src/pkg/cli/client/byoc/aws/testdata/build-failure.events diff --git a/src/pkg/cli/client/byoc/aws/test_ecs_events/failure-then-success-c1v6g2m5qlvm.json b/src/pkg/cli/client/byoc/aws/testdata/failure-then-success-c1v6g2m5qlvm.json similarity index 100% rename from src/pkg/cli/client/byoc/aws/test_ecs_events/failure-then-success-c1v6g2m5qlvm.json rename to src/pkg/cli/client/byoc/aws/testdata/failure-then-success-c1v6g2m5qlvm.json diff --git a/src/pkg/cli/client/byoc/aws/test_ecs_events/failure-then-success.events b/src/pkg/cli/client/byoc/aws/testdata/failure-then-success.events similarity index 100% rename from src/pkg/cli/client/byoc/aws/test_ecs_events/failure-then-success.events rename to src/pkg/cli/client/byoc/aws/testdata/failure-then-success.events diff --git a/src/pkg/cli/client/byoc/aws/test_ecs_events/healthcheck-failure-8ui3h0yf5xqg.json b/src/pkg/cli/client/byoc/aws/testdata/healthcheck-failure-8ui3h0yf5xqg.json similarity index 100% rename from src/pkg/cli/client/byoc/aws/test_ecs_events/healthcheck-failure-8ui3h0yf5xqg.json rename to src/pkg/cli/client/byoc/aws/testdata/healthcheck-failure-8ui3h0yf5xqg.json diff --git a/src/pkg/cli/client/byoc/aws/test_ecs_events/healthcheck-failure.events b/src/pkg/cli/client/byoc/aws/testdata/healthcheck-failure.events similarity index 100% rename from src/pkg/cli/client/byoc/aws/test_ecs_events/healthcheck-failure.events rename to src/pkg/cli/client/byoc/aws/testdata/healthcheck-failure.events diff --git a/src/pkg/cli/client/byoc/aws/test_ecs_events/processexit-failure-se3n0qmzhzpm.json b/src/pkg/cli/client/byoc/aws/testdata/processexit-failure-se3n0qmzhzpm.json similarity index 100% rename from src/pkg/cli/client/byoc/aws/test_ecs_events/processexit-failure-se3n0qmzhzpm.json rename to src/pkg/cli/client/byoc/aws/testdata/processexit-failure-se3n0qmzhzpm.json diff --git a/src/pkg/cli/client/byoc/aws/test_ecs_events/processexit-failure.events b/src/pkg/cli/client/byoc/aws/testdata/processexit-failure.events similarity index 100% rename from src/pkg/cli/client/byoc/aws/test_ecs_events/processexit-failure.events rename to src/pkg/cli/client/byoc/aws/testdata/processexit-failure.events diff --git a/src/pkg/cli/client/byoc/aws/test_ecs_events/success-f249u7ap07ef.json b/src/pkg/cli/client/byoc/aws/testdata/success-f249u7ap07ef.json similarity index 100% rename from src/pkg/cli/client/byoc/aws/test_ecs_events/success-f249u7ap07ef.json rename to src/pkg/cli/client/byoc/aws/testdata/success-f249u7ap07ef.json diff --git a/src/pkg/cli/client/byoc/aws/test_ecs_events/success.events b/src/pkg/cli/client/byoc/aws/testdata/success.events similarity index 100% rename from src/pkg/cli/client/byoc/aws/test_ecs_events/success.events rename to src/pkg/cli/client/byoc/aws/testdata/success.events diff --git a/src/pkg/cli/client/byoc/baseclient.go b/src/pkg/cli/client/byoc/baseclient.go index bb621a77a..258434124 100644 --- a/src/pkg/cli/client/byoc/baseclient.go +++ b/src/pkg/cli/client/byoc/baseclient.go @@ -2,7 +2,6 @@ package byoc import ( "context" - "encoding/json" "errors" "fmt" "os" @@ -168,50 +167,3 @@ func (b *ByocBaseClient) GetProjectDomain(projectName, zone string) string { func GetPrivateDomain(projectName string) string { return DnsSafeLabel(projectName) + ".internal" } - -type Obj interface { - Name() string - Size() int64 -} - -func (b *ByocBaseClient) ParsePulumiStackObject(ctx context.Context, obj Obj, bucket, prefix string, objLoader func(ctx context.Context, bucket, object string) ([]byte, error)) (string, error) { - // The JSON file for an empty stack is ~600 bytes; we add a margin of 100 bytes to account for the length of the stack/project names - if !strings.HasSuffix(obj.Name(), ".json") || obj.Size() < 700 { - return "", nil - } - // Cut off the prefix and the .json suffix - stack := (obj.Name())[len(prefix) : len(obj.Name())-5] - // Check the contents of the JSON file, because the size is not a reliable indicator of a valid stack - data, err := objLoader(ctx, bucket, obj.Name()) - if err != nil { - return "", fmt.Errorf("failed to get Pulumi state object %q: %w", obj.Name(), err) - } - var state struct { - Version int `json:"version"` - Checkpoint struct { - // Stack string `json:"stack"` TODO: could use this instead of deriving the stack name from the key - Latest struct { - Resources []struct{} `json:"resources,omitempty"` - PendingOperations []struct { - Resource struct { - Urn string `json:"urn"` - } - } `json:"pending_operations,omitempty"` - } - } - } - if err := json.Unmarshal(data, &state); err != nil { - return "", fmt.Errorf("failed to decode Pulumi state %q: %w", obj.Name(), err) - } else if state.Version != 3 { - term.Debug("Skipping Pulumi state with version", state.Version) - } else if len(state.Checkpoint.Latest.PendingOperations) > 0 { - for _, op := range state.Checkpoint.Latest.PendingOperations { - parts := strings.Split(op.Resource.Urn, "::") // prefix::project::type::resource => urn:provider:stack::project::plugin:file:class::name - stack += fmt.Sprintf(" (pending %q)", parts[3]) - } - } else if len(state.Checkpoint.Latest.Resources) == 0 { - return "", nil // skip: no resources and no pending operations - } - - return stack, nil -} diff --git a/src/pkg/cli/client/byoc/do/byoc.go b/src/pkg/cli/client/byoc/do/byoc.go index 2f222f553..2eec4c9c0 100644 --- a/src/pkg/cli/client/byoc/do/byoc.go +++ b/src/pkg/cli/client/byoc/do/byoc.go @@ -52,9 +52,13 @@ var ( type ByocDo struct { *byoc.ByocBaseClient - buildRepo string - client *godo.Client - driver *appPlatform.DoApp + buildRepo string + client *godo.Client + driver *appPlatform.DoApp + lastCdAppID string + lastCdDeploymentID string + lastCdEtag types.ETag + // lastCdStart time.Time } var _ client.Provider = (*ByocDo)(nil) @@ -76,12 +80,12 @@ func NewByocProvider(ctx context.Context, tenantName types.TenantName) *ByocDo { } func (b *ByocDo) GetProjectUpdate(ctx context.Context, projectName string) (*defangv1.ProjectUpdate, error) { - client, err := b.driver.CreateS3Client() + s3client, err := b.driver.CreateS3Client() if err != nil { return nil, err } - bucketName, err := b.driver.GetBucketName(ctx, client) + bucketName, err := b.driver.GetBucketName(ctx, s3client) if err != nil { return nil, err } @@ -92,7 +96,7 @@ func (b *ByocDo) GetProjectUpdate(ctx context.Context, projectName string) (*def } path := fmt.Sprintf("projects/%s/%s/project.pb", projectName, b.PulumiStack) - getObjectOutput, err := client.GetObject(ctx, &s3.GetObjectInput{ + getObjectOutput, err := s3client.GetObject(ctx, &s3.GetObjectInput{ Bucket: &bucketName, Key: &path, }) @@ -204,6 +208,7 @@ func (b *ByocDo) deploy(ctx context.Context, req *defangv1.DeployRequest, cmd st return nil, err } + b.lastCdEtag = etag return &defangv1.DeployResponse{ Services: serviceInfos, Etag: etag, @@ -219,28 +224,24 @@ func (b *ByocDo) BootstrapCommand(ctx context.Context, req client.BootstrapComma if err != nil { return "", err } - etag := pkg.RandomID() + etag := pkg.RandomID() + b.lastCdEtag = etag return etag, nil } func (b *ByocDo) BootstrapList(ctx context.Context) ([]string, error) { - // Use DO api to query which apps (or projects) exist based on defang constant - - var projectList []string - - projects, _, err := b.client.Projects.List(ctx, &godo.ListOptions{}) + s3client, err := b.driver.CreateS3Client() if err != nil { return nil, err } - for _, project := range projects { - if strings.Contains(project.Name, "Defang") { - projectList = append(projectList, project.Name) - } + bucketName, err := b.driver.GetBucketName(ctx, s3client) + if bucketName == "" { + return nil, err } - return projectList, nil + return awsbyoc.ListPulumiStacks(ctx, s3client, bucketName) } func (b *ByocDo) CreateUploadURL(ctx context.Context, req *defangv1.UploadURLRequest) (*defangv1.UploadURLResponse, error) { @@ -299,7 +300,7 @@ func (b *ByocDo) GetService(ctx context.Context, s *defangv1.GetRequest) (*defan for _, service := range app.Spec.Services { if service.Name == s.Name { - serviceInfo = b.processServiceInfo(service, s.Project) + serviceInfo = processServiceInfo(service, s.Project) } } @@ -314,7 +315,7 @@ func (b *ByocDo) getProjectInfo(ctx context.Context, services *[]*defangv1.Servi } for _, service := range app.Spec.Services { - serviceInfo := b.processServiceInfo(service, projectName) + serviceInfo := processServiceInfo(service, projectName) *services = append(*services, serviceInfo) } @@ -373,19 +374,30 @@ func (b *ByocDo) PutConfig(ctx context.Context, config *defangv1.PutConfigReques } func (b *ByocDo) Follow(ctx context.Context, req *defangv1.TailRequest) (client.ServerStream[defangv1.TailResponse], error) { - //Look up the CD app directly instead of relying on the etag - cdApp, err := b.getAppByName(ctx, appPlatform.CdName) - if err != nil { - return nil, err - } + var appID, deploymentID string - var deploymentID string - if cdApp.PendingDeployment != nil { - deploymentID = cdApp.PendingDeployment.GetID() + if req.Etag != "" && req.Etag == b.lastCdEtag { + // Use the last known app and deployment ID from the last CD command + appID = b.lastCdAppID + deploymentID = b.lastCdDeploymentID } - if deploymentID == "" && cdApp.ActiveDeployment != nil { - deploymentID = cdApp.ActiveDeployment.GetID() + if deploymentID == "" || appID == "" { + //Look up the CD app directly instead of relying on the etag + term.Debug("Fetching app and deployment ID for app", appPlatform.CdName) + cdApp, err := b.getAppByName(ctx, appPlatform.CdName) + if err != nil { + return nil, err + } + appID = cdApp.ID + switch { + case cdApp.PendingDeployment != nil: + deploymentID = cdApp.PendingDeployment.ID + case cdApp.InProgressDeployment != nil: + deploymentID = cdApp.InProgressDeployment.ID + case cdApp.ActiveDeployment != nil: + deploymentID = cdApp.ActiveDeployment.ID + } } if deploymentID == "" { @@ -394,7 +406,7 @@ func (b *ByocDo) Follow(ctx context.Context, req *defangv1.TailRequest) (client. term.Info("Waiting for CD command to finish gathering logs") for { - deploymentInfo, _, err := b.client.Apps.GetDeployment(ctx, cdApp.ID, deploymentID) + deploymentInfo, _, err := b.client.Apps.GetDeployment(ctx, appID, deploymentID) if err != nil { return nil, err } @@ -408,7 +420,8 @@ func (b *ByocDo) Follow(ctx context.Context, req *defangv1.TailRequest) (client. case godo.DeploymentPhase_Error, godo.DeploymentPhase_Canceled: if logType.Has(logs.LogTypeBuild) { - logs, _, err := b.client.Apps.GetLogs(ctx, cdApp.ID, deploymentID, "", godo.AppLogTypeDeploy, true, 50) + // TODO: provide component name + logs, _, err := b.client.Apps.GetLogs(ctx, appID, deploymentID, "", godo.AppLogTypeDeploy, true, 50) if err != nil { return nil, err } @@ -418,7 +431,7 @@ func (b *ByocDo) Follow(ctx context.Context, req *defangv1.TailRequest) (client. case godo.DeploymentPhase_Active: if logType.Has(logs.LogTypeBuild) { - logs, _, err := b.client.Apps.GetLogs(ctx, cdApp.ID, deploymentID, "", godo.AppLogTypeDeploy, true, 50) + logs, _, err := b.client.Apps.GetLogs(ctx, appID, deploymentID, "", godo.AppLogTypeDeploy, true, 50) if err != nil { return nil, err } @@ -446,12 +459,12 @@ func (b *ByocDo) TearDown(ctx context.Context) error { return err } - _, err = b.client.Registry.Delete(ctx) + _, err = b.client.Apps.Delete(ctx, app.ID) if err != nil { return err } - _, err = b.client.Apps.Delete(ctx, app.ID) + _, err = b.client.Registry.Delete(ctx) if err != nil { return err } @@ -501,9 +514,92 @@ func (i DoAccountInfo) Details() string { return "" } -func (b *ByocDo) Subscribe(context.Context, *defangv1.SubscribeRequest) (client.ServerStream[defangv1.SubscribeResponse], error) { - //optional - return nil, errors.New("please check the Activity tab in the DigitalOcean App Platform console") +func (b *ByocDo) Subscribe(ctx context.Context, req *defangv1.SubscribeRequest) (client.ServerStream[defangv1.SubscribeResponse], error) { + if req.Etag != b.lastCdEtag || b.lastCdAppID == "" { + return nil, errors.ErrUnsupported // TODO: fetch the deployment ID for the given etag + } + ctx, cancel := context.WithCancel(ctx) // canceled by subscribeStream.Close() + return &subscribeStream{ + appID: b.lastCdAppID, + b: b, + deploymentID: b.lastCdDeploymentID, + ctx: ctx, + cancel: cancel, + queue: make(chan *defangv1.SubscribeResponse, 10), + }, nil +} + +type subscribeStream struct { + appID string + b *ByocDo + ctx context.Context + cancel context.CancelFunc + deploymentID string + err error + queue chan *defangv1.SubscribeResponse + msg *defangv1.SubscribeResponse +} + +func phaseToState(phase godo.DeploymentPhase) defangv1.ServiceState { + switch phase { + case godo.DeploymentPhase_Building: + return defangv1.ServiceState_BUILD_RUNNING + case godo.DeploymentPhase_Active: + return defangv1.ServiceState_DEPLOYMENT_COMPLETED + case godo.DeploymentPhase_Canceled: + return defangv1.ServiceState_DEPLOYMENT_SCALED_IN + case godo.DeploymentPhase_Error: + return defangv1.ServiceState_DEPLOYMENT_FAILED + case godo.DeploymentPhase_PendingBuild: + return defangv1.ServiceState_BUILD_QUEUED + case godo.DeploymentPhase_PendingDeploy: + return defangv1.ServiceState_UPDATE_QUEUED + case godo.DeploymentPhase_Deploying: + return defangv1.ServiceState_DEPLOYMENT_PENDING + default: + return defangv1.ServiceState_NOT_SPECIFIED + } +} + +func (s *subscribeStream) Receive() bool { + select { + case <-s.ctx.Done(): + s.err = s.ctx.Err() + s.msg = nil + return false + case r := <-s.queue: + s.msg = r + return true + default: + } + deployment, _, err := s.b.client.Apps.GetDeployment(s.ctx, s.appID, s.deploymentID) + if err != nil { + s.msg = nil + s.err = err + return false + } + for _, service := range deployment.Spec.Services { + s.queue <- &defangv1.SubscribeResponse{ + Name: service.Name, + Status: string(deployment.Phase), + State: phaseToState(deployment.Phase), + } + } + s.msg = <-s.queue + return err == nil +} + +func (s *subscribeStream) Msg() *defangv1.SubscribeResponse { + return s.msg +} + +func (s *subscribeStream) Err() error { + return s.err +} + +func (s *subscribeStream) Close() error { + s.cancel() + return nil } func (b *ByocDo) Query(ctx context.Context, req *defangv1.DebugRequest) error { @@ -527,7 +623,13 @@ func (b *ByocDo) runCdCommand(ctx context.Context, projectName, delegateDomain s } } app, err := b.driver.Run(ctx, env, b.CDImage, append([]string{"node", "lib/index.js"}, cmd...)...) - return app, err + if err != nil { + return nil, err + } + + b.lastCdAppID = app.ID + b.lastCdDeploymentID = app.PendingDeployment.ID + return app, nil } func (b *ByocDo) environment(projectName, delegateDomain string) []*godo.AppVariableDefinition { @@ -652,7 +754,7 @@ func (b *ByocDo) setUp(ctx context.Context) error { return nil } - if err := b.driver.SetUp(ctx); err != nil { + if err := b.driver.SetUpBucket(ctx); err != nil { return err } @@ -701,14 +803,12 @@ func (b *ByocDo) getAppByName(ctx context.Context, name string) (*godo.App, erro return nil, fmt.Errorf("app not found: %s", appName) } -func (b *ByocDo) processServiceInfo(service *godo.AppServiceSpec, projectName string) *defangv1.ServiceInfo { +func processServiceInfo(service *godo.AppServiceSpec, projectName string) *defangv1.ServiceInfo { serviceInfo := &defangv1.ServiceInfo{ Project: projectName, - Etag: pkg.RandomID(), + Etag: pkg.RandomID(), // TODO: get the real etag from spec somehow Service: &defangv1.Service{ Name: service.Name, - // Image: service.Image.Digest, - // Environment: getServiceEnv(service.Envs), }, } diff --git a/src/pkg/cli/client/byoc/gcp/byoc.go b/src/pkg/cli/client/byoc/gcp/byoc.go index 6466ad8b7..a9489d29d 100644 --- a/src/pkg/cli/client/byoc/gcp/byoc.go +++ b/src/pkg/cli/client/byoc/gcp/byoc.go @@ -229,7 +229,7 @@ func (b *ByocGcp) BootstrapList(ctx context.Context) ([]string, error) { var stacks []string err = b.driver.IterateBucketObjects(ctx, bucketName, prefix, func(obj *storage.ObjectAttrs) error { - stack, err := b.ParsePulumiStackObject(ctx, gcpObj{obj}, bucketName, prefix, b.driver.GetBucketObject) + stack, err := byoc.ParsePulumiStackObject(ctx, gcpObj{obj}, bucketName, prefix, b.driver.GetBucketObject) if err != nil { return err } diff --git a/src/pkg/cli/client/byoc/parse.go b/src/pkg/cli/client/byoc/parse.go new file mode 100644 index 000000000..0b9263127 --- /dev/null +++ b/src/pkg/cli/client/byoc/parse.go @@ -0,0 +1,57 @@ +package byoc + +import ( + "context" + "encoding/json" + "fmt" + "strings" + + "github.com/DefangLabs/defang/src/pkg/term" +) + +type Obj interface { + Name() string + Size() int64 +} + +func ParsePulumiStackObject(ctx context.Context, obj Obj, bucket, prefix string, objLoader func(ctx context.Context, bucket, object string) ([]byte, error)) (string, error) { + // The JSON file for an empty stack is ~600 bytes; we add a margin of 100 bytes to account for the length of the stack/project names + if !strings.HasSuffix(obj.Name(), ".json") || obj.Size() < 700 { + return "", nil + } + // Cut off the prefix and the .json suffix + stack := (obj.Name())[len(prefix) : len(obj.Name())-5] + // Check the contents of the JSON file, because the size is not a reliable indicator of a valid stack + data, err := objLoader(ctx, bucket, obj.Name()) + if err != nil { + return "", fmt.Errorf("failed to get Pulumi state object %q: %w", obj.Name(), err) + } + var state struct { + Version int `json:"version"` + Checkpoint struct { + // Stack string `json:"stack"` TODO: could use this instead of deriving the stack name from the key + Latest struct { + Resources []struct{} `json:"resources,omitempty"` + PendingOperations []struct { + Resource struct { + Urn string `json:"urn"` + } + } `json:"pending_operations,omitempty"` + } + } + } + if err := json.Unmarshal(data, &state); err != nil { + return "", fmt.Errorf("failed to decode Pulumi state %q: %w", obj.Name(), err) + } else if state.Version != 3 { + term.Debug("Skipping Pulumi state with version", state.Version) + } else if len(state.Checkpoint.Latest.PendingOperations) > 0 { + for _, op := range state.Checkpoint.Latest.PendingOperations { + parts := strings.Split(op.Resource.Urn, "::") // prefix::project::type::resource => urn:provider:stack::project::plugin:file:class::name + stack += fmt.Sprintf(" (pending %q)", parts[3]) + } + } else if len(state.Checkpoint.Latest.Resources) == 0 { + return "", nil // skip: no resources and no pending operations + } + + return stack, nil +} diff --git a/src/pkg/cli/getServices.go b/src/pkg/cli/getServices.go index 476287a51..897d21a66 100644 --- a/src/pkg/cli/getServices.go +++ b/src/pkg/cli/getServices.go @@ -19,12 +19,9 @@ func (e ErrNoServices) Error() string { return fmt.Sprintf("no services found in project %q", e.ProjectName) } -type PrintService struct { - Name string - Etag string - PublicFqdn string - PrivateFqdn string - Status string +type printService struct { + Service string + *defangv1.ServiceInfo } func GetServices(ctx context.Context, projectName string, provider client.Provider, long bool) error { @@ -54,17 +51,14 @@ func GetServices(ctx context.Context, projectName string, provider client.Provid return PrintObject("", servicesResponse) } - printServices := make([]PrintService, numServices) + printServices := make([]printService, numServices) for i, si := range servicesResponse.Services { - printServices[i] = PrintService{ - Name: si.Service.Name, - Etag: si.Etag, - PublicFqdn: si.PublicFqdn, - PrivateFqdn: si.PrivateFqdn, - Status: si.Status, + printServices[i] = printService{ + Service: si.Service.Name, + ServiceInfo: si, } servicesResponse.Services[i] = nil } - return term.Table(printServices, []string{"Name", "Etag", "PublicFqdn", "PrivateFqdn", "Status"}) + return term.Table(printServices, []string{"Service", "Etag", "PublicFqdn", "PrivateFqdn", "Status"}) } diff --git a/src/pkg/cli/getServices_test.go b/src/pkg/cli/getServices_test.go index 841e62b90..efb8a5b55 100644 --- a/src/pkg/cli/getServices_test.go +++ b/src/pkg/cli/getServices_test.go @@ -82,8 +82,8 @@ func TestGetServices(t *testing.T) { if err != nil { t.Fatalf("GetServices() error = %v", err) } - expectedOutput := `Name Etag PublicFqdn PrivateFqdn Status -foo a1b2c3 test-foo.prod1.defang.dev UNKNOWN + expectedOutput := `Service Etag PublicFqdn PrivateFqdn Status +foo a1b2c3 test-foo.prod1.defang.dev UNKNOWN ` receivedLines := strings.Split(stdout.String(), "\n") diff --git a/src/pkg/clouds/do/appPlatform/setup.go b/src/pkg/clouds/do/appPlatform/setup.go index 8997f0b1b..d7c4b904d 100644 --- a/src/pkg/clouds/do/appPlatform/setup.go +++ b/src/pkg/clouds/do/appPlatform/setup.go @@ -67,7 +67,7 @@ func (d *DoApp) GetBucketName(ctx context.Context, s3Client *s3.Client) (string, return bucketName, nil } -func (d *DoApp) SetUp(ctx context.Context) error { +func (d *DoApp) SetUpBucket(ctx context.Context) error { s3Client, err := d.CreateS3Client() if err != nil { return err @@ -167,7 +167,8 @@ func (d DoApp) Run(ctx context.Context, env []*godo.AppVariableDefinition, cdIma if currentCd.Spec != nil && currentCd.Spec.Name != "" { term.Debugf("Updating existing CD app") currentCd, _, err = client.Apps.Update(ctx, currentCd.ID, &godo.AppUpdateRequest{ - Spec: appJobSpec, + Spec: appJobSpec, + UpdateAllSourceVersions: true, // force update of the CD image }) if err != nil {