(TFECO-7366) Stacks: parse terraform-sources.json to support remote component sources

.changes/unreleased/ENHANCEMENTS-20241028-090453.yaml

@@ -0,0 +1,6 @@
+kind: ENHANCEMENTS
+body: 'Stacks: parse terraform-sources.json to support remote component sources'
+time: 2024-10-28T09:04:53.004252+01:00
+custom:
+    Issue: "1836"
+    Repository: terraform-ls

go.mod

@@ -10,6 +10,7 @@ require (
 	github.com/hashicorp/go-cleanhttp v0.5.2
 	github.com/hashicorp/go-memdb v1.3.4
 	github.com/hashicorp/go-multierror v1.1.1
+	github.com/hashicorp/go-slug v0.16.0
 	github.com/hashicorp/go-uuid v1.0.3
 	github.com/hashicorp/go-version v1.7.0
 	github.com/hashicorp/hc-install v0.9.0
@@ -18,7 +19,7 @@ require (
 	github.com/hashicorp/terraform-exec v0.21.0
 	github.com/hashicorp/terraform-json v0.23.0
 	github.com/hashicorp/terraform-registry-address v0.2.3
-	github.com/hashicorp/terraform-schema v0.0.0-20241115125457-9097f9e5a82e
+	github.com/hashicorp/terraform-schema v0.0.0-20241128095320-1392f740c4fe
 	github.com/mcuadros/go-defaults v1.2.0
 	github.com/mh-cbon/go-fmt-fail v0.0.0-20160815164508-67765b3fbcb5
 	github.com/mitchellh/cli v1.1.5
@@ -37,6 +38,7 @@ require (
 
 require (
 	github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect
+	github.com/apparentlymart/go-versions v1.0.1 // indirect
 	github.com/felixge/httpsnoop v1.0.4 // indirect
 	github.com/go-logr/logr v1.4.2 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect

go.sum

@@ -63,6 +63,8 @@ github.com/apparentlymart/go-textseg v1.0.0 h1:rRmlIsPEEhUTIKQb7T++Nz/A5Q6C9IuX2
 github.com/apparentlymart/go-textseg v1.0.0/go.mod h1:z96Txxhf3xSFMPmb5X/1W05FF/Nj9VFpLOpjS5yuumk=
 github.com/apparentlymart/go-textseg/v15 v15.0.0 h1:uYvfpb3DyLSCGWnctWKGj857c6ew1u1fNQOlOtuGxQY=
 github.com/apparentlymart/go-textseg/v15 v15.0.0/go.mod h1:K8XmNZdhEBkdlyDdvbmmsvpAG721bKi0joRfFdHIWJ4=
+github.com/apparentlymart/go-versions v1.0.1 h1:ECIpSn0adcYNsBfSRwdDdz9fWlL+S/6EUd9+irwkBgU=
+github.com/apparentlymart/go-versions v1.0.1/go.mod h1:YF5j7IQtrOAOnsGkniupEA5bfCjzd7i14yu0shZavyM=
 github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
 github.com/armon/go-radix v1.0.0 h1:F4z6KzEeeQIMeLFa97iZU6vupzoecKdU5TX24SNppXI=
 github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
@@ -130,6 +132,7 @@ github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY=
 github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
 github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
 github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
+github.com/go-test/deep v1.0.1/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA=
 github.com/go-test/deep v1.0.3 h1:ZrJSEWsXzPOxaZnFteGEfooLba+ju3FYIbOrS+rQd68=
 github.com/go-test/deep v1.0.3/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA=
 github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
@@ -210,6 +213,8 @@ github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+l
 github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
 github.com/hashicorp/go-retryablehttp v0.7.7 h1:C8hUCYzor8PIfXHa4UrZkU4VvK8o9ISHxT2Q8+VepXU=
 github.com/hashicorp/go-retryablehttp v0.7.7/go.mod h1:pkQpWZeYWskR+D1tR2O5OcBFOxfA7DoAO6xtkuQnHTk=
+github.com/hashicorp/go-slug v0.16.0 h1:S/ko9fms1gf6305ktJNUKGxFmscZ+yWvAtsas0SYUyA=
+github.com/hashicorp/go-slug v0.16.0/go.mod h1:THWVTAXwJEinbsp4/bBRcmbaO5EYNLTqxbG4tZ3gCYQ=
 github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
 github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8=
 github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
@@ -233,8 +238,8 @@ github.com/hashicorp/terraform-json v0.23.0 h1:sniCkExU4iKtTADReHzACkk8fnpQXrdD2
 github.com/hashicorp/terraform-json v0.23.0/go.mod h1:MHdXbBAbSg0GvzuWazEGKAn/cyNfIB7mN6y7KJN6y2c=
 github.com/hashicorp/terraform-registry-address v0.2.3 h1:2TAiKJ1A3MAkZlH1YI/aTVcLZRu7JseiXNRHbOAyoTI=
 github.com/hashicorp/terraform-registry-address v0.2.3/go.mod h1:lFHA76T8jfQteVfT7caREqguFrW3c4MFSPhZB7HHgUM=
-github.com/hashicorp/terraform-schema v0.0.0-20241115125457-9097f9e5a82e h1:yozV7l40vatIUPCYTeTqpPvOKsPDOd00kg8Tetf4VeQ=
-github.com/hashicorp/terraform-schema v0.0.0-20241115125457-9097f9e5a82e/go.mod h1:hwYMiQp/tVcJtYfbNSxEEK+ilauXwwtZgpLXmeUBVGg=
+github.com/hashicorp/terraform-schema v0.0.0-20241128095320-1392f740c4fe h1:2pVtzihaLjn6PTIyKom+X491QlLupxGoLRf5Ik8zpYM=
+github.com/hashicorp/terraform-schema v0.0.0-20241128095320-1392f740c4fe/go.mod h1:3vDqHlpaMuTeBXSC4LWDM/m2QdEe9DmC90IgyuhdgZw=
 github.com/hashicorp/terraform-svchost v0.1.1 h1:EZZimZ1GxdqFRinZ1tpJwVxxt49xc/S52uzrw4x0jKQ=
 github.com/hashicorp/terraform-svchost v0.1.1/go.mod h1:mNsjQfZyf/Jhz35v6/0LWcv26+X7JPS+buii2c9/ctc=
 github.com/hexops/autogold v1.3.1 h1:YgxF9OHWbEIUjhDbpnLhgVsjUDsiHDTyDfy2lrfdlzo=
@@ -279,6 +284,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348 h1:MtvEpTB6LX3vkb4ax0b5D2DHbNAUsen0Gx5wZoq3lV4=
+github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k=
 github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY=
 github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0=
 github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=

internal/features/rootmodules/events.go

@@ -5,6 +5,7 @@ package rootmodules
 
 import (
 	"context"
+	"path/filepath"
 
 	"github.com/hashicorp/terraform-ls/internal/document"
 	"github.com/hashicorp/terraform-ls/internal/features/rootmodules/ast"
@@ -85,6 +86,18 @@ func (f *RootModulesFeature) didOpen(ctx context.Context, dir document.DirHandle
 	}
 	ids = append(ids, modManifestId)
 
+	terraformSourcesId, err := f.stateStore.JobStore.EnqueueJob(ctx, job.Job{
+		Dir: dir,
+		Func: func(ctx context.Context) error {
+			return jobs.ParseTerraformSources(ctx, f.fs, f.Store, dir.Path())
+		},
+		Type: op.OpTypeParseTerraformSources.String(),
+	})
+	if err != nil {
+		return ids, err
+	}
+	ids = append(ids, terraformSourcesId)
+
 	pSchemaVerId, err := f.stateStore.JobStore.EnqueueJob(ctx, job.Job{
 		Dir: dir,
 		Func: func(ctx context.Context) error {
@@ -168,6 +181,9 @@ func (f *RootModulesFeature) manifestChange(ctx context.Context, dir document.Di
 	if changeType == protocol.Deleted {
 		// Manifest is deleted, so we clear the manifest from the store
 		f.Store.UpdateModManifest(path, nil, nil)
+		// We also delete the Terraform Sources (if they exist), since delete changes can also happen if the
+		// entire .terraform directory is deleted and there should only be either a manifest or terraform sources anyway
+		f.Store.UpdateTerraformSources(path, nil, nil)
 		return ids, nil
 	}
 
@@ -186,6 +202,21 @@ func (f *RootModulesFeature) manifestChange(ctx context.Context, dir document.Di
 	}
 	ids = append(ids, modManifestId)
 
+	terraformSourcesId, err := f.stateStore.JobStore.EnqueueJob(ctx, job.Job{
+		Dir: dir,
+		Func: func(ctx context.Context) error {
+			return jobs.ParseTerraformSources(ctx, f.fs, f.Store, dir.Path())
+		},
+		Type: op.OpTypeParseTerraformSources.String(),
+		Defer: func(ctx context.Context, jobErr error) (job.IDs, error) {
+			return f.indexTerraformSourcesDirs(ctx, dir)
+		},
+	})
+	if err != nil {
+		return ids, err
+	}
+	ids = append(ids, terraformSourcesId)
+
 	return ids, nil
 }
 
@@ -204,3 +235,14 @@ func (f *RootModulesFeature) indexInstalledModuleCalls(ctx context.Context, dir
 
 	return jobIds, nil
 }
+
+func (f *RootModulesFeature) indexTerraformSourcesDirs(ctx context.Context, dir document.DirHandle) (job.IDs, error) {
+	jobIds := make(job.IDs, 0)
+
+	for _, subDir := range f.Store.TerraformSourcesDirectories(dir.Path()) {
+		dh := document.DirHandleFromPath(filepath.Join(dir.Path(), subDir))
+		f.stateStore.WalkerPaths.EnqueueDir(ctx, dh)
+	}
+
+	return jobIds, nil
+}

internal/features/rootmodules/jobs/terraform_sources.go

@@ -0,0 +1,56 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: MPL-2.0
+
+package jobs
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/hashicorp/terraform-ls/internal/document"
+	"github.com/hashicorp/terraform-ls/internal/features/rootmodules/state"
+	"github.com/hashicorp/terraform-ls/internal/job"
+	"github.com/hashicorp/terraform-ls/internal/terraform/datadir"
+	op "github.com/hashicorp/terraform-ls/internal/terraform/module/operation"
+)
+
+// ParseTerraformSources parses the NEW* "module manifest" which
+// contains records of installed modules, e.g. where they're
+// installed on the filesystem.
+// This is useful for processing any modules which are not local
+// nor hosted in the Registry (which would be handled by
+// [GetModuleDataFromRegistry]).
+// NEW* as there is a new terraform-sources.json file format which currently only exists for stacks.
+func ParseTerraformSources(ctx context.Context, fs ReadOnlyFS, rootStore *state.RootStore, modPath string) error {
+	mod, err := rootStore.RootRecordByPath(modPath)
+	if err != nil {
+		return err
+	}
+
+	// Avoid parsing if it is already in progress or already known
+	if mod.TerraformSourcesState != op.OpStateUnknown && !job.IgnoreState(ctx) {
+		return job.StateNotChangedErr{Dir: document.DirHandleFromPath(modPath)}
+	}
+
+	err = rootStore.SetTerraformSourcesState(modPath, op.OpStateLoading)
+	if err != nil {
+		return err
+	}
+
+	tfs, err := datadir.ParseTerraformSourcesFromFile(modPath)
+	if err != nil {
+		err := fmt.Errorf("failed to parse terraform sources: %w", err)
+		sErr := rootStore.UpdateTerraformSources(modPath, nil, err)
+		if sErr != nil {
+			return sErr
+		}
+		return err
+	}
+
+	sErr := rootStore.UpdateTerraformSources(modPath, tfs, err)
+
+	if sErr != nil {
+		return sErr
+	}
+	return err
+}