From 824e1b1d423b3ddb11b154cc1f3aafd8dd116753 Mon Sep 17 00:00:00 2001 From: Ansgar Mertens Date: Wed, 25 Sep 2024 17:55:28 +0200 Subject: [PATCH 1/8] support parsing terraform-sources.json files created by tfstacks cli to support completions for remote modules in stacks projects --- go.mod | 2 + go.sum | 7 ++ internal/features/rootmodules/events.go | 42 +++++++++++ .../rootmodules/jobs/terraform_sources.go | 66 ++++++++++++++++ .../rootmodules/state/installed_modules.go | 34 ++++++++- .../features/rootmodules/state/root_record.go | 9 +++ .../features/rootmodules/state/root_store.go | 72 ++++++++++++++++++ internal/features/stacks/events.go | 26 +++++-- internal/features/stacks/jobs/references.go | 8 +- .../handlers/did_change_watched_files.go | 13 ++++ internal/terraform/datadir/datadir.go | 4 + internal/terraform/datadir/paths.go | 8 ++ .../terraform/datadir/terraform_sources.go | 75 +++++++++++++++++++ .../module/operation/op_type_string.go | 44 ++++++----- .../terraform/module/operation/operation.go | 1 + 15 files changed, 379 insertions(+), 32 deletions(-) create mode 100644 internal/features/rootmodules/jobs/terraform_sources.go create mode 100644 internal/terraform/datadir/terraform_sources.go diff --git a/go.mod b/go.mod index f8e52e0b..208d66da 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( github.com/hashicorp/go-cleanhttp v0.5.2 github.com/hashicorp/go-memdb v1.3.4 github.com/hashicorp/go-multierror v1.1.1 + github.com/hashicorp/go-slug v0.16.0 github.com/hashicorp/go-uuid v1.0.3 github.com/hashicorp/go-version v1.7.0 github.com/hashicorp/hc-install v0.9.0 @@ -37,6 +38,7 @@ require ( require ( github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect + github.com/apparentlymart/go-versions v1.0.1 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect diff --git a/go.sum b/go.sum index bb6db492..55d9c23a 100644 --- a/go.sum +++ b/go.sum @@ -63,6 +63,8 @@ github.com/apparentlymart/go-textseg v1.0.0 h1:rRmlIsPEEhUTIKQb7T++Nz/A5Q6C9IuX2 github.com/apparentlymart/go-textseg v1.0.0/go.mod h1:z96Txxhf3xSFMPmb5X/1W05FF/Nj9VFpLOpjS5yuumk= github.com/apparentlymart/go-textseg/v15 v15.0.0 h1:uYvfpb3DyLSCGWnctWKGj857c6ew1u1fNQOlOtuGxQY= github.com/apparentlymart/go-textseg/v15 v15.0.0/go.mod h1:K8XmNZdhEBkdlyDdvbmmsvpAG721bKi0joRfFdHIWJ4= +github.com/apparentlymart/go-versions v1.0.1 h1:ECIpSn0adcYNsBfSRwdDdz9fWlL+S/6EUd9+irwkBgU= +github.com/apparentlymart/go-versions v1.0.1/go.mod h1:YF5j7IQtrOAOnsGkniupEA5bfCjzd7i14yu0shZavyM= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/armon/go-radix v1.0.0 h1:F4z6KzEeeQIMeLFa97iZU6vupzoecKdU5TX24SNppXI= github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= @@ -130,6 +132,7 @@ github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-test/deep v1.0.1/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= github.com/go-test/deep v1.0.3 h1:ZrJSEWsXzPOxaZnFteGEfooLba+ju3FYIbOrS+rQd68= github.com/go-test/deep v1.0.3/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= @@ -210,6 +213,8 @@ github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+l github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hashicorp/go-retryablehttp v0.7.7 h1:C8hUCYzor8PIfXHa4UrZkU4VvK8o9ISHxT2Q8+VepXU= github.com/hashicorp/go-retryablehttp v0.7.7/go.mod h1:pkQpWZeYWskR+D1tR2O5OcBFOxfA7DoAO6xtkuQnHTk= +github.com/hashicorp/go-slug v0.16.0 h1:S/ko9fms1gf6305ktJNUKGxFmscZ+yWvAtsas0SYUyA= +github.com/hashicorp/go-slug v0.16.0/go.mod h1:THWVTAXwJEinbsp4/bBRcmbaO5EYNLTqxbG4tZ3gCYQ= github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8= github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= @@ -279,6 +284,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348 h1:MtvEpTB6LX3vkb4ax0b5D2DHbNAUsen0Gx5wZoq3lV4= +github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= diff --git a/internal/features/rootmodules/events.go b/internal/features/rootmodules/events.go index 79ed64d3..1f4ecfe4 100644 --- a/internal/features/rootmodules/events.go +++ b/internal/features/rootmodules/events.go @@ -5,6 +5,7 @@ package rootmodules import ( "context" + "path/filepath" "github.com/hashicorp/terraform-ls/internal/document" "github.com/hashicorp/terraform-ls/internal/features/rootmodules/ast" @@ -85,6 +86,18 @@ func (f *RootModulesFeature) didOpen(ctx context.Context, dir document.DirHandle } ids = append(ids, modManifestId) + terraformSourcesId, err := f.stateStore.JobStore.EnqueueJob(ctx, job.Job{ + Dir: dir, + Func: func(ctx context.Context) error { + return jobs.ParseTerraformSources(ctx, f.fs, f.Store, dir.Path()) + }, + Type: op.OpTypeParseTerraformSources.String(), + }) + if err != nil { + return ids, err + } + ids = append(ids, terraformSourcesId) + pSchemaVerId, err := f.stateStore.JobStore.EnqueueJob(ctx, job.Job{ Dir: dir, Func: func(ctx context.Context) error { @@ -168,6 +181,9 @@ func (f *RootModulesFeature) manifestChange(ctx context.Context, dir document.Di if changeType == protocol.Deleted { // Manifest is deleted, so we clear the manifest from the store f.Store.UpdateModManifest(path, nil, nil) + // We also delete the Terraform Sources (if they exist), since delete changes can also happen if the + // entire .terraform directory is deleted and there should only be either a manifest or terraform sources anyway + f.Store.UpdateTerraformSources(path, nil, nil) return ids, nil } @@ -186,6 +202,21 @@ func (f *RootModulesFeature) manifestChange(ctx context.Context, dir document.Di } ids = append(ids, modManifestId) + terraformSourcesId, err := f.stateStore.JobStore.EnqueueJob(ctx, job.Job{ + Dir: dir, + Func: func(ctx context.Context) error { + return jobs.ParseTerraformSources(ctx, f.fs, f.Store, dir.Path()) + }, + Type: op.OpTypeParseTerraformSources.String(), + Defer: func(ctx context.Context, jobErr error) (job.IDs, error) { + return f.indexTerraformSourcesDirs(ctx, dir) + }, + }) + if err != nil { + return ids, err + } + ids = append(ids, terraformSourcesId) + return ids, nil } @@ -204,3 +235,14 @@ func (f *RootModulesFeature) indexInstalledModuleCalls(ctx context.Context, dir return jobIds, nil } + +func (f *RootModulesFeature) indexTerraformSourcesDirs(ctx context.Context, dir document.DirHandle) (job.IDs, error) { + jobIds := make(job.IDs, 0) + + for _, subDir := range f.Store.TerraformSourcesDirectories(dir.Path()) { + dh := document.DirHandleFromPath(filepath.Join(dir.Path(), subDir)) + f.stateStore.WalkerPaths.EnqueueDir(ctx, dh) + } + + return jobIds, nil +} diff --git a/internal/features/rootmodules/jobs/terraform_sources.go b/internal/features/rootmodules/jobs/terraform_sources.go new file mode 100644 index 00000000..104e473d --- /dev/null +++ b/internal/features/rootmodules/jobs/terraform_sources.go @@ -0,0 +1,66 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +package jobs + +import ( + "context" + "fmt" + + "github.com/hashicorp/terraform-ls/internal/document" + "github.com/hashicorp/terraform-ls/internal/features/rootmodules/state" + "github.com/hashicorp/terraform-ls/internal/job" + "github.com/hashicorp/terraform-ls/internal/terraform/datadir" + op "github.com/hashicorp/terraform-ls/internal/terraform/module/operation" +) + +// ParseTerraformSources parses the NEW* "module manifest" which +// contains records of installed modules, e.g. where they're +// installed on the filesystem. +// This is useful for processing any modules which are not local +// nor hosted in the Registry (which would be handled by +// [GetModuleDataFromRegistry]). +// NEW* as there is a new terraform-sources.json file format which currently only exists for stacks. +func ParseTerraformSources(ctx context.Context, fs ReadOnlyFS, rootStore *state.RootStore, modPath string) error { + mod, err := rootStore.RootRecordByPath(modPath) + if err != nil { + return err + } + + // Avoid parsing if it is already in progress or already known + if mod.TerraformSourcesState != op.OpStateUnknown && !job.IgnoreState(ctx) { + return job.StateNotChangedErr{Dir: document.DirHandleFromPath(modPath)} + } + + err = rootStore.SetTerraformSourcesState(modPath, op.OpStateLoading) + if err != nil { + return err + } + + terraformSourcesPath, ok := datadir.TerraformSourcesDirPath(fs, modPath) + if !ok { + err := fmt.Errorf("%s: terraform sources file does not exist", modPath) + sErr := rootStore.UpdateTerraformSources(modPath, nil, err) + if sErr != nil { + return sErr + } + return err + } + + tfs, err := datadir.ParseTerraformSourcesFromFile(terraformSourcesPath) + if err != nil { + err := fmt.Errorf("failed to parse terraform sources: %w", err) + sErr := rootStore.UpdateTerraformSources(modPath, nil, err) + if sErr != nil { + return sErr + } + return err + } + + sErr := rootStore.UpdateTerraformSources(modPath, tfs, err) + + if sErr != nil { + return sErr + } + return err +} diff --git a/internal/features/rootmodules/state/installed_modules.go b/internal/features/rootmodules/state/installed_modules.go index 87640985..9ec4d31f 100644 --- a/internal/features/rootmodules/state/installed_modules.go +++ b/internal/features/rootmodules/state/installed_modules.go @@ -3,7 +3,12 @@ package state -import "github.com/hashicorp/terraform-ls/internal/terraform/datadir" +import ( + "path/filepath" + + "github.com/hashicorp/terraform-ls/internal/terraform/datadir" + tfmod "github.com/hashicorp/terraform-schema/module" +) // InstalledModules is a map of normalized source addresses from the // manifest to the path of the local directory where the module is installed @@ -24,3 +29,30 @@ func InstalledModulesFromManifest(manifest *datadir.ModuleManifest) InstalledMod return installedModules } + +func InstalledModulesFromTerraformSources(path string, sources *datadir.TerraformSources) InstalledModules { + if sources == nil { + return nil + } + + // map raw source address to local directory + + installedModules := make(InstalledModules) + + for _, remote := range sources.RemotePackages() { + absDir, err := sources.LocalPathForSource(remote.SourceAddr("")) + if err != nil { + continue // TODO: log error + } + // installed modules expects a relative dir + dir, err := filepath.Rel(path, absDir) + if err != nil { + continue // TODO: log error + } + + normalizedSource := tfmod.ParseModuleSourceAddr(remote.String()) + installedModules[normalizedSource.String()] = dir + } + + return installedModules +} diff --git a/internal/features/rootmodules/state/root_record.go b/internal/features/rootmodules/state/root_record.go index 7e459932..ee0a922c 100644 --- a/internal/features/rootmodules/state/root_record.go +++ b/internal/features/rootmodules/state/root_record.go @@ -23,6 +23,10 @@ type RootRecord struct { ModManifestErr error ModManifestState op.OpState + TerraformSources *datadir.TerraformSources + TerraformSourcesErr error + TerraformSourcesState op.OpState + // InstalledModules is a map of normalized source addresses from the // manifest to the path of the local directory where the module is installed InstalledModules InstalledModules @@ -50,6 +54,10 @@ func (m *RootRecord) Copy() *RootRecord { ModManifestErr: m.ModManifestErr, ModManifestState: m.ModManifestState, + TerraformSources: m.TerraformSources.Copy(), + TerraformSourcesErr: m.TerraformSourcesErr, + TerraformSourcesState: m.TerraformSourcesState, + // version.Version is practically immutable once parsed TerraformVersion: m.TerraformVersion, TerraformVersionErr: m.TerraformVersionErr, @@ -86,6 +94,7 @@ func newRootRecord(path string) *RootRecord { path: path, ProviderSchemaState: op.OpStateUnknown, ModManifestState: op.OpStateUnknown, + TerraformSourcesState: op.OpStateUnknown, TerraformVersionState: op.OpStateUnknown, InstalledProvidersState: op.OpStateUnknown, } diff --git a/internal/features/rootmodules/state/root_store.go b/internal/features/rootmodules/state/root_store.go index 7d87e09a..0e366c66 100644 --- a/internal/features/rootmodules/state/root_store.go +++ b/internal/features/rootmodules/state/root_store.go @@ -274,6 +274,57 @@ func (s *RootStore) UpdateModManifest(path string, manifest *datadir.ModuleManif return nil } +func (s *RootStore) SetTerraformSourcesState(path string, state op.OpState) error { + txn := s.db.Txn(true) + defer txn.Abort() + + record, err := rootRecordCopyByPath(txn, path) + if err != nil { + return err + } + + record.TerraformSourcesState = state + + err = txn.Insert(s.tableName, record) + if err != nil { + return err + } + + txn.Commit() + return nil +} + +func (s *RootStore) UpdateTerraformSources(path string, manifest *datadir.TerraformSources, mErr error) error { + txn := s.db.Txn(true) + txn.Defer(func() { + s.SetTerraformSourcesState(path, op.OpStateLoaded) + }) + defer txn.Abort() + + record, err := rootRecordCopyByPath(txn, path) + if err != nil { + return err + } + + record.TerraformSources = manifest + record.TerraformSourcesErr = mErr + // this races with modules.json files, but that's okay as they should not exist at the same time + record.InstalledModules = InstalledModulesFromTerraformSources(path, manifest) + + err = txn.Insert(s.tableName, record) + if err != nil { + return err + } + + err = s.queueRecordChange(nil, record) + if err != nil { + return err + } + + txn.Commit() + return nil +} + func (s *RootStore) SetTerraformVersionState(path string, state op.OpState) error { txn := s.db.Txn(true) defer txn.Abort() @@ -350,6 +401,8 @@ func (s *RootStore) CallersOfModule(path string) ([]string, error) { if record.ModManifest.ContainsLocalModule(path) { callers = append(callers, record.path) } + + // TODO: support TerraformSources as well here } return callers, nil @@ -379,6 +432,25 @@ func (s *RootStore) InstalledModuleCalls(path string) (map[string]tfmod.Installe return installed, err } +func (s *RootStore) TerraformSourcesDirectories(path string) []string { + dirs := make([]string, 0) + + record, err := s.RootRecordByPath(path) + if err != nil { + return dirs + } + + // If terraform-sources.json file was loaded, we assume that InstalledModules + // contains them as modules.json and terraform-sources.json are not expected to exist at the same time + if record.TerraformSourcesState == op.OpStateLoaded { + for _, dir := range record.InstalledModules { + dirs = append(dirs, dir) + } + } + + return dirs +} + func (s *RootStore) queueRecordChange(oldRecord, newRecord *RootRecord) error { changes := globalState.Changes{} diff --git a/internal/features/stacks/events.go b/internal/features/stacks/events.go index 94bcc661..3a920862 100644 --- a/internal/features/stacks/events.go +++ b/internal/features/stacks/events.go @@ -21,7 +21,7 @@ import ( globalAst "github.com/hashicorp/terraform-ls/internal/terraform/ast" "github.com/hashicorp/terraform-ls/internal/terraform/module/operation" tfaddr "github.com/hashicorp/terraform-registry-address" - "github.com/hashicorp/terraform-schema/module" + tfmod "github.com/hashicorp/terraform-schema/module" ) func (f *StacksFeature) discover(path string, files []string) error { @@ -216,7 +216,7 @@ func (f *StacksFeature) decodeStack(ctx context.Context, dir document.DirHandle, f.logger.Printf("loading module metadata returned error: %s", jobErr) } - componentIds, err := loadStackComponentSources(ctx, f.store, f.bus, path) + componentIds, err := f.decodeStackComponentSources(ctx, f.store, f.bus, path) deferIds = append(deferIds, componentIds...) if err != nil { f.logger.Printf("loading stack component sources returned error: %s", err) @@ -330,8 +330,8 @@ func (f *StacksFeature) removeIndexedStack(rawPath string) { } } -// loadStackComponentSources will trigger parsing the local terraform modules for a stack in the ModulesFeature -func loadStackComponentSources(ctx context.Context, stackStore *state.StackStore, bus *eventbus.EventBus, stackPath string) (job.IDs, error) { +// decodeStackComponentSources will trigger parsing the local terraform modules for a stack in the ModulesFeature +func (f *StacksFeature) decodeStackComponentSources(ctx context.Context, stackStore *state.StackStore, bus *eventbus.EventBus, stackPath string) (job.IDs, error) { ids := make(job.IDs, 0) record, err := stackStore.StackRecordByPath(stackPath) @@ -349,12 +349,22 @@ func loadStackComponentSources(ctx context.Context, stackStore *state.StackStore var fullPath string // detect if component.Source is a local module switch component.SourceAddr.(type) { - case module.LocalSourceAddr: + case tfmod.LocalSourceAddr: fullPath = filepath.Join(stackPath, filepath.FromSlash(component.Source)) + // For registry modules, we need to find the local installation path (if installed) case tfaddr.Module: - continue - case module.RemoteSourceAddr: - continue + installedDir, ok := f.rootFeature.InstalledModulePath(stackPath, component.SourceAddr.String()) + if !ok { + continue + } + fullPath = filepath.Join(stackPath, filepath.FromSlash(installedDir)) + // For other remote modules, we need to find the local installation path (if installed) + case tfmod.RemoteSourceAddr: + installedDir, ok := f.rootFeature.InstalledModulePath(stackPath, component.SourceAddr.String()) + if !ok { + continue + } + fullPath = filepath.Join(stackPath, filepath.FromSlash(installedDir)) default: // Unknown source address, we can't resolve the path continue diff --git a/internal/features/stacks/jobs/references.go b/internal/features/stacks/jobs/references.go index 54f6eda8..9f7a5779 100644 --- a/internal/features/stacks/jobs/references.go +++ b/internal/features/stacks/jobs/references.go @@ -26,7 +26,7 @@ import ( // For example it tells us that variable block between certain LOC // can be referred to as var.foobar. This is useful e.g. during completion, // go-to-definition or go-to-references. -func DecodeReferenceTargets(ctx context.Context, stackStore *state.StackStore, moduleReader sdecoder.ModuleReader, rootFeature sdecoder.RootReader, stackPath string) error { +func DecodeReferenceTargets(ctx context.Context, stackStore *state.StackStore, moduleReader sdecoder.ModuleReader, rootReader sdecoder.RootReader, stackPath string) error { mod, err := stackStore.StackRecordByPath(stackPath) if err != nil { return err @@ -47,7 +47,7 @@ func DecodeReferenceTargets(ctx context.Context, stackStore *state.StackStore, m d := decoder.NewDecoder(&sdecoder.PathReader{ StateReader: stackStore, ModuleReader: moduleReader, - RootReader: rootFeature, + RootReader: rootReader, }) d.SetContext(idecoder.DecoderContext(ctx)) @@ -96,7 +96,7 @@ func DecodeReferenceTargets(ctx context.Context, stackStore *state.StackStore, m // For example it tells us that there is a reference address var.foobar // at a particular LOC. This can be later matched with targets // (as obtained via [DecodeReferenceTargets]) during hover or go-to-definition. -func DecodeReferenceOrigins(ctx context.Context, stackStore *state.StackStore, moduleReader sdecoder.ModuleReader, rootFeature sdecoder.RootReader, stackPath string) error { +func DecodeReferenceOrigins(ctx context.Context, stackStore *state.StackStore, moduleReader sdecoder.ModuleReader, rootReader sdecoder.RootReader, stackPath string) error { mod, err := stackStore.StackRecordByPath(stackPath) if err != nil { return err @@ -117,7 +117,7 @@ func DecodeReferenceOrigins(ctx context.Context, stackStore *state.StackStore, m d := decoder.NewDecoder(&sdecoder.PathReader{ StateReader: stackStore, ModuleReader: moduleReader, - RootReader: rootFeature, + RootReader: rootReader, }) d.SetContext(idecoder.DecoderContext(ctx)) diff --git a/internal/langserver/handlers/did_change_watched_files.go b/internal/langserver/handlers/did_change_watched_files.go index 2bb8ccb4..d517afa8 100644 --- a/internal/langserver/handlers/did_change_watched_files.go +++ b/internal/langserver/handlers/did_change_watched_files.go @@ -76,6 +76,19 @@ func (svc *service) DidChangeWatchedFiles(ctx context.Context, params lsp.DidCha continue } + // If the .terraform/modules/terraform-sources.json file changes + if modUri, ok := datadir.ModuleUriFromTerraformSourcesFile(rawURI); ok { + modHandle := document.DirHandleFromURI(modUri) + // manifest change event handles terraform-sources.json as well + svc.eventBus.ManifestChange(eventbus.ManifestChangeEvent{ + Context: ctx, // We pass the context for data here + Dir: modHandle, + ChangeType: change.Type, + }) + + continue + } + rawPath, err := uri.PathFromURI(rawURI) if err != nil { svc.logger.Printf("error parsing %q: %s", rawURI, err) diff --git a/internal/terraform/datadir/datadir.go b/internal/terraform/datadir/datadir.go index 25667cea..d42b71e6 100644 --- a/internal/terraform/datadir/datadir.go +++ b/internal/terraform/datadir/datadir.go @@ -45,6 +45,10 @@ func ModulePath(filePath string) (string, bool) { if strings.HasSuffix(filePath, manifestSuffix) { return strings.TrimSuffix(filePath, manifestSuffix), true } + terraformSourcesSuffix := filepath.Join(terraformSourcesDirElements...) + if strings.HasSuffix(filePath, terraformSourcesSuffix) { + return strings.TrimSuffix(filePath, terraformSourcesSuffix), true + } for _, pathElems := range pluginLockFilePathElements { suffix := filepath.Join(pathElems...) diff --git a/internal/terraform/datadir/paths.go b/internal/terraform/datadir/paths.go index f6c76281..c009c593 100644 --- a/internal/terraform/datadir/paths.go +++ b/internal/terraform/datadir/paths.go @@ -85,3 +85,11 @@ func ModuleUriFromModuleLockFile(rawUri string) (string, bool) { } return "", false } + +func ModuleUriFromTerraformSourcesFile(rawUri string) (string, bool) { + suffix := "/" + path.Join(terraformSourcesPathElements...) + if strings.HasSuffix(rawUri, suffix) { + return strings.TrimSuffix(rawUri, suffix), true + } + return "", false +} diff --git a/internal/terraform/datadir/terraform_sources.go b/internal/terraform/datadir/terraform_sources.go new file mode 100644 index 00000000..2baa525f --- /dev/null +++ b/internal/terraform/datadir/terraform_sources.go @@ -0,0 +1,75 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +package datadir + +import ( + "fmt" + "io/fs" + "path/filepath" + + "github.com/hashicorp/go-slug/sourcebundle" +) + +var terraformSourcesDirElements = []string{ + DataDirName, "modules", +} +var terraformSourcesPathElements = []string{ + DataDirName, "modules", "terraform-sources.json", +} + +func TerraformSourcesDirPath(fs fs.StatFS, modulePath string) (string, bool) { + terraformSourcesPath := filepath.Join( + append([]string{modulePath}, + terraformSourcesPathElements...)...) + terraformSourcesDirPath := filepath.Join( + append([]string{modulePath}, + terraformSourcesDirElements...)...) + + fi, err := fs.Stat(terraformSourcesPath) + if err == nil && fi.Mode().IsRegular() { + return terraformSourcesDirPath, true // TODO: this is a bit weird and misleading, maybe we should just use the bundle thing reading the dir and catch the proper error or sth like that + } + return "", false +} + +type TerraformSources struct { + sourcebundle.Bundle + rootDir string // we need to duplicate this as our rootDir is different from the bundle's +} + +func (mm *TerraformSources) Copy() *TerraformSources { + if mm == nil { + return nil + } + + newTfS := &TerraformSources{ + Bundle: mm.Bundle, + rootDir: mm.rootDir, + } + + return newTfS +} + +func (tfs *TerraformSources) RootDir() string { + return tfs.rootDir +} + +func ParseTerraformSourcesFromFile(path string) (*TerraformSources, error) { + bundle, err := sourcebundle.OpenDir(path) + if err != nil { + return nil, err + } + + tfs := &TerraformSources{ + Bundle: *bundle, + } + + rootDir, ok := ModulePath(path) + if !ok { + return nil, fmt.Errorf("failed to detect module path: %s", path) + } + tfs.rootDir = filepath.Clean(rootDir) + + return tfs, nil +} diff --git a/internal/terraform/module/operation/op_type_string.go b/internal/terraform/module/operation/op_type_string.go index afa48295..a78cdde7 100644 --- a/internal/terraform/module/operation/op_type_string.go +++ b/internal/terraform/module/operation/op_type_string.go @@ -15,28 +15,34 @@ func _() { _ = x[OpTypeParseModuleConfiguration-4] _ = x[OpTypeParseVariables-5] _ = x[OpTypeParseModuleManifest-6] - _ = x[OpTypeLoadModuleMetadata-7] - _ = x[OpTypeDecodeReferenceTargets-8] - _ = x[OpTypeDecodeReferenceOrigins-9] - _ = x[OpTypeDecodeVarsReferences-10] - _ = x[OpTypeGetModuleDataFromRegistry-11] - _ = x[OpTypeParseProviderVersions-12] - _ = x[OpTypePreloadEmbeddedSchema-13] - _ = x[OpTypeStacksPreloadEmbeddedSchema-14] - _ = x[OpTypeSchemaModuleValidation-15] - _ = x[OpTypeSchemaStackValidation-16] - _ = x[OpTypeSchemaVarsValidation-17] - _ = x[OpTypeReferenceValidation-18] - _ = x[OpTypeReferenceStackValidation-19] - _ = x[OpTypeTerraformValidate-20] - _ = x[OpTypeParseStackConfiguration-21] - _ = x[OpTypeLoadStackMetadata-22] - _ = x[OpTypeLoadStackRequiredTerraformVersion-23] + _ = x[OpTypeParseTerraformSources-7] + _ = x[OpTypeLoadModuleMetadata-8] + _ = x[OpTypeDecodeReferenceTargets-9] + _ = x[OpTypeDecodeReferenceOrigins-10] + _ = x[OpTypeDecodeVarsReferences-11] + _ = x[OpTypeGetModuleDataFromRegistry-12] + _ = x[OpTypeParseProviderVersions-13] + _ = x[OpTypePreloadEmbeddedSchema-14] + _ = x[OpTypeStacksPreloadEmbeddedSchema-15] + _ = x[OpTypeSchemaModuleValidation-16] + _ = x[OpTypeSchemaStackValidation-17] + _ = x[OpTypeSchemaVarsValidation-18] + _ = x[OpTypeReferenceValidation-19] + _ = x[OpTypeReferenceStackValidation-20] + _ = x[OpTypeTerraformValidate-21] + _ = x[OpTypeParseStackConfiguration-22] + _ = x[OpTypeLoadStackMetadata-23] + _ = x[OpTypeLoadStackRequiredTerraformVersion-24] + _ = x[OpTypeParseTestConfiguration-25] + _ = x[OpTypeLoadTestMetadata-26] + _ = x[OpTypeDecodeTestReferenceTargets-27] + _ = x[OpTypeDecodeTestReferenceOrigins-28] + _ = x[OpTypeSchemaTestValidation-29] } -const _OpType_name = "OpTypeUnknownOpTypeGetTerraformVersionOpTypeGetInstalledTerraformVersionOpTypeObtainSchemaOpTypeParseModuleConfigurationOpTypeParseVariablesOpTypeParseModuleManifestOpTypeLoadModuleMetadataOpTypeDecodeReferenceTargetsOpTypeDecodeReferenceOriginsOpTypeDecodeVarsReferencesOpTypeGetModuleDataFromRegistryOpTypeParseProviderVersionsOpTypePreloadEmbeddedSchemaOpTypeStacksPreloadEmbeddedSchemaOpTypeSchemaModuleValidationOpTypeSchemaStackValidationOpTypeSchemaVarsValidationOpTypeReferenceValidationOpTypeReferenceStackValidationOpTypeTerraformValidateOpTypeParseStackConfigurationOpTypeLoadStackMetadataOpTypeLoadStackRequiredTerraformVersion" +const _OpType_name = "OpTypeUnknownOpTypeGetTerraformVersionOpTypeGetInstalledTerraformVersionOpTypeObtainSchemaOpTypeParseModuleConfigurationOpTypeParseVariablesOpTypeParseModuleManifestOpTypeParseTerraformSourcesOpTypeLoadModuleMetadataOpTypeDecodeReferenceTargetsOpTypeDecodeReferenceOriginsOpTypeDecodeVarsReferencesOpTypeGetModuleDataFromRegistryOpTypeParseProviderVersionsOpTypePreloadEmbeddedSchemaOpTypeStacksPreloadEmbeddedSchemaOpTypeSchemaModuleValidationOpTypeSchemaStackValidationOpTypeSchemaVarsValidationOpTypeReferenceValidationOpTypeReferenceStackValidationOpTypeTerraformValidateOpTypeParseStackConfigurationOpTypeLoadStackMetadataOpTypeLoadStackRequiredTerraformVersionOpTypeParseTestConfigurationOpTypeLoadTestMetadataOpTypeDecodeTestReferenceTargetsOpTypeDecodeTestReferenceOriginsOpTypeSchemaTestValidation" -var _OpType_index = [...]uint16{0, 13, 38, 72, 90, 120, 140, 165, 189, 217, 245, 271, 302, 329, 356, 389, 417, 444, 470, 495, 525, 548, 577, 600, 639} +var _OpType_index = [...]uint16{0, 13, 38, 72, 90, 120, 140, 165, 192, 216, 244, 272, 298, 329, 356, 383, 416, 444, 471, 497, 522, 552, 575, 604, 627, 666, 694, 716, 748, 780, 806} func (i OpType) String() string { if i >= OpType(len(_OpType_index)-1) { diff --git a/internal/terraform/module/operation/operation.go b/internal/terraform/module/operation/operation.go index 784dd5bc..be574440 100644 --- a/internal/terraform/module/operation/operation.go +++ b/internal/terraform/module/operation/operation.go @@ -24,6 +24,7 @@ const ( OpTypeParseModuleConfiguration OpTypeParseVariables OpTypeParseModuleManifest + OpTypeParseTerraformSources OpTypeLoadModuleMetadata OpTypeDecodeReferenceTargets OpTypeDecodeReferenceOrigins From 327aac6e0ed426aca44dddcfaf530b2fcc66bd58 Mon Sep 17 00:00:00 2001 From: Ansgar Mertens Date: Wed, 23 Oct 2024 16:41:20 +0200 Subject: [PATCH 2/8] log errors --- internal/features/rootmodules/state/installed_modules.go | 9 ++++++--- internal/features/rootmodules/state/root_store.go | 2 +- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/internal/features/rootmodules/state/installed_modules.go b/internal/features/rootmodules/state/installed_modules.go index 9ec4d31f..3aaf99bf 100644 --- a/internal/features/rootmodules/state/installed_modules.go +++ b/internal/features/rootmodules/state/installed_modules.go @@ -4,6 +4,7 @@ package state import ( + "log" "path/filepath" "github.com/hashicorp/terraform-ls/internal/terraform/datadir" @@ -30,7 +31,7 @@ func InstalledModulesFromManifest(manifest *datadir.ModuleManifest) InstalledMod return installedModules } -func InstalledModulesFromTerraformSources(path string, sources *datadir.TerraformSources) InstalledModules { +func InstalledModulesFromTerraformSources(path string, sources *datadir.TerraformSources, logger *log.Logger) InstalledModules { if sources == nil { return nil } @@ -42,12 +43,14 @@ func InstalledModulesFromTerraformSources(path string, sources *datadir.Terrafor for _, remote := range sources.RemotePackages() { absDir, err := sources.LocalPathForSource(remote.SourceAddr("")) if err != nil { - continue // TODO: log error + logger.Printf("Error getting local path for source %s: %s", remote.String(), err) + continue } // installed modules expects a relative dir dir, err := filepath.Rel(path, absDir) if err != nil { - continue // TODO: log error + logger.Printf("Error getting relative path for source %s and path %s and absolute dir %s: %s", remote.String(), path, absDir, err) + continue } normalizedSource := tfmod.ParseModuleSourceAddr(remote.String()) diff --git a/internal/features/rootmodules/state/root_store.go b/internal/features/rootmodules/state/root_store.go index 0e366c66..4128674d 100644 --- a/internal/features/rootmodules/state/root_store.go +++ b/internal/features/rootmodules/state/root_store.go @@ -309,7 +309,7 @@ func (s *RootStore) UpdateTerraformSources(path string, manifest *datadir.Terraf record.TerraformSources = manifest record.TerraformSourcesErr = mErr // this races with modules.json files, but that's okay as they should not exist at the same time - record.InstalledModules = InstalledModulesFromTerraformSources(path, manifest) + record.InstalledModules = InstalledModulesFromTerraformSources(path, manifest, s.logger) err = txn.Insert(s.tableName, record) if err != nil { From 927a8f593fb803440696a5a129408c4d182edcb0 Mon Sep 17 00:00:00 2001 From: Ansgar Mertens Date: Mon, 28 Oct 2024 08:59:56 +0100 Subject: [PATCH 3/8] add tests --- .../jobs/terraform_sources_test.go | 221 ++++++++++++++++++ .../features/rootmodules/state/root_store.go | 2 +- 2 files changed, 222 insertions(+), 1 deletion(-) create mode 100644 internal/features/rootmodules/jobs/terraform_sources_test.go diff --git a/internal/features/rootmodules/jobs/terraform_sources_test.go b/internal/features/rootmodules/jobs/terraform_sources_test.go new file mode 100644 index 00000000..3f5b5f79 --- /dev/null +++ b/internal/features/rootmodules/jobs/terraform_sources_test.go @@ -0,0 +1,221 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +package jobs + +import ( + "context" + "fmt" + "io/fs" + "os" + "path/filepath" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/hashicorp/terraform-ls/internal/features/rootmodules/state" + globalState "github.com/hashicorp/terraform-ls/internal/state" + "github.com/hashicorp/terraform-ls/internal/terraform/module/operation" +) + +// should we expose this from internal/filesystem/filesystem.go instead? +type osFs struct{} + +func (osfs osFs) Open(name string) (fs.File, error) { + return os.Open(name) +} + +func (osfs osFs) Stat(name string) (fs.FileInfo, error) { + return os.Stat(name) +} + +func (osfs osFs) ReadDir(name string) ([]fs.DirEntry, error) { + return os.ReadDir(name) +} + +func (osfs osFs) ReadFile(name string) ([]byte, error) { + return os.ReadFile(name) +} + +func TestParseTerraformSources(t *testing.T) { + modPath := t.TempDir() + manifestDir := filepath.Join(modPath, ".terraform", "modules") + err := os.MkdirAll(manifestDir, 0755) + if err != nil { + t.Fatal(err) + } + + err = os.WriteFile(filepath.Join(manifestDir, "terraform-sources.json"), []byte(`{ + "terraform_source_bundle": 1, + "packages": [ + { + "source": "git::https://github.com/shernandez5/terraform-kubernetes-crd-demo-module?ref=f6cf642c8671262aac30f0af6e62b6ee85a54204", + "local": "UmN8ypf1BrY_efIIl4pzoutkgPaJClzCskrS6IWxDfI", + "meta": {} + }, + { + "source": "git::https://github.com/shernandez5/terraforming-stacks.git", + "local": "m9Di4tJSWWxjddtdLEPk1u9uhAdx6uuzJWzUIds_1BQ", + "meta": {} + } + ], + "registry": [ + { + "source": "registry.terraform.io/shernandez5/crd-demo-module/kubernetes", + "versions": { + "0.1.0": { + "source": "git::https://github.com/shernandez5/terraform-kubernetes-crd-demo-module?ref=f6cf642c8671262aac30f0af6e62b6ee85a54204", + "deprecation": null + } + } + } + ] + }`), 0755) + + if err != nil { + t.Fatal(err) + } + + gs, err := globalState.NewStateStore() + if err != nil { + t.Fatal(err) + } + rs, err := state.NewRootStore(gs.ChangeStore, gs.ProviderSchemas) + if err != nil { + t.Fatal(err) + } + + err = rs.Add(modPath) + if err != nil { + t.Fatal(err) + } + + ctx := context.Background() + err = ParseTerraformSources(ctx, osFs{}, rs, modPath) + if err != nil { + t.Fatal(err) + } + + mod, err := rs.RootRecordByPath(modPath) + if err != nil { + t.Fatal(err) + } + + if mod.TerraformSourcesState != operation.OpStateLoaded { + t.Fatalf("expected state to be loaded, %q given", mod.TerraformSourcesState) + } + + if mod.TerraformSourcesErr != nil { + t.Fatalf("unexpected error: %s", mod.TerraformSourcesErr) + } + + expectedInstalledModules := state.InstalledModules{ + "git::https://github.com/shernandez5/terraform-kubernetes-crd-demo-module?ref=f6cf642c8671262aac30f0af6e62b6ee85a54204": ".terraform/modules/UmN8ypf1BrY_efIIl4pzoutkgPaJClzCskrS6IWxDfI", + "git::https://github.com/shernandez5/terraforming-stacks.git": ".terraform/modules/m9Di4tJSWWxjddtdLEPk1u9uhAdx6uuzJWzUIds_1BQ", + } + if diff := cmp.Diff(expectedInstalledModules, mod.InstalledModules); diff != "" { + t.Fatalf("unexpected installed modules: %s", diff) + } +} + +func TestParseTerraformSources_no_sources_file(t *testing.T) { + modPath := t.TempDir() + manifestDir := filepath.Join(modPath, ".terraform", "modules") + err := os.MkdirAll(manifestDir, 0755) + if err != nil { + t.Fatal(err) + } + + // not writing any sources file + + gs, err := globalState.NewStateStore() + if err != nil { + t.Fatal(err) + } + rs, err := state.NewRootStore(gs.ChangeStore, gs.ProviderSchemas) + if err != nil { + t.Fatal(err) + } + + err = rs.Add(modPath) + if err != nil { + t.Fatal(err) + } + + ctx := context.Background() + err = ParseTerraformSources(ctx, osFs{}, rs, modPath) + if err == nil { + t.Fatal("expected error for missing sources file") + } + + mod, err := rs.RootRecordByPath(modPath) + if err != nil { + t.Fatal(err) + } + + if mod.TerraformSourcesState != operation.OpStateLoaded { + t.Fatalf("expected state to be loaded, %q given", mod.TerraformSourcesState) + } + + if mod.TerraformSourcesErr == nil { + t.Fatal("expected error for missing sources file") + } + + if mod.TerraformSourcesErr.Error() != fmt.Sprintf("%s: terraform sources file does not exist", modPath) { + t.Fatalf("unexpected error: %s", mod.TerraformSourcesErr) + } +} + +func TestParseTerraformSources_invalid_sources_file(t *testing.T) { + modPath := t.TempDir() + manifestDir := filepath.Join(modPath, ".terraform", "modules") + err := os.MkdirAll(manifestDir, 0755) + if err != nil { + t.Fatal(err) + } + + err = os.WriteFile(filepath.Join(manifestDir, "terraform-sources.json"), []byte(`{ + "terraform_source_bundle": 0 + }`), 0755) + + if err != nil { + t.Fatal(err) + } + + gs, err := globalState.NewStateStore() + if err != nil { + t.Fatal(err) + } + rs, err := state.NewRootStore(gs.ChangeStore, gs.ProviderSchemas) + if err != nil { + t.Fatal(err) + } + + err = rs.Add(modPath) + if err != nil { + t.Fatal(err) + } + + ctx := context.Background() + err = ParseTerraformSources(ctx, osFs{}, rs, modPath) + if err == nil { + t.Fatal("expected error for invalid sources file") + } + + mod, err := rs.RootRecordByPath(modPath) + if err != nil { + t.Fatal(err) + } + + if mod.TerraformSourcesState != operation.OpStateLoaded { + t.Fatalf("expected state to be loaded, %q given", mod.TerraformSourcesState) + } + + if mod.TerraformSourcesErr == nil { + t.Fatal("expected error for invalid sources file") + } + + if mod.TerraformSourcesErr.Error() != "failed to parse terraform sources: invalid manifest: unsupported format version 0" { + t.Fatalf("unexpected error: %s", mod.TerraformSourcesErr) + } + +} diff --git a/internal/features/rootmodules/state/root_store.go b/internal/features/rootmodules/state/root_store.go index 4128674d..644343bc 100644 --- a/internal/features/rootmodules/state/root_store.go +++ b/internal/features/rootmodules/state/root_store.go @@ -402,7 +402,7 @@ func (s *RootStore) CallersOfModule(path string) ([]string, error) { callers = append(callers, record.path) } - // TODO: support TerraformSources as well here + // TODO: support TerraformSources as well here -> they don't contain local modules though, so we'd need to do this different here for stacks? } return callers, nil From baead0c73535087ffe73aa23d63efeda82a32f4b Mon Sep 17 00:00:00 2001 From: Ansgar Mertens Date: Mon, 28 Oct 2024 09:15:16 +0100 Subject: [PATCH 4/8] add changie entry --- .changes/unreleased/ENHANCEMENTS-20241028-090453.yaml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .changes/unreleased/ENHANCEMENTS-20241028-090453.yaml diff --git a/.changes/unreleased/ENHANCEMENTS-20241028-090453.yaml b/.changes/unreleased/ENHANCEMENTS-20241028-090453.yaml new file mode 100644 index 00000000..0a936d03 --- /dev/null +++ b/.changes/unreleased/ENHANCEMENTS-20241028-090453.yaml @@ -0,0 +1,6 @@ +kind: ENHANCEMENTS +body: 'Stacks: parse terraform-sources.json to support remote component sources' +time: 2024-10-28T09:04:53.004252+01:00 +custom: + Issue: "1836" + Repository: terraform-ls From 55ead5396d30a7721c182e0dfe811ce0983c31ec Mon Sep 17 00:00:00 2001 From: Ansgar Mertens Date: Tue, 29 Oct 2024 14:27:32 +0100 Subject: [PATCH 5/8] fix: ensure test also succeeds on windows using backslashes in paths --- internal/features/rootmodules/jobs/terraform_sources_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/features/rootmodules/jobs/terraform_sources_test.go b/internal/features/rootmodules/jobs/terraform_sources_test.go index 3f5b5f79..2cfe72e9 100644 --- a/internal/features/rootmodules/jobs/terraform_sources_test.go +++ b/internal/features/rootmodules/jobs/terraform_sources_test.go @@ -109,8 +109,8 @@ func TestParseTerraformSources(t *testing.T) { } expectedInstalledModules := state.InstalledModules{ - "git::https://github.com/shernandez5/terraform-kubernetes-crd-demo-module?ref=f6cf642c8671262aac30f0af6e62b6ee85a54204": ".terraform/modules/UmN8ypf1BrY_efIIl4pzoutkgPaJClzCskrS6IWxDfI", - "git::https://github.com/shernandez5/terraforming-stacks.git": ".terraform/modules/m9Di4tJSWWxjddtdLEPk1u9uhAdx6uuzJWzUIds_1BQ", + "git::https://github.com/shernandez5/terraform-kubernetes-crd-demo-module?ref=f6cf642c8671262aac30f0af6e62b6ee85a54204": filepath.FromSlash(".terraform/modules/UmN8ypf1BrY_efIIl4pzoutkgPaJClzCskrS6IWxDfI"), + "git::https://github.com/shernandez5/terraforming-stacks.git": filepath.FromSlash(".terraform/modules/m9Di4tJSWWxjddtdLEPk1u9uhAdx6uuzJWzUIds_1BQ"), } if diff := cmp.Diff(expectedInstalledModules, mod.InstalledModules); diff != "" { t.Fatalf("unexpected installed modules: %s", diff) From 09e5e0ac79cff7cde952abf6f21bae7b66bdf464 Mon Sep 17 00:00:00 2001 From: Ansgar Mertens Date: Wed, 27 Nov 2024 16:09:16 +0100 Subject: [PATCH 6/8] simplify terraform sources parsing code --- .../rootmodules/jobs/terraform_sources.go | 12 +------- .../jobs/terraform_sources_test.go | 5 +++- .../terraform/datadir/terraform_sources.go | 29 ++++--------------- 3 files changed, 11 insertions(+), 35 deletions(-) diff --git a/internal/features/rootmodules/jobs/terraform_sources.go b/internal/features/rootmodules/jobs/terraform_sources.go index 104e473d..622e4c2d 100644 --- a/internal/features/rootmodules/jobs/terraform_sources.go +++ b/internal/features/rootmodules/jobs/terraform_sources.go @@ -37,17 +37,7 @@ func ParseTerraformSources(ctx context.Context, fs ReadOnlyFS, rootStore *state. return err } - terraformSourcesPath, ok := datadir.TerraformSourcesDirPath(fs, modPath) - if !ok { - err := fmt.Errorf("%s: terraform sources file does not exist", modPath) - sErr := rootStore.UpdateTerraformSources(modPath, nil, err) - if sErr != nil { - return sErr - } - return err - } - - tfs, err := datadir.ParseTerraformSourcesFromFile(terraformSourcesPath) + tfs, err := datadir.ParseTerraformSourcesFromFile(modPath) if err != nil { err := fmt.Errorf("failed to parse terraform sources: %w", err) sErr := rootStore.UpdateTerraformSources(modPath, nil, err) diff --git a/internal/features/rootmodules/jobs/terraform_sources_test.go b/internal/features/rootmodules/jobs/terraform_sources_test.go index 2cfe72e9..5a1b3b8d 100644 --- a/internal/features/rootmodules/jobs/terraform_sources_test.go +++ b/internal/features/rootmodules/jobs/terraform_sources_test.go @@ -160,7 +160,10 @@ func TestParseTerraformSources_no_sources_file(t *testing.T) { t.Fatal("expected error for missing sources file") } - if mod.TerraformSourcesErr.Error() != fmt.Sprintf("%s: terraform sources file does not exist", modPath) { + // ( ಠ ʖ̯ ಠ) + errorUnix := fmt.Sprintf("failed to parse terraform sources: cannot read manifest: open %s: The system cannot find the file specified.", filepath.FromSlash(modPath+"/.terraform/modules/terraform-sources.json")) + errorWindows := fmt.Sprintf("failed to parse terraform sources: cannot read manifest: open %s: no such file or directory", filepath.FromSlash(modPath+"/.terraform/modules/terraform-sources.json")) + if mod.TerraformSourcesErr.Error() != errorUnix && mod.TerraformSourcesErr.Error() != errorWindows { t.Fatalf("unexpected error: %s", mod.TerraformSourcesErr) } } diff --git a/internal/terraform/datadir/terraform_sources.go b/internal/terraform/datadir/terraform_sources.go index 2baa525f..3113d518 100644 --- a/internal/terraform/datadir/terraform_sources.go +++ b/internal/terraform/datadir/terraform_sources.go @@ -4,8 +4,6 @@ package datadir import ( - "fmt" - "io/fs" "path/filepath" "github.com/hashicorp/go-slug/sourcebundle" @@ -18,21 +16,6 @@ var terraformSourcesPathElements = []string{ DataDirName, "modules", "terraform-sources.json", } -func TerraformSourcesDirPath(fs fs.StatFS, modulePath string) (string, bool) { - terraformSourcesPath := filepath.Join( - append([]string{modulePath}, - terraformSourcesPathElements...)...) - terraformSourcesDirPath := filepath.Join( - append([]string{modulePath}, - terraformSourcesDirElements...)...) - - fi, err := fs.Stat(terraformSourcesPath) - if err == nil && fi.Mode().IsRegular() { - return terraformSourcesDirPath, true // TODO: this is a bit weird and misleading, maybe we should just use the bundle thing reading the dir and catch the proper error or sth like that - } - return "", false -} - type TerraformSources struct { sourcebundle.Bundle rootDir string // we need to duplicate this as our rootDir is different from the bundle's @@ -55,7 +38,11 @@ func (tfs *TerraformSources) RootDir() string { return tfs.rootDir } -func ParseTerraformSourcesFromFile(path string) (*TerraformSources, error) { +func ParseTerraformSourcesFromFile(modulePath string) (*TerraformSources, error) { + path := filepath.Join( + append([]string{modulePath}, + terraformSourcesDirElements...)...) + bundle, err := sourcebundle.OpenDir(path) if err != nil { return nil, err @@ -65,11 +52,7 @@ func ParseTerraformSourcesFromFile(path string) (*TerraformSources, error) { Bundle: *bundle, } - rootDir, ok := ModulePath(path) - if !ok { - return nil, fmt.Errorf("failed to detect module path: %s", path) - } - tfs.rootDir = filepath.Clean(rootDir) + tfs.rootDir = filepath.Clean(modulePath) return tfs, nil } From 4813e29dc0a344c49302d612b691a36b4751f606 Mon Sep 17 00:00:00 2001 From: Ansgar Mertens Date: Wed, 27 Nov 2024 16:09:43 +0100 Subject: [PATCH 7/8] parse registry modules from terraform-sources.json as well --- .../jobs/terraform_sources_test.go | 8 +++--- .../rootmodules/state/installed_modules.go | 26 +++++++++++++++++++ .../features/rootmodules/state/root_store.go | 2 -- 3 files changed, 29 insertions(+), 7 deletions(-) diff --git a/internal/features/rootmodules/jobs/terraform_sources_test.go b/internal/features/rootmodules/jobs/terraform_sources_test.go index 5a1b3b8d..e0c9c6b6 100644 --- a/internal/features/rootmodules/jobs/terraform_sources_test.go +++ b/internal/features/rootmodules/jobs/terraform_sources_test.go @@ -5,7 +5,7 @@ package jobs import ( "context" - "fmt" + "errors" "io/fs" "os" "path/filepath" @@ -111,6 +111,7 @@ func TestParseTerraformSources(t *testing.T) { expectedInstalledModules := state.InstalledModules{ "git::https://github.com/shernandez5/terraform-kubernetes-crd-demo-module?ref=f6cf642c8671262aac30f0af6e62b6ee85a54204": filepath.FromSlash(".terraform/modules/UmN8ypf1BrY_efIIl4pzoutkgPaJClzCskrS6IWxDfI"), "git::https://github.com/shernandez5/terraforming-stacks.git": filepath.FromSlash(".terraform/modules/m9Di4tJSWWxjddtdLEPk1u9uhAdx6uuzJWzUIds_1BQ"), + "registry.terraform.io/shernandez5/crd-demo-module/kubernetes": filepath.FromSlash(".terraform/modules/UmN8ypf1BrY_efIIl4pzoutkgPaJClzCskrS6IWxDfI"), } if diff := cmp.Diff(expectedInstalledModules, mod.InstalledModules); diff != "" { t.Fatalf("unexpected installed modules: %s", diff) @@ -160,10 +161,7 @@ func TestParseTerraformSources_no_sources_file(t *testing.T) { t.Fatal("expected error for missing sources file") } - // ( ಠ ʖ̯ ಠ) - errorUnix := fmt.Sprintf("failed to parse terraform sources: cannot read manifest: open %s: The system cannot find the file specified.", filepath.FromSlash(modPath+"/.terraform/modules/terraform-sources.json")) - errorWindows := fmt.Sprintf("failed to parse terraform sources: cannot read manifest: open %s: no such file or directory", filepath.FromSlash(modPath+"/.terraform/modules/terraform-sources.json")) - if mod.TerraformSourcesErr.Error() != errorUnix && mod.TerraformSourcesErr.Error() != errorWindows { + if !errors.Is(mod.TerraformSourcesErr, os.ErrNotExist) { t.Fatalf("unexpected error: %s", mod.TerraformSourcesErr) } } diff --git a/internal/features/rootmodules/state/installed_modules.go b/internal/features/rootmodules/state/installed_modules.go index 3aaf99bf..d915f3f3 100644 --- a/internal/features/rootmodules/state/installed_modules.go +++ b/internal/features/rootmodules/state/installed_modules.go @@ -57,5 +57,31 @@ func InstalledModulesFromTerraformSources(path string, sources *datadir.Terrafor installedModules[normalizedSource.String()] = dir } + for _, pkg := range sources.RegistryPackages() { + for _, version := range sources.RegistryPackageVersions(pkg) { + addr, ok := sources.RegistryPackageSourceAddr(pkg, version) + + if !ok { + logger.Printf("Error getting source address for package %s and version %s", pkg.String(), version.String()) + continue + } + + absDir, err := sources.LocalPathForSource(addr) + if err != nil { + logger.Printf("Error getting local path for source %s: %s", pkg.String(), err) + continue + } + // installed modules expects a relative dir + dir, err := filepath.Rel(path, absDir) + if err != nil { + logger.Printf("Error getting relative path for source %s and path %s and absolute dir %s: %s", pkg.String(), path, absDir, err) + continue + } + + normalizedSource := tfmod.ParseModuleSourceAddr(pkg.String()) + installedModules[normalizedSource.String()] = dir + } + } + return installedModules } diff --git a/internal/features/rootmodules/state/root_store.go b/internal/features/rootmodules/state/root_store.go index 644343bc..fc0b8779 100644 --- a/internal/features/rootmodules/state/root_store.go +++ b/internal/features/rootmodules/state/root_store.go @@ -401,8 +401,6 @@ func (s *RootStore) CallersOfModule(path string) ([]string, error) { if record.ModManifest.ContainsLocalModule(path) { callers = append(callers, record.path) } - - // TODO: support TerraformSources as well here -> they don't contain local modules though, so we'd need to do this different here for stacks? } return callers, nil From 55dd53ff2a1789dbb132e0e1614da76d54d41219 Mon Sep 17 00:00:00 2001 From: Ansgar Mertens Date: Thu, 28 Nov 2024 10:53:55 +0100 Subject: [PATCH 8/8] Bump terraform-schema to `1392f74` --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 208d66da..b582a804 100644 --- a/go.mod +++ b/go.mod @@ -19,7 +19,7 @@ require ( github.com/hashicorp/terraform-exec v0.21.0 github.com/hashicorp/terraform-json v0.23.0 github.com/hashicorp/terraform-registry-address v0.2.3 - github.com/hashicorp/terraform-schema v0.0.0-20241115125457-9097f9e5a82e + github.com/hashicorp/terraform-schema v0.0.0-20241128095320-1392f740c4fe github.com/mcuadros/go-defaults v1.2.0 github.com/mh-cbon/go-fmt-fail v0.0.0-20160815164508-67765b3fbcb5 github.com/mitchellh/cli v1.1.5 diff --git a/go.sum b/go.sum index 55d9c23a..8287aa66 100644 --- a/go.sum +++ b/go.sum @@ -238,8 +238,8 @@ github.com/hashicorp/terraform-json v0.23.0 h1:sniCkExU4iKtTADReHzACkk8fnpQXrdD2 github.com/hashicorp/terraform-json v0.23.0/go.mod h1:MHdXbBAbSg0GvzuWazEGKAn/cyNfIB7mN6y7KJN6y2c= github.com/hashicorp/terraform-registry-address v0.2.3 h1:2TAiKJ1A3MAkZlH1YI/aTVcLZRu7JseiXNRHbOAyoTI= github.com/hashicorp/terraform-registry-address v0.2.3/go.mod h1:lFHA76T8jfQteVfT7caREqguFrW3c4MFSPhZB7HHgUM= -github.com/hashicorp/terraform-schema v0.0.0-20241115125457-9097f9e5a82e h1:yozV7l40vatIUPCYTeTqpPvOKsPDOd00kg8Tetf4VeQ= -github.com/hashicorp/terraform-schema v0.0.0-20241115125457-9097f9e5a82e/go.mod h1:hwYMiQp/tVcJtYfbNSxEEK+ilauXwwtZgpLXmeUBVGg= +github.com/hashicorp/terraform-schema v0.0.0-20241128095320-1392f740c4fe h1:2pVtzihaLjn6PTIyKom+X491QlLupxGoLRf5Ik8zpYM= +github.com/hashicorp/terraform-schema v0.0.0-20241128095320-1392f740c4fe/go.mod h1:3vDqHlpaMuTeBXSC4LWDM/m2QdEe9DmC90IgyuhdgZw= github.com/hashicorp/terraform-svchost v0.1.1 h1:EZZimZ1GxdqFRinZ1tpJwVxxt49xc/S52uzrw4x0jKQ= github.com/hashicorp/terraform-svchost v0.1.1/go.mod h1:mNsjQfZyf/Jhz35v6/0LWcv26+X7JPS+buii2c9/ctc= github.com/hexops/autogold v1.3.1 h1:YgxF9OHWbEIUjhDbpnLhgVsjUDsiHDTyDfy2lrfdlzo=