feat: parse .terraform-version files for stacks dirs

.changes/v0.34.0-alpha20240611.md

@@ -0,0 +1,15 @@
+## 0.34.0-alpha20240611 (11 June 2024)
+
+ENHANCEMENTS:
+
+* Add documentation for using vim with YouCompleteMe ([#1718](https://github.com/hashicorp/terraform-ls/issues/1718))
+* Re-architect the language server for improved performance and resource utilization ([#1667](https://github.com/hashicorp/terraform-ls/issues/1667))
+
+This marks the completion of a major refactoring effort. The language server will now start up much faster and use less resources, especially on larger workspaces. We achieve this by doing less work during the initial walk of a workspace. Instead, we only parse modules with open files. Whenever a file of a module is opened, we schedule all the jobs needed to understand the contents of that directory (and the referenced modules).
+
+We have tested this with workspaces and configurations of different sizes, but still expect some bugs. Please give this preview a try and let us know how it works for you.
+
+INTERNAL:
+
+* Split internal modules state into separate features ([#1667](https://github.com/hashicorp/terraform-ls/issues/1667))
+

.copywrite.hcl

@@ -8,7 +8,10 @@ project {
   # files or folders should be ignored
   header_ignore = [
     "**/testdata/**",
+    "**/testdata-initialize/**",
     ".github/ISSUE_TEMPLATE/**",
     ".changes/**",
+    "internal/schemas/gen-workspace/**",
+    "internal/schemas/tf-plugin-cache/**",
   ]
 }

CHANGELOG.md

@@ -1,3 +1,18 @@
+## 0.34.0-alpha20240611 (11 June 2024)
+
+ENHANCEMENTS:
+
+* Add documentation for using vim with YouCompleteMe ([#1718](https://github.com/hashicorp/terraform-ls/issues/1718))
+* Re-architect the language server for improved performance and resource utilization ([#1667](https://github.com/hashicorp/terraform-ls/issues/1667))
+
+This marks the completion of a major refactoring effort. The language server will now start up much faster and use less resources, especially on larger workspaces. We achieve this by doing less work during the initial walk of a workspace. Instead, we only parse modules with open files. Whenever a file of a module is opened, we schedule all the jobs needed to understand the contents of that directory (and the referenced modules).
+
+We have tested this with workspaces and configurations of different sizes, but still expect some bugs. Please give this preview a try and let us know how it works for you.
+
+INTERNAL:
+
+* Split internal modules state into separate features ([#1667](https://github.com/hashicorp/terraform-ls/issues/1667))
+
 ## 0.33.2 (06 June 2024)
 
 BUG FIXES:

go.mod

@@ -18,7 +18,7 @@ require (
 	github.com/hashicorp/terraform-exec v0.21.0
 	github.com/hashicorp/terraform-json v0.22.1
 	github.com/hashicorp/terraform-registry-address v0.2.3
-	github.com/hashicorp/terraform-schema v0.0.0-20240527093557-661c6794495e
+	github.com/hashicorp/terraform-schema v0.0.0-20240607143625-26a6f401ff0c
 	github.com/mcuadros/go-defaults v1.2.0
 	github.com/mh-cbon/go-fmt-fail v0.0.0-20160815164508-67765b3fbcb5
 	github.com/mitchellh/cli v1.1.5

go.sum

@@ -230,8 +230,8 @@ github.com/hashicorp/terraform-json v0.22.1 h1:xft84GZR0QzjPVWs4lRUwvTcPnegqlyS7
 github.com/hashicorp/terraform-json v0.22.1/go.mod h1:JbWSQCLFSXFFhg42T7l9iJwdGXBYV8fmmD6o/ML4p3A=
 github.com/hashicorp/terraform-registry-address v0.2.3 h1:2TAiKJ1A3MAkZlH1YI/aTVcLZRu7JseiXNRHbOAyoTI=
 github.com/hashicorp/terraform-registry-address v0.2.3/go.mod h1:lFHA76T8jfQteVfT7caREqguFrW3c4MFSPhZB7HHgUM=
-github.com/hashicorp/terraform-schema v0.0.0-20240527093557-661c6794495e h1:H8s/5oVHR+jlMILG4qbG4OycOr+8piyGUOpL+kqx24k=
-github.com/hashicorp/terraform-schema v0.0.0-20240527093557-661c6794495e/go.mod h1:lLCq9hyDL4yO7tcAu0Qj7MIwpw3StgB/DVcJM9r1ymA=
+github.com/hashicorp/terraform-schema v0.0.0-20240607143625-26a6f401ff0c h1:+ku2UJbLniAXN+WHNpmDosYOlCe6IcwuuJv2kmZli9U=
+github.com/hashicorp/terraform-schema v0.0.0-20240607143625-26a6f401ff0c/go.mod h1:lLCq9hyDL4yO7tcAu0Qj7MIwpw3StgB/DVcJM9r1ymA=
 github.com/hashicorp/terraform-svchost v0.1.1 h1:EZZimZ1GxdqFRinZ1tpJwVxxt49xc/S52uzrw4x0jKQ=
 github.com/hashicorp/terraform-svchost v0.1.1/go.mod h1:mNsjQfZyf/Jhz35v6/0LWcv26+X7JPS+buii2c9/ctc=
 github.com/hexops/autogold v1.3.1 h1:YgxF9OHWbEIUjhDbpnLhgVsjUDsiHDTyDfy2lrfdlzo=

internal/decoder/decoder.go

@@ -7,88 +7,12 @@ import (
 	"context"
 
 	"github.com/hashicorp/hcl-lang/decoder"
-	"github.com/hashicorp/hcl-lang/reference"
-	"github.com/hashicorp/hcl/v2"
 	"github.com/hashicorp/terraform-ls/internal/codelens"
 	ilsp "github.com/hashicorp/terraform-ls/internal/lsp"
 	lsp "github.com/hashicorp/terraform-ls/internal/protocol"
-	"github.com/hashicorp/terraform-ls/internal/state"
-	"github.com/hashicorp/terraform-ls/internal/terraform/ast"
 	"github.com/hashicorp/terraform-ls/internal/utm"
-	tfschema "github.com/hashicorp/terraform-schema/schema"
 )
 
-func modulePathContext(mod *state.Module, schemaReader state.SchemaReader, modReader ModuleReader) (*decoder.PathContext, error) {
-	schema, err := schemaForModule(mod, schemaReader, modReader)
-	if err != nil {
-		return nil, err
-	}
-	functions, err := functionsForModule(mod, schemaReader)
-	if err != nil {
-		return nil, err
-	}
-
-	pathCtx := &decoder.PathContext{
-		Schema:           schema,
-		ReferenceOrigins: make(reference.Origins, 0),
-		ReferenceTargets: make(reference.Targets, 0),
-		Files:            make(map[string]*hcl.File, 0),
-		Functions:        functions,
-		Validators:       moduleValidators,
-	}
-
-	for _, origin := range mod.RefOrigins {
-		if ast.IsModuleFilename(origin.OriginRange().Filename) {
-			pathCtx.ReferenceOrigins = append(pathCtx.ReferenceOrigins, origin)
-		}
-	}
-	for _, target := range mod.RefTargets {
-		if target.RangePtr != nil && ast.IsModuleFilename(target.RangePtr.Filename) {
-			pathCtx.ReferenceTargets = append(pathCtx.ReferenceTargets, target)
-		} else if target.RangePtr == nil {
-			pathCtx.ReferenceTargets = append(pathCtx.ReferenceTargets, target)
-		}
-	}
-
-	for name, f := range mod.ParsedModuleFiles {
-		pathCtx.Files[name.String()] = f
-	}
-
-	return pathCtx, nil
-}
-
-func varsPathContext(mod *state.Module) (*decoder.PathContext, error) {
-	schema, err := tfschema.SchemaForVariables(mod.Meta.Variables, mod.Path)
-	if err != nil {
-		return nil, err
-	}
-
-	pathCtx := &decoder.PathContext{
-		Schema:           schema,
-		ReferenceOrigins: make(reference.Origins, 0),
-		ReferenceTargets: make(reference.Targets, 0),
-		Files:            make(map[string]*hcl.File),
-	}
-
-	if len(mod.ParsedModuleFiles) > 0 {
-		// Only validate if this is actually a module
-		// as we may come across standalone tfvars files
-		// for which we have no context.
-		pathCtx.Validators = varsValidators
-	}
-
-	for _, origin := range mod.VarsRefOrigins {
-		if ast.IsVarsFilename(origin.OriginRange().Filename) {
-			pathCtx.ReferenceOrigins = append(pathCtx.ReferenceOrigins, origin)
-		}
-	}
-
-	for name, f := range mod.ParsedVarsFiles {
-		pathCtx.Files[name.String()] = f
-	}
-	return pathCtx, nil
-}
-
 func DecoderContext(ctx context.Context) decoder.DecoderContext {
 	dCtx := decoder.NewDecoderContext()
 	dCtx.UtmSource = utm.UtmSource

internal/decoder/path_reader.go

@@ -7,66 +7,34 @@ import (
 	"context"
 	"fmt"
 
-	"github.com/hashicorp/go-version"
 	"github.com/hashicorp/hcl-lang/decoder"
 	"github.com/hashicorp/hcl-lang/lang"
-	ilsp "github.com/hashicorp/terraform-ls/internal/lsp"
-	"github.com/hashicorp/terraform-ls/internal/state"
-	tfaddr "github.com/hashicorp/terraform-registry-address"
-	tfmod "github.com/hashicorp/terraform-schema/module"
-	"github.com/hashicorp/terraform-schema/registry"
 )
 
-type ModuleReader interface {
-	ModuleByPath(modPath string) (*state.Module, error)
-	List() ([]*state.Module, error)
-	ModuleCalls(modPath string) (tfmod.ModuleCalls, error)
-	LocalModuleMeta(modPath string) (*tfmod.Meta, error)
-	RegistryModuleMeta(addr tfaddr.Module, cons version.Constraints) (*registry.ModuleData, error)
-}
+type PathReaderMap map[string]decoder.PathReader
 
-type PathReader struct {
-	ModuleReader ModuleReader
-	SchemaReader state.SchemaReader
+// GlobalPathReader is a PathReader that delegates language specific PathReaders
+// that usually come from features.
+type GlobalPathReader struct {
+	PathReaderMap PathReaderMap
 }
 
-var _ decoder.PathReader = &PathReader{}
+var _ decoder.PathReader = &GlobalPathReader{}
 
-func (mr *PathReader) Paths(ctx context.Context) []lang.Path {
+func (mr *GlobalPathReader) Paths(ctx context.Context) []lang.Path {
 	paths := make([]lang.Path, 0)
 
-	modList, err := mr.ModuleReader.List()
-	if err != nil {
-		return paths
-	}
-	for _, mod := range modList {
-		paths = append(paths, lang.Path{
-			Path:       mod.Path,
-			LanguageID: ilsp.Terraform.String(),
-		})
-		if len(mod.ParsedVarsFiles) > 0 {
-			paths = append(paths, lang.Path{
-				Path:       mod.Path,
-				LanguageID: ilsp.Tfvars.String(),
-			})
-		}
+	for _, feature := range mr.PathReaderMap {
+		paths = append(paths, feature.Paths(ctx)...)
 	}
 
 	return paths
 }
 
-func (mr *PathReader) PathContext(path lang.Path) (*decoder.PathContext, error) {
-	mod, err := mr.ModuleReader.ModuleByPath(path.Path)
-	if err != nil {
-		return nil, err
-	}
-
-	switch path.LanguageID {
-	case ilsp.Terraform.String():
-		return modulePathContext(mod, mr.SchemaReader, mr.ModuleReader)
-	case ilsp.Tfvars.String():
-		return varsPathContext(mod)
+func (mr *GlobalPathReader) PathContext(path lang.Path) (*decoder.PathContext, error) {
+	if feature, ok := mr.PathReaderMap[path.LanguageID]; ok {
+		return feature.PathContext(path)
 	}
 
-	return nil, fmt.Errorf("unknown language ID: %q", path.LanguageID)
+	return nil, fmt.Errorf("no feature found for language %s", path.LanguageID)
 }

internal/eventbus/bus.go

@@ -0,0 +1,99 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: MPL-2.0
+
+package eventbus
+
+import (
+	"io"
+	"log"
+	"sync"
+)
+
+const ChannelSize = 10
+
+var discardLogger = log.New(io.Discard, "", 0)
+
+// EventBus is a simple event bus that allows for subscribing to and publishing
+// events of a specific type.
+//
+// It has a static list of topics. Each topic can have multiple subscribers.
+// When an event is published to a topic, it is sent to all subscribers.
+type EventBus struct {
+	logger *log.Logger
+
+	didOpenTopic          *Topic[DidOpenEvent]
+	didChangeTopic        *Topic[DidChangeEvent]
+	didChangeWatchedTopic *Topic[DidChangeWatchedEvent]
+	discoverTopic         *Topic[DiscoverEvent]
+
+	manifestChangeTopic   *Topic[ManifestChangeEvent]
+	pluginLockChangeTopic *Topic[PluginLockChangeEvent]
+}
+
+func NewEventBus() *EventBus {
+	return &EventBus{
+		logger:                discardLogger,
+		didOpenTopic:          NewTopic[DidOpenEvent](),
+		didChangeTopic:        NewTopic[DidChangeEvent](),
+		didChangeWatchedTopic: NewTopic[DidChangeWatchedEvent](),
+		discoverTopic:         NewTopic[DiscoverEvent](),
+		manifestChangeTopic:   NewTopic[ManifestChangeEvent](),
+		pluginLockChangeTopic: NewTopic[PluginLockChangeEvent](),
+	}
+}
+
+func (eb *EventBus) SetLogger(logger *log.Logger) {
+	eb.logger = logger
+}
+
+// Topic represents a generic subscription topic
+type Topic[T any] struct {
+	subscribers []Subscriber[T]
+	mutex       sync.Mutex
+}
+
+// Subscriber represents a subscriber to a topic
+type Subscriber[T any] struct {
+	// channel is the channel to which all events of the topic are sent
+	channel chan<- T
+
+	// doneChannel is an optional channel that the subscriber can use to signal
+	// that it is done processing the event
+	doneChannel <-chan struct{}
+}
+
+// NewTopic creates a new topic
+func NewTopic[T any]() *Topic[T] {
+	return &Topic[T]{
+		subscribers: make([]Subscriber[T], 0),
+	}
+}
+
+// Subscribe adds a subscriber to a topic
+func (eb *Topic[T]) Subscribe(doneChannel <-chan struct{}) <-chan T {
+	channel := make(chan T, ChannelSize)
+	eb.mutex.Lock()
+	defer eb.mutex.Unlock()
+
+	eb.subscribers = append(eb.subscribers, Subscriber[T]{
+		channel:     channel,
+		doneChannel: doneChannel,
+	})
+	return channel
+}
+
+// Publish sends an event to all subscribers of a specific topic
+func (eb *Topic[T]) Publish(event T) {
+	eb.mutex.Lock()
+	defer eb.mutex.Unlock()
+
+	for _, subscriber := range eb.subscribers {
+		// Send the event to the subscriber
+		subscriber.channel <- event
+
+		if subscriber.doneChannel != nil {
+			// And wait until the subscriber is done processing it
+			<-subscriber.doneChannel
+		}
+	}
+}

internal/eventbus/did_change.go

@@ -0,0 +1,31 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: MPL-2.0
+
+package eventbus
+
+import (
+	"context"
+
+	"github.com/hashicorp/terraform-ls/internal/document"
+)
+
+// DidChangeEvent is an event to signal that a file in directory has changed.
+//
+// It is usually emitted when a document is changed via a language server
+// text synchronization event.
+type DidChangeEvent struct {
+	Context context.Context
+
+	Dir        document.DirHandle
+	LanguageID string
+}
+
+func (n *EventBus) OnDidChange(identifier string, doneChannel <-chan struct{}) <-chan DidChangeEvent {
+	n.logger.Printf("bus: %q subscribed to OnDidChange", identifier)
+	return n.didChangeTopic.Subscribe(doneChannel)
+}
+
+func (n *EventBus) DidChange(e DidChangeEvent) {
+	n.logger.Printf("bus: -> DidChange %s", e.Dir)
+	n.didChangeTopic.Publish(e)
+}