From 60e8997d12669ff0d98902f3674ab14bec8f4bac Mon Sep 17 00:00:00 2001 From: Sarah French Date: Mon, 20 Oct 2025 18:40:26 +0100 Subject: [PATCH 01/23] Allow the builtin terraform provider to contain multiple PSS implementations, pass through requests to matching store based on type name. --- .../builtin/providers/terraform/provider.go | 40 ++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/internal/builtin/providers/terraform/provider.go b/internal/builtin/providers/terraform/provider.go index a6bf9c0592fa..f3c0504b3d13 100644 --- a/internal/builtin/providers/terraform/provider.go +++ b/internal/builtin/providers/terraform/provider.go @@ -14,7 +14,13 @@ import ( ) // Provider is an implementation of providers.Interface -type Provider struct{} +type Provider struct { + + // State storage implementations + stores map[string]providers.Interface +} + +var _ providers.Interface = &Provider{} // NewProvider returns a new terraform provider func NewProvider() providers.Interface { @@ -284,48 +290,80 @@ func (p *Provider) ListResource(req providers.ListResourceRequest) providers.Lis } func (p *Provider) ValidateStateStoreConfig(req providers.ValidateStateStoreConfigRequest) providers.ValidateStateStoreConfigResponse { + if s, ok := p.stores[req.TypeName]; ok { + return s.ValidateStateStoreConfig(req) + } + var resp providers.ValidateStateStoreConfigResponse resp.Diagnostics.Append(fmt.Errorf("unsupported state store type %q", req.TypeName)) return resp } func (p *Provider) ConfigureStateStore(req providers.ConfigureStateStoreRequest) providers.ConfigureStateStoreResponse { + if s, ok := p.stores[req.TypeName]; ok { + return s.ConfigureStateStore(req) + } + var resp providers.ConfigureStateStoreResponse resp.Diagnostics.Append(fmt.Errorf("unsupported state store type %q", req.TypeName)) return resp } func (p *Provider) ReadStateBytes(req providers.ReadStateBytesRequest) providers.ReadStateBytesResponse { + if s, ok := p.stores[req.TypeName]; ok { + return s.ReadStateBytes(req) + } + var resp providers.ReadStateBytesResponse resp.Diagnostics.Append(fmt.Errorf("unsupported state store type %q", req.TypeName)) return resp } func (p *Provider) WriteStateBytes(req providers.WriteStateBytesRequest) providers.WriteStateBytesResponse { + if s, ok := p.stores[req.TypeName]; ok { + return s.WriteStateBytes(req) + } + var resp providers.WriteStateBytesResponse resp.Diagnostics.Append(fmt.Errorf("unsupported state store type %q", req.TypeName)) return resp } func (p *Provider) LockState(req providers.LockStateRequest) providers.LockStateResponse { + if s, ok := p.stores[req.TypeName]; ok { + return s.LockState(req) + } + var resp providers.LockStateResponse resp.Diagnostics.Append(fmt.Errorf("unsupported state store type %q", req.TypeName)) return resp } func (p *Provider) UnlockState(req providers.UnlockStateRequest) providers.UnlockStateResponse { + if s, ok := p.stores[req.TypeName]; ok { + return s.UnlockState(req) + } + var resp providers.UnlockStateResponse resp.Diagnostics.Append(fmt.Errorf("unsupported state store type %q", req.TypeName)) return resp } func (p *Provider) GetStates(req providers.GetStatesRequest) providers.GetStatesResponse { + if s, ok := p.stores[req.TypeName]; ok { + return s.GetStates(req) + } + var resp providers.GetStatesResponse resp.Diagnostics.Append(fmt.Errorf("unsupported state store type %q", req.TypeName)) return resp } func (p *Provider) DeleteState(req providers.DeleteStateRequest) providers.DeleteStateResponse { + if s, ok := p.stores[req.TypeName]; ok { + return s.DeleteState(req) + } + var resp providers.DeleteStateResponse resp.Diagnostics.Append(fmt.Errorf("unsupported state store type %q", req.TypeName)) return resp From 306d896df884a3a05cc5e42409155487a73c8ec8 Mon Sep 17 00:00:00 2001 From: Sarah French Date: Mon, 20 Oct 2025 18:43:11 +0100 Subject: [PATCH 02/23] Make it easier to use provider.Interface as interface for state store implementation; add embeddable unimplementedProviderInterface struct --- .../terraform/state_store_unimplemented.go | 120 ++++++++++++++++++ 1 file changed, 120 insertions(+) create mode 100644 internal/builtin/providers/terraform/state_store_unimplemented.go diff --git a/internal/builtin/providers/terraform/state_store_unimplemented.go b/internal/builtin/providers/terraform/state_store_unimplemented.go new file mode 100644 index 000000000000..f83821b42b16 --- /dev/null +++ b/internal/builtin/providers/terraform/state_store_unimplemented.go @@ -0,0 +1,120 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package terraform + +import "github.com/hashicorp/terraform/internal/providers" + +// unimplementedProviderInterface implements all methods of provider.Interface but they panic and report they aren't implemented. +// +// This allows state store implementations to be passed around as provider.Interface instances in the builtin provider. +// To use unimplementedProviderInterface embed it into a struct implementing state storage. +type unimplementedProviderInterface struct{} + +var _ providers.Interface = &InMemStoreSingle{} + +func (unimplementedProviderInterface) GetProviderSchema() providers.GetProviderSchemaResponse { + panic("GetProviderSchema isn't implemented") +} +func (unimplementedProviderInterface) GetResourceIdentitySchemas() providers.GetResourceIdentitySchemasResponse { + panic("GetResourceIdentitySchemas isn't implemented") +} +func (unimplementedProviderInterface) ValidateProviderConfig(providers.ValidateProviderConfigRequest) providers.ValidateProviderConfigResponse { + panic("ValidateProviderConfig isn't implemented") +} +func (unimplementedProviderInterface) ValidateResourceConfig(providers.ValidateResourceConfigRequest) providers.ValidateResourceConfigResponse { + panic("ValidateResourceConfig isn't implemented") +} +func (unimplementedProviderInterface) ValidateDataResourceConfig(providers.ValidateDataResourceConfigRequest) providers.ValidateDataResourceConfigResponse { + panic("ValidateDataResourceConfig isn't implemented") +} +func (unimplementedProviderInterface) ValidateEphemeralResourceConfig(providers.ValidateEphemeralResourceConfigRequest) providers.ValidateEphemeralResourceConfigResponse { + panic("ValidateEphemeralResourceConfig isn't implemented") +} +func (unimplementedProviderInterface) ValidateListResourceConfig(providers.ValidateListResourceConfigRequest) providers.ValidateListResourceConfigResponse { + panic("ValidateListResourceConfig isn't implemented") +} +func (unimplementedProviderInterface) UpgradeResourceState(providers.UpgradeResourceStateRequest) providers.UpgradeResourceStateResponse { + panic("UpgradeResourceState isn't implemented") +} +func (unimplementedProviderInterface) UpgradeResourceIdentity(providers.UpgradeResourceIdentityRequest) providers.UpgradeResourceIdentityResponse { + panic("UpgradeResourceIdentity isn't implemented") +} +func (unimplementedProviderInterface) ConfigureProvider(providers.ConfigureProviderRequest) providers.ConfigureProviderResponse { + panic("ConfigureProvider isn't implemented") +} +func (unimplementedProviderInterface) Stop() error { + panic("Stop isn't implemented") +} +func (unimplementedProviderInterface) ReadResource(providers.ReadResourceRequest) providers.ReadResourceResponse { + panic("ReadResource isn't implemented") +} +func (unimplementedProviderInterface) PlanResourceChange(providers.PlanResourceChangeRequest) providers.PlanResourceChangeResponse { + panic("PlanResourceChange isn't implemented") +} +func (unimplementedProviderInterface) ApplyResourceChange(providers.ApplyResourceChangeRequest) providers.ApplyResourceChangeResponse { + panic("ApplyResourceChange isn't implemented") +} +func (unimplementedProviderInterface) ImportResourceState(providers.ImportResourceStateRequest) providers.ImportResourceStateResponse { + panic("ImportResourceState isn't implemented") +} +func (unimplementedProviderInterface) GenerateResourceConfig(providers.GenerateResourceConfigRequest) providers.GenerateResourceConfigResponse { + panic("GenerateResourceConfig isn't implemented") +} +func (unimplementedProviderInterface) MoveResourceState(providers.MoveResourceStateRequest) providers.MoveResourceStateResponse { + panic("MoveResourceState isn't implemented") +} +func (unimplementedProviderInterface) ReadDataSource(providers.ReadDataSourceRequest) providers.ReadDataSourceResponse { + panic("ReadDataSource isn't implemented") +} +func (unimplementedProviderInterface) OpenEphemeralResource(providers.OpenEphemeralResourceRequest) providers.OpenEphemeralResourceResponse { + panic("OpenEphemeralResource isn't implemented") +} +func (unimplementedProviderInterface) RenewEphemeralResource(providers.RenewEphemeralResourceRequest) providers.RenewEphemeralResourceResponse { + panic("RenewEphemeralResource isn't implemented") +} +func (unimplementedProviderInterface) CloseEphemeralResource(providers.CloseEphemeralResourceRequest) providers.CloseEphemeralResourceResponse { + panic("CloseEphemeralResource isn't implemented") +} +func (unimplementedProviderInterface) CallFunction(providers.CallFunctionRequest) providers.CallFunctionResponse { + panic("CallFunction isn't implemented") +} +func (unimplementedProviderInterface) ListResource(providers.ListResourceRequest) providers.ListResourceResponse { + panic("ListResource isn't implemented") +} +func (unimplementedProviderInterface) ValidateStateStoreConfig(providers.ValidateStateStoreConfigRequest) providers.ValidateStateStoreConfigResponse { + panic("ValidateStateStoreConfig isn't implemented") +} +func (unimplementedProviderInterface) ConfigureStateStore(providers.ConfigureStateStoreRequest) providers.ConfigureStateStoreResponse { + panic("ConfigureStateStore isn't implemented") +} +func (unimplementedProviderInterface) ReadStateBytes(providers.ReadStateBytesRequest) providers.ReadStateBytesResponse { + panic("ReadStateBytes isn't implemented") +} +func (unimplementedProviderInterface) WriteStateBytes(providers.WriteStateBytesRequest) providers.WriteStateBytesResponse { + panic("WriteStateBytes isn't implemented") +} +func (unimplementedProviderInterface) LockState(providers.LockStateRequest) providers.LockStateResponse { + panic("LockState isn't implemented") +} +func (unimplementedProviderInterface) UnlockState(providers.UnlockStateRequest) providers.UnlockStateResponse { + panic("UnlockState isn't implemented") +} +func (unimplementedProviderInterface) GetStates(providers.GetStatesRequest) providers.GetStatesResponse { + panic("GetStates isn't implemented") +} +func (unimplementedProviderInterface) DeleteState(providers.DeleteStateRequest) providers.DeleteStateResponse { + panic("DeleteState isn't implemented") +} +func (unimplementedProviderInterface) PlanAction(providers.PlanActionRequest) providers.PlanActionResponse { + panic("PlanAction isn't implemented") +} +func (unimplementedProviderInterface) InvokeAction(providers.InvokeActionRequest) providers.InvokeActionResponse { + panic("InvokeAction isn't implemented") +} +func (unimplementedProviderInterface) ValidateActionConfig(providers.ValidateActionConfigRequest) providers.ValidateActionConfigResponse { + panic("ValidateActionConfig isn't implemented") +} +func (unimplementedProviderInterface) Close() error { + panic("Close isn't implemented") +} From 9f104faa403e483ecd3591b6db2b34310bef7ced Mon Sep 17 00:00:00 2001 From: Sarah French Date: Mon, 20 Oct 2025 20:02:30 +0100 Subject: [PATCH 03/23] Add initial implementation of inmem state store to the builtin terraform provider. Stop people using it unless TF_ACC is set. --- .../builtin/providers/terraform/provider.go | 14 +- .../providers/terraform/state_store_inmem.go | 229 ++++++++++++++++++ 2 files changed, 242 insertions(+), 1 deletion(-) create mode 100644 internal/builtin/providers/terraform/state_store_inmem.go diff --git a/internal/builtin/providers/terraform/provider.go b/internal/builtin/providers/terraform/provider.go index f3c0504b3d13..a5da221aa80e 100644 --- a/internal/builtin/providers/terraform/provider.go +++ b/internal/builtin/providers/terraform/provider.go @@ -6,6 +6,7 @@ package terraform import ( "fmt" "log" + "os" tfaddr "github.com/hashicorp/terraform-registry-address" "github.com/zclconf/go-cty/cty" @@ -24,7 +25,11 @@ var _ providers.Interface = &Provider{} // NewProvider returns a new terraform provider func NewProvider() providers.Interface { - return &Provider{} + return &Provider{ + stores: map[string]providers.Interface{ + "terraform_inmem": &InMemStoreSingle{}, + }, + } } // GetSchema returns the complete schema for the provider. @@ -83,6 +88,13 @@ func (p *Provider) GetProviderSchema() providers.GetProviderSchemaResponse { Actions: map[string]providers.ActionSchema{}, } providers.SchemaCache.Set(tfaddr.NewProvider(tfaddr.BuiltInProviderHost, tfaddr.BuiltInProviderNamespace, "terraform"), resp) + + // Only include the inmem state store in the provider when `TF_ACC` is set in the environment + // Excluding this from the schemas is sufficient to block usage. + if v := os.Getenv("TF_ACC"); v != "" { + resp.StateStores["terraform_inmem"] = stateStoreInMemGetSchema() + } + return resp } diff --git a/internal/builtin/providers/terraform/state_store_inmem.go b/internal/builtin/providers/terraform/state_store_inmem.go new file mode 100644 index 000000000000..84af01d24fcd --- /dev/null +++ b/internal/builtin/providers/terraform/state_store_inmem.go @@ -0,0 +1,229 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package terraform + +import ( + "errors" + "fmt" + "sort" + "sync" + + "github.com/hashicorp/go-uuid" + "github.com/hashicorp/terraform/internal/backend" + "github.com/hashicorp/terraform/internal/configs/configschema" + "github.com/hashicorp/terraform/internal/providers" + "github.com/hashicorp/terraform/internal/tfdiags" + "github.com/zclconf/go-cty/cty" +) + +// Matches default in command package. +const defaultStateStoreChunkSize int64 = 8 << 20 // 8 MB + +// InMemStoreSingle allows 'storing' state in memory for the purpose of testing. +// +// "Single" reflects the fact that this implementation does not use any global scope vars +// in its implementation, unlike the current inmem backend. HOWEVER, you can test whether locking +// blocks multiple clients trying to access the same state at once by creating multiple instances +// of backend.Backend that wrap the same provider.Interface instance. +type InMemStoreSingle struct { + states stateMap + locks lockMap + + unimplementedProviderInterface +} + +var _ providers.Interface = &InMemStoreSingle{} + +func stateStoreInMemGetSchema() providers.Schema { + return providers.Schema{ + Body: &configschema.Block{ + Attributes: map[string]*configschema.Attribute{ + "lock_id": { + Type: cty.String, + Optional: true, + Description: "initializes the state in a locked configuration", + }, + }, + }, + } +} + +func (m *InMemStoreSingle) ValidateStateStoreConfig(req providers.ValidateStateStoreConfigRequest) providers.ValidateStateStoreConfigResponse { + // Not implemented in original inmem backend. + // The inmem used default logic in the backendbase package that cannot be replicated here easily. + // Instead, here is a rough implementation of validation: + var resp providers.ValidateStateStoreConfigResponse + + attrs := req.Config.AsValueMap() + if v, ok := attrs["lock_id"]; ok { + if !v.IsKnown() { + resp.Diagnostics = resp.Diagnostics.Append(errors.New("the attribute \"lock_id\" cannot be an unknown value")) + return resp + } + } + + return resp +} + +func (m *InMemStoreSingle) ConfigureStateStore(req providers.ConfigureStateStoreRequest) providers.ConfigureStateStoreResponse { + resp := providers.ConfigureStateStoreResponse{} + + m.states.Lock() + defer m.states.Unlock() + + // set the default client lock info per the test config + configVal := req.Config + if v := configVal.GetAttr("lock_id"); !v.IsNull() { + m.locks.lock(backend.DefaultStateName, v.AsString()) + } + + // We need to return a suggested chunk size; use default value + resp.Capabilities.ChunkSize = defaultStateStoreChunkSize + return resp +} + +func (m *InMemStoreSingle) ReadStateBytes(req providers.ReadStateBytesRequest) providers.ReadStateBytesResponse { + resp := providers.ReadStateBytesResponse{} + + v, ok := m.states.m[req.StateId] + if !ok { + // Does not exist, so return no bytes + + resp.Diagnostics = resp.Diagnostics.Append(tfdiags.Sourceless( + tfdiags.Warning, + "State doesn't exist, yet", + fmt.Sprintf("There's no state for workspace %q yet", req.StateId), + )) + return resp + } + + resp.Bytes = v + return resp +} + +func (m *InMemStoreSingle) WriteStateBytes(req providers.WriteStateBytesRequest) providers.WriteStateBytesResponse { + resp := providers.WriteStateBytesResponse{} + + if m.states.m == nil { + m.states.m = make(map[string][]byte, 1) + } + + m.states.m[req.StateId] = req.Bytes + + return resp +} + +func (m *InMemStoreSingle) LockState(req providers.LockStateRequest) providers.LockStateResponse { + resp := providers.LockStateResponse{} + + lockIdBytes, err := uuid.GenerateUUID() + if err != nil { + resp.Diagnostics = resp.Diagnostics.Append(fmt.Errorf("error creating random lock uuid: %w", err)) + return resp + } + + lockId := string(lockIdBytes) + returnedLockId, err := m.locks.lock(req.StateId, lockId) + if err != nil { + resp.Diagnostics = resp.Diagnostics.Append(err) + } + + resp.LockId = string(returnedLockId) + return resp +} + +func (m *InMemStoreSingle) UnlockState(req providers.UnlockStateRequest) providers.UnlockStateResponse { + resp := providers.UnlockStateResponse{} + + err := m.locks.unlock(req.StateId, req.LockId) + if err != nil { + resp.Diagnostics = resp.Diagnostics.Append(fmt.Errorf("error when unlocking state %q: %w", req.StateId, err)) + return resp + } + + return resp +} + +func (m *InMemStoreSingle) GetStates(req providers.GetStatesRequest) providers.GetStatesResponse { + m.states.Lock() + defer m.states.Unlock() + + resp := providers.GetStatesResponse{} + + var workspaces []string + + for s := range m.states.m { + workspaces = append(workspaces, s) + } + + sort.Strings(workspaces) + resp.States = workspaces + return resp +} + +func (m *InMemStoreSingle) DeleteState(req providers.DeleteStateRequest) providers.DeleteStateResponse { + m.states.Lock() + defer m.states.Unlock() + + resp := providers.DeleteStateResponse{} + + if req.StateId == backend.DefaultStateName || req.StateId == "" { + resp.Diagnostics = resp.Diagnostics.Append(fmt.Errorf("can't delete default state")) + return resp + } + + delete(m.states.m, req.StateId) + return resp +} + +type stateMap struct { + sync.Mutex + m map[string][]byte // key=state name/workspace, value=state +} + +type lockMap struct { + sync.Mutex + m map[string]string // key=state name/workspace, value=lock_id +} + +func (l *lockMap) lock(name string, lockId string) (string, error) { + l.Lock() + defer l.Unlock() + + lock, ok := l.m[name] + if ok { + // Error; lock already exists for that state/workspace + return "", fmt.Errorf("state %q is already locked with lock id %q", name, lock) + } + + if l.m == nil { + l.m = make(map[string]string, 1) + } + + l.m[name] = lockId + + return lockId, nil +} + +func (l *lockMap) unlock(name, id string) error { + l.Lock() + defer l.Unlock() + + lockId, ok := l.m[name] + + if !ok { + return errors.New("state not locked") + } + + if id != lockId { + return fmt.Errorf("invalid lock id: %q was locked with lock id %q, but tried to unlock with lock id %q", + name, + lockId, + id, + ) + } + + delete(l.m, name) + return nil +} From b79de8d23a03fb494e8cdd1f5dabddffffb9fc7e Mon Sep 17 00:00:00 2001 From: Sarah French Date: Mon, 20 Oct 2025 20:03:52 +0100 Subject: [PATCH 04/23] Add method for getting an inmem store instance that has default workspace state already. --- .../builtin/providers/terraform/provider.go | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/internal/builtin/providers/terraform/provider.go b/internal/builtin/providers/terraform/provider.go index a5da221aa80e..7bb06df7074e 100644 --- a/internal/builtin/providers/terraform/provider.go +++ b/internal/builtin/providers/terraform/provider.go @@ -4,6 +4,7 @@ package terraform import ( + "bytes" "fmt" "log" "os" @@ -11,7 +12,9 @@ import ( tfaddr "github.com/hashicorp/terraform-registry-address" "github.com/zclconf/go-cty/cty" + "github.com/hashicorp/terraform/internal/backend" "github.com/hashicorp/terraform/internal/providers" + "github.com/hashicorp/terraform/internal/states/statefile" ) // Provider is an implementation of providers.Interface @@ -32,6 +35,33 @@ func NewProvider() providers.Interface { } } +// NewProvider returns a new terraform provider where the internal +// state store(s) all have the default workspace already existing +func NewProviderWithDefaultState() providers.Interface { + // Get the empty state file as bytes + f := statefile.New(nil, "", 0) + + var buf bytes.Buffer + err := statefile.Write(f, &buf) + if err != nil { + panic(err) + } + emptyStateBytes := buf.Bytes() + + // Return a provider where all state stores have existing default workspaces + return &Provider{ + stores: map[string]providers.Interface{ + "terraform_inmem": &InMemStoreSingle{ + states: stateMap{ + m: map[string][]byte{ + backend.DefaultStateName: emptyStateBytes, + }, + }, + }, + }, + } +} + // GetSchema returns the complete schema for the provider. func (p *Provider) GetProviderSchema() providers.GetProviderSchemaResponse { resp := providers.GetProviderSchemaResponse{ From f54f95b41e925379e3c0033d11b5184f8dc5a99a Mon Sep 17 00:00:00 2001 From: Sarah French Date: Mon, 20 Oct 2025 20:05:50 +0100 Subject: [PATCH 05/23] Copy some tests from the inmem backend and use for the inmem state store. --- .../remote-state/inmem/backend_test.go | 2 +- .../terraform/state_store_inmem_test.go | 96 +++++++++++++++++++ 2 files changed, 97 insertions(+), 1 deletion(-) create mode 100644 internal/builtin/providers/terraform/state_store_inmem_test.go diff --git a/internal/backend/remote-state/inmem/backend_test.go b/internal/backend/remote-state/inmem/backend_test.go index 516177c16db7..ae6debc604d3 100644 --- a/internal/backend/remote-state/inmem/backend_test.go +++ b/internal/backend/remote-state/inmem/backend_test.go @@ -65,7 +65,7 @@ func TestBackendLocked(t *testing.T) { backend.TestBackendStateLocks(t, b1, b2) } -// use the this backen to test the remote.State implementation +// use this backend to test the remote.State implementation func TestRemoteState(t *testing.T) { defer Reset() b := backend.TestBackendConfig(t, New(), hcl.EmptyBody()) diff --git a/internal/builtin/providers/terraform/state_store_inmem_test.go b/internal/builtin/providers/terraform/state_store_inmem_test.go new file mode 100644 index 000000000000..738ca09b503e --- /dev/null +++ b/internal/builtin/providers/terraform/state_store_inmem_test.go @@ -0,0 +1,96 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package terraform + +import ( + "testing" + + "github.com/hashicorp/hcl/v2" + "github.com/hashicorp/terraform/internal/addrs" + "github.com/hashicorp/terraform/internal/backend" + "github.com/hashicorp/terraform/internal/backend/pluggable" + "github.com/hashicorp/terraform/internal/states" + "github.com/zclconf/go-cty/cty" +) + +func TestBackendLocked(t *testing.T) { + t.Setenv("TF_ACC", "1") // enable using the inmem state store + + storeName := "terraform_inmem" + // Use NewProviderWithDefaultState so default workspace exists already, + // because backend.TestBackendStateLocks assumes they exist by default. + provider := NewProviderWithDefaultState() + + plug1, err := pluggable.NewPluggable(provider, storeName) + if err != nil { + t.Fatal(err) + } + plug2, err := pluggable.NewPluggable(provider, storeName) + if err != nil { + t.Fatal(err) + } + + b1 := backend.TestBackendConfig(t, plug1, nil) + b2 := backend.TestBackendConfig(t, plug2, nil) + + backend.TestBackendStateLocks(t, b1, b2) +} + +func TestRemoteState(t *testing.T) { + t.Setenv("TF_ACC", "1") // enable using the inmem state store + + storeName := "terraform_inmem" + provider := NewProvider() + + plug, err := pluggable.NewPluggable(provider, storeName) + if err != nil { + t.Fatal(err) + } + + b := backend.TestBackendConfig(t, plug, hcl.EmptyBody()) + + // The default workspace doesn't exist by default + // (Note that this depends on the factory method used to get the provider above) + workspaces, wDiags := b.Workspaces() + if wDiags.HasErrors() { + t.Fatal(wDiags.Err()) + } + if len(workspaces) != 0 { + t.Fatalf("unexpected response from Workspaces method: %#v", workspaces) + } + + // create a new workspace in this backend + workspace := "workspace" + emptyState := states.NewState() + + sMgr, sDiags := b.StateMgr(workspace) + if sDiags.HasErrors() { + t.Fatal(sDiags.Err()) + } + if err := sMgr.WriteState(emptyState); err != nil { + t.Fatal(err) + } + if err := sMgr.PersistState(nil); err != nil { + t.Fatal(err) + } + + // force overwriting the remote state + newState := states.NewState() + newState.SetOutputValue( + addrs.OutputValue{Name: "foo"}.Absolute(addrs.RootModuleInstance), + cty.StringVal("bar"), + false) + + if err := sMgr.WriteState(newState); err != nil { + t.Fatal(err) + } + + if err := sMgr.PersistState(nil); err != nil { + t.Fatal(err) + } + + if err := sMgr.RefreshState(); err != nil { + t.Fatal(err) + } +} From a3ffebda98f9b0ca16ace54be8a55e0009d2cb90 Mon Sep 17 00:00:00 2001 From: Sarah French Date: Tue, 21 Oct 2025 17:25:56 +0100 Subject: [PATCH 06/23] Simplify code to avoid map of state stores, for now --- .../builtin/providers/terraform/provider.go | 52 ++++---- .../providers/terraform/state_store_inmem.go | 6 +- .../terraform/state_store_inmem_test.go | 8 +- .../terraform/state_store_unimplemented.go | 120 ------------------ 4 files changed, 29 insertions(+), 157 deletions(-) delete mode 100644 internal/builtin/providers/terraform/state_store_unimplemented.go diff --git a/internal/builtin/providers/terraform/provider.go b/internal/builtin/providers/terraform/provider.go index 7bb06df7074e..54c9ca031127 100644 --- a/internal/builtin/providers/terraform/provider.go +++ b/internal/builtin/providers/terraform/provider.go @@ -20,8 +20,8 @@ import ( // Provider is an implementation of providers.Interface type Provider struct { - // State storage implementations - stores map[string]providers.Interface + // State storage implementation(s) + inMem *InMemStoreSingle // terraform_inmem } var _ providers.Interface = &Provider{} @@ -29,9 +29,7 @@ var _ providers.Interface = &Provider{} // NewProvider returns a new terraform provider func NewProvider() providers.Interface { return &Provider{ - stores: map[string]providers.Interface{ - "terraform_inmem": &InMemStoreSingle{}, - }, + inMem: &InMemStoreSingle{}, } } @@ -50,12 +48,10 @@ func NewProviderWithDefaultState() providers.Interface { // Return a provider where all state stores have existing default workspaces return &Provider{ - stores: map[string]providers.Interface{ - "terraform_inmem": &InMemStoreSingle{ - states: stateMap{ - m: map[string][]byte{ - backend.DefaultStateName: emptyStateBytes, - }, + inMem: &InMemStoreSingle{ + states: stateMap{ + m: map[string][]byte{ + backend.DefaultStateName: emptyStateBytes, }, }, }, @@ -122,7 +118,7 @@ func (p *Provider) GetProviderSchema() providers.GetProviderSchemaResponse { // Only include the inmem state store in the provider when `TF_ACC` is set in the environment // Excluding this from the schemas is sufficient to block usage. if v := os.Getenv("TF_ACC"); v != "" { - resp.StateStores["terraform_inmem"] = stateStoreInMemGetSchema() + resp.StateStores[inMemStoreName] = stateStoreInMemGetSchema() } return resp @@ -332,8 +328,8 @@ func (p *Provider) ListResource(req providers.ListResourceRequest) providers.Lis } func (p *Provider) ValidateStateStoreConfig(req providers.ValidateStateStoreConfigRequest) providers.ValidateStateStoreConfigResponse { - if s, ok := p.stores[req.TypeName]; ok { - return s.ValidateStateStoreConfig(req) + if req.TypeName == inMemStoreName { + return p.inMem.ValidateStateStoreConfig(req) } var resp providers.ValidateStateStoreConfigResponse @@ -342,8 +338,8 @@ func (p *Provider) ValidateStateStoreConfig(req providers.ValidateStateStoreConf } func (p *Provider) ConfigureStateStore(req providers.ConfigureStateStoreRequest) providers.ConfigureStateStoreResponse { - if s, ok := p.stores[req.TypeName]; ok { - return s.ConfigureStateStore(req) + if req.TypeName == inMemStoreName { + return p.inMem.ConfigureStateStore(req) } var resp providers.ConfigureStateStoreResponse @@ -352,8 +348,8 @@ func (p *Provider) ConfigureStateStore(req providers.ConfigureStateStoreRequest) } func (p *Provider) ReadStateBytes(req providers.ReadStateBytesRequest) providers.ReadStateBytesResponse { - if s, ok := p.stores[req.TypeName]; ok { - return s.ReadStateBytes(req) + if req.TypeName == inMemStoreName { + return p.inMem.ReadStateBytes(req) } var resp providers.ReadStateBytesResponse @@ -362,8 +358,8 @@ func (p *Provider) ReadStateBytes(req providers.ReadStateBytesRequest) providers } func (p *Provider) WriteStateBytes(req providers.WriteStateBytesRequest) providers.WriteStateBytesResponse { - if s, ok := p.stores[req.TypeName]; ok { - return s.WriteStateBytes(req) + if req.TypeName == inMemStoreName { + return p.inMem.WriteStateBytes(req) } var resp providers.WriteStateBytesResponse @@ -372,8 +368,8 @@ func (p *Provider) WriteStateBytes(req providers.WriteStateBytesRequest) provide } func (p *Provider) LockState(req providers.LockStateRequest) providers.LockStateResponse { - if s, ok := p.stores[req.TypeName]; ok { - return s.LockState(req) + if req.TypeName == inMemStoreName { + return p.inMem.LockState(req) } var resp providers.LockStateResponse @@ -382,8 +378,8 @@ func (p *Provider) LockState(req providers.LockStateRequest) providers.LockState } func (p *Provider) UnlockState(req providers.UnlockStateRequest) providers.UnlockStateResponse { - if s, ok := p.stores[req.TypeName]; ok { - return s.UnlockState(req) + if req.TypeName == inMemStoreName { + return p.inMem.UnlockState(req) } var resp providers.UnlockStateResponse @@ -392,8 +388,8 @@ func (p *Provider) UnlockState(req providers.UnlockStateRequest) providers.Unloc } func (p *Provider) GetStates(req providers.GetStatesRequest) providers.GetStatesResponse { - if s, ok := p.stores[req.TypeName]; ok { - return s.GetStates(req) + if req.TypeName == inMemStoreName { + return p.inMem.GetStates(req) } var resp providers.GetStatesResponse @@ -402,8 +398,8 @@ func (p *Provider) GetStates(req providers.GetStatesRequest) providers.GetStates } func (p *Provider) DeleteState(req providers.DeleteStateRequest) providers.DeleteStateResponse { - if s, ok := p.stores[req.TypeName]; ok { - return s.DeleteState(req) + if req.TypeName == inMemStoreName { + return p.inMem.DeleteState(req) } var resp providers.DeleteStateResponse diff --git a/internal/builtin/providers/terraform/state_store_inmem.go b/internal/builtin/providers/terraform/state_store_inmem.go index 84af01d24fcd..6f3860808f58 100644 --- a/internal/builtin/providers/terraform/state_store_inmem.go +++ b/internal/builtin/providers/terraform/state_store_inmem.go @@ -17,6 +17,8 @@ import ( "github.com/zclconf/go-cty/cty" ) +const inMemStoreName = "terraform_inmem" + // Matches default in command package. const defaultStateStoreChunkSize int64 = 8 << 20 // 8 MB @@ -29,12 +31,8 @@ const defaultStateStoreChunkSize int64 = 8 << 20 // 8 MB type InMemStoreSingle struct { states stateMap locks lockMap - - unimplementedProviderInterface } -var _ providers.Interface = &InMemStoreSingle{} - func stateStoreInMemGetSchema() providers.Schema { return providers.Schema{ Body: &configschema.Block{ diff --git a/internal/builtin/providers/terraform/state_store_inmem_test.go b/internal/builtin/providers/terraform/state_store_inmem_test.go index 738ca09b503e..224b4707658c 100644 --- a/internal/builtin/providers/terraform/state_store_inmem_test.go +++ b/internal/builtin/providers/terraform/state_store_inmem_test.go @@ -17,16 +17,15 @@ import ( func TestBackendLocked(t *testing.T) { t.Setenv("TF_ACC", "1") // enable using the inmem state store - storeName := "terraform_inmem" // Use NewProviderWithDefaultState so default workspace exists already, // because backend.TestBackendStateLocks assumes they exist by default. provider := NewProviderWithDefaultState() - plug1, err := pluggable.NewPluggable(provider, storeName) + plug1, err := pluggable.NewPluggable(provider, inMemStoreName) if err != nil { t.Fatal(err) } - plug2, err := pluggable.NewPluggable(provider, storeName) + plug2, err := pluggable.NewPluggable(provider, inMemStoreName) if err != nil { t.Fatal(err) } @@ -40,10 +39,9 @@ func TestBackendLocked(t *testing.T) { func TestRemoteState(t *testing.T) { t.Setenv("TF_ACC", "1") // enable using the inmem state store - storeName := "terraform_inmem" provider := NewProvider() - plug, err := pluggable.NewPluggable(provider, storeName) + plug, err := pluggable.NewPluggable(provider, inMemStoreName) if err != nil { t.Fatal(err) } diff --git a/internal/builtin/providers/terraform/state_store_unimplemented.go b/internal/builtin/providers/terraform/state_store_unimplemented.go deleted file mode 100644 index f83821b42b16..000000000000 --- a/internal/builtin/providers/terraform/state_store_unimplemented.go +++ /dev/null @@ -1,120 +0,0 @@ -// Copyright (c) HashiCorp, Inc. -// SPDX-License-Identifier: BUSL-1.1 - -package terraform - -import "github.com/hashicorp/terraform/internal/providers" - -// unimplementedProviderInterface implements all methods of provider.Interface but they panic and report they aren't implemented. -// -// This allows state store implementations to be passed around as provider.Interface instances in the builtin provider. -// To use unimplementedProviderInterface embed it into a struct implementing state storage. -type unimplementedProviderInterface struct{} - -var _ providers.Interface = &InMemStoreSingle{} - -func (unimplementedProviderInterface) GetProviderSchema() providers.GetProviderSchemaResponse { - panic("GetProviderSchema isn't implemented") -} -func (unimplementedProviderInterface) GetResourceIdentitySchemas() providers.GetResourceIdentitySchemasResponse { - panic("GetResourceIdentitySchemas isn't implemented") -} -func (unimplementedProviderInterface) ValidateProviderConfig(providers.ValidateProviderConfigRequest) providers.ValidateProviderConfigResponse { - panic("ValidateProviderConfig isn't implemented") -} -func (unimplementedProviderInterface) ValidateResourceConfig(providers.ValidateResourceConfigRequest) providers.ValidateResourceConfigResponse { - panic("ValidateResourceConfig isn't implemented") -} -func (unimplementedProviderInterface) ValidateDataResourceConfig(providers.ValidateDataResourceConfigRequest) providers.ValidateDataResourceConfigResponse { - panic("ValidateDataResourceConfig isn't implemented") -} -func (unimplementedProviderInterface) ValidateEphemeralResourceConfig(providers.ValidateEphemeralResourceConfigRequest) providers.ValidateEphemeralResourceConfigResponse { - panic("ValidateEphemeralResourceConfig isn't implemented") -} -func (unimplementedProviderInterface) ValidateListResourceConfig(providers.ValidateListResourceConfigRequest) providers.ValidateListResourceConfigResponse { - panic("ValidateListResourceConfig isn't implemented") -} -func (unimplementedProviderInterface) UpgradeResourceState(providers.UpgradeResourceStateRequest) providers.UpgradeResourceStateResponse { - panic("UpgradeResourceState isn't implemented") -} -func (unimplementedProviderInterface) UpgradeResourceIdentity(providers.UpgradeResourceIdentityRequest) providers.UpgradeResourceIdentityResponse { - panic("UpgradeResourceIdentity isn't implemented") -} -func (unimplementedProviderInterface) ConfigureProvider(providers.ConfigureProviderRequest) providers.ConfigureProviderResponse { - panic("ConfigureProvider isn't implemented") -} -func (unimplementedProviderInterface) Stop() error { - panic("Stop isn't implemented") -} -func (unimplementedProviderInterface) ReadResource(providers.ReadResourceRequest) providers.ReadResourceResponse { - panic("ReadResource isn't implemented") -} -func (unimplementedProviderInterface) PlanResourceChange(providers.PlanResourceChangeRequest) providers.PlanResourceChangeResponse { - panic("PlanResourceChange isn't implemented") -} -func (unimplementedProviderInterface) ApplyResourceChange(providers.ApplyResourceChangeRequest) providers.ApplyResourceChangeResponse { - panic("ApplyResourceChange isn't implemented") -} -func (unimplementedProviderInterface) ImportResourceState(providers.ImportResourceStateRequest) providers.ImportResourceStateResponse { - panic("ImportResourceState isn't implemented") -} -func (unimplementedProviderInterface) GenerateResourceConfig(providers.GenerateResourceConfigRequest) providers.GenerateResourceConfigResponse { - panic("GenerateResourceConfig isn't implemented") -} -func (unimplementedProviderInterface) MoveResourceState(providers.MoveResourceStateRequest) providers.MoveResourceStateResponse { - panic("MoveResourceState isn't implemented") -} -func (unimplementedProviderInterface) ReadDataSource(providers.ReadDataSourceRequest) providers.ReadDataSourceResponse { - panic("ReadDataSource isn't implemented") -} -func (unimplementedProviderInterface) OpenEphemeralResource(providers.OpenEphemeralResourceRequest) providers.OpenEphemeralResourceResponse { - panic("OpenEphemeralResource isn't implemented") -} -func (unimplementedProviderInterface) RenewEphemeralResource(providers.RenewEphemeralResourceRequest) providers.RenewEphemeralResourceResponse { - panic("RenewEphemeralResource isn't implemented") -} -func (unimplementedProviderInterface) CloseEphemeralResource(providers.CloseEphemeralResourceRequest) providers.CloseEphemeralResourceResponse { - panic("CloseEphemeralResource isn't implemented") -} -func (unimplementedProviderInterface) CallFunction(providers.CallFunctionRequest) providers.CallFunctionResponse { - panic("CallFunction isn't implemented") -} -func (unimplementedProviderInterface) ListResource(providers.ListResourceRequest) providers.ListResourceResponse { - panic("ListResource isn't implemented") -} -func (unimplementedProviderInterface) ValidateStateStoreConfig(providers.ValidateStateStoreConfigRequest) providers.ValidateStateStoreConfigResponse { - panic("ValidateStateStoreConfig isn't implemented") -} -func (unimplementedProviderInterface) ConfigureStateStore(providers.ConfigureStateStoreRequest) providers.ConfigureStateStoreResponse { - panic("ConfigureStateStore isn't implemented") -} -func (unimplementedProviderInterface) ReadStateBytes(providers.ReadStateBytesRequest) providers.ReadStateBytesResponse { - panic("ReadStateBytes isn't implemented") -} -func (unimplementedProviderInterface) WriteStateBytes(providers.WriteStateBytesRequest) providers.WriteStateBytesResponse { - panic("WriteStateBytes isn't implemented") -} -func (unimplementedProviderInterface) LockState(providers.LockStateRequest) providers.LockStateResponse { - panic("LockState isn't implemented") -} -func (unimplementedProviderInterface) UnlockState(providers.UnlockStateRequest) providers.UnlockStateResponse { - panic("UnlockState isn't implemented") -} -func (unimplementedProviderInterface) GetStates(providers.GetStatesRequest) providers.GetStatesResponse { - panic("GetStates isn't implemented") -} -func (unimplementedProviderInterface) DeleteState(providers.DeleteStateRequest) providers.DeleteStateResponse { - panic("DeleteState isn't implemented") -} -func (unimplementedProviderInterface) PlanAction(providers.PlanActionRequest) providers.PlanActionResponse { - panic("PlanAction isn't implemented") -} -func (unimplementedProviderInterface) InvokeAction(providers.InvokeActionRequest) providers.InvokeActionResponse { - panic("InvokeAction isn't implemented") -} -func (unimplementedProviderInterface) ValidateActionConfig(providers.ValidateActionConfigRequest) providers.ValidateActionConfigResponse { - panic("ValidateActionConfig isn't implemented") -} -func (unimplementedProviderInterface) Close() error { - panic("Close isn't implemented") -} From c4e027414cf6bc14bd35406f87438ccf7d2122ab Mon Sep 17 00:00:00 2001 From: Sarah French Date: Wed, 22 Oct 2025 14:58:23 +0100 Subject: [PATCH 07/23] WIP simple --- internal/command/e2etest/primary_test.go | 141 +++++++++++ .../main.tf | 23 ++ internal/provider-simple-v6/provider.go | 15 +- .../provider-simple-v6/state_store_inmem.go | 227 ++++++++++++++++++ 4 files changed, 405 insertions(+), 1 deletion(-) create mode 100644 internal/command/e2etest/testdata/full-workflow-null-with-state-store/main.tf create mode 100644 internal/provider-simple-v6/state_store_inmem.go diff --git a/internal/command/e2etest/primary_test.go b/internal/command/e2etest/primary_test.go index c7b6eaa1ee24..adcd8e900de6 100644 --- a/internal/command/e2etest/primary_test.go +++ b/internal/command/e2etest/primary_test.go @@ -4,6 +4,7 @@ package e2etest import ( + "os" "path/filepath" "reflect" "sort" @@ -12,6 +13,7 @@ import ( "github.com/davecgh/go-spew/spew" "github.com/hashicorp/terraform/internal/e2e" + "github.com/hashicorp/terraform/internal/getproviders" "github.com/hashicorp/terraform/internal/plans" "github.com/zclconf/go-cty/cty" ) @@ -230,3 +232,142 @@ func TestPrimaryChdirOption(t *testing.T) { t.Errorf("incorrect destroy tally; want 0 destroyed:\n%s", stdout) } } + +// Requires TF_TEST_EXPERIMENT and TF_ACC to be set in the environment +func TestPrimary_stateStore(t *testing.T) { + if !canRunGoBuild { + // We're running in a separate-build-then-run context, so we can't + // currently execute this test which depends on being able to build + // new executable at runtime. + // + // (See the comment on canRunGoBuild's declaration for more information.) + t.Skip("can't run without building a new provider executable") + } + t.Parallel() + + tf := e2e.NewBinary(t, terraformBin, "testdata/full-workflow-null-with-state-store") + + // In order to do a decent end-to-end test for this case we will need a real + // enough provider plugin to try to run and make sure we are able to + // actually run it. Here will build the simple and simple6 (built with + // protocol v6) providers. + simple6Provider := filepath.Join(tf.WorkDir(), "terraform-provider-simple6") + simple6ProviderExe := e2e.GoBuild("github.com/hashicorp/terraform/internal/provider-simple-v6/main", simple6Provider) + + // Move the provider binaries into a directory that we will point terraform + // to using the -plugin-dir cli flag. + platform := getproviders.CurrentPlatform.String() + hashiDir := "cache/registry.terraform.io/hashicorp/" + if err := os.MkdirAll(tf.Path(hashiDir, "simple6/0.0.1/", platform), os.ModePerm); err != nil { + t.Fatal(err) + } + if err := os.Rename(simple6ProviderExe, tf.Path(hashiDir, "simple6/0.0.1/", platform, "terraform-provider-simple6")); err != nil { + t.Fatal(err) + } + + //// INIT + _, stderr, err := tf.Run("init", "-enable-pluggable-state-storage-experiment=true", "-plugin-dir=cache") + if err != nil { + t.Fatalf("unexpected init error: %s\nstderr:\n%s", err, stderr) + } + + //// PLAN + _, stderr, err = tf.Run("plan", "-out=tfplan") + if err != nil { + t.Fatalf("unexpected plan error: %s\nstderr:\n%s", err, stderr) + } + + //// APPLY + stdout, stderr, err := tf.Run("apply", "tfplan") + if err != nil { + t.Fatalf("unexpected apply error: %s\nstderr:\n%s", err, stderr) + } + + if !strings.Contains(stdout, "Apply complete! Resources: 2 added, 0 changed, 0 destroyed.") { + t.Fatalf("wrong output:\nstdout:%s\nstderr%s", stdout, stderr) + } + + /// DESTROY + stdout, stderr, err = tf.Run("destroy", "-auto-approve") + if err != nil { + t.Fatalf("unexpected apply error: %s\nstderr:\n%s", err, stderr) + } + + if !strings.Contains(stdout, "Resources: 2 destroyed") { + t.Fatalf("wrong destroy output\nstdout:%s\nstderr:%s", stdout, stderr) + } + + // //// INIT + // stdout, stderr, err := tf.Run("init", "-enable-pluggable-state-storage-experiment=true", "-plugin-dir=cache") + // if err != nil { + // t.Fatalf("unexpected init error: %s\nstderr:\n%s", err, stderr) + // } + + // // Make sure we actually downloaded the plugins, rather than picking up + // // copies that might be already installed globally on the system. + // if !strings.Contains(stdout, "Installing hashicorp/template v") { + // t.Errorf("template provider download message is missing from init output:\n%s", stdout) + // t.Logf("(this can happen if you have a copy of the plugin in one of the global plugin search dirs)") + // } + // if !strings.Contains(stdout, "Installing hashicorp/null v") { + // t.Errorf("null provider download message is missing from init output:\n%s", stdout) + // t.Logf("(this can happen if you have a copy of the plugin in one of the global plugin search dirs)") + // } + + // //// PLAN + // // No separate plan step; this test lets the apply make a plan. + + // //// APPLY + // stdout, stderr, err = tf.Run("apply") + // if err != nil { + // t.Fatalf("unexpected apply error: %s\nstderr:\n%s", err, stderr) + // } + + // if !strings.Contains(stdout, "Resources: 1 added, 0 changed, 0 destroyed") { + // t.Errorf("incorrect apply tally; want 1 added:\n%s", stdout) + // } + + // state, err := tf.LocalState() + // if err != nil { + // t.Fatalf("failed to read state file: %s", err) + // } + + // stateResources := state.RootModule().Resources + // var gotResources []string + // for n := range stateResources { + // gotResources = append(gotResources, n) + // } + // sort.Strings(gotResources) + + // wantResources := []string{ + // "data.template_file.test", + // "null_resource.test", + // } + + // if !reflect.DeepEqual(gotResources, wantResources) { + // t.Errorf("wrong resources in state\ngot: %#v\nwant: %#v", gotResources, wantResources) + // } + + // //// DESTROY + // stdout, stderr, err = tf.Run("destroy", "-auto-approve") + // if err != nil { + // t.Fatalf("unexpected destroy error: %s\nstderr:\n%s", err, stderr) + // } + + // if !strings.Contains(stdout, "Resources: 1 destroyed") { + // t.Errorf("incorrect destroy tally; want 1 destroyed:\n%s", stdout) + // } + + // state, err = tf.LocalState() + // if err != nil { + // t.Fatalf("failed to read state file after destroy: %s", err) + // } + + // stateResources = state.RootModule().Resources + // if len(stateResources) != 0 { + // t.Errorf("wrong resources in state after destroy; want none, but still have:%s", spew.Sdump(stateResources)) + // } + +} + +// TODO: TestPrimarySeparatePlan_stateStore diff --git a/internal/command/e2etest/testdata/full-workflow-null-with-state-store/main.tf b/internal/command/e2etest/testdata/full-workflow-null-with-state-store/main.tf new file mode 100644 index 000000000000..41c39ffd798c --- /dev/null +++ b/internal/command/e2etest/testdata/full-workflow-null-with-state-store/main.tf @@ -0,0 +1,23 @@ +terraform { + required_providers { + simple6 = { + source = "registry.terraform.io/hashicorp/simple6" + } + } + + state_store "simple6_inmem" { + provider "simple6" {} + } +} + +variable "name" { + default = "world" +} + +data "terraform_data" "my-data" { + input = "hello ${var.name}" +} + +output "greeting" { + value = data.terraform_data.output +} diff --git a/internal/provider-simple-v6/provider.go b/internal/provider-simple-v6/provider.go index 9d7ce7f2456a..9cc896abe24e 100644 --- a/internal/provider-simple-v6/provider.go +++ b/internal/provider-simple-v6/provider.go @@ -8,6 +8,7 @@ import ( "errors" "fmt" "log" + "os" "time" "github.com/zclconf/go-cty/cty" @@ -19,6 +20,8 @@ import ( type simple struct { schema providers.GetProviderSchemaResponse + + inMem *InMemStoreSingle } func Provider() providers.Interface { @@ -46,7 +49,7 @@ func Provider() providers.Interface { }, } - return simple{ + provider := simple{ schema: providers.GetProviderSchemaResponse{ Provider: providers.Schema{ Body: &configschema.Block{ @@ -97,7 +100,17 @@ func Provider() providers.Interface { }, }, }, + + inMem: &InMemStoreSingle{}, // default workspace doesn't exist by default here; needs explicit creation via init command } + + // Only include the inmem state store in the provider when `TF_ACC` is set in the environment + // Excluding this from the schemas is sufficient to block usage. + if v := os.Getenv("TF_ACC"); v != "" { + provider.schema.StateStores[inMemStoreName] = stateStoreInMemGetSchema() + } + + return provider } func (s simple) GetProviderSchema() providers.GetProviderSchemaResponse { diff --git a/internal/provider-simple-v6/state_store_inmem.go b/internal/provider-simple-v6/state_store_inmem.go new file mode 100644 index 000000000000..266ec6af72a5 --- /dev/null +++ b/internal/provider-simple-v6/state_store_inmem.go @@ -0,0 +1,227 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package simple + +import ( + "errors" + "fmt" + "sort" + "sync" + + "github.com/hashicorp/go-uuid" + "github.com/hashicorp/terraform/internal/backend" + "github.com/hashicorp/terraform/internal/configs/configschema" + "github.com/hashicorp/terraform/internal/providers" + "github.com/hashicorp/terraform/internal/tfdiags" + "github.com/zclconf/go-cty/cty" +) + +const inMemStoreName = "simple6_inmem" + +// Matches default in command package. +const defaultStateStoreChunkSize int64 = 8 << 20 // 8 MB + +// InMemStoreSingle allows 'storing' state in memory for the purpose of testing. +// +// "Single" reflects the fact that this implementation does not use any global scope vars +// in its implementation, unlike the current inmem backend. HOWEVER, you can test whether locking +// blocks multiple clients trying to access the same state at once by creating multiple instances +// of backend.Backend that wrap the same provider.Interface instance. +type InMemStoreSingle struct { + states stateMap + locks lockMap +} + +func stateStoreInMemGetSchema() providers.Schema { + return providers.Schema{ + Body: &configschema.Block{ + Attributes: map[string]*configschema.Attribute{ + "lock_id": { + Type: cty.String, + Optional: true, + Description: "initializes the state in a locked configuration", + }, + }, + }, + } +} + +func (m *InMemStoreSingle) ValidateStateStoreConfig(req providers.ValidateStateStoreConfigRequest) providers.ValidateStateStoreConfigResponse { + // Not implemented in original inmem backend. + // The inmem used default logic in the backendbase package that cannot be replicated here easily. + // Instead, here is a rough implementation of validation: + var resp providers.ValidateStateStoreConfigResponse + + attrs := req.Config.AsValueMap() + if v, ok := attrs["lock_id"]; ok { + if !v.IsKnown() { + resp.Diagnostics = resp.Diagnostics.Append(errors.New("the attribute \"lock_id\" cannot be an unknown value")) + return resp + } + } + + return resp +} + +func (m *InMemStoreSingle) ConfigureStateStore(req providers.ConfigureStateStoreRequest) providers.ConfigureStateStoreResponse { + resp := providers.ConfigureStateStoreResponse{} + + m.states.Lock() + defer m.states.Unlock() + + // set the default client lock info per the test config + configVal := req.Config + if v := configVal.GetAttr("lock_id"); !v.IsNull() { + m.locks.lock(backend.DefaultStateName, v.AsString()) + } + + // We need to return a suggested chunk size; use default value + resp.Capabilities.ChunkSize = defaultStateStoreChunkSize + return resp +} + +func (m *InMemStoreSingle) ReadStateBytes(req providers.ReadStateBytesRequest) providers.ReadStateBytesResponse { + resp := providers.ReadStateBytesResponse{} + + v, ok := m.states.m[req.StateId] + if !ok { + // Does not exist, so return no bytes + + resp.Diagnostics = resp.Diagnostics.Append(tfdiags.Sourceless( + tfdiags.Warning, + "State doesn't exist, yet", + fmt.Sprintf("There's no state for workspace %q yet", req.StateId), + )) + return resp + } + + resp.Bytes = v + return resp +} + +func (m *InMemStoreSingle) WriteStateBytes(req providers.WriteStateBytesRequest) providers.WriteStateBytesResponse { + resp := providers.WriteStateBytesResponse{} + + if m.states.m == nil { + m.states.m = make(map[string][]byte, 1) + } + + m.states.m[req.StateId] = req.Bytes + + return resp +} + +func (m *InMemStoreSingle) LockState(req providers.LockStateRequest) providers.LockStateResponse { + resp := providers.LockStateResponse{} + + lockIdBytes, err := uuid.GenerateUUID() + if err != nil { + resp.Diagnostics = resp.Diagnostics.Append(fmt.Errorf("error creating random lock uuid: %w", err)) + return resp + } + + lockId := string(lockIdBytes) + returnedLockId, err := m.locks.lock(req.StateId, lockId) + if err != nil { + resp.Diagnostics = resp.Diagnostics.Append(err) + } + + resp.LockId = string(returnedLockId) + return resp +} + +func (m *InMemStoreSingle) UnlockState(req providers.UnlockStateRequest) providers.UnlockStateResponse { + resp := providers.UnlockStateResponse{} + + err := m.locks.unlock(req.StateId, req.LockId) + if err != nil { + resp.Diagnostics = resp.Diagnostics.Append(fmt.Errorf("error when unlocking state %q: %w", req.StateId, err)) + return resp + } + + return resp +} + +func (m *InMemStoreSingle) GetStates(req providers.GetStatesRequest) providers.GetStatesResponse { + m.states.Lock() + defer m.states.Unlock() + + resp := providers.GetStatesResponse{} + + var workspaces []string + + for s := range m.states.m { + workspaces = append(workspaces, s) + } + + sort.Strings(workspaces) + resp.States = workspaces + return resp +} + +func (m *InMemStoreSingle) DeleteState(req providers.DeleteStateRequest) providers.DeleteStateResponse { + m.states.Lock() + defer m.states.Unlock() + + resp := providers.DeleteStateResponse{} + + if req.StateId == backend.DefaultStateName || req.StateId == "" { + resp.Diagnostics = resp.Diagnostics.Append(fmt.Errorf("can't delete default state")) + return resp + } + + delete(m.states.m, req.StateId) + return resp +} + +type stateMap struct { + sync.Mutex + m map[string][]byte // key=state name/workspace, value=state +} + +type lockMap struct { + sync.Mutex + m map[string]string // key=state name/workspace, value=lock_id +} + +func (l *lockMap) lock(name string, lockId string) (string, error) { + l.Lock() + defer l.Unlock() + + lock, ok := l.m[name] + if ok { + // Error; lock already exists for that state/workspace + return "", fmt.Errorf("state %q is already locked with lock id %q", name, lock) + } + + if l.m == nil { + l.m = make(map[string]string, 1) + } + + l.m[name] = lockId + + return lockId, nil +} + +func (l *lockMap) unlock(name, id string) error { + l.Lock() + defer l.Unlock() + + lockId, ok := l.m[name] + + if !ok { + return errors.New("state not locked") + } + + if id != lockId { + return fmt.Errorf("invalid lock id: %q was locked with lock id %q, but tried to unlock with lock id %q", + name, + lockId, + id, + ) + } + + delete(l.m, name) + return nil +} From 759df3992693d7ca03a4922c6a0f4dc8462b9d54 Mon Sep 17 00:00:00 2001 From: Sarah French Date: Thu, 23 Oct 2025 13:08:53 +0100 Subject: [PATCH 08/23] Implement inmem state store in provider-simple-v6, remove from builtin terraform provider --- .../builtin/providers/terraform/provider.go | 78 +----- .../providers/terraform/state_store_inmem.go | 227 ------------------ internal/provider-simple-v6/provider.go | 98 +++++++- .../provider-simple-v6/state_store_inmem.go | 7 +- .../state_store_inmem_test.go | 10 +- 5 files changed, 98 insertions(+), 322 deletions(-) delete mode 100644 internal/builtin/providers/terraform/state_store_inmem.go rename internal/{builtin/providers/terraform => provider-simple-v6}/state_store_inmem_test.go (90%) diff --git a/internal/builtin/providers/terraform/provider.go b/internal/builtin/providers/terraform/provider.go index 54c9ca031127..06974d23cbfb 100644 --- a/internal/builtin/providers/terraform/provider.go +++ b/internal/builtin/providers/terraform/provider.go @@ -4,58 +4,23 @@ package terraform import ( - "bytes" "fmt" "log" - "os" tfaddr "github.com/hashicorp/terraform-registry-address" "github.com/zclconf/go-cty/cty" - "github.com/hashicorp/terraform/internal/backend" "github.com/hashicorp/terraform/internal/providers" - "github.com/hashicorp/terraform/internal/states/statefile" ) // Provider is an implementation of providers.Interface -type Provider struct { - - // State storage implementation(s) - inMem *InMemStoreSingle // terraform_inmem -} +type Provider struct{} var _ providers.Interface = &Provider{} // NewProvider returns a new terraform provider func NewProvider() providers.Interface { - return &Provider{ - inMem: &InMemStoreSingle{}, - } -} - -// NewProvider returns a new terraform provider where the internal -// state store(s) all have the default workspace already existing -func NewProviderWithDefaultState() providers.Interface { - // Get the empty state file as bytes - f := statefile.New(nil, "", 0) - - var buf bytes.Buffer - err := statefile.Write(f, &buf) - if err != nil { - panic(err) - } - emptyStateBytes := buf.Bytes() - - // Return a provider where all state stores have existing default workspaces - return &Provider{ - inMem: &InMemStoreSingle{ - states: stateMap{ - m: map[string][]byte{ - backend.DefaultStateName: emptyStateBytes, - }, - }, - }, - } + return &Provider{} } // GetSchema returns the complete schema for the provider. @@ -114,13 +79,6 @@ func (p *Provider) GetProviderSchema() providers.GetProviderSchemaResponse { Actions: map[string]providers.ActionSchema{}, } providers.SchemaCache.Set(tfaddr.NewProvider(tfaddr.BuiltInProviderHost, tfaddr.BuiltInProviderNamespace, "terraform"), resp) - - // Only include the inmem state store in the provider when `TF_ACC` is set in the environment - // Excluding this from the schemas is sufficient to block usage. - if v := os.Getenv("TF_ACC"); v != "" { - resp.StateStores[inMemStoreName] = stateStoreInMemGetSchema() - } - return resp } @@ -328,80 +286,48 @@ func (p *Provider) ListResource(req providers.ListResourceRequest) providers.Lis } func (p *Provider) ValidateStateStoreConfig(req providers.ValidateStateStoreConfigRequest) providers.ValidateStateStoreConfigResponse { - if req.TypeName == inMemStoreName { - return p.inMem.ValidateStateStoreConfig(req) - } - var resp providers.ValidateStateStoreConfigResponse resp.Diagnostics.Append(fmt.Errorf("unsupported state store type %q", req.TypeName)) return resp } func (p *Provider) ConfigureStateStore(req providers.ConfigureStateStoreRequest) providers.ConfigureStateStoreResponse { - if req.TypeName == inMemStoreName { - return p.inMem.ConfigureStateStore(req) - } - var resp providers.ConfigureStateStoreResponse resp.Diagnostics.Append(fmt.Errorf("unsupported state store type %q", req.TypeName)) return resp } func (p *Provider) ReadStateBytes(req providers.ReadStateBytesRequest) providers.ReadStateBytesResponse { - if req.TypeName == inMemStoreName { - return p.inMem.ReadStateBytes(req) - } - var resp providers.ReadStateBytesResponse resp.Diagnostics.Append(fmt.Errorf("unsupported state store type %q", req.TypeName)) return resp } func (p *Provider) WriteStateBytes(req providers.WriteStateBytesRequest) providers.WriteStateBytesResponse { - if req.TypeName == inMemStoreName { - return p.inMem.WriteStateBytes(req) - } - var resp providers.WriteStateBytesResponse resp.Diagnostics.Append(fmt.Errorf("unsupported state store type %q", req.TypeName)) return resp } func (p *Provider) LockState(req providers.LockStateRequest) providers.LockStateResponse { - if req.TypeName == inMemStoreName { - return p.inMem.LockState(req) - } - var resp providers.LockStateResponse resp.Diagnostics.Append(fmt.Errorf("unsupported state store type %q", req.TypeName)) return resp } func (p *Provider) UnlockState(req providers.UnlockStateRequest) providers.UnlockStateResponse { - if req.TypeName == inMemStoreName { - return p.inMem.UnlockState(req) - } - var resp providers.UnlockStateResponse resp.Diagnostics.Append(fmt.Errorf("unsupported state store type %q", req.TypeName)) return resp } func (p *Provider) GetStates(req providers.GetStatesRequest) providers.GetStatesResponse { - if req.TypeName == inMemStoreName { - return p.inMem.GetStates(req) - } - var resp providers.GetStatesResponse resp.Diagnostics.Append(fmt.Errorf("unsupported state store type %q", req.TypeName)) return resp } func (p *Provider) DeleteState(req providers.DeleteStateRequest) providers.DeleteStateResponse { - if req.TypeName == inMemStoreName { - return p.inMem.DeleteState(req) - } - var resp providers.DeleteStateResponse resp.Diagnostics.Append(fmt.Errorf("unsupported state store type %q", req.TypeName)) return resp diff --git a/internal/builtin/providers/terraform/state_store_inmem.go b/internal/builtin/providers/terraform/state_store_inmem.go deleted file mode 100644 index 6f3860808f58..000000000000 --- a/internal/builtin/providers/terraform/state_store_inmem.go +++ /dev/null @@ -1,227 +0,0 @@ -// Copyright (c) HashiCorp, Inc. -// SPDX-License-Identifier: BUSL-1.1 - -package terraform - -import ( - "errors" - "fmt" - "sort" - "sync" - - "github.com/hashicorp/go-uuid" - "github.com/hashicorp/terraform/internal/backend" - "github.com/hashicorp/terraform/internal/configs/configschema" - "github.com/hashicorp/terraform/internal/providers" - "github.com/hashicorp/terraform/internal/tfdiags" - "github.com/zclconf/go-cty/cty" -) - -const inMemStoreName = "terraform_inmem" - -// Matches default in command package. -const defaultStateStoreChunkSize int64 = 8 << 20 // 8 MB - -// InMemStoreSingle allows 'storing' state in memory for the purpose of testing. -// -// "Single" reflects the fact that this implementation does not use any global scope vars -// in its implementation, unlike the current inmem backend. HOWEVER, you can test whether locking -// blocks multiple clients trying to access the same state at once by creating multiple instances -// of backend.Backend that wrap the same provider.Interface instance. -type InMemStoreSingle struct { - states stateMap - locks lockMap -} - -func stateStoreInMemGetSchema() providers.Schema { - return providers.Schema{ - Body: &configschema.Block{ - Attributes: map[string]*configschema.Attribute{ - "lock_id": { - Type: cty.String, - Optional: true, - Description: "initializes the state in a locked configuration", - }, - }, - }, - } -} - -func (m *InMemStoreSingle) ValidateStateStoreConfig(req providers.ValidateStateStoreConfigRequest) providers.ValidateStateStoreConfigResponse { - // Not implemented in original inmem backend. - // The inmem used default logic in the backendbase package that cannot be replicated here easily. - // Instead, here is a rough implementation of validation: - var resp providers.ValidateStateStoreConfigResponse - - attrs := req.Config.AsValueMap() - if v, ok := attrs["lock_id"]; ok { - if !v.IsKnown() { - resp.Diagnostics = resp.Diagnostics.Append(errors.New("the attribute \"lock_id\" cannot be an unknown value")) - return resp - } - } - - return resp -} - -func (m *InMemStoreSingle) ConfigureStateStore(req providers.ConfigureStateStoreRequest) providers.ConfigureStateStoreResponse { - resp := providers.ConfigureStateStoreResponse{} - - m.states.Lock() - defer m.states.Unlock() - - // set the default client lock info per the test config - configVal := req.Config - if v := configVal.GetAttr("lock_id"); !v.IsNull() { - m.locks.lock(backend.DefaultStateName, v.AsString()) - } - - // We need to return a suggested chunk size; use default value - resp.Capabilities.ChunkSize = defaultStateStoreChunkSize - return resp -} - -func (m *InMemStoreSingle) ReadStateBytes(req providers.ReadStateBytesRequest) providers.ReadStateBytesResponse { - resp := providers.ReadStateBytesResponse{} - - v, ok := m.states.m[req.StateId] - if !ok { - // Does not exist, so return no bytes - - resp.Diagnostics = resp.Diagnostics.Append(tfdiags.Sourceless( - tfdiags.Warning, - "State doesn't exist, yet", - fmt.Sprintf("There's no state for workspace %q yet", req.StateId), - )) - return resp - } - - resp.Bytes = v - return resp -} - -func (m *InMemStoreSingle) WriteStateBytes(req providers.WriteStateBytesRequest) providers.WriteStateBytesResponse { - resp := providers.WriteStateBytesResponse{} - - if m.states.m == nil { - m.states.m = make(map[string][]byte, 1) - } - - m.states.m[req.StateId] = req.Bytes - - return resp -} - -func (m *InMemStoreSingle) LockState(req providers.LockStateRequest) providers.LockStateResponse { - resp := providers.LockStateResponse{} - - lockIdBytes, err := uuid.GenerateUUID() - if err != nil { - resp.Diagnostics = resp.Diagnostics.Append(fmt.Errorf("error creating random lock uuid: %w", err)) - return resp - } - - lockId := string(lockIdBytes) - returnedLockId, err := m.locks.lock(req.StateId, lockId) - if err != nil { - resp.Diagnostics = resp.Diagnostics.Append(err) - } - - resp.LockId = string(returnedLockId) - return resp -} - -func (m *InMemStoreSingle) UnlockState(req providers.UnlockStateRequest) providers.UnlockStateResponse { - resp := providers.UnlockStateResponse{} - - err := m.locks.unlock(req.StateId, req.LockId) - if err != nil { - resp.Diagnostics = resp.Diagnostics.Append(fmt.Errorf("error when unlocking state %q: %w", req.StateId, err)) - return resp - } - - return resp -} - -func (m *InMemStoreSingle) GetStates(req providers.GetStatesRequest) providers.GetStatesResponse { - m.states.Lock() - defer m.states.Unlock() - - resp := providers.GetStatesResponse{} - - var workspaces []string - - for s := range m.states.m { - workspaces = append(workspaces, s) - } - - sort.Strings(workspaces) - resp.States = workspaces - return resp -} - -func (m *InMemStoreSingle) DeleteState(req providers.DeleteStateRequest) providers.DeleteStateResponse { - m.states.Lock() - defer m.states.Unlock() - - resp := providers.DeleteStateResponse{} - - if req.StateId == backend.DefaultStateName || req.StateId == "" { - resp.Diagnostics = resp.Diagnostics.Append(fmt.Errorf("can't delete default state")) - return resp - } - - delete(m.states.m, req.StateId) - return resp -} - -type stateMap struct { - sync.Mutex - m map[string][]byte // key=state name/workspace, value=state -} - -type lockMap struct { - sync.Mutex - m map[string]string // key=state name/workspace, value=lock_id -} - -func (l *lockMap) lock(name string, lockId string) (string, error) { - l.Lock() - defer l.Unlock() - - lock, ok := l.m[name] - if ok { - // Error; lock already exists for that state/workspace - return "", fmt.Errorf("state %q is already locked with lock id %q", name, lock) - } - - if l.m == nil { - l.m = make(map[string]string, 1) - } - - l.m[name] = lockId - - return lockId, nil -} - -func (l *lockMap) unlock(name, id string) error { - l.Lock() - defer l.Unlock() - - lockId, ok := l.m[name] - - if !ok { - return errors.New("state not locked") - } - - if id != lockId { - return fmt.Errorf("invalid lock id: %q was locked with lock id %q, but tried to unlock with lock id %q", - name, - lockId, - id, - ) - } - - delete(l.m, name) - return nil -} diff --git a/internal/provider-simple-v6/provider.go b/internal/provider-simple-v6/provider.go index 9cc896abe24e..1004798a3d2b 100644 --- a/internal/provider-simple-v6/provider.go +++ b/internal/provider-simple-v6/provider.go @@ -5,6 +5,7 @@ package simple import ( + "bytes" "errors" "fmt" "log" @@ -14,8 +15,10 @@ import ( "github.com/zclconf/go-cty/cty" ctyjson "github.com/zclconf/go-cty/cty/json" + "github.com/hashicorp/terraform/internal/backend" "github.com/hashicorp/terraform/internal/configs/configschema" "github.com/hashicorp/terraform/internal/providers" + "github.com/hashicorp/terraform/internal/states/statefile" ) type simple struct { @@ -24,7 +27,35 @@ type simple struct { inMem *InMemStoreSingle } +// Provider returns an instance of providers.Interface func Provider() providers.Interface { + return provider() +} + +// ProviderWithDefaultWorkspace returns an instance of providers.Interface, +// where the underlying simple struct has been changed to indicate that the +// default workspace has already been created as an empty state file. +func ProviderWithDefaultWorkspace() providers.Interface { + // Get the empty state file as bytes + f := statefile.New(nil, "", 0) + + var buf bytes.Buffer + err := statefile.Write(f, &buf) + if err != nil { + panic(err) + } + emptyStateBytes := buf.Bytes() + + p := provider() + + p.inMem.states.m = make(map[string][]byte, 1) + p.inMem.states.m[backend.DefaultStateName] = emptyStateBytes + + return p +} + +// provider returns an instance of simple +func provider() simple { simpleResource := providers.Schema{ Body: &configschema.Block{ Attributes: map[string]*configschema.Attribute{ @@ -77,7 +108,8 @@ func Provider() providers.Interface { }, }, }, - Actions: map[string]providers.ActionSchema{}, + Actions: map[string]providers.ActionSchema{}, + StateStores: map[string]providers.Schema{}, ServerCapabilities: providers.ServerCapabilities{ PlanDestroy: true, GetProviderSchemaOptional: true, @@ -323,35 +355,83 @@ func (s simple) ListResource(req providers.ListResourceRequest) (resp providers. } func (s simple) ValidateStateStoreConfig(req providers.ValidateStateStoreConfigRequest) providers.ValidateStateStoreConfigResponse { - panic("not implemented") + if req.TypeName == inMemStoreName { + return s.inMem.ValidateStateStoreConfig(req) + } + + var resp providers.ValidateStateStoreConfigResponse + resp.Diagnostics.Append(fmt.Errorf("unsupported state store type %q", req.TypeName)) + return resp } func (s simple) ConfigureStateStore(req providers.ConfigureStateStoreRequest) providers.ConfigureStateStoreResponse { - panic("not implemented") + if req.TypeName == inMemStoreName { + return s.inMem.ConfigureStateStore(req) + } + + var resp providers.ConfigureStateStoreResponse + resp.Diagnostics.Append(fmt.Errorf("unsupported state store type %q", req.TypeName)) + return resp } func (s simple) ReadStateBytes(req providers.ReadStateBytesRequest) providers.ReadStateBytesResponse { - panic("not implemented") + if req.TypeName == inMemStoreName { + return s.inMem.ReadStateBytes(req) + } + + var resp providers.ReadStateBytesResponse + resp.Diagnostics.Append(fmt.Errorf("unsupported state store type %q", req.TypeName)) + return resp } func (s simple) WriteStateBytes(req providers.WriteStateBytesRequest) providers.WriteStateBytesResponse { - panic("not implemented") + if req.TypeName == inMemStoreName { + return s.inMem.WriteStateBytes(req) + } + + var resp providers.WriteStateBytesResponse + resp.Diagnostics.Append(fmt.Errorf("unsupported state store type %q", req.TypeName)) + return resp } func (s simple) LockState(req providers.LockStateRequest) providers.LockStateResponse { - panic("not implemented") + if req.TypeName == inMemStoreName { + return s.inMem.LockState(req) + } + + var resp providers.LockStateResponse + resp.Diagnostics.Append(fmt.Errorf("unsupported state store type %q", req.TypeName)) + return resp } func (s simple) UnlockState(req providers.UnlockStateRequest) providers.UnlockStateResponse { - panic("not implemented") + if req.TypeName == inMemStoreName { + return s.inMem.UnlockState(req) + } + + var resp providers.UnlockStateResponse + resp.Diagnostics.Append(fmt.Errorf("unsupported state store type %q", req.TypeName)) + return resp } func (s simple) GetStates(req providers.GetStatesRequest) providers.GetStatesResponse { - panic("not implemented") + if req.TypeName == inMemStoreName { + return s.inMem.GetStates(req) + } + + var resp providers.GetStatesResponse + resp.Diagnostics.Append(fmt.Errorf("unsupported state store type %q", req.TypeName)) + return resp } func (s simple) DeleteState(req providers.DeleteStateRequest) providers.DeleteStateResponse { - panic("not implemented") + if req.TypeName == inMemStoreName { + return s.inMem.DeleteState(req) + } + + var resp providers.DeleteStateResponse + resp.Diagnostics.Append(fmt.Errorf("unsupported state store type %q", req.TypeName)) + return resp } func (s simple) PlanAction(providers.PlanActionRequest) providers.PlanActionResponse { diff --git a/internal/provider-simple-v6/state_store_inmem.go b/internal/provider-simple-v6/state_store_inmem.go index 266ec6af72a5..1e0ff5bfb4e3 100644 --- a/internal/provider-simple-v6/state_store_inmem.go +++ b/internal/provider-simple-v6/state_store_inmem.go @@ -19,9 +19,6 @@ import ( const inMemStoreName = "simple6_inmem" -// Matches default in command package. -const defaultStateStoreChunkSize int64 = 8 << 20 // 8 MB - // InMemStoreSingle allows 'storing' state in memory for the purpose of testing. // // "Single" reflects the fact that this implementation does not use any global scope vars @@ -76,8 +73,8 @@ func (m *InMemStoreSingle) ConfigureStateStore(req providers.ConfigureStateStore m.locks.lock(backend.DefaultStateName, v.AsString()) } - // We need to return a suggested chunk size; use default value - resp.Capabilities.ChunkSize = defaultStateStoreChunkSize + // We need to return a suggested chunk size; use the value suggested by Core + resp.Capabilities.ChunkSize = req.Capabilities.ChunkSize return resp } diff --git a/internal/builtin/providers/terraform/state_store_inmem_test.go b/internal/provider-simple-v6/state_store_inmem_test.go similarity index 90% rename from internal/builtin/providers/terraform/state_store_inmem_test.go rename to internal/provider-simple-v6/state_store_inmem_test.go index 224b4707658c..ef2e9352d216 100644 --- a/internal/builtin/providers/terraform/state_store_inmem_test.go +++ b/internal/provider-simple-v6/state_store_inmem_test.go @@ -1,7 +1,7 @@ // Copyright (c) HashiCorp, Inc. // SPDX-License-Identifier: BUSL-1.1 -package terraform +package simple import ( "testing" @@ -17,9 +17,9 @@ import ( func TestBackendLocked(t *testing.T) { t.Setenv("TF_ACC", "1") // enable using the inmem state store - // Use NewProviderWithDefaultState so default workspace exists already, - // because backend.TestBackendStateLocks assumes they exist by default. - provider := NewProviderWithDefaultState() + // backend.TestBackendStateLocks assumes the default workspace exists + // by default, so we need to make it exist using the method below. + provider := ProviderWithDefaultWorkspace() plug1, err := pluggable.NewPluggable(provider, inMemStoreName) if err != nil { @@ -39,7 +39,7 @@ func TestBackendLocked(t *testing.T) { func TestRemoteState(t *testing.T) { t.Setenv("TF_ACC", "1") // enable using the inmem state store - provider := NewProvider() + provider := Provider() plug, err := pluggable.NewPluggable(provider, inMemStoreName) if err != nil { From c6cea63bc3dd11b3bc3d1ba887c2132eb7b2f063 Mon Sep 17 00:00:00 2001 From: Sarah French Date: Thu, 23 Oct 2025 14:06:13 +0100 Subject: [PATCH 09/23] Move PSS chunking-related constants into the `pluggable` package, so they can be reused. --- internal/backend/pluggable/chunks.go | 18 ++++++++++++++++++ internal/command/meta_backend.go | 26 ++++++-------------------- 2 files changed, 24 insertions(+), 20 deletions(-) create mode 100644 internal/backend/pluggable/chunks.go diff --git a/internal/backend/pluggable/chunks.go b/internal/backend/pluggable/chunks.go new file mode 100644 index 000000000000..af181ebe82d4 --- /dev/null +++ b/internal/backend/pluggable/chunks.go @@ -0,0 +1,18 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package pluggable + +const ( + // DefaultStateStoreChunkSize is the default chunk size proposed + // to the provider. + // This can be tweaked but should provide reasonable performance + // trade-offs for average network conditions and state file sizes. + DefaultStateStoreChunkSize int64 = 8 << 20 // 8 MB + + // MaxStateStoreChunkSize is the highest chunk size provider may choose + // which we still consider reasonable/safe. + // This reflects terraform-plugin-go's max. RPC message size of 256MB + // and leaves plenty of space for other variable data like diagnostics. + MaxStateStoreChunkSize int64 = 128 << 20 // 128 MB +) diff --git a/internal/command/meta_backend.go b/internal/command/meta_backend.go index 4f3ad3972979..e3d3bd9de924 100644 --- a/internal/command/meta_backend.go +++ b/internal/command/meta_backend.go @@ -50,20 +50,6 @@ import ( tfversion "github.com/hashicorp/terraform/version" ) -const ( - // defaultStateStoreChunkSize is the default chunk size proposed - // to the provider. - // This can be tweaked but should provide reasonable performance - // trade-offs for average network conditions and state file sizes. - defaultStateStoreChunkSize int64 = 8 << 20 // 8 MB - - // maxStateStoreChunkSize is the highest chunk size provider may choose - // which we still consider reasonable/safe. - // This reflects terraform-plugin-go's max. RPC message size of 256MB - // and leaves plenty of space for other variable data like diagnostics. - maxStateStoreChunkSize int64 = 128 << 20 // 128 MB -) - // BackendOpts are the options used to initialize a backendrun.OperationsBackend. type BackendOpts struct { // BackendConfig is a representation of the backend configuration block given in @@ -2085,7 +2071,7 @@ func (m *Meta) savedStateStore(sMgr *clistate.LocalState, factory providers.Fact TypeName: s.StateStore.Type, Config: stateStoreConfigVal, Capabilities: providers.StateStoreClientCapabilities{ - ChunkSize: defaultStateStoreChunkSize, + ChunkSize: backendPluggable.DefaultStateStoreChunkSize, }, }) diags = diags.Append(cfgStoreResp.Diagnostics) @@ -2094,10 +2080,10 @@ func (m *Meta) savedStateStore(sMgr *clistate.LocalState, factory providers.Fact } chunkSize := cfgStoreResp.Capabilities.ChunkSize - if chunkSize == 0 || chunkSize > maxStateStoreChunkSize { + if chunkSize == 0 || chunkSize > backendPluggable.MaxStateStoreChunkSize { diags = diags.Append(fmt.Errorf("Failed to negotiate acceptable chunk size. "+ "Expected size > 0 and <= %d bytes, provider wants %d bytes", - maxStateStoreChunkSize, chunkSize, + backendPluggable.MaxStateStoreChunkSize, chunkSize, )) return nil, diags } @@ -2362,7 +2348,7 @@ func (m *Meta) stateStoreInitFromConfig(c *configs.StateStore, factory providers TypeName: c.Type, Config: stateStoreConfigVal, Capabilities: providers.StateStoreClientCapabilities{ - ChunkSize: defaultStateStoreChunkSize, + ChunkSize: backendPluggable.DefaultStateStoreChunkSize, }, }) diags = diags.Append(cfgStoreResp.Diagnostics) @@ -2371,10 +2357,10 @@ func (m *Meta) stateStoreInitFromConfig(c *configs.StateStore, factory providers } chunkSize := cfgStoreResp.Capabilities.ChunkSize - if chunkSize == 0 || chunkSize > maxStateStoreChunkSize { + if chunkSize == 0 || chunkSize > backendPluggable.MaxStateStoreChunkSize { diags = diags.Append(fmt.Errorf("Failed to negotiate acceptable chunk size. "+ "Expected size > 0 and <= %d bytes, provider wants %d bytes", - maxStateStoreChunkSize, chunkSize, + backendPluggable.MaxStateStoreChunkSize, chunkSize, )) return nil, cty.NilVal, cty.NilVal, diags } From 320fa96d11e1ce46cce4913735d11bdfc5fac267 Mon Sep 17 00:00:00 2001 From: Sarah French Date: Fri, 24 Oct 2025 11:15:45 +0100 Subject: [PATCH 10/23] Implement PSS-related methods in grpcwrap package --- internal/grpcwrap/provider6.go | 253 +++++++++++++++++++++++++++++++-- 1 file changed, 245 insertions(+), 8 deletions(-) diff --git a/internal/grpcwrap/provider6.go b/internal/grpcwrap/provider6.go index 106d8cbacb52..20a52a77a2af 100644 --- a/internal/grpcwrap/provider6.go +++ b/internal/grpcwrap/provider6.go @@ -4,8 +4,11 @@ package grpcwrap import ( + "bytes" "context" + "errors" "fmt" + "io" "github.com/zclconf/go-cty/cty" "github.com/zclconf/go-cty/cty/function" @@ -15,6 +18,9 @@ import ( "google.golang.org/grpc/status" "google.golang.org/protobuf/types/known/timestamppb" + proto6 "github.com/hashicorp/terraform/internal/tfplugin6" + + backendPluggable "github.com/hashicorp/terraform/internal/backend/pluggable" "github.com/hashicorp/terraform/internal/plugin6/convert" "github.com/hashicorp/terraform/internal/providers" "github.com/hashicorp/terraform/internal/tfplugin6" @@ -31,11 +37,15 @@ func Provider6(p providers.Interface) tfplugin6.ProviderServer { } } +var _ tfplugin6.ProviderServer = &provider6{} + type provider6 struct { provider providers.Interface schema providers.GetProviderSchemaResponse identitySchemas providers.GetResourceIdentitySchemasResponse + chunkSize int64 + tfplugin6.UnimplementedProviderServer } @@ -902,35 +912,262 @@ func (p *provider6) ListResource(req *tfplugin6.ListResource_Request, res tfplug } func (p *provider6) ValidateStateStoreConfig(ctx context.Context, req *tfplugin6.ValidateStateStore_Request) (*tfplugin6.ValidateStateStore_Response, error) { - panic("not implemented") + resp := &tfplugin6.ValidateStateStore_Response{} + + s, ok := p.schema.StateStores[req.TypeName] + if !ok { + diag := &tfplugin6.Diagnostic{ + Severity: tfplugin6.Diagnostic_ERROR, + Summary: "Unsupported state store type", + Detail: fmt.Sprintf("This provider doesn't include a state store called %q", req.TypeName), + } + resp.Diagnostics = append(resp.Diagnostics, diag) + return resp, nil + } + ty := s.Body.ImpliedType() + + configVal, err := decodeDynamicValue6(req.Config, ty) + if err != nil { + resp.Diagnostics = convert.AppendProtoDiag(resp.Diagnostics, err) + return resp, nil + } + + prepareResp := p.provider.ValidateStateStoreConfig(providers.ValidateStateStoreConfigRequest{ + TypeName: req.TypeName, + Config: configVal, + }) + + resp.Diagnostics = convert.AppendProtoDiag(resp.Diagnostics, prepareResp.Diagnostics) + return resp, nil } func (p *provider6) ConfigureStateStore(ctx context.Context, req *tfplugin6.ConfigureStateStore_Request) (*tfplugin6.ConfigureStateStore_Response, error) { - panic("not implemented") + resp := &tfplugin6.ConfigureStateStore_Response{} + + s, ok := p.schema.StateStores[req.TypeName] + if !ok { + diag := &tfplugin6.Diagnostic{ + Severity: tfplugin6.Diagnostic_ERROR, + Summary: "Unsupported state store type", + Detail: fmt.Sprintf("This provider doesn't include a state store called %q", req.TypeName), + } + resp.Diagnostics = append(resp.Diagnostics, diag) + return resp, nil + } + ty := s.Body.ImpliedType() + + configVal, err := decodeDynamicValue6(req.Config, ty) + if err != nil { + resp.Diagnostics = convert.AppendProtoDiag(resp.Diagnostics, err) + return resp, nil + } + + configureResp := p.provider.ConfigureStateStore(providers.ConfigureStateStoreRequest{ + TypeName: req.TypeName, + Config: configVal, + Capabilities: providers.StateStoreClientCapabilities{ + ChunkSize: backendPluggable.DefaultStateStoreChunkSize, + }, + }) + + // Validate the returned chunk size value + if configureResp.Capabilities.ChunkSize == 0 || configureResp.Capabilities.ChunkSize > backendPluggable.MaxStateStoreChunkSize { + diag := &tfplugin6.Diagnostic{ + Severity: tfplugin6.Diagnostic_ERROR, + Summary: "Failed to negotiate acceptable chunk size", + Detail: fmt.Sprintf("Expected size > 0 and <= %d bytes, provider wants %d bytes", + backendPluggable.MaxStateStoreChunkSize, configureResp.Capabilities.ChunkSize), + } + resp.Diagnostics = append(resp.Diagnostics, diag) + return resp, nil + } + p.chunkSize = configureResp.Capabilities.ChunkSize + + resp.Diagnostics = convert.AppendProtoDiag(resp.Diagnostics, configureResp.Diagnostics) + resp.Capabilities = &tfplugin6.StateStoreServerCapabilities{ + ChunkSize: configureResp.Capabilities.ChunkSize, + } + return resp, nil } func (p *provider6) ReadStateBytes(req *tfplugin6.ReadStateBytes_Request, srv tfplugin6.Provider_ReadStateBytesServer) error { - panic("not implemented") + stateReadResp := p.provider.ReadStateBytes(providers.ReadStateBytesRequest{ + TypeName: req.TypeName, + StateId: req.StateId, + }) + + state := stateReadResp.Bytes + reader := bytes.NewReader(state) + totalLength := reader.Size() // length in bytes + rangeStart := 0 + + for { + var diags []*proto6.Diagnostic + readBytes := make([]byte, p.chunkSize) + byteCount, err := reader.Read(readBytes) + if err != nil && !errors.Is(err, io.EOF) { + diags := []*proto6.Diagnostic{ + { + Severity: proto6.Diagnostic_ERROR, + Summary: "Error reading from state file", + Detail: fmt.Sprintf("PSS experienced an error when reading from the state file for workspace %s: %s", + req.StateId, + err, + ), + }, + } + err := srv.Send(&proto6.ReadStateBytes_Response{ + // Zero values accompany the error diagnostic + Bytes: nil, + TotalLength: 0, + Range: &proto6.StateRange{ + Start: 0, + End: 0, + }, + Diagnostics: diags, + }) + if err != nil { + return err + } + } + + if byteCount == 0 { + // The previous iteration read the last byte of the data. + return nil + } + + err = srv.Send(&proto6.ReadStateBytes_Response{ + Bytes: readBytes[0:byteCount], + TotalLength: int64(totalLength), + Range: &proto6.StateRange{ + Start: int64(rangeStart), + End: int64(rangeStart + byteCount), + }, + Diagnostics: diags, + }) + if err != nil { + return err + } + + // Track progress to ensure Range values are correct. + rangeStart += byteCount + } } func (p *provider6) WriteStateBytes(srv tfplugin6.Provider_WriteStateBytesServer) error { - panic("not implemented") + var typeName string + var stateId string + + state := bytes.Buffer{} + var grpcErr error + var totalReceivedBytes int + var expectedTotalLength int64 + for { + chunk, err := srv.Recv() + if err == io.EOF { + break + } + if err != nil { + grpcErr = fmt.Errorf("wrapped err: %w", err) + break + } + if expectedTotalLength == 0 { + // On the first iteration + expectedTotalLength = chunk.TotalLength // record expected length + if chunk.Meta != nil { + // We expect the Meta to be set on the first message, only + typeName = chunk.Meta.TypeName + stateId = chunk.Meta.StateId + } else { + panic("expected Meta to be set on first chunk sent to WriteStateBytes") + } + } + + n, err := state.Write(chunk.Bytes) + if err != nil { + return fmt.Errorf("error writing state: %w", err) + } + totalReceivedBytes += n + } + + if grpcErr != nil { + return grpcErr + } + + if int64(totalReceivedBytes) != expectedTotalLength { + return fmt.Errorf("expected to receive state in %d bytes, actually received %d bytes", expectedTotalLength, totalReceivedBytes) + } + + if totalReceivedBytes == 0 { + // Even an empty state file has content; no bytes is not valid + return errors.New("No state data received from Terraform: No state data was received from Terraform. This is a bug and should be reported.") + } + + resp := p.provider.WriteStateBytes(providers.WriteStateBytesRequest{ + StateId: stateId, + TypeName: typeName, + Bytes: state.Bytes(), + }) + + err := srv.SendAndClose(&proto6.WriteStateBytes_Response{ + Diagnostics: convert.AppendProtoDiag([]*proto6.Diagnostic{}, resp.Diagnostics), + }) + + return err } func (p *provider6) LockState(ctx context.Context, req *tfplugin6.LockState_Request) (*tfplugin6.LockState_Response, error) { - panic("not implemented") + lockResp := p.provider.LockState(providers.LockStateRequest{ + TypeName: req.TypeName, + StateId: req.StateId, + Operation: req.Operation, + }) + + resp := &tfplugin6.LockState_Response{ + LockId: lockResp.LockId, + Diagnostics: convert.AppendProtoDiag([]*proto6.Diagnostic{}, lockResp.Diagnostics), + } + + return resp, nil } func (p *provider6) UnlockState(ctx context.Context, req *tfplugin6.UnlockState_Request) (*tfplugin6.UnlockState_Response, error) { - panic("not implemented") + unlockResp := p.provider.UnlockState(providers.UnlockStateRequest{ + TypeName: req.TypeName, + StateId: req.StateId, + LockId: req.LockId, + }) + + resp := &tfplugin6.UnlockState_Response{ + Diagnostics: convert.AppendProtoDiag([]*proto6.Diagnostic{}, unlockResp.Diagnostics), + } + + return resp, nil } func (p *provider6) GetStates(ctx context.Context, req *tfplugin6.GetStates_Request) (*tfplugin6.GetStates_Response, error) { - panic("not implemented") + getStatesResp := p.provider.GetStates(providers.GetStatesRequest{ + TypeName: req.TypeName, + }) + + resp := &tfplugin6.GetStates_Response{ + StateId: getStatesResp.States, + Diagnostics: convert.AppendProtoDiag([]*tfplugin6.Diagnostic{}, getStatesResp.Diagnostics), + } + + return resp, nil } func (p *provider6) DeleteState(ctx context.Context, req *tfplugin6.DeleteState_Request) (*tfplugin6.DeleteState_Response, error) { - panic("not implemented") + deleteStatesResp := p.provider.DeleteState(providers.DeleteStateRequest{ + TypeName: req.TypeName, + }) + + resp := &tfplugin6.DeleteState_Response{ + Diagnostics: convert.AppendProtoDiag([]*tfplugin6.Diagnostic{}, deleteStatesResp.Diagnostics), + } + + return resp, nil } func (p *provider6) PlanAction(_ context.Context, req *tfplugin6.PlanAction_Request) (*tfplugin6.PlanAction_Response, error) { From 992eb53c020fc868222f83802b32db4ed3433a5d Mon Sep 17 00:00:00 2001 From: Sarah French Date: Fri, 24 Oct 2025 11:17:05 +0100 Subject: [PATCH 11/23] Update e2e test - works as expected but is blocked at apply step by other work to allow PSS to be used with other commands. --- internal/command/e2etest/primary_test.go | 112 ++++++----------------- 1 file changed, 27 insertions(+), 85 deletions(-) diff --git a/internal/command/e2etest/primary_test.go b/internal/command/e2etest/primary_test.go index adcd8e900de6..ba18d3cbe44b 100644 --- a/internal/command/e2etest/primary_test.go +++ b/internal/command/e2etest/primary_test.go @@ -266,108 +266,50 @@ func TestPrimary_stateStore(t *testing.T) { } //// INIT - _, stderr, err := tf.Run("init", "-enable-pluggable-state-storage-experiment=true", "-plugin-dir=cache") + stdout, stderr, err := tf.Run("init", "-enable-pluggable-state-storage-experiment=true", "-plugin-dir=cache", "-no-color") if err != nil { t.Fatalf("unexpected init error: %s\nstderr:\n%s", err, stderr) } - //// PLAN - _, stderr, err = tf.Run("plan", "-out=tfplan") - if err != nil { - t.Fatalf("unexpected plan error: %s\nstderr:\n%s", err, stderr) + if !strings.Contains(stdout, "Terraform created an empty state file for the default workspace") { + t.Errorf("notice about creating the default workspace is missing from init output:\n%s", stdout) } + //// PLAN + // No separate plan step; this test lets the apply make a plan. + //// APPLY - stdout, stderr, err := tf.Run("apply", "tfplan") + stdout, stderr, err = tf.Run("apply", "-auto-approve", "-no-color") if err != nil { t.Fatalf("unexpected apply error: %s\nstderr:\n%s", err, stderr) } - if !strings.Contains(stdout, "Apply complete! Resources: 2 added, 0 changed, 0 destroyed.") { - t.Fatalf("wrong output:\nstdout:%s\nstderr%s", stdout, stderr) + if !strings.Contains(stdout, "Resources: 1 added, 0 changed, 0 destroyed") { + t.Errorf("incorrect apply tally; want 1 added:\n%s", stdout) } - /// DESTROY + // INSPECT STATE + stdout, stderr, err = tf.Run("state", "list") + + //// DESTROY stdout, stderr, err = tf.Run("destroy", "-auto-approve") if err != nil { - t.Fatalf("unexpected apply error: %s\nstderr:\n%s", err, stderr) + t.Fatalf("unexpected destroy error: %s\nstderr:\n%s", err, stderr) } - if !strings.Contains(stdout, "Resources: 2 destroyed") { - t.Fatalf("wrong destroy output\nstdout:%s\nstderr:%s", stdout, stderr) - } - - // //// INIT - // stdout, stderr, err := tf.Run("init", "-enable-pluggable-state-storage-experiment=true", "-plugin-dir=cache") - // if err != nil { - // t.Fatalf("unexpected init error: %s\nstderr:\n%s", err, stderr) - // } - - // // Make sure we actually downloaded the plugins, rather than picking up - // // copies that might be already installed globally on the system. - // if !strings.Contains(stdout, "Installing hashicorp/template v") { - // t.Errorf("template provider download message is missing from init output:\n%s", stdout) - // t.Logf("(this can happen if you have a copy of the plugin in one of the global plugin search dirs)") - // } - // if !strings.Contains(stdout, "Installing hashicorp/null v") { - // t.Errorf("null provider download message is missing from init output:\n%s", stdout) - // t.Logf("(this can happen if you have a copy of the plugin in one of the global plugin search dirs)") - // } - - // //// PLAN - // // No separate plan step; this test lets the apply make a plan. - - // //// APPLY - // stdout, stderr, err = tf.Run("apply") - // if err != nil { - // t.Fatalf("unexpected apply error: %s\nstderr:\n%s", err, stderr) - // } - - // if !strings.Contains(stdout, "Resources: 1 added, 0 changed, 0 destroyed") { - // t.Errorf("incorrect apply tally; want 1 added:\n%s", stdout) - // } - - // state, err := tf.LocalState() - // if err != nil { - // t.Fatalf("failed to read state file: %s", err) - // } - - // stateResources := state.RootModule().Resources - // var gotResources []string - // for n := range stateResources { - // gotResources = append(gotResources, n) - // } - // sort.Strings(gotResources) - - // wantResources := []string{ - // "data.template_file.test", - // "null_resource.test", - // } - - // if !reflect.DeepEqual(gotResources, wantResources) { - // t.Errorf("wrong resources in state\ngot: %#v\nwant: %#v", gotResources, wantResources) - // } - - // //// DESTROY - // stdout, stderr, err = tf.Run("destroy", "-auto-approve") - // if err != nil { - // t.Fatalf("unexpected destroy error: %s\nstderr:\n%s", err, stderr) - // } - - // if !strings.Contains(stdout, "Resources: 1 destroyed") { - // t.Errorf("incorrect destroy tally; want 1 destroyed:\n%s", stdout) - // } - - // state, err = tf.LocalState() - // if err != nil { - // t.Fatalf("failed to read state file after destroy: %s", err) - // } - - // stateResources = state.RootModule().Resources - // if len(stateResources) != 0 { - // t.Errorf("wrong resources in state after destroy; want none, but still have:%s", spew.Sdump(stateResources)) - // } + if !strings.Contains(stdout, "Resources: 1 destroyed") { + t.Errorf("incorrect destroy tally; want 1 destroyed:\n%s", stdout) + } + state, err := tf.LocalState() + if err != nil { + t.Fatalf("failed to read state file after destroy: %s", err) + } + + stateResources := state.RootModule().Resources + if len(stateResources) != 0 { + t.Errorf("wrong resources in state after destroy; want none, but still have:%s", spew.Sdump(stateResources)) + } } -// TODO: TestPrimarySeparatePlan_stateStore +// TODO: TestPrimarySeparatePlan_stateStore - once support for PSS in plan files is implemented From eeaaac8601cd82036566f20ca35dfcc977acdf73 Mon Sep 17 00:00:00 2001 From: Sarah French Date: Fri, 24 Oct 2025 12:02:45 +0100 Subject: [PATCH 12/23] Fix issues in test fixture, rename it --- internal/command/e2etest/primary_test.go | 2 +- .../main.tf | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) rename internal/command/e2etest/testdata/{full-workflow-null-with-state-store => full-workflow-with-state-store}/main.tf (76%) diff --git a/internal/command/e2etest/primary_test.go b/internal/command/e2etest/primary_test.go index ba18d3cbe44b..237a99228714 100644 --- a/internal/command/e2etest/primary_test.go +++ b/internal/command/e2etest/primary_test.go @@ -245,7 +245,7 @@ func TestPrimary_stateStore(t *testing.T) { } t.Parallel() - tf := e2e.NewBinary(t, terraformBin, "testdata/full-workflow-null-with-state-store") + tf := e2e.NewBinary(t, terraformBin, "testdata/full-workflow-with-state-store") // In order to do a decent end-to-end test for this case we will need a real // enough provider plugin to try to run and make sure we are able to diff --git a/internal/command/e2etest/testdata/full-workflow-null-with-state-store/main.tf b/internal/command/e2etest/testdata/full-workflow-with-state-store/main.tf similarity index 76% rename from internal/command/e2etest/testdata/full-workflow-null-with-state-store/main.tf rename to internal/command/e2etest/testdata/full-workflow-with-state-store/main.tf index 41c39ffd798c..14142c851de4 100644 --- a/internal/command/e2etest/testdata/full-workflow-null-with-state-store/main.tf +++ b/internal/command/e2etest/testdata/full-workflow-with-state-store/main.tf @@ -14,10 +14,10 @@ variable "name" { default = "world" } -data "terraform_data" "my-data" { +resource "terraform_data" "my-data" { input = "hello ${var.name}" } output "greeting" { - value = data.terraform_data.output + value = resource.terraform_data.my-data.output } From 099b2068bcba2beb656c03b3df5b52edaf59d6e2 Mon Sep 17 00:00:00 2001 From: Sarah French Date: Fri, 24 Oct 2025 12:03:11 +0100 Subject: [PATCH 13/23] Update test - remove steps that are impossible to perform with inmem state storage. --- internal/command/e2etest/primary_test.go | 30 ++++-------------------- 1 file changed, 4 insertions(+), 26 deletions(-) diff --git a/internal/command/e2etest/primary_test.go b/internal/command/e2etest/primary_test.go index 237a99228714..201c1d3594bb 100644 --- a/internal/command/e2etest/primary_test.go +++ b/internal/command/e2etest/primary_test.go @@ -247,10 +247,8 @@ func TestPrimary_stateStore(t *testing.T) { tf := e2e.NewBinary(t, terraformBin, "testdata/full-workflow-with-state-store") - // In order to do a decent end-to-end test for this case we will need a real - // enough provider plugin to try to run and make sure we are able to - // actually run it. Here will build the simple and simple6 (built with - // protocol v6) providers. + // In order to test integration with PSS we need a provider plugin implementing a state store. + // Here will build the simple6 (built with protocol v6) provider, which implements PSS. simple6Provider := filepath.Join(tf.WorkDir(), "terraform-provider-simple6") simple6ProviderExe := e2e.GoBuild("github.com/hashicorp/terraform/internal/provider-simple-v6/main", simple6Provider) @@ -288,28 +286,8 @@ func TestPrimary_stateStore(t *testing.T) { t.Errorf("incorrect apply tally; want 1 added:\n%s", stdout) } - // INSPECT STATE - stdout, stderr, err = tf.Run("state", "list") - - //// DESTROY - stdout, stderr, err = tf.Run("destroy", "-auto-approve") - if err != nil { - t.Fatalf("unexpected destroy error: %s\nstderr:\n%s", err, stderr) - } - - if !strings.Contains(stdout, "Resources: 1 destroyed") { - t.Errorf("incorrect destroy tally; want 1 destroyed:\n%s", stdout) - } - - state, err := tf.LocalState() - if err != nil { - t.Fatalf("failed to read state file after destroy: %s", err) - } - - stateResources := state.RootModule().Resources - if len(stateResources) != 0 { - t.Errorf("wrong resources in state after destroy; want none, but still have:%s", spew.Sdump(stateResources)) - } + // We cannot inspect state or perform a destroy here, as the state isn't persisted between steps + // when we use the simple6_inmem state store. } // TODO: TestPrimarySeparatePlan_stateStore - once support for PSS in plan files is implemented From 40b99306294590ef1aed95751ddf9d0b341f55e1 Mon Sep 17 00:00:00 2001 From: Sarah French Date: Fri, 24 Oct 2025 12:27:46 +0100 Subject: [PATCH 14/23] Stop gating the inMem state store behind TF_ACC in the simple6 provider This provider is not accessible to users so it's unnecessary. --- internal/command/e2etest/primary_test.go | 2 +- internal/provider-simple-v6/provider.go | 13 ++++--------- .../provider-simple-v6/state_store_inmem_test.go | 4 ---- 3 files changed, 5 insertions(+), 14 deletions(-) diff --git a/internal/command/e2etest/primary_test.go b/internal/command/e2etest/primary_test.go index 201c1d3594bb..6d1a1ed09bd4 100644 --- a/internal/command/e2etest/primary_test.go +++ b/internal/command/e2etest/primary_test.go @@ -233,7 +233,7 @@ func TestPrimaryChdirOption(t *testing.T) { } } -// Requires TF_TEST_EXPERIMENT and TF_ACC to be set in the environment +// Requires TF_TEST_EXPERIMENT to be set in the environment func TestPrimary_stateStore(t *testing.T) { if !canRunGoBuild { // We're running in a separate-build-then-run context, so we can't diff --git a/internal/provider-simple-v6/provider.go b/internal/provider-simple-v6/provider.go index 1004798a3d2b..06317260cdd6 100644 --- a/internal/provider-simple-v6/provider.go +++ b/internal/provider-simple-v6/provider.go @@ -9,7 +9,6 @@ import ( "errors" "fmt" "log" - "os" "time" "github.com/zclconf/go-cty/cty" @@ -108,8 +107,10 @@ func provider() simple { }, }, }, - Actions: map[string]providers.ActionSchema{}, - StateStores: map[string]providers.Schema{}, + Actions: map[string]providers.ActionSchema{}, + StateStores: map[string]providers.Schema{ + inMemStoreName: stateStoreInMemGetSchema(), + }, ServerCapabilities: providers.ServerCapabilities{ PlanDestroy: true, GetProviderSchemaOptional: true, @@ -136,12 +137,6 @@ func provider() simple { inMem: &InMemStoreSingle{}, // default workspace doesn't exist by default here; needs explicit creation via init command } - // Only include the inmem state store in the provider when `TF_ACC` is set in the environment - // Excluding this from the schemas is sufficient to block usage. - if v := os.Getenv("TF_ACC"); v != "" { - provider.schema.StateStores[inMemStoreName] = stateStoreInMemGetSchema() - } - return provider } diff --git a/internal/provider-simple-v6/state_store_inmem_test.go b/internal/provider-simple-v6/state_store_inmem_test.go index ef2e9352d216..fcb7856593b4 100644 --- a/internal/provider-simple-v6/state_store_inmem_test.go +++ b/internal/provider-simple-v6/state_store_inmem_test.go @@ -15,8 +15,6 @@ import ( ) func TestBackendLocked(t *testing.T) { - t.Setenv("TF_ACC", "1") // enable using the inmem state store - // backend.TestBackendStateLocks assumes the default workspace exists // by default, so we need to make it exist using the method below. provider := ProviderWithDefaultWorkspace() @@ -37,8 +35,6 @@ func TestBackendLocked(t *testing.T) { } func TestRemoteState(t *testing.T) { - t.Setenv("TF_ACC", "1") // enable using the inmem state store - provider := Provider() plug, err := pluggable.NewPluggable(provider, inMemStoreName) From 4705e5202629a9a0d1007bd68afa2187840e3c0f Mon Sep 17 00:00:00 2001 From: Sarah French Date: Fri, 24 Oct 2025 12:30:38 +0100 Subject: [PATCH 15/23] Skip E2E test for PSS unless experiments enabled --- internal/command/e2etest/primary_test.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/internal/command/e2etest/primary_test.go b/internal/command/e2etest/primary_test.go index 6d1a1ed09bd4..d09e6055547d 100644 --- a/internal/command/e2etest/primary_test.go +++ b/internal/command/e2etest/primary_test.go @@ -235,6 +235,10 @@ func TestPrimaryChdirOption(t *testing.T) { // Requires TF_TEST_EXPERIMENT to be set in the environment func TestPrimary_stateStore(t *testing.T) { + if v := os.Getenv("TF_TEST_EXPERIMENT"); v == "" { + t.Skip("can't run without enabling experiments in the executable terraform binary, enable with TF_TEST_EXPERIMENT=1") + } + if !canRunGoBuild { // We're running in a separate-build-then-run context, so we can't // currently execute this test which depends on being able to build From bd0fa59cf62b6ad139fb46ceb66b95493bafe576 Mon Sep 17 00:00:00 2001 From: Sarah French Date: Fri, 24 Oct 2025 12:30:54 +0100 Subject: [PATCH 16/23] Enable experiments in E2E tests in automation --- .github/workflows/checks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 7f8c7992bb90..f9a8c80da502 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -108,7 +108,7 @@ jobs: - name: "End-to-end tests" run: | - TF_ACC=1 go test -v ./internal/command/e2etest + TF_TEST_EXPERIMENT=1 TF_ACC=1 go test -v ./internal/command/e2etest consistency-checks: name: "Code Consistency Checks" From 5f0581fbca92ab8d14b9f98b31cd8394221ced38 Mon Sep 17 00:00:00 2001 From: Sarah French Date: Fri, 24 Oct 2025 14:57:58 +0100 Subject: [PATCH 17/23] Add missing space to error message text --- internal/command/arguments/init.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/command/arguments/init.go b/internal/command/arguments/init.go index 968650b07223..6b2cb008d0c1 100644 --- a/internal/command/arguments/init.go +++ b/internal/command/arguments/init.go @@ -146,7 +146,7 @@ func ParseInit(args []string, experimentsEnabled bool) (*Init, tfdiags.Diagnosti diags = diags.Append(tfdiags.Sourceless( tfdiags.Error, "Cannot use -enable-pluggable-state-storage-experiment flag without experiments enabled", - "Terraform cannot use the-enable-pluggable-state-storage-experiment flag (or TF_ENABLE_PLUGGABLE_STATE_STORAGE environment variable) unless experiments are enabled.", + "Terraform cannot use the -enable-pluggable-state-storage-experiment flag (or TF_ENABLE_PLUGGABLE_STATE_STORAGE environment variable) unless experiments are enabled.", )) } if !init.CreateDefaultWorkspace { From b986709cacd69ef1f57c2bac3a8ad9eb8c2a7d14 Mon Sep 17 00:00:00 2001 From: Sarah French Date: Thu, 30 Oct 2025 14:08:27 +0000 Subject: [PATCH 18/23] Fix TF_TEST_EXPERIMENT => TF_TEST_EXPERIMENTS --- .github/workflows/checks.yml | 2 +- internal/command/e2etest/primary_test.go | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index f9a8c80da502..ac31e8681236 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -108,7 +108,7 @@ jobs: - name: "End-to-end tests" run: | - TF_TEST_EXPERIMENT=1 TF_ACC=1 go test -v ./internal/command/e2etest + TF_TEST_EXPERIMENTS=1 TF_ACC=1 go test -v ./internal/command/e2etest consistency-checks: name: "Code Consistency Checks" diff --git a/internal/command/e2etest/primary_test.go b/internal/command/e2etest/primary_test.go index d09e6055547d..4e873e5cc830 100644 --- a/internal/command/e2etest/primary_test.go +++ b/internal/command/e2etest/primary_test.go @@ -233,10 +233,10 @@ func TestPrimaryChdirOption(t *testing.T) { } } -// Requires TF_TEST_EXPERIMENT to be set in the environment +// Requires TF_TEST_EXPERIMENTS to be set in the environment func TestPrimary_stateStore(t *testing.T) { - if v := os.Getenv("TF_TEST_EXPERIMENT"); v == "" { - t.Skip("can't run without enabling experiments in the executable terraform binary, enable with TF_TEST_EXPERIMENT=1") + if v := os.Getenv("TF_TEST_EXPERIMENTS"); v == "" { + t.Skip("can't run without enabling experiments in the executable terraform binary, enable with TF_TEST_EXPERIMENTS=1") } if !canRunGoBuild { From 214630723b1c1ce855b5f9d8a7e1a184d8383fbb Mon Sep 17 00:00:00 2001 From: Sarah French Date: Thu, 30 Oct 2025 16:00:18 +0000 Subject: [PATCH 19/23] Ensure state stores are configured with a suggested chunk size from Core --- internal/backend/pluggable/pluggable.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/internal/backend/pluggable/pluggable.go b/internal/backend/pluggable/pluggable.go index 279d963becf9..61bdcc404d25 100644 --- a/internal/backend/pluggable/pluggable.go +++ b/internal/backend/pluggable/pluggable.go @@ -105,6 +105,9 @@ func (p *Pluggable) Configure(config cty.Value) tfdiags.Diagnostics { req := providers.ConfigureStateStoreRequest{ TypeName: p.typeName, Config: config, + Capabilities: providers.StateStoreClientCapabilities{ + ChunkSize: DefaultStateStoreChunkSize, + }, } resp := p.provider.ConfigureStateStore(req) return resp.Diagnostics From 0cc10c5d41894f01b87e98c646e9758b81cb7d00 Mon Sep 17 00:00:00 2001 From: Sarah French Date: Thu, 30 Oct 2025 16:43:43 +0000 Subject: [PATCH 20/23] Small changes to inmem E2E test --- internal/command/e2etest/primary_test.go | 4 +--- .../full-workflow-with-state-store-fs/main.tf | 23 +++++++++++++++++++ .../main.tf | 0 .../state_store_inmem_test.go | 4 ++-- 4 files changed, 26 insertions(+), 5 deletions(-) create mode 100644 internal/command/e2etest/testdata/full-workflow-with-state-store-fs/main.tf rename internal/command/e2etest/testdata/{full-workflow-with-state-store => full-workflow-with-state-store-inmem}/main.tf (100%) diff --git a/internal/command/e2etest/primary_test.go b/internal/command/e2etest/primary_test.go index 4e873e5cc830..ee40dc65e953 100644 --- a/internal/command/e2etest/primary_test.go +++ b/internal/command/e2etest/primary_test.go @@ -249,7 +249,7 @@ func TestPrimary_stateStore(t *testing.T) { } t.Parallel() - tf := e2e.NewBinary(t, terraformBin, "testdata/full-workflow-with-state-store") + tf := e2e.NewBinary(t, terraformBin, "testdata/full-workflow-with-state-store-inmem") // In order to test integration with PSS we need a provider plugin implementing a state store. // Here will build the simple6 (built with protocol v6) provider, which implements PSS. @@ -293,5 +293,3 @@ func TestPrimary_stateStore(t *testing.T) { // We cannot inspect state or perform a destroy here, as the state isn't persisted between steps // when we use the simple6_inmem state store. } - -// TODO: TestPrimarySeparatePlan_stateStore - once support for PSS in plan files is implemented diff --git a/internal/command/e2etest/testdata/full-workflow-with-state-store-fs/main.tf b/internal/command/e2etest/testdata/full-workflow-with-state-store-fs/main.tf new file mode 100644 index 000000000000..d2f5c9b4446f --- /dev/null +++ b/internal/command/e2etest/testdata/full-workflow-with-state-store-fs/main.tf @@ -0,0 +1,23 @@ +terraform { + required_providers { + simple6 = { + source = "registry.terraform.io/hashicorp/simple6" + } + } + + state_store "simple6_fs" { + provider "simple6" {} + } +} + +variable "name" { + default = "world" +} + +resource "terraform_data" "my-data" { + input = "hello ${var.name}" +} + +output "greeting" { + value = resource.terraform_data.my-data.output +} diff --git a/internal/command/e2etest/testdata/full-workflow-with-state-store/main.tf b/internal/command/e2etest/testdata/full-workflow-with-state-store-inmem/main.tf similarity index 100% rename from internal/command/e2etest/testdata/full-workflow-with-state-store/main.tf rename to internal/command/e2etest/testdata/full-workflow-with-state-store-inmem/main.tf diff --git a/internal/provider-simple-v6/state_store_inmem_test.go b/internal/provider-simple-v6/state_store_inmem_test.go index fcb7856593b4..2f0a6142c66f 100644 --- a/internal/provider-simple-v6/state_store_inmem_test.go +++ b/internal/provider-simple-v6/state_store_inmem_test.go @@ -14,7 +14,7 @@ import ( "github.com/zclconf/go-cty/cty" ) -func TestBackendLocked(t *testing.T) { +func TestInMemStoreLocked(t *testing.T) { // backend.TestBackendStateLocks assumes the default workspace exists // by default, so we need to make it exist using the method below. provider := ProviderWithDefaultWorkspace() @@ -34,7 +34,7 @@ func TestBackendLocked(t *testing.T) { backend.TestBackendStateLocks(t, b1, b2) } -func TestRemoteState(t *testing.T) { +func TestInMemStoreRemoteState(t *testing.T) { provider := Provider() plug, err := pluggable.NewPluggable(provider, inMemStoreName) From 9979fe14618a49af49a7703a61470f7456ade20d Mon Sep 17 00:00:00 2001 From: Sarah French Date: Thu, 30 Oct 2025 16:46:56 +0000 Subject: [PATCH 21/23] Add filesystem state store (no locking) and E2E test --- internal/command/e2etest/primary_test.go | 86 ++++++ internal/provider-simple-v6/provider.go | 32 ++- internal/provider-simple-v6/state_store_fs.go | 262 ++++++++++++++++++ .../provider-simple-v6/state_store_fs_test.go | 75 +++++ 4 files changed, 453 insertions(+), 2 deletions(-) create mode 100644 internal/provider-simple-v6/state_store_fs.go create mode 100644 internal/provider-simple-v6/state_store_fs_test.go diff --git a/internal/command/e2etest/primary_test.go b/internal/command/e2etest/primary_test.go index ee40dc65e953..7e6745d44e19 100644 --- a/internal/command/e2etest/primary_test.go +++ b/internal/command/e2etest/primary_test.go @@ -15,6 +15,7 @@ import ( "github.com/hashicorp/terraform/internal/e2e" "github.com/hashicorp/terraform/internal/getproviders" "github.com/hashicorp/terraform/internal/plans" + "github.com/hashicorp/terraform/internal/states/statefile" "github.com/zclconf/go-cty/cty" ) @@ -249,6 +250,85 @@ func TestPrimary_stateStore(t *testing.T) { } t.Parallel() + tf := e2e.NewBinary(t, terraformBin, "testdata/full-workflow-with-state-store-fs") + + // In order to test integration with PSS we need a provider plugin implementing a state store. + // Here will build the simple6 (built with protocol v6) provider, which implements PSS. + simple6Provider := filepath.Join(tf.WorkDir(), "terraform-provider-simple6") + simple6ProviderExe := e2e.GoBuild("github.com/hashicorp/terraform/internal/provider-simple-v6/main", simple6Provider) + + // Move the provider binaries into a directory that we will point terraform + // to using the -plugin-dir cli flag. + platform := getproviders.CurrentPlatform.String() + hashiDir := "cache/registry.terraform.io/hashicorp/" + if err := os.MkdirAll(tf.Path(hashiDir, "simple6/0.0.1/", platform), os.ModePerm); err != nil { + t.Fatal(err) + } + if err := os.Rename(simple6ProviderExe, tf.Path(hashiDir, "simple6/0.0.1/", platform, "terraform-provider-simple6")); err != nil { + t.Fatal(err) + } + + //// INIT + stdout, stderr, err := tf.Run("init", "-enable-pluggable-state-storage-experiment=true", "-plugin-dir=cache", "-no-color") + if err != nil { + t.Fatalf("unexpected init error: %s\nstderr:\n%s", err, stderr) + } + + if !strings.Contains(stdout, "Terraform created an empty state file for the default workspace") { + t.Errorf("notice about creating the default workspace is missing from init output:\n%s", stdout) + } + + //// PLAN + // No separate plan step; this test lets the apply make a plan. + + //// APPLY + stdout, stderr, err = tf.Run("apply", "-auto-approve", "-no-color") + if err != nil { + t.Fatalf("unexpected apply error: %s\nstderr:\n%s", err, stderr) + } + + if !strings.Contains(stdout, "Resources: 1 added, 0 changed, 0 destroyed") { + t.Errorf("incorrect apply tally; want 1 added:\n%s", stdout) + } + + // Check the statefile saved by the fs state store. + path := "terraform.tfstate.d/default/terraform.tfstate" + f, err := tf.OpenFile(path) + if err != nil { + t.Fatalf("unexpected error opening state file %s: %s\nstderr:\n%s", path, err, stderr) + } + defer f.Close() + + stateFile, err := statefile.Read(f) + if err != nil { + t.Fatalf("unexpected error reading statefile %s: %s\nstderr:\n%s", path, err, stderr) + } + + r := stateFile.State.RootModule().Resources + if len(r) != 1 { + t.Fatalf("expected state to include one resource, but got %d", len(r)) + } + if _, ok := r["terraform_data.my-data"]; !ok { + t.Fatalf("expected state to include terraform_data.my-data but it's missing") + } +} + +// Requires TF_TEST_EXPERIMENTS to be set in the environment +func TestPrimary_stateStore_inMem(t *testing.T) { + if v := os.Getenv("TF_TEST_EXPERIMENTS"); v == "" { + t.Skip("can't run without enabling experiments in the executable terraform binary, enable with TF_TEST_EXPERIMENTS=1") + } + + if !canRunGoBuild { + // We're running in a separate-build-then-run context, so we can't + // currently execute this test which depends on being able to build + // new executable at runtime. + // + // (See the comment on canRunGoBuild's declaration for more information.) + t.Skip("can't run without building a new provider executable") + } + t.Parallel() + tf := e2e.NewBinary(t, terraformBin, "testdata/full-workflow-with-state-store-inmem") // In order to test integration with PSS we need a provider plugin implementing a state store. @@ -268,6 +348,9 @@ func TestPrimary_stateStore(t *testing.T) { } //// INIT + // + // Note - the inmem PSS implementation means that the default workspace state created during init + // is lost as soon as the command completes. stdout, stderr, err := tf.Run("init", "-enable-pluggable-state-storage-experiment=true", "-plugin-dir=cache", "-no-color") if err != nil { t.Fatalf("unexpected init error: %s\nstderr:\n%s", err, stderr) @@ -281,6 +364,9 @@ func TestPrimary_stateStore(t *testing.T) { // No separate plan step; this test lets the apply make a plan. //// APPLY + // + // Note - the inmem PSS implementation means that writing to the default workspace during apply + // is creating the default state file for the first time. stdout, stderr, err = tf.Run("apply", "-auto-approve", "-no-color") if err != nil { t.Fatalf("unexpected apply error: %s\nstderr:\n%s", err, stderr) diff --git a/internal/provider-simple-v6/provider.go b/internal/provider-simple-v6/provider.go index 06317260cdd6..1ee7e44a5cf2 100644 --- a/internal/provider-simple-v6/provider.go +++ b/internal/provider-simple-v6/provider.go @@ -24,6 +24,7 @@ type simple struct { schema providers.GetProviderSchemaResponse inMem *InMemStoreSingle + fs *FsStore } // Provider returns an instance of providers.Interface @@ -109,7 +110,8 @@ func provider() simple { }, Actions: map[string]providers.ActionSchema{}, StateStores: map[string]providers.Schema{ - inMemStoreName: stateStoreInMemGetSchema(), + inMemStoreName: stateStoreInMemGetSchema(), // simple6_inmem + fsStoreName: stateStoreFsGetSchema(), // simple6_fs }, ServerCapabilities: providers.ServerCapabilities{ PlanDestroy: true, @@ -134,7 +136,9 @@ func provider() simple { }, }, - inMem: &InMemStoreSingle{}, // default workspace doesn't exist by default here; needs explicit creation via init command + // default workspaces doesn't exist by default here; needs explicit creation via init command + inMem: &InMemStoreSingle{}, + fs: &FsStore{}, } return provider @@ -353,6 +357,9 @@ func (s simple) ValidateStateStoreConfig(req providers.ValidateStateStoreConfigR if req.TypeName == inMemStoreName { return s.inMem.ValidateStateStoreConfig(req) } + if req.TypeName == fsStoreName { + return s.fs.ValidateStateStoreConfig(req) + } var resp providers.ValidateStateStoreConfigResponse resp.Diagnostics.Append(fmt.Errorf("unsupported state store type %q", req.TypeName)) @@ -363,6 +370,9 @@ func (s simple) ConfigureStateStore(req providers.ConfigureStateStoreRequest) pr if req.TypeName == inMemStoreName { return s.inMem.ConfigureStateStore(req) } + if req.TypeName == fsStoreName { + return s.fs.ConfigureStateStore(req) + } var resp providers.ConfigureStateStoreResponse resp.Diagnostics.Append(fmt.Errorf("unsupported state store type %q", req.TypeName)) @@ -373,6 +383,9 @@ func (s simple) ReadStateBytes(req providers.ReadStateBytesRequest) providers.Re if req.TypeName == inMemStoreName { return s.inMem.ReadStateBytes(req) } + if req.TypeName == fsStoreName { + return s.fs.ReadStateBytes(req) + } var resp providers.ReadStateBytesResponse resp.Diagnostics.Append(fmt.Errorf("unsupported state store type %q", req.TypeName)) @@ -383,6 +396,9 @@ func (s simple) WriteStateBytes(req providers.WriteStateBytesRequest) providers. if req.TypeName == inMemStoreName { return s.inMem.WriteStateBytes(req) } + if req.TypeName == fsStoreName { + return s.fs.WriteStateBytes(req) + } var resp providers.WriteStateBytesResponse resp.Diagnostics.Append(fmt.Errorf("unsupported state store type %q", req.TypeName)) @@ -393,6 +409,9 @@ func (s simple) LockState(req providers.LockStateRequest) providers.LockStateRes if req.TypeName == inMemStoreName { return s.inMem.LockState(req) } + if req.TypeName == fsStoreName { + return s.fs.LockState(req) + } var resp providers.LockStateResponse resp.Diagnostics.Append(fmt.Errorf("unsupported state store type %q", req.TypeName)) @@ -403,6 +422,9 @@ func (s simple) UnlockState(req providers.UnlockStateRequest) providers.UnlockSt if req.TypeName == inMemStoreName { return s.inMem.UnlockState(req) } + if req.TypeName == fsStoreName { + return s.fs.UnlockState(req) + } var resp providers.UnlockStateResponse resp.Diagnostics.Append(fmt.Errorf("unsupported state store type %q", req.TypeName)) @@ -413,6 +435,9 @@ func (s simple) GetStates(req providers.GetStatesRequest) providers.GetStatesRes if req.TypeName == inMemStoreName { return s.inMem.GetStates(req) } + if req.TypeName == fsStoreName { + return s.fs.GetStates(req) + } var resp providers.GetStatesResponse resp.Diagnostics.Append(fmt.Errorf("unsupported state store type %q", req.TypeName)) @@ -423,6 +448,9 @@ func (s simple) DeleteState(req providers.DeleteStateRequest) providers.DeleteSt if req.TypeName == inMemStoreName { return s.inMem.DeleteState(req) } + if req.TypeName == fsStoreName { + return s.fs.DeleteState(req) + } var resp providers.DeleteStateResponse resp.Diagnostics.Append(fmt.Errorf("unsupported state store type %q", req.TypeName)) diff --git a/internal/provider-simple-v6/state_store_fs.go b/internal/provider-simple-v6/state_store_fs.go new file mode 100644 index 000000000000..f2a8f9ccb7b7 --- /dev/null +++ b/internal/provider-simple-v6/state_store_fs.go @@ -0,0 +1,262 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package simple + +import ( + "bytes" + "errors" + "fmt" + "io" + "log" + "os" + "path" + "path/filepath" + "sort" + + "github.com/hashicorp/terraform/internal/backend" + "github.com/hashicorp/terraform/internal/configs/configschema" + "github.com/hashicorp/terraform/internal/providers" + "github.com/hashicorp/terraform/internal/states/statemgr" + "github.com/hashicorp/terraform/internal/tfdiags" + "github.com/zclconf/go-cty/cty" +) + +const fsStoreName = "simple6_fs" +const defaultWorkspaceDir = "terraform.tfstate.d" + +// FsStore allows storing state in the local filesystem. +// +// This state storage implementation differs from the old "local" backend in core, +// by storing all states in the custom, or default, workspace directory. In the "local" +// backend the default state was a special case and was handled differently to custom workspaces. +type FsStore struct { + // Configured values + workspaceDir string + chunkSize int64 + + states map[string]*statemgr.Filesystem +} + +func stateStoreFsGetSchema() providers.Schema { + return providers.Schema{ + Body: &configschema.Block{ + Attributes: map[string]*configschema.Attribute{ + "workspace_dir": { + Type: cty.String, + Optional: true, + Description: "The directory where state files will be created. When unset the value will default to terraform.tfstate.d", + }, + }, + }, + } +} + +func (f *FsStore) ValidateStateStoreConfig(req providers.ValidateStateStoreConfigRequest) providers.ValidateStateStoreConfigResponse { + var resp providers.ValidateStateStoreConfigResponse + + attrs := req.Config.AsValueMap() + if v, ok := attrs["workspace_dir"]; ok { + if !v.IsKnown() { + resp.Diagnostics = resp.Diagnostics.Append(errors.New("the attribute \"workspace_dir\" cannot be an unknown value")) + return resp + } + } + + return resp +} + +func (f *FsStore) ConfigureStateStore(req providers.ConfigureStateStoreRequest) providers.ConfigureStateStoreResponse { + resp := providers.ConfigureStateStoreResponse{} + + configVal := req.Config + if v := configVal.GetAttr("workspace_dir"); !v.IsNull() { + f.workspaceDir = v.AsString() + } else { + f.workspaceDir = defaultWorkspaceDir + } + + if f.states == nil { + f.states = make(map[string]*statemgr.Filesystem) + } + + // We need to select return a suggested chunk size; use the value suggested by Core + resp.Capabilities.ChunkSize = req.Capabilities.ChunkSize + f.chunkSize = req.Capabilities.ChunkSize + + return resp +} + +func (f *FsStore) LockState(req providers.LockStateRequest) providers.LockStateResponse { + resp := providers.LockStateResponse{} + resp.Diagnostics = resp.Diagnostics.Append(tfdiags.Sourceless( + tfdiags.Warning, + "Locking not implemented", + fmt.Sprintf("Could not lock state %q; state locking isn't implemented", req.StateId), + )) + return resp +} + +func (f *FsStore) UnlockState(req providers.UnlockStateRequest) providers.UnlockStateResponse { + resp := providers.UnlockStateResponse{} + resp.Diagnostics = resp.Diagnostics.Append(tfdiags.Sourceless( + tfdiags.Warning, + "Unlocking not implemented", + fmt.Sprintf("Could not unlock state %q; state locking isn't implemented", req.StateId), + )) + return resp +} + +func (f *FsStore) GetStates(req providers.GetStatesRequest) providers.GetStatesResponse { + resp := providers.GetStatesResponse{} + + entries, err := os.ReadDir(f.workspaceDir) + // no error if there's no envs configured + if os.IsNotExist(err) { + return resp + } + if err != nil { + resp.Diagnostics = resp.Diagnostics.Append(err) + return resp + } + + var envs []string + for _, entry := range entries { + if entry.IsDir() { + envs = append(envs, filepath.Base(entry.Name())) + } + } + + sort.Strings(envs) + resp.States = envs + return resp +} + +func (f *FsStore) DeleteState(req providers.DeleteStateRequest) providers.DeleteStateResponse { + resp := providers.DeleteStateResponse{} + + if req.StateId == "" { + resp.Diagnostics = resp.Diagnostics.Append(errors.New("empty state name")) + return resp + } + + if req.StateId == backend.DefaultStateName { + resp.Diagnostics = resp.Diagnostics.Append(errors.New("cannot delete default state")) + return resp + } + + delete(f.states, req.StateId) + err := os.RemoveAll(filepath.Join(f.workspaceDir, req.StateId)) + if err != nil { + resp.Diagnostics = resp.Diagnostics.Append(fmt.Errorf("error deleting workspace %s: %w", req.StateId, err)) + return resp + } + + return resp +} + +func (f *FsStore) getStatePath(stateId string) string { + return path.Join(f.workspaceDir, stateId, "terraform.tfstate") +} + +func (f *FsStore) getStateDir(stateId string) string { + return path.Join(f.workspaceDir, stateId) +} + +func (f *FsStore) ReadStateBytes(req providers.ReadStateBytesRequest) providers.ReadStateBytesResponse { + log.Printf("[DEBUG] ReadStateBytes: reading state from workspace %q", req.StateId) + resp := providers.ReadStateBytesResponse{} + + // E.g. terraform.tfstate.d/foobar/terraform.tfstate + path := f.getStatePath(req.StateId) + file, err := os.Open(path) + fileExists := true + + if err != nil { + if _, ok := err.(*os.PathError); ok { + fileExists = false + } else { + // Error other than the file not existing + resp.Diagnostics = resp.Diagnostics.Append(fmt.Errorf("error opening state file %q: %w", path, err)) + return resp + } + } + defer file.Close() + + buf := bytes.Buffer{} + var processedBytes int + if fileExists { + for { + b := make([]byte, f.chunkSize) + n, err := file.Read(b) + if err == io.EOF { + break + } + if err != nil { + resp.Diagnostics = resp.Diagnostics.Append(fmt.Errorf("error reading from state file %q: %w", path, err)) + return resp + } + buf.Write(b[0:n]) + processedBytes += n + } + } + log.Printf("[DEBUG] ReadStateBytes: read %d bytes of data from state file %q", processedBytes, path) + + if processedBytes == 0 { + // Does not exist, so return no bytes + resp.Diagnostics = resp.Diagnostics.Append(tfdiags.Sourceless( + tfdiags.Warning, + "State doesn't exist, yet", + fmt.Sprintf("There's no state for workspace %q yet", req.StateId), + )) + } + + resp.Bytes = buf.Bytes() + return resp +} + +func (f *FsStore) WriteStateBytes(req providers.WriteStateBytesRequest) providers.WriteStateBytesResponse { + log.Printf("[DEBUG] WriteStateBytes: writing state to workspace %q", req.StateId) + resp := providers.WriteStateBytesResponse{} + + // E.g. terraform.tfstate.d/foobar/terraform.tfstate + path := f.getStatePath(req.StateId) + + // Create or open state file + dir := f.getStateDir(req.StateId) + err := os.MkdirAll(dir, 0755) + if err != nil { + resp.Diagnostics = resp.Diagnostics.Append(fmt.Errorf("error creating state file directory %q: %w", dir, err)) + } + + file, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, 0666) + if err != nil { + resp.Diagnostics = resp.Diagnostics.Append(fmt.Errorf("error opening state file %q: %w", path, err)) + } + + buf := bytes.NewBuffer(req.Bytes) + var processedBytes int + if f.chunkSize == 0 { + panic("WriteStateBytes: chunk size zero. This is an error in Terraform and should be reported") + } + for { + data := buf.Next(int(f.chunkSize)) + if len(data) == 0 { + break + } + n, err := file.Write(data) + if err != nil { + resp.Diagnostics = resp.Diagnostics.Append(fmt.Errorf("error writing to state file %q: %w", path, err)) + return resp + } + + processedBytes += n + } + log.Printf("[DEBUG] WriteStateBytes: wrote %d bytes of data to state file %q", processedBytes, path) + + if processedBytes == 0 { + resp.Diagnostics = resp.Diagnostics.Append(fmt.Errorf("missing state data: write action wrote %d bytes of data to file %q.", processedBytes, path)) + } + + return resp +} diff --git a/internal/provider-simple-v6/state_store_fs_test.go b/internal/provider-simple-v6/state_store_fs_test.go new file mode 100644 index 000000000000..bdec5c3da16a --- /dev/null +++ b/internal/provider-simple-v6/state_store_fs_test.go @@ -0,0 +1,75 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package simple + +import ( + "testing" + + "github.com/hashicorp/hcl/v2" + "github.com/hashicorp/terraform/internal/addrs" + "github.com/hashicorp/terraform/internal/backend" + "github.com/hashicorp/terraform/internal/backend/pluggable" + "github.com/hashicorp/terraform/internal/states" + "github.com/zclconf/go-cty/cty" +) + +// No testing of locking with 2 clients, as locking isn't fully implemented. + +func TestFsStoreRemoteState(t *testing.T) { + td := t.TempDir() + t.Chdir(td) + + provider := Provider() + + plug, err := pluggable.NewPluggable(provider, fsStoreName) + if err != nil { + t.Fatal(err) + } + + b := backend.TestBackendConfig(t, plug, hcl.EmptyBody()) + + // The default workspace doesn't exist by default + // (Note that this depends on the factory method used to get the provider above) + workspaces, wDiags := b.Workspaces() + if wDiags.HasErrors() { + t.Fatal(wDiags.Err()) + } + if len(workspaces) != 0 { + t.Fatalf("unexpected response from Workspaces method: %#v", workspaces) + } + + // create a new workspace in this backend + workspace := "workspace" + emptyState := states.NewState() + + sMgr, sDiags := b.StateMgr(workspace) + if sDiags.HasErrors() { + t.Fatal(sDiags.Err()) + } + if err := sMgr.WriteState(emptyState); err != nil { + t.Fatal(err) + } + if err := sMgr.PersistState(nil); err != nil { + t.Fatal(err) + } + + // force overwriting the remote state + newState := states.NewState() + newState.SetOutputValue( + addrs.OutputValue{Name: "foo"}.Absolute(addrs.RootModuleInstance), + cty.StringVal("bar"), + false) + + if err := sMgr.WriteState(newState); err != nil { + t.Fatal(err) + } + + if err := sMgr.PersistState(nil); err != nil { + t.Fatal(err) + } + + if err := sMgr.RefreshState(); err != nil { + t.Fatal(err) + } +} From 316d5b542399bfa8b02875be7b398078221260e3 Mon Sep 17 00:00:00 2001 From: Sarah French Date: Mon, 3 Nov 2025 09:42:59 +0000 Subject: [PATCH 22/23] Replace ioutil.ReadDir with os.ReadDir Seeing as I was in this part of the code during this branch's work --- internal/backend/local/backend.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/internal/backend/local/backend.go b/internal/backend/local/backend.go index 28004661a616..79c5b7bc2ad0 100644 --- a/internal/backend/local/backend.go +++ b/internal/backend/local/backend.go @@ -7,7 +7,6 @@ import ( "context" "errors" "fmt" - "io/ioutil" "log" "os" "path/filepath" @@ -207,7 +206,7 @@ func (b *Local) Workspaces() ([]string, tfdiags.Diagnostics) { // the listing always start with "default" envs := []string{backend.DefaultStateName} - entries, err := ioutil.ReadDir(b.stateWorkspaceDir()) + entries, err := os.ReadDir(b.stateWorkspaceDir()) // no error if there's no envs configured if os.IsNotExist(err) { return envs, nil From 9088fe59a923b7bfc3ea2668ae966528afbb2cda Mon Sep 17 00:00:00 2001 From: Sarah French Date: Thu, 6 Nov 2025 16:53:15 +0000 Subject: [PATCH 23/23] fix: Pass through id of state to delete in grpcwrap's DeleteState method. --- internal/grpcwrap/provider6.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/grpcwrap/provider6.go b/internal/grpcwrap/provider6.go index 20a52a77a2af..a91c2cc0df2a 100644 --- a/internal/grpcwrap/provider6.go +++ b/internal/grpcwrap/provider6.go @@ -1161,6 +1161,7 @@ func (p *provider6) GetStates(ctx context.Context, req *tfplugin6.GetStates_Requ func (p *provider6) DeleteState(ctx context.Context, req *tfplugin6.DeleteState_Request) (*tfplugin6.DeleteState_Response, error) { deleteStatesResp := p.provider.DeleteState(providers.DeleteStateRequest{ TypeName: req.TypeName, + StateId: req.StateId, }) resp := &tfplugin6.DeleteState_Response{