From ee5cec0014c7b89debbc87f753fec0da21f74dea Mon Sep 17 00:00:00 2001 From: Daniel Schmidt Date: Fri, 7 Nov 2025 14:01:23 +0100 Subject: [PATCH 1/2] init: skip dependencies declared in development override --- .../v1.15/ENHANCEMENTS-20251107-140221.yaml | 5 ++ internal/command/e2etest/provider_dev_test.go | 79 ++++++++++++++++++- .../pkgdir/.exists | 1 + .../provider-dev-override.tf | 22 ++++++ internal/command/init.go | 3 +- internal/command/meta_providers.go | 17 +++- 6 files changed, 121 insertions(+), 6 deletions(-) create mode 100644 .changes/v1.15/ENHANCEMENTS-20251107-140221.yaml create mode 100644 internal/command/e2etest/testdata/provider-dev-override-with-existing/pkgdir/.exists create mode 100644 internal/command/e2etest/testdata/provider-dev-override-with-existing/provider-dev-override.tf diff --git a/.changes/v1.15/ENHANCEMENTS-20251107-140221.yaml b/.changes/v1.15/ENHANCEMENTS-20251107-140221.yaml new file mode 100644 index 000000000000..fdef6f32cacb --- /dev/null +++ b/.changes/v1.15/ENHANCEMENTS-20251107-140221.yaml @@ -0,0 +1,5 @@ +kind: ENHANCEMENTS +body: "init: skip dependencies declared in development override. This allows you to use `terraform init` with developer overrides and install dependencies that are not declared in the override file." +time: 2025-11-07T14:02:21.847382+01:00 +custom: + Issue: "27459" diff --git a/internal/command/e2etest/provider_dev_test.go b/internal/command/e2etest/provider_dev_test.go index 04561d02567c..ba280c464a1c 100644 --- a/internal/command/e2etest/provider_dev_test.go +++ b/internal/command/e2etest/provider_dev_test.go @@ -12,6 +12,7 @@ import ( "testing" "github.com/hashicorp/terraform/internal/e2e" + "github.com/hashicorp/terraform/internal/getproviders" ) // TestProviderDevOverrides is a test for the special dev_overrides setting @@ -86,13 +87,83 @@ func TestProviderDevOverrides(t *testing.T) { } stdout, stderr, err = tf.Run("init") - if err == nil { - t.Fatal("expected error: Failed to query available provider packages") + if err != nil { + t.Fatalf("unexpected error: %e", err) + } + if got, want := stdout, `Provider development overrides are in effect`; !strings.Contains(got, want) { + t.Errorf("stdout doesn't include the warning about development overrides\nwant: %s\n%s", want, got) + } +} + +func TestProviderDevOverridesWithProviderToDownload(t *testing.T) { + if !canRunGoBuild { + // We're running in a separate-build-then-run context, so we can't + // currently execute this test which depends on being able to build + // new executable at runtime. + // + // (See the comment on canRunGoBuild's declaration for more information.) + t.Skip("can't run without building a new provider executable") + } + t.Parallel() + + // This test reaches out to releases.hashicorp.com to download the + // null provider, so it can only run if network access is allowed. + skipIfCannotAccessNetwork(t) + + tf := e2e.NewBinary(t, terraformBin, "testdata/provider-dev-override-with-existing") + + // In order to do a decent end-to-end test for this case we will need a + // real enough provider plugin to try to run and make sure we are able + // to actually run it. For now we'll use the "test" provider for that, + // because it happens to be in this repository and therefore allows + // us to avoid drawing in anything external, but we might revisit this + // strategy in future if other needs cause us to evolve the test + // provider in a way that makes it less suitable for this particular test, + // such as if it stops being buildable into an independent executable. + providerExeDir := filepath.Join(tf.WorkDir(), "pkgdir") + providerExePrefix := filepath.Join(providerExeDir, "terraform-provider-test_") + providerExe := e2e.GoBuild("github.com/hashicorp/terraform/internal/provider-simple/main", providerExePrefix) + t.Logf("temporary provider executable is %s", providerExe) + + err := ioutil.WriteFile(filepath.Join(tf.WorkDir(), "dev.tfrc"), []byte(fmt.Sprintf(` + provider_installation { + dev_overrides { + "example.com/test/test" = %q + } + direct {} + } + `, providerExeDir)), os.ModePerm) + if err != nil { + t.Fatal(err) + } + + tf.AddEnv("TF_CLI_CONFIG_FILE=dev.tfrc") + + stdout, stderr, err := tf.Run("providers") + if err != nil { + t.Fatalf("unexpected error: %s\n%s", err, stderr) + } + if got, want := stdout, `provider[example.com/test/test]`; !strings.Contains(got, want) { + t.Errorf("configuration should depend on %s, but doesn't\n%s", want, got) + } + + stdout, stderr, err = tf.Run("init") + if err != nil { + t.Fatalf("unexpected error: %e", err) } if got, want := stdout, `Provider development overrides are in effect`; !strings.Contains(got, want) { t.Errorf("stdout doesn't include the warning about development overrides\nwant: %s\n%s", want, got) } - if got, want := stderr, `Failed to query available provider packages`; !strings.Contains(got, want) { - t.Errorf("stderr doesn't include the error about listing unavailable development provider\nwant: %s\n%s", want, got) + + // Check if the null provider has been installed + const providerVersion = "3.1.0" // must match the version in the fixture config + pluginDir := filepath.Join(tf.WorkDir(), ".terraform", "providers", "registry.terraform.io", "hashicorp", "null", providerVersion, getproviders.CurrentPlatform.String()) + pluginExe := filepath.Join(pluginDir, "terraform-provider-null_v"+providerVersion+"_x5") + if getproviders.CurrentPlatform.OS == "windows" { + pluginExe += ".exe" // ugh + } + + if _, err := os.Stat(pluginExe); os.IsNotExist(err) { + t.Fatalf("expected plugin executable %s to exist, but it does not", pluginExe) } } diff --git a/internal/command/e2etest/testdata/provider-dev-override-with-existing/pkgdir/.exists b/internal/command/e2etest/testdata/provider-dev-override-with-existing/pkgdir/.exists new file mode 100644 index 000000000000..052e1ad06c5f --- /dev/null +++ b/internal/command/e2etest/testdata/provider-dev-override-with-existing/pkgdir/.exists @@ -0,0 +1 @@ +This is where the test will place the temporary build of the test provider. diff --git a/internal/command/e2etest/testdata/provider-dev-override-with-existing/provider-dev-override.tf b/internal/command/e2etest/testdata/provider-dev-override-with-existing/provider-dev-override.tf new file mode 100644 index 000000000000..63f6c5dcbaad --- /dev/null +++ b/internal/command/e2etest/testdata/provider-dev-override-with-existing/provider-dev-override.tf @@ -0,0 +1,22 @@ +terraform { + required_providers { + # this one is overwritten by dev override + simple = { + source = "example.com/test/test" + version = "2.0.0" + } + + # this one should still be loaded + null = { + # Our version is intentionally fixed so that we have a fixed + # test case here, though we might have to update this in future + # if e.g. Terraform stops supporting plugin protocol 5, or if + # the null provider is yanked from the registry for some reason. + source = "hashicorp/null" + version = "3.1.0" + } + } +} + +data "simple_resource" "test" { +} diff --git a/internal/command/init.go b/internal/command/init.go index 033df62c8a13..4c77dd4a8790 100644 --- a/internal/command/init.go +++ b/internal/command/init.go @@ -258,11 +258,12 @@ func (c *InitCommand) getProviders(ctx context.Context, config *configs.Config, if hclDiags.HasErrors() { return false, true, diags } + + reqs = c.removeDevOverrides(reqs) if state != nil { stateReqs := state.ProviderRequirements() reqs = reqs.Merge(stateReqs) } - for providerAddr := range reqs { if providerAddr.IsLegacy() { diags = diags.Append(tfdiags.Sourceless( diff --git a/internal/command/meta_providers.go b/internal/command/meta_providers.go index c80c5bacb94c..54175a3704b0 100644 --- a/internal/command/meta_providers.go +++ b/internal/command/meta_providers.go @@ -18,6 +18,7 @@ import ( builtinProviders "github.com/hashicorp/terraform/internal/builtin/providers" "github.com/hashicorp/terraform/internal/depsfile" "github.com/hashicorp/terraform/internal/getproviders" + "github.com/hashicorp/terraform/internal/getproviders/providerreqs" "github.com/hashicorp/terraform/internal/logging" tfplugin "github.com/hashicorp/terraform/internal/plugin" tfplugin6 "github.com/hashicorp/terraform/internal/plugin6" @@ -176,7 +177,6 @@ func (m *Meta) providerDevOverrideInitWarnings() tfdiags.Diagnostics { for addr, path := range m.ProviderDevOverrides { detailMsg.WriteString(fmt.Sprintf(" - %s in %s\n", addr.ForDisplay(), path)) } - detailMsg.WriteString("\nSkip terraform init when using provider development overrides. It is not necessary and may error unexpectedly.") return tfdiags.Diagnostics{ tfdiags.Sourceless( tfdiags.Warning, @@ -186,6 +186,21 @@ func (m *Meta) providerDevOverrideInitWarnings() tfdiags.Diagnostics { } } +func (m *Meta) removeDevOverrides(reqs providerreqs.Requirements) providerreqs.Requirements { + // Deep copy the requirements to avoid mutating the input + copiedReqs := make(providerreqs.Requirements) + for provider, versions := range reqs { + // Only copy if the provider is not overridden + if _, overridden := m.ProviderDevOverrides[provider]; !overridden { + copiedVersions := make(providerreqs.VersionConstraints, len(versions)) + copy(copiedVersions, versions) + copiedReqs[provider] = copiedVersions + } + } + + return copiedReqs +} + // providerDevOverrideRuntimeWarnings returns a diagnostics that contains at // least one warning if and only if there is at least one provider development // override in effect. If not, the result is always empty. The result never From 862b6ba4f522cde5028456688d07b77aa4413bd4 Mon Sep 17 00:00:00 2001 From: Daniel Schmidt Date: Fri, 7 Nov 2025 17:04:02 +0100 Subject: [PATCH 2/2] add additional warning that providers won't be downloaded --- internal/command/e2etest/provider_dev_test.go | 13 ++++++++++--- internal/command/meta_providers.go | 1 + 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/internal/command/e2etest/provider_dev_test.go b/internal/command/e2etest/provider_dev_test.go index ba280c464a1c..f96481d56e88 100644 --- a/internal/command/e2etest/provider_dev_test.go +++ b/internal/command/e2etest/provider_dev_test.go @@ -86,13 +86,17 @@ func TestProviderDevOverrides(t *testing.T) { t.Errorf("stdout doesn't include the warning about development overrides\nwant: %s\n%s", want, got) } - stdout, stderr, err = tf.Run("init") + stdout, _, _ = tf.Run("init") if err != nil { t.Fatalf("unexpected error: %e", err) } if got, want := stdout, `Provider development overrides are in effect`; !strings.Contains(got, want) { t.Errorf("stdout doesn't include the warning about development overrides\nwant: %s\n%s", want, got) } + + if got, want := stdout, "These providers are not installed as part of init since they"; !strings.Contains(got, want) { + t.Errorf("stdout doesn't include init specific warning about consequences of overrides \nwant: %s\n%s", want, got) + } } func TestProviderDevOverridesWithProviderToDownload(t *testing.T) { @@ -139,7 +143,7 @@ func TestProviderDevOverridesWithProviderToDownload(t *testing.T) { tf.AddEnv("TF_CLI_CONFIG_FILE=dev.tfrc") - stdout, stderr, err := tf.Run("providers") + stdout, stderr, _ := tf.Run("providers") if err != nil { t.Fatalf("unexpected error: %s\n%s", err, stderr) } @@ -147,13 +151,16 @@ func TestProviderDevOverridesWithProviderToDownload(t *testing.T) { t.Errorf("configuration should depend on %s, but doesn't\n%s", want, got) } - stdout, stderr, err = tf.Run("init") + stdout, _, err = tf.Run("init") if err != nil { t.Fatalf("unexpected error: %e", err) } if got, want := stdout, `Provider development overrides are in effect`; !strings.Contains(got, want) { t.Errorf("stdout doesn't include the warning about development overrides\nwant: %s\n%s", want, got) } + if got, want := stdout, "These providers are not installed as part of init since they were"; !strings.Contains(got, want) { + t.Errorf("stdout doesn't include init specific warning about consequences of overrides \nwant: %s\n got: %s", want, got) + } // Check if the null provider has been installed const providerVersion = "3.1.0" // must match the version in the fixture config diff --git a/internal/command/meta_providers.go b/internal/command/meta_providers.go index 54175a3704b0..9bb0380b4d6b 100644 --- a/internal/command/meta_providers.go +++ b/internal/command/meta_providers.go @@ -177,6 +177,7 @@ func (m *Meta) providerDevOverrideInitWarnings() tfdiags.Diagnostics { for addr, path := range m.ProviderDevOverrides { detailMsg.WriteString(fmt.Sprintf(" - %s in %s\n", addr.ForDisplay(), path)) } + detailMsg.WriteString("\nThese providers are not installed as part of init since they were overwritten. If this is unintentional please re-run without the development overrides set.") return tfdiags.Diagnostics{ tfdiags.Sourceless( tfdiags.Warning,