Incorporate go-getter support for 'git::' filepaths

The go-getters library now has support for local file system paths to
Git repositories, specified with the 'git::' forcing token. The feature
works for both absolute and relative filepaths, and supports all the
usual go-getter goodies including '//' delimited subdirs and URI-style
query parameters.[0][1]

We incorporate that capability into Terraform, which allows users to
specify paths to locally present Git repositories from which to clone
other Terrform modules on which they are dependent. When coupled with
Git submodules, this creates a powerful way to manage Terraform modules
at specific versions without requiring those modules to be available on
the network (e.g., on GitHub):

    module "my_module" {
        source = "git::../git-submodules/tf-modules/some-tf-module?ref=v0.1.0"
        // ...
    }

From the perspective of Terraform, such Git repositories are "remote" in
the same way that repositories on GitHub are.

Note that within a Terraform module "call" block, the filepaths
specified are relative to the directory in which the *.tf file lives,
not relative to the current working directory of the Terraform
process. In order to support this feature, Terraform needs to supply
that contextual information to go-getter to allow relative filepath
resolution to work. In order to do so, we needed to switch over to using
go-getter's new "Contextual Detector" API. It works in the same basic
way as the traditional Detector API, but allows us to provide this
additional information.

In keeping with the "keep things simple" comment in the commit message
of 2b2ac1f, we are here maintaining our custom go-getter detectors
in two places. Only now each is called FooCtxDetector rather than
FooDetector. Nevertheless, all except the GitCtxDetector do little more
than "pass through" delegation to its analogous FooDetector counterpart.

Fixes hashicorp#25488
Fixes hashicorp#21107

[0] hashicorp/go-getter#268
[1] hashicorp/go-getter#269

Author: salewski

configs/configload/getter.go

@@ -17,13 +17,13 @@ import (
 // any meddling that might be done by other go-getter callers linked into our
 // executable.
 
-var goGetterDetectors = []getter.Detector{
-	new(getter.GitHubDetector),
-	new(getter.GitDetector),
-	new(getter.BitBucketDetector),
-	new(getter.GCSDetector),
-	new(getter.S3Detector),
-	new(getter.FileDetector),
+var goGetterCtxDetectors = []getter.CtxDetector{
+	new(getter.GitHubCtxDetector),
+	new(getter.GitCtxDetector),
+	new(getter.BitBucketCtxDetector),
+	new(getter.GCSCtxDetector),
+	new(getter.S3CtxDetector),
+	new(getter.FileCtxDetector),
 }
 
 var goGetterNoDetectors = []getter.Detector{}
@@ -80,12 +80,12 @@ type reusingGetter map[string]string
 // end-user-actionable error messages. At this time we do not have any
 // reasonable way to improve these error messages at this layer because
 // the underlying errors are not separatelyr recognizable.
-func (g reusingGetter) getWithGoGetter(instPath, addr string) (string, error) {
+func (g reusingGetter) getWithGoGetter(instPath, addr, srcAbs string) (string, error) {
 	packageAddr, subDir := splitAddrSubdir(addr)
 
 	log.Printf("[DEBUG] will download %q to %s", packageAddr, instPath)
 
-	realAddr, err := getter.Detect(packageAddr, instPath, goGetterDetectors)
+	realAddr, err := getter.CtxDetect(packageAddr, instPath, srcAbs, goGetterCtxDetectors)
 	if err != nil {
 		return "", err
 	}

internal/initwd/getter.go

@@ -20,13 +20,13 @@ import (
 // any meddling that might be done by other go-getter callers linked into our
 // executable.
 
-var goGetterDetectors = []getter.Detector{
-	new(getter.GitHubDetector),
-	new(getter.GitDetector),
-	new(getter.BitBucketDetector),
-	new(getter.GCSDetector),
-	new(getter.S3Detector),
-	new(getter.FileDetector),
+var goGetterCtxDetectors = []getter.CtxDetector{
+	new(getter.GitHubCtxDetector),
+	new(getter.GitCtxDetector),
+	new(getter.BitBucketCtxDetector),
+	new(getter.GCSCtxDetector),
+	new(getter.S3CtxDetector),
+	new(getter.FileCtxDetector),
 }
 
 var goGetterNoDetectors = []getter.Detector{}
@@ -83,12 +83,12 @@ type reusingGetter map[string]string
 // end-user-actionable error messages. At this time we do not have any
 // reasonable way to improve these error messages at this layer because
 // the underlying errors are not separately recognizable.
-func (g reusingGetter) getWithGoGetter(instPath, addr string) (string, error) {
+func (g reusingGetter) getWithGoGetter(instPath, addr, srcAbs string) (string, error) {
 	packageAddr, subDir := splitAddrSubdir(addr)
 
 	log.Printf("[DEBUG] will download %q to %s", packageAddr, instPath)
 
-	realAddr, err := getter.Detect(packageAddr, instPath, goGetterDetectors)
+	realAddr, err := getter.CtxDetect(packageAddr, instPath, srcAbs, goGetterCtxDetectors)
 	if err != nil {
 		return "", err
 	}

internal/initwd/module_install.go

@@ -450,7 +450,13 @@ func (i *ModuleInstaller) installRegistryModule(req *earlyconfig.ModuleRequest,
 
 	log.Printf("[TRACE] ModuleInstaller: %s %s %s is available at %q", key, addr, latestMatch, dlAddr)
 
-	modDir, err := getter.getWithGoGetter(instPath, dlAddr)
+	srcDirAbs, sdDiags := i.calledFromSourceDirAbs(req)
+	if srcDirAbs == "" {
+		diags = append(diags, sdDiags...)
+		return nil, nil, diags
+	}
+
+	modDir, err := getter.getWithGoGetter(instPath, dlAddr, srcDirAbs)
 	if err != nil {
 		// Errors returned by go-getter have very inconsistent quality as
 		// end-user error messages, but for now we're accepting that because
@@ -521,7 +527,13 @@ func (i *ModuleInstaller) installGoGetterModule(req *earlyconfig.ModuleRequest,
 		return nil, diags
 	}
 
-	modDir, err := getter.getWithGoGetter(instPath, req.SourceAddr)
+	srcDirAbs, sdDiags := i.calledFromSourceDirAbs(req)
+	if srcDirAbs == "" {
+		diags = append(diags, sdDiags...)
+		return nil, diags
+	}
+
+	modDir, err := getter.getWithGoGetter(instPath, req.SourceAddr, srcDirAbs)
 	if err != nil {
 		if _, ok := err.(*MaybeRelativePathErr); ok {
 			log.Printf(
@@ -588,3 +600,46 @@ func (i *ModuleInstaller) installGoGetterModule(req *earlyconfig.ModuleRequest,
 func (i *ModuleInstaller) packageInstallPath(modulePath addrs.Module) string {
 	return filepath.Join(i.modsDir, strings.Join(modulePath, "."))
 }
+
+// A called module can exist in a (possibly remote) source code repository,
+// and go-getter will attempt to retrieve the modules from the repo if that is
+// the case.
+//
+// There is special case notion of "remote" for Terraform modules that live in
+// source code repositories that are "outside of the user's current Terraform
+// project", but which are expected to be present on the local file system. In
+// particular, Git submodules (see git-submodule(1)) can be used to keep such
+// repos nearby, in a location from which they can be cloned relative to
+// top-level directory of the current project.
+//
+// The location to such local repos can only be known in advance by the
+// relative path **from the Terraform module** that is "calling" a module
+// whose source comes from such a repo:
+//
+//     module "my_module" {
+//         source = "git::../../../git-submodules/tf-modules/my-tf-module//some/subdir?rev=v1.2.3"
+//         // ...
+//     }
+//
+// To locate such repos, we must provide go-getter with the absolute path to
+// the directory for the Terraform module that contains such references.
+//
+func (i *ModuleInstaller) calledFromSourceDirAbs(req *earlyconfig.ModuleRequest) (string, tfdiags.Diagnostics) {
+	callingModSourceDir := filepath.Dir(req.CallPos.Filename) // dirname
+
+	var diags tfdiags.Diagnostics
+
+	abs, err := filepath.Abs(callingModSourceDir)
+	if err != nil {
+		diags = diags.Append(tfdiags.Sourceless(
+			tfdiags.Error,
+			"Unable to resolve absolute filepath of Terraform module",
+			fmt.Sprintf(
+				"Terraform tried resolve the absolute filepath of source file \"%s\", but encountered an error: %s",
+				callingModSourceDir, err,
+			),
+		))
+	}
+
+	return abs, diags
+}