feat: Add autodiscover mode

.node-version

@@ -1 +1 @@
-20.8.1
+20.9.0

cmd/server.go

@@ -54,6 +54,7 @@ const (
 	AllowForkPRsFlag                 = "allow-fork-prs"
 	AllowRepoConfigFlag              = "allow-repo-config"
 	AtlantisURLFlag                  = "atlantis-url"
+	AutoDiscoverModeFlag             = "autodiscover-mode"
 	AutomergeFlag                    = "automerge"
 	ParallelPlanFlag                 = "parallel-plan"
 	ParallelApplyFlag                = "parallel-apply"
@@ -152,6 +153,7 @@ const (
 	DefaultADBasicUser                  = ""
 	DefaultADBasicPassword              = ""
 	DefaultADHostname                   = "dev.azure.com"
+	DefaultAutoDiscoverMode             = "auto"
 	DefaultAutoplanFileList             = "**/*.tf,**/*.tfvars,**/*.tfvars.json,**/terragrunt.hcl,**/.terraform.lock.hcl"
 	DefaultAllowCommands                = "version,plan,apply,unlock,approve_policies"
 	DefaultCheckoutStrategy             = CheckoutStrategyBranch
@@ -211,6 +213,12 @@ var stringFlags = map[string]stringFlag{
 	AtlantisURLFlag: {
 		description: "URL that Atlantis can be reached at. Defaults to http://$(hostname):$port where $port is from --" + PortFlag + ". Supports a base path ex. https://example.com/basepath.",
 	},
+	AutoDiscoverModeFlag: {
+		description: "Auto discover mode controls whether projects in a repo are discovered by Atlantis. Defaults to \"auto\" which " +
+			"means projects will be discovered when no explicit projects are defined in repo config. Also supports \"enabled\" (always " +
+			"discover projects) and \"disabled\" (never discover projects).",
+		defaultValue: DefaultAutoDiscoverMode,
+	},
 	AutoplanModulesFromProjects: {
 		description: "Comma separated list of file patterns to select projects Atlantis will index for module dependencies." +
 			" Indexed projects will automatically be planned if a module they depend on is modified." +
@@ -884,6 +892,9 @@ func (s *ServerCmd) setDefaults(c *server.UserConfig) {
 	if c.WebPassword == "" {
 		c.WebPassword = DefaultWebPassword
 	}
+	if c.AutoDiscoverModeFlag == "" {
+		c.AutoDiscoverModeFlag = DefaultAutoDiscoverMode
+	}
 }
 
 func (s *ServerCmd) validate(userConfig server.UserConfig) error {

cmd/server_test.go

@@ -60,6 +60,7 @@ var testFlags = map[string]interface{}{
 	AllowCommandsFlag:                "version,plan,unlock,import,approve_policies", // apply is disabled by DisableApply
 	AllowForkPRsFlag:                 true,
 	AllowRepoConfigFlag:              true,
+	AutoDiscoverModeFlag:             "auto",
 	AutomergeFlag:                    true,
 	AutoplanFileListFlag:             "**/*.tf,**/*.yml",
 	BitbucketBaseURLFlag:             "https://bitbucket-base-url.com",

go.mod

@@ -14,7 +14,7 @@ require (
 	github.com/golang-jwt/jwt/v5 v5.0.0
 	github.com/google/go-github/v54 v54.0.0
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510
-	github.com/google/uuid v1.3.1
+	github.com/google/uuid v1.4.0
 	github.com/gorilla/mux v1.8.0
 	github.com/gorilla/websocket v1.5.0
 	github.com/hashicorp/go-getter/v2 v2.2.1
@@ -43,7 +43,7 @@ require (
 	github.com/urfave/negroni/v3 v3.0.0
 	github.com/warrensbox/terraform-switcher v0.1.1-0.20221027055942-201c8e92e997
 	github.com/xanzy/go-gitlab v0.93.2
-	go.etcd.io/bbolt v1.3.7
+	go.etcd.io/bbolt v1.3.8
 	go.uber.org/zap v1.26.0
 	golang.org/x/term v0.13.0
 	golang.org/x/text v0.13.0

go.sum

@@ -228,8 +228,8 @@ github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaU
 github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ=
 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4=
-github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4=
+github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
 github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g=
@@ -467,8 +467,8 @@ github.com/yuin/gopher-lua v1.1.0 h1:BojcDhfyDWgU2f2TOzYK/g5p2gxMrku8oupLDqlnSqE
 github.com/yuin/gopher-lua v1.1.0/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw=
 github.com/zclconf/go-cty v1.13.2 h1:4GvrUxe/QUDYuJKAav4EYqdM47/kZa672LwmXFmEKT0=
 github.com/zclconf/go-cty v1.13.2/go.mod h1:YKQzy/7pZ7iq2jNFzy5go57xdxdWoLLpaEp4u238AE0=
-go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ=
-go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw=
+go.etcd.io/bbolt v1.3.8 h1:xs88BrvEv273UsB79e0hcVrlUWmS0a8upikMFhSyAtA=
+go.etcd.io/bbolt v1.3.8/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
 go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=

runatlantis.io/docs/repo-level-atlantis-yaml.md

@@ -34,19 +34,27 @@ By default, this is not allowed.
 :::
 
 ::: warning
-Once an `atlantis.yaml` file exists in a repo, Atlantis won't try to determine
-where to run plan automatically. Instead it will just follow the project configuration.
-This means that you'll need to define each project in your repo.
+Once an `atlantis.yaml` file exists in a repo and one or more `projects` are configured,
+Atlantis won't try to determine where to run plan automatically. Instead it will just
+follow the project configuration. This means that you'll need to define each project
+in your repo.
 
 If you have many directories with Terraform configuration, each directory will
 need to be defined.
+
+This behavior can be overriden by setting `autodiscover.mode` to
+`enabled` in which case Atlantis will still try to discover projects which were not
+explicitly configured. If the directory of any discovered project conflicts with a
+manually configured project, the manually configured project will take precedence.
 :::
 
 ## Example Using All Keys
 
 ```yaml
 version: 3
 automerge: true
+autodiscover:
+  mode: auto
 delete_source_branch_on_merge: true
 parallel_plan: true
 parallel_apply: true
@@ -281,6 +289,34 @@ in each group one by one.
 If any plan/apply fails and `abort_on_execution_order_fail` is set to true on a repo level, all the 
 following groups will be aborted. For this example, if project2 fails then project1 will not run.
 
+### Autodiscovery Config
+```yaml
+autodiscover:
+  mode: "auto"
+```
+The above is the default configuration for `autodiscover.mode`. When `autodiscover.mode` is auto,
+projects will be discovered only if the repo has no `projects` configured.
+
+```yaml
+autodiscover:
+  mode: "disabled"
+```
+With the config above, Atlantis will never try to discover projects, even when there are no
+`projects` configured. This is useful if dynamically generating Atlantis config in pre_workflow hooks.
+See [Dynamic Repo Config Generation](pre-workflow-hooks.html#dynamic-repo-config-generation).
+
+```yaml
+autodiscover:
+  mode: "enabled"
+```
+With the config above, Atlantis will unconditionally try to discover projects based on modified_files,
+even when the directory of the project is missing from the configured `projects` in the repo configuration.
+If a discovered project has the same directory as a project which was manually configured in `projects`,
+the manual configuration will take precedence.
+
+Use this feature when some projects require specific configuration in a repo with many projects yet
+it's still desirable for Atlantis to plan/apply for projects not enumerated in the config.
+
 ### Custom Backend Config
 See [Custom Workflow Use Cases: Custom Backend Config](custom-workflows.html#custom-backend-config)
 

runatlantis.io/docs/server-configuration.md

@@ -106,6 +106,22 @@ Values are chosen in this order:
   * If a load balancer with a non http/https port (not the one defined in the `--port` flag) is used, update the URL to include the port like in the example above.
    * This URL is used as the `details` link next to each atlantis job to view the job's logs.
 
+### `--autodiscover-mode`
+  ```bash
+  atlantis server --autodiscover-mode="<auto|enabled|disabled>"
+  # or
+  ATLANTIS_AUTODISCOVER_MODE="<auto|enabled|disabled>"
+  ```
+  Sets auto discover mode, default is "auto". When set to "auto", projects in a repo will be discovered by
+  Atlantis when there are no projects configured in the repo config. If one or more projects are defined
+  in the repo config then auto discovery will be completely disabled.
+
+  When set to "enabled" projects will be discovered unconditionally. If an auto discovered project is already
+  defined in the projects section of the repo config, the project from the repo config will take precedence over
+  the auto discovered project.
+
+  When set to "disabled" projects will never be discovered, even if there are no projects configured in the repo config.
+
 ### `--automerge`
   ```bash
   atlantis server --automerge

runatlantis.io/docs/server-side-repo-config.md

@@ -88,6 +88,10 @@ repos:
   # policy_check defines if policy checking should be enable on this repository.
   policy_check: false
 
+  # autodiscover defines how atlantis should automatically discover projects in this repository.
+  autodiscover:
+    mode: auto
+
   # id can also be an exact match.
 - id: github.com/myorg/specific-repo
 
@@ -496,6 +500,7 @@ If you set a workflow with the key `default`, it will override this.
 | repo_locking                  | bool     | false   | no       | Whether or not to get a lock.                                                                                                                                                                                                                                                                             |
 | policy_check                  | bool     | false   | no       | Whether or not to run policy checks on this repository.                                                                                                                                                                                                                                                   |
 | custom_policy_check                  | bool     | false   | no       | Whether or not to enable custom policy check tools outside of Conftest on this repository.                                                                                                                                                                                                       |
+| autodiscover                  | AutoDiscover     | none   | no       | Auto discover settings for this repo
 
 
 :::tip Notes

server/controllers/events/events_controller_e2e_test.go

@@ -1326,6 +1326,7 @@ func setupE2E(t *testing.T, repoDir string, opt setupOption) (events_controllers
 		false,
 		false,
 		false,
+		"auto",
 		statsScope,
 		logger,
 		terraformClient,

server/core/config/parser_validator_test.go

@@ -1329,6 +1329,22 @@ func TestParseGlobalCfg(t *testing.T) {
   import_requirements: [invalid]`,
 			expErr: "repos: (0: (import_requirements: \"invalid\" is not a valid import_requirement, only \"approved\", \"mergeable\" and \"undiverged\" are supported.).).",
 		},
+		"disable autodiscover": {
+			input: `repos: 
+- id: /.*/
+  autodiscover:
+    mode: disabled`,
+			exp: valid.GlobalCfg{
+				Repos: []valid.Repo{
+					defaultCfg.Repos[0],
+					{
+						IDRegex:      regexp.MustCompile(".*"),
+						AutoDiscover: &valid.AutoDiscover{Mode: valid.AutoDiscoverDisabledMode},
+					},
+				},
+				Workflows: defaultCfg.Workflows,
+			},
+		},
 		"no workflows key": {
 			input: `repos: []`,
 			exp:   defaultCfg,
@@ -1404,13 +1420,17 @@ repos:
   allowed_overrides: [plan_requirements, apply_requirements, import_requirements, workflow, delete_source_branch_on_merge]
   allow_custom_workflows: true
   policy_check: true
+  autodiscover:
+    mode: enabled
 - id: /.*/
   branch: /(master|main)/
   pre_workflow_hooks:
     - run: custom workflow command
   post_workflow_hooks:
     - run: custom workflow command
   policy_check: false
+  autodiscover:
+    mode: disabled
 workflows:
   custom1:
     plan:
@@ -1457,13 +1477,15 @@ policies:
 						AllowedOverrides:     []string{"plan_requirements", "apply_requirements", "import_requirements", "workflow", "delete_source_branch_on_merge"},
 						AllowCustomWorkflows: Bool(true),
 						PolicyCheck:          Bool(true),
+						AutoDiscover:         &valid.AutoDiscover{Mode: valid.AutoDiscoverEnabledMode},
 					},
 					{
 						IDRegex:           regexp.MustCompile(".*"),
 						BranchRegex:       regexp.MustCompile("(master|main)"),
 						PreWorkflowHooks:  preWorkflowHooks,
 						PostWorkflowHooks: postWorkflowHooks,
 						PolicyCheck:       Bool(false),
+						AutoDiscover:      &valid.AutoDiscover{Mode: valid.AutoDiscoverDisabledMode},
 					},
 				},
 				Workflows: map[string]valid.Workflow{
@@ -1574,6 +1596,7 @@ workflows:
 						RepoLocking:               Bool(true),
 						PolicyCheck:               Bool(false),
 						CustomPolicyCheck:         Bool(false),
+						AutoDiscover:              raw.DefaultAutoDiscover(),
 					},
 				},
 				Workflows: map[string]valid.Workflow{
@@ -1727,7 +1750,10 @@ func TestParserValidator_ParseGlobalCfgJSON(t *testing.T) {
       "allowed_workflows": ["custom"],
       "apply_requirements": ["mergeable", "approved"],
       "allowed_overrides": ["workflow", "apply_requirements"],
-      "allow_custom_workflows": true
+      "allow_custom_workflows": true,
+      "autodiscover": {
+        "mode": "enabled"
+      }
     },
     {
       "id": "github.com/owner/repo"
@@ -1792,13 +1818,15 @@ func TestParserValidator_ParseGlobalCfgJSON(t *testing.T) {
 						AllowedWorkflows:     []string{"custom"},
 						AllowedOverrides:     []string{"workflow", "apply_requirements"},
 						AllowCustomWorkflows: Bool(true),
+						AutoDiscover:         &valid.AutoDiscover{Mode: valid.AutoDiscoverEnabledMode},
 					},
 					{
 						ID:                   "github.com/owner/repo",
 						IDRegex:              nil,
 						ApplyRequirements:    nil,
 						AllowedOverrides:     nil,
 						AllowCustomWorkflows: nil,
+						AutoDiscover:         nil,
 					},
 				},
 				Workflows: map[string]valid.Workflow{

server/core/config/raw/autodiscover.go

@@ -0,0 +1,38 @@
+package raw
+
+import (
+	validation "github.com/go-ozzo/ozzo-validation"
+	"github.com/runatlantis/atlantis/server/core/config/valid"
+)
+
+var DefaultAutoDiscoverMode = valid.AutoDiscoverAutoMode
+
+type AutoDiscover struct {
+	Mode *valid.AutoDiscoverMode `yaml:"mode,omitempty"`
+}
+
+func (a AutoDiscover) ToValid() *valid.AutoDiscover {
+	var v valid.AutoDiscover
+
+	if a.Mode != nil {
+		v.Mode = *a.Mode
+	} else {
+		v.Mode = DefaultAutoDiscoverMode
+	}
+
+	return &v
+}
+
+func (a AutoDiscover) Validate() error {
+	res := validation.ValidateStruct(&a,
+		// If a.Mode is nil, this should still pass validation.
+		validation.Field(&a.Mode, validation.In(valid.AutoDiscoverAutoMode, valid.AutoDiscoverDisabledMode, valid.AutoDiscoverEnabledMode)),
+	)
+	return res
+}
+
+func DefaultAutoDiscover() *valid.AutoDiscover {
+	return &valid.AutoDiscover{
+		Mode: DefaultAutoDiscoverMode,
+	}
+}