feat: remote taskfiles (HTTP) (#1152)

* feat: remote taskfiles over http

* feat: allow insecure connections when --insecure flag is provided

* feat: better error handling for fetch errors

* fix: ensure cache directory always exists

* fix: setup logger before everything else

* feat: put remote taskfiles behind an experiment

* feat: --download and --offline flags for remote taskfiles

* feat: node.Read accepts a context

* feat: experiment docs

* chore: changelog

* chore: remove unused optional param from Node interface

* chore: tidy up and generalise NewNode function

* fix: use sha256 in remote checksum

* feat: --download by itself will not run a task

* feat: custom error if remote taskfiles experiment is not enabled

* refactor: BaseNode functional options and simplified FileNode

* fix: use hex encoding for checksum instead of b64

CHANGELOG.md

@@ -2,7 +2,10 @@
 
 ## Unreleased
 
-- Prep work for remote Taskfiles (#1316 by @pd93).
+- Prep work for Remote Taskfiles (#1316 by @pd93).
+- Added the
+  [Remote Taskfiles experiment](https://taskfile.dev/experiments/remote-taskfiles)
+  as a draft (#1152, #1317 by @pd93).
 
 ## v3.29.1 - 2023-08-26
 
@@ -42,7 +45,8 @@
 - Bug fixes were made to the
   [npm installation method](https://taskfile.dev/installation/#npm). (#1190, by
   @sounisi5011).
-- Added the [gentle force experiment](https://taskfile.dev/experiments) as a
+- Added the
+  [gentle force experiment](https://taskfile.dev/experiments/gentle-force) as a
   draft (#1200, #1216 by @pd93).
 - Added an `--experiments` flag to allow you to see which experiments are
   enabled (#1242 by @pd93).

args/args.go

@@ -8,7 +8,7 @@ import (
 
 // ParseV3 parses command line argument: tasks and global variables
 func ParseV3(args ...string) ([]taskfile.Call, *taskfile.Vars) {
-	var calls []taskfile.Call
+	calls := []taskfile.Call{}
 	globals := &taskfile.Vars{}
 
 	for _, arg := range args {
@@ -21,16 +21,12 @@ func ParseV3(args ...string) ([]taskfile.Call, *taskfile.Vars) {
 		globals.Set(name, taskfile.Var{Static: value})
 	}
 
-	if len(calls) == 0 {
-		calls = append(calls, taskfile.Call{Task: "default", Direct: true})
-	}
-
 	return calls, globals
 }
 
 // ParseV2 parses command line argument: tasks and vars of each task
 func ParseV2(args ...string) ([]taskfile.Call, *taskfile.Vars) {
-	var calls []taskfile.Call
+	calls := []taskfile.Call{}
 	globals := &taskfile.Vars{}
 
 	for _, arg := range args {
@@ -51,10 +47,6 @@ func ParseV2(args ...string) ([]taskfile.Call, *taskfile.Vars) {
 		}
 	}
 
-	if len(calls) == 0 {
-		calls = append(calls, taskfile.Call{Task: "default", Direct: true})
-	}
-
 	return calls, globals
 }
 

args/args_test.go

@@ -73,22 +73,16 @@ func TestArgsV3(t *testing.T) {
 			},
 		},
 		{
-			Args: nil,
-			ExpectedCalls: []taskfile.Call{
-				{Task: "default", Direct: true},
-			},
+			Args:          nil,
+			ExpectedCalls: []taskfile.Call{},
 		},
 		{
-			Args: []string{},
-			ExpectedCalls: []taskfile.Call{
-				{Task: "default", Direct: true},
-			},
+			Args:          []string{},
+			ExpectedCalls: []taskfile.Call{},
 		},
 		{
-			Args: []string{"FOO=bar", "BAR=baz"},
-			ExpectedCalls: []taskfile.Call{
-				{Task: "default", Direct: true},
-			},
+			Args:          []string{"FOO=bar", "BAR=baz"},
+			ExpectedCalls: []taskfile.Call{},
 			ExpectedGlobals: &taskfile.Vars{
 				OrderedMap: orderedmap.FromMapWithOrder(
 					map[string]taskfile.Var{
@@ -191,22 +185,16 @@ func TestArgsV2(t *testing.T) {
 			},
 		},
 		{
-			Args: nil,
-			ExpectedCalls: []taskfile.Call{
-				{Task: "default", Direct: true},
-			},
+			Args:          nil,
+			ExpectedCalls: []taskfile.Call{},
 		},
 		{
-			Args: []string{},
-			ExpectedCalls: []taskfile.Call{
-				{Task: "default", Direct: true},
-			},
+			Args:          []string{},
+			ExpectedCalls: []taskfile.Call{},
 		},
 		{
-			Args: []string{"FOO=bar", "BAR=baz"},
-			ExpectedCalls: []taskfile.Call{
-				{Task: "default", Direct: true},
-			},
+			Args:          []string{"FOO=bar", "BAR=baz"},
+			ExpectedCalls: []taskfile.Call{},
 			ExpectedGlobals: &taskfile.Vars{
 				OrderedMap: orderedmap.FromMapWithOrder(
 					map[string]taskfile.Var{

cmd/task/task.go

@@ -53,6 +53,7 @@ var flags struct {
 	listJson    bool
 	taskSort    string
 	status      bool
+	insecure    bool
 	force       bool
 	forceAll    bool
 	watch       bool
@@ -71,6 +72,8 @@ var flags struct {
 	interval    time.Duration
 	global      bool
 	experiments bool
+	download    bool
+	offline     bool
 }
 
 func main() {
@@ -112,6 +115,7 @@ func run() error {
 	pflag.BoolVarP(&flags.listJson, "json", "j", false, "Formats task list as JSON.")
 	pflag.StringVar(&flags.taskSort, "sort", "", "Changes the order of the tasks when listed. [default|alphanumeric|none].")
 	pflag.BoolVar(&flags.status, "status", false, "Exits with non-zero exit code if any of the given tasks is not up-to-date.")
+	pflag.BoolVar(&flags.insecure, "insecure", false, "Forces Task to download Taskfiles over insecure connections.")
 	pflag.BoolVarP(&flags.watch, "watch", "w", false, "Enables watch of the given task.")
 	pflag.BoolVarP(&flags.verbose, "verbose", "v", false, "Enables verbose mode.")
 	pflag.BoolVarP(&flags.silent, "silent", "s", false, "Disables echoing.")
@@ -140,6 +144,12 @@ func run() error {
 		pflag.BoolVarP(&flags.forceAll, "force", "f", false, "Forces execution even when the task is up-to-date.")
 	}
 
+	// Remote Taskfiles experiment will adds the "download" and "offline" flags
+	if experiments.RemoteTaskfiles {
+		pflag.BoolVar(&flags.download, "download", false, "Downloads a cached version of a remote Taskfile.")
+		pflag.BoolVar(&flags.offline, "offline", false, "Forces Task to only use local or cached Taskfiles.")
+	}
+
 	pflag.Parse()
 
 	if flags.version {
@@ -173,6 +183,10 @@ func run() error {
 		return nil
 	}
 
+	if flags.download && flags.offline {
+		return errors.New("task: You can't set both --download and --offline flags")
+	}
+
 	if flags.global && flags.dir != "" {
 		log.Fatal("task: You can't set both --global and --dir")
 		return nil
@@ -216,6 +230,9 @@ func run() error {
 	e := task.Executor{
 		Force:       flags.force,
 		ForceAll:    flags.forceAll,
+		Insecure:    flags.insecure,
+		Download:    flags.download,
+		Offline:     flags.offline,
 		Watch:       flags.watch,
 		Verbose:     flags.verbose,
 		Silent:      flags.silent,
@@ -278,6 +295,13 @@ func run() error {
 		calls, globals = args.ParseV2(tasksAndVars...)
 	}
 
+	// If there are no calls, run the default task instead
+	// Unless the download flag is specified, in which case we want to download
+	// the Taskfile and do nothing else
+	if len(calls) == 0 && !flags.download {
+		calls = append(calls, taskfile.Call{Task: "default", Direct: true})
+	}
+
 	globals.Set("CLI_ARGS", taskfile.Var{Static: cliArgs})
 	e.Taskfile.Vars.Merge(globals)
 

docs/docs/experiments/remote_taskfiles.md

@@ -0,0 +1,81 @@
+---
+slug: /experiments/remote-taskfiles/
+---
+
+# Remote Taskfiles
+
+- Issue: [#1317][remote-taskfiles-experiment]
+- Environment variable: `TASK_X_REMOTE_TASKFILES=1`
+
+This experiment allows you to specify a remote Taskfile URL when including a
+Taskfile. For example:
+
+```yaml
+version: '3'
+
+include:
+  my-remote-namespace: https://raw.githubusercontent.com/my-org/my-repo/main/Taskfile.yml
+```
+
+This works exactly the same way that including a local file does. Any tasks in
+the remote Taskfile will be available to run from your main Taskfile via the
+namespace `my-remote-namespace`. For example, if the remote file contains the
+following:
+
+```yaml
+version: '3'
+
+tasks:
+  hello:
+    silent: true
+    cmds:
+      - echo "Hello from the remote Taskfile!"
+```
+
+and you run `task my-remote-namespace:hello`, it will print the text: "Hello
+from the remote Taskfile!" to your console.
+
+## Security
+
+Running commands from sources that you do not control is always a potential
+security risk. For this reason, we have added some checks when using remote
+Taskfiles:
+
+1. When running a task from a remote Taskfile for the first time, Task will
+   print a warning to the console asking you to check that you are sure that you
+   trust the source of the Taskfile. If you do not accept the prompt, then Task
+   will exit with code `104` (not trusted) and nothing will run. If you accept
+   the prompt, the remote Taskfile will run and further calls to the remote
+   Taskfile will not prompt you again.
+2. Whenever you run a remote Taskfile, Task will create and store a checksum of
+   the file that you are running. If the checksum changes, then Task will print
+   another warning to the console to inform you that the contents of the remote
+   file has changed. If you do not accept the prompt, then Task will exit with
+   code `104` (not trusted) and nothing will run. If you accept the prompt, the
+   checksum will be updated and the remote Taskfile will run.
+
+Task currently supports both `http` and `https` URLs. However, the `http`
+requests will not execute by default unless you run the task with the
+`--insecure` flag. This is to protect you from accidentally running a remote
+Taskfile that is hosted on and unencrypted connection. Sources that are not
+protected by TLS are vulnerable to [man-in-the-middle
+attacks][man-in-the-middle-attacks] and should be avoided unless you know what
+you are doing.
+
+## Caching & Running Offline
+
+If for whatever reason, you don't have access to the internet, but you still
+need to be able to run your tasks, you are able to use the `--download` flag to
+store a cached copy of the remote Taskfile.
+
+<!-- TODO: The following behavior may change -->
+
+If Task detects that you have a local copy of the remote Taskfile, it will use
+your local copy instead of downloading the remote file. You can force Task to
+work offline by using the `--offline` flag. This will prevent Task from making
+any calls to remote sources.
+
+<!-- prettier-ignore-start -->
+[remote-taskfiles-experiment]: https://github.com/go-task/task/issues/1317
+[man-in-the-middle-attacks]: https://en.wikipedia.org/wiki/Man-in-the-middle_attack
+<!-- prettier-ignore-end -->

errors/errors.go

@@ -13,6 +13,10 @@ const (
 	CodeTaskfileNotFound int = iota + 100
 	CodeTaskfileAlreadyExists
 	CodeTaskfileInvalid
+	CodeTaskfileFetchFailed
+	CodeTaskfileNotTrusted
+	CodeTaskfileNotSecure
+	CodeTaskfileCacheNotFound
 )
 
 // Task related exit codes
@@ -40,3 +44,13 @@ type TaskError interface {
 func New(text string) error {
 	return errors.New(text)
 }
+
+// Is wraps the standard errors.Is function so that we don't need to alias that package.
+func Is(err, target error) bool {
+	return errors.Is(err, target)
+}
+
+// As wraps the standard errors.As function so that we don't need to alias that package.
+func As(err error, target any) bool {
+	return errors.As(err, target)
+}

errors/errors_taskfile.go

@@ -2,6 +2,7 @@ package errors
 
 import (
 	"fmt"
+	"net/http"
 )
 
 // TaskfileNotFoundError is returned when no appropriate Taskfile is found when
@@ -16,7 +17,7 @@ func (err TaskfileNotFoundError) Error() string {
 	if err.Walk {
 		walkText = " (or any of the parent directories)"
 	}
-	return fmt.Sprintf(`task: No Taskfile found at "%s"%s`, err.URI, walkText)
+	return fmt.Sprintf(`task: No Taskfile found at %q%s`, err.URI, walkText)
 }
 
 func (err TaskfileNotFoundError) Code() int {
@@ -49,3 +50,73 @@ func (err TaskfileInvalidError) Error() string {
 func (err TaskfileInvalidError) Code() int {
 	return CodeTaskfileInvalid
 }
+
+// TaskfileFetchFailedError is returned when no appropriate Taskfile is found when
+// searching the filesystem.
+type TaskfileFetchFailedError struct {
+	URI            string
+	HTTPStatusCode int
+}
+
+func (err TaskfileFetchFailedError) Error() string {
+	var statusText string
+	if err.HTTPStatusCode != 0 {
+		statusText = fmt.Sprintf(" with status code %d (%s)", err.HTTPStatusCode, http.StatusText(err.HTTPStatusCode))
+	}
+	return fmt.Sprintf(`task: Download of %q failed%s`, err.URI, statusText)
+}
+
+func (err TaskfileFetchFailedError) Code() int {
+	return CodeTaskfileFetchFailed
+}
+
+// TaskfileNotTrustedError is returned when the user does not accept the trust
+// prompt when downloading a remote Taskfile.
+type TaskfileNotTrustedError struct {
+	URI string
+}
+
+func (err *TaskfileNotTrustedError) Error() string {
+	return fmt.Sprintf(
+		`task: Taskfile %q not trusted by user`,
+		err.URI,
+	)
+}
+
+func (err *TaskfileNotTrustedError) Code() int {
+	return CodeTaskfileNotTrusted
+}
+
+// TaskfileNotSecureError is returned when the user attempts to download a
+// remote Taskfile over an insecure connection.
+type TaskfileNotSecureError struct {
+	URI string
+}
+
+func (err *TaskfileNotSecureError) Error() string {
+	return fmt.Sprintf(
+		`task: Taskfile %q cannot be downloaded over an insecure connection. You can override this by using the --insecure flag`,
+		err.URI,
+	)
+}
+
+func (err *TaskfileNotSecureError) Code() int {
+	return CodeTaskfileNotSecure
+}
+
+// TaskfileCacheNotFound is returned when the user attempts to use an offline
+// (cached) Taskfile but the files does not exist in the cache.
+type TaskfileCacheNotFound struct {
+	URI string
+}
+
+func (err *TaskfileCacheNotFound) Error() string {
+	return fmt.Sprintf(
+		`task: Taskfile %q was not found in the cache. Remove the --offline flag to use a remote copy or download it using the --download flag`,
+		err.URI,
+	)
+}
+
+func (err *TaskfileCacheNotFound) Code() int {
+	return CodeTaskfileCacheNotFound
+}