Add functionality to copy to/from the droplet with a custom API, add …

…documentation

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,15 @@
+## Type of Change
+<!-- What part of the source are you modifying? -->
+
+## What OS are you on?
+<!-- Please place your operating systen here -->
+
+## What issue does this relate to?
+<!-- Use a GitHub keyword ('resolves #xx', 'fixes #xx', 'closes #xx') to automatically close the relevant issue. -->
+
+### What should this PR do?
+<!-- Write a quick bullet point summary of the changes this PR should be making. -->
+
+### What are the acceptance criteria?
+<!-- Write a list of what should reviewers be checking before they approve this PR. -->
+<!-- If there are UI changes, include before and after screenshots in a table for comparison. -->

.github/workflows/build.yml

@@ -17,4 +17,4 @@ jobs:
         uses: actions/upload-artifact@v1
         with:
           name: builds
-          path: dist
+          path: cli-dist

.gitignore

@@ -16,3 +16,7 @@
 
 # IDEA data.
 .idea/
+
+# Dist's
+cli-dist
+droplet-tools-dist

README.md

@@ -9,4 +9,36 @@ The following sub-commands are included:
 - `setsize`: Allows you to modify the droplet size. Note that you need to go through the setup with do-disposable auth first (that will also configure this for the first time).
 - `up`: Allows you to start up a new disposable droplet.
 
-Note that by default, `up` will use the default values from your configuration and the latest Debian release for your distro. However, you can override this with flags (see `do-disposable up -h` for details).
+Additionally, when deploying the droplet, the following commands are deployed to the droplet:
+- `copyfrom <host file/folder path> [droplet save location]`: Allows you to copy a file/folder from the host to the droplet.
+- `copyback <droplet file/folder path> [host save location]`: Allows you to copy a file/folder back from the droplet.
+
+## Authentication
+To authenticate on first boot (or to change your SSH key/token), you can run `do-disposable auth`. When you run this, you will be prompted for your DigitalOcean token:
+
+![token_prompt](https://i.imgur.com/lEnuaSL.png)
+
+You can get this from the API section of the DigitalOcean dashboard:
+![api](https://i.imgur.com/naOZtuJ.png) ![token](https://i.imgur.com/VRWmHsH.png)
+
+If this is your first launch, this will also run [set region](#set-region) and [set size](#set-size) This will also automatically generate the SSH key which do-disposable will use internally.
+
+## Set region
+To set the region outside of the first boot, you can use `do-disposable setregion`. This will prompt you to hit the key of the region which you wish to use:
+
+![setregion](https://i.imgur.com/5t28FT0.png)
+
+## Set size
+To set the size outside of the first boot, you can use `do-disposable setsize`. This will prompt you to hit the key of the region which you wish to use:
+
+![setsize](https://i.imgur.com/Ao8mXFu.png)
+
+## Starting the droplet
+To start the droplet, you can use `do-disposable up`. Note that by default, `up` will use the default values from your configuration and the latest Debian release for your distro. The following flags can be used:
+- `distro`: Allows you to override the distro slug with another one from the DigitalOcean API. This defaults to the latest Debian release.
+- `region`: Allows you to override the region slug with another one. Will default to the one set above.
+- `size`: Allows you to override the size slug with another one. Will default to the one set above.
+
+From here, you can run Linux commands (including `copyfrom` and `copyback`) and then you can exit the droplet. Exiting will destroy the droplet:
+
+![session](https://i.imgur.com/UXxEv3w.png)

build.sh

@@ -1,16 +1,18 @@
 #!/usr/bin/env bash
 # Heavily based off https://www.digitalocean.com/community/tutorials/how-to-build-go-executables-for-multiple-platforms-on-ubuntu-16-04
 
-mkdir dist
+# 1) Build the do-disposable binary
 
-platforms=("windows/amd64" "windows/386" "windows/arm" "freebsd/amd64" "freebsd/386" "freebsd/arm" "freebsd/arm64" "linux/amd64" "linux/386" "linux/arm" "linux/arm64" "darwin/amd64")
+mkdir cli-dist
+
+platforms=("windows/amd64" "windows/386" "windows/arm" "freebsd/amd64" "freebsd/386" "freebsd/arm" "linux/amd64" "linux/386" "linux/arm" "linux/arm64" "darwin/amd64")
 
 for platform in "${platforms[@]}"
 do
     platform_split=(${platform//\// })
     GOOS=${platform_split[0]}
     GOARCH=${platform_split[1]}
-    output_name='./dist/do-disposable_'$GOOS'-'$GOARCH
+    output_name='./cli-dist/do-disposable_'$GOOS'-'$GOARCH
     if [ $GOOS = "windows" ]; then
         output_name+='.exe'
     fi
@@ -21,3 +23,34 @@ do
         exit 1
     fi
 done
+
+# 2) Build the copyback/copyfrom binaries
+
+platforms=("freebsd/amd64" "linux/amd64")
+
+mkdir droplet-tools-dist
+
+for platform in "${platforms[@]}"
+do
+    platform_split=(${platform//\// })
+    GOOS=${platform_split[0]}
+    GOARCH=${platform_split[1]}
+    output_name='../droplet-tools-dist/copyback_'$GOOS
+
+    cd copyback
+    env GOOS=$GOOS GOARCH=$GOARCH go build -o $output_name .
+    if [ $? -ne 0 ]; then
+        echo 'An error has occurred! Aborting the script execution...'
+        exit 1
+    fi
+    cd ..
+
+    output_name='../droplet-tools-dist/copyfrom_'$GOOS
+    cd copyback
+    env GOOS=$GOOS GOARCH=$GOARCH go build -o $output_name .
+    if [ $? -ne 0 ]; then
+        echo 'An error has occurred! Aborting the script execution...'
+        exit 1
+    fi
+    cd ..
+done

client_init.go

@@ -10,7 +10,7 @@ import (
 	"github.com/google/uuid"
 	"golang.org/x/crypto/ssh"
 	"os"
-	"path"
+	"path/filepath"
 	"strings"
 )
 
@@ -47,7 +47,7 @@ func loadConfig() (string, bool) {
 		// There's not much we can do here except crash.
 		panic(err)
 	}
-	fp := path.Join(homedir, ".do-disposable")
+	fp := filepath.Join(homedir, ".do-disposable")
 	f, err := os.Open(fp)
 
 	// Handle file being existent/readable.

copyback/README.md

@@ -0,0 +1,2 @@
+# copyback
+A simple tool for inside the Linux/FreeBSD droplet to copy data back to the host.

copyback/copyback.go

@@ -0,0 +1,154 @@
+package main
+
+import (
+	"bytes"
+	"encoding/gob"
+	"github.com/jakemakesstuff/structuredhttp"
+	"io"
+	"os"
+	"path/filepath"
+)
+
+// Used to gracefully error the application.
+func gracefulError(message string) {
+	println(message)
+	os.Exit(1)
+}
+
+// Used to handle the transfer to the host.
+func transferToHost(hostRelPath string, r io.Reader, len uint, perm os.FileMode) {
+	// Initialise the transfer.
+	type transferInit struct {
+		LocalPath string
+		TotalBytes uint
+		Perm os.FileMode
+	}
+	buf := &bytes.Buffer{}
+	encoder := gob.NewEncoder(buf)
+	err := encoder.Encode(&transferInit{
+		LocalPath:  hostRelPath,
+		TotalBytes: len,
+		Perm:       perm,
+	})
+	if err != nil {
+		panic(err)
+	}
+	resp, err := structuredhttp.POST("http://127.0.0.1:8190/v1/StartTransferSession").Reader(buf).Run()
+	if err != nil {
+		panic(err)
+	}
+	err = resp.RaiseForStatus()
+	if err != nil {
+		gracefulError(func() (text string) { text, _ = resp.Text(); return }())
+	}
+	transferId, err := resp.Text()
+	if err != nil {
+		panic(err)
+	}
+
+	// Chunk the transfer into 1MB blocks.
+	block := make([]byte, 1000000)
+	for {
+		// Read 1MB maximum.
+		n, err := r.Read(block)
+		if err != nil {
+			if err == io.EOF {
+				break
+			}
+			panic(err)
+		}
+		resizedBlock := block[:n]
+
+		// Upload the chunk.
+		resp, err := structuredhttp.POST("http://127.0.0.1:8190/v1/HandleFragment").Bytes(resizedBlock).Header("Transfer-ID", transferId).Run()
+		if err != nil {
+			panic(err)
+		}
+		err = resp.RaiseForStatus()
+		if err != nil {
+			gracefulError(func() (text string) { text, _ = resp.Text(); return }())
+		}
+	}
+}
+
+// Shows the command usage.
+func usage()  {
+	println("copyback - copy a file/folder back from the droplet")
+	println("usage: copyback <droplet file/folder path> [host save location]")
+	os.Exit(0)
+}
+
+// Used to process a file.
+func processFile(s os.FileInfo, dropletAbsPath, hostRelPath string) {
+	// Get the size for later.
+	size := s.Size()
+
+	// Create a reader for the file.
+	r, err := os.Open(dropletAbsPath)
+	if err != nil {
+		panic(err)
+	}
+	defer r.Close()
+
+	// Upload the file.
+	transferToHost(hostRelPath, r, uint(size), s.Mode())
+}
+
+// The main function.
+func main() {
+	// Check the arg count.
+	if len(os.Args) == 1 {
+		usage()
+	}
+
+	// Get the droplet path.
+	dropletPath := os.Args[1]
+	if dropletPath == "-h" {
+		usage()
+	}
+
+	// Stat the droplet file.
+	dropletAbsPath, err := filepath.Abs(dropletPath)
+	if err != nil {
+		gracefulError(err.Error())
+	}
+
+	// Check if the file exists.
+	s, err := os.Stat(dropletAbsPath)
+	if err != nil {
+		gracefulError(err.Error())
+		return
+	}
+
+	// Get the host relative path.
+	var hostRelPath string
+	if len(os.Args) > 2 {
+		hostRelPath = os.Args[2]
+	} else {
+		if filepath.Base(dropletAbsPath) == "." {
+			hostRelPath = "."
+		} else {
+			hostRelPath = "./"+filepath.Base(dropletAbsPath)
+		}
+	}
+
+	// Check if this is a folder.
+	if s.IsDir() {
+		err := filepath.Walk(dropletAbsPath, func(path string, s os.FileInfo, err error) error {
+			if s.IsDir() {
+				return nil
+			}
+			diff, err := filepath.Rel(dropletAbsPath, path)
+			if err != nil {
+				return err
+			}
+			processFile(s, path, filepath.Join(hostRelPath, diff))
+			return nil
+		})
+		if err != nil {
+			panic(err)
+		}
+	} else {
+		processFile(s, dropletAbsPath, hostRelPath)
+	}
+}

copyback/go.mod

@@ -0,0 +1,5 @@
+module github.com/JakeMakesStuff/do-disposable/copyback
+
+go 1.14
+
+require github.com/jakemakesstuff/structuredhttp v0.0.0-20200614104234-f8e4b2aebe68

copyback/go.sum

@@ -0,0 +1,2 @@
+github.com/jakemakesstuff/structuredhttp v0.0.0-20200614104234-f8e4b2aebe68 h1:vkIG58xJJFlq0UlieHO1YEGISr922HaN61dPPEyAVts=
+github.com/jakemakesstuff/structuredhttp v0.0.0-20200614104234-f8e4b2aebe68/go.mod h1:yWWc7Ao4LkCeYfEgk4nGTd3oLszff89CFyKS7gA9HMc=

copyfrom/README.md

@@ -0,0 +1,2 @@
+# copyfrom
+A simple tool for inside the Linux/FreeBSD droplet to copy data from the host.