From 56135b556176f79777c1cbce3c63df80ca02eded Mon Sep 17 00:00:00 2001 From: Ankit <4nkitd@gmail.com> Date: Fri, 24 Jun 2022 12:49:20 +0530 Subject: [PATCH] application --- .github/workflows/go.yml | 26 ++++++ .github/workflows/goreleaser.yml | 25 ++++++ .gitignore | 20 +++++ DEVELOPMENT | 29 ++++++ Makefile | 7 ++ README.md | 110 +++++++++++++++++++++++ archive/archive.go | 63 +++++++++++++ archive/artchive_test.go | 42 +++++++++ compressor/base.go | 65 ++++++++++++++ compressor/base_test.go | 41 +++++++++ compressor/tar.go | 36 ++++++++ compressor/tar_test.go | 20 +++++ compressor/tgz.go | 36 ++++++++ compressor/tgz_test.go | 20 +++++ config/config.go | 150 +++++++++++++++++++++++++++++++ config/config_test.go | 72 +++++++++++++++ database/base.go | 85 ++++++++++++++++++ database/base_test.go | 58 ++++++++++++ database/mongodb.go | 113 +++++++++++++++++++++++ database/mongodb_test.go | 60 +++++++++++++ database/mysql.go | 88 ++++++++++++++++++ database/mysql_test.go | 93 +++++++++++++++++++ database/postgresql.go | 83 +++++++++++++++++ database/redis.go | 146 ++++++++++++++++++++++++++++++ encryptor/base.go | 53 +++++++++++ encryptor/open_ssl.go | 53 +++++++++++ encryptor/open_ssl_test.go | 26 ++++++ go.mod | 30 +++++++ go.sum | 89 ++++++++++++++++++ gobackup_test.yml | 104 +++++++++++++++++++++ goreleaser.yml | 13 +++ helper/exec.go | 53 +++++++++++ helper/exec_test.go | 29 ++++++ helper/filepath.go | 35 ++++++++ helper/filepath_test.go | 44 +++++++++ helper/utils.go | 29 ++++++ helper/utils_test.go | 23 +++++ install | 17 ++++ logger/logger.go | 54 +++++++++++ main.go | 78 ++++++++++++++++ model/model.go | 75 ++++++++++++++++ storage/base.go | 80 +++++++++++++++++ storage/base_test.go | 19 ++++ storage/cycler.go | 109 ++++++++++++++++++++++ storage/cycler_test.go | 44 +++++++++ storage/ftp.go | 91 +++++++++++++++++++ storage/local.go | 39 ++++++++ storage/oss.go | 102 +++++++++++++++++++++ storage/s3.go | 92 +++++++++++++++++++ storage/scp.go | 113 +++++++++++++++++++++++ 50 files changed, 2982 insertions(+) create mode 100644 .github/workflows/go.yml create mode 100644 .github/workflows/goreleaser.yml create mode 100644 .gitignore create mode 100644 DEVELOPMENT create mode 100644 Makefile create mode 100644 README.md create mode 100644 archive/archive.go create mode 100644 archive/artchive_test.go create mode 100644 compressor/base.go create mode 100644 compressor/base_test.go create mode 100644 compressor/tar.go create mode 100644 compressor/tar_test.go create mode 100644 compressor/tgz.go create mode 100644 compressor/tgz_test.go create mode 100644 config/config.go create mode 100644 config/config_test.go create mode 100644 database/base.go create mode 100644 database/base_test.go create mode 100644 database/mongodb.go create mode 100644 database/mongodb_test.go create mode 100644 database/mysql.go create mode 100644 database/mysql_test.go create mode 100644 database/postgresql.go create mode 100644 database/redis.go create mode 100644 encryptor/base.go create mode 100644 encryptor/open_ssl.go create mode 100644 encryptor/open_ssl_test.go create mode 100644 go.mod create mode 100644 go.sum create mode 100644 gobackup_test.yml create mode 100644 goreleaser.yml create mode 100644 helper/exec.go create mode 100644 helper/exec_test.go create mode 100644 helper/filepath.go create mode 100644 helper/filepath_test.go create mode 100644 helper/utils.go create mode 100644 helper/utils_test.go create mode 100644 install create mode 100644 logger/logger.go create mode 100644 main.go create mode 100644 model/model.go create mode 100644 storage/base.go create mode 100644 storage/base_test.go create mode 100644 storage/cycler.go create mode 100644 storage/cycler_test.go create mode 100644 storage/ftp.go create mode 100644 storage/local.go create mode 100644 storage/oss.go create mode 100644 storage/s3.go create mode 100644 storage/scp.go diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml new file mode 100644 index 0000000..996632a --- /dev/null +++ b/.github/workflows/go.yml @@ -0,0 +1,26 @@ +name: Go +on: [push] +jobs: + + build: + name: Build + runs-on: ubuntu-latest + env: + GO_ENV: test + steps: + + - name: Set up Go + uses: actions/setup-go@v1 + with: + go-version: 1.13 + id: go + + - name: Check out code into the Go module directory + uses: actions/checkout@v1 + + - name: Get dependencies + run: | + go get -v -t -d ./... + + - name: Test + run: go test ./... \ No newline at end of file diff --git a/.github/workflows/goreleaser.yml b/.github/workflows/goreleaser.yml new file mode 100644 index 0000000..6ba3392 --- /dev/null +++ b/.github/workflows/goreleaser.yml @@ -0,0 +1,25 @@ +name: goreleaser + +on: + push: + tags: + - "*" +jobs: + goreleaser: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Set up Go + uses: actions/setup-go@v2 + with: + go-version: 1.16 + - name: Run GoReleaser + uses: goreleaser/goreleaser-action@v2 + with: + version: latest + args: release --rm-dist + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..38bb028 --- /dev/null +++ b/.gitignore @@ -0,0 +1,20 @@ +# Binaries for programs and plugins +*.exe +*.dll +*.so +*.dylib + +# Test binary, build with `go test -c` +*.test + +# Output of the go coverage tool, specifically when used with LiteIDE +*.out + +# Project-local glide cache, RE: https://github.com/Masterminds/glide/issues/736 +.glide/ +gobackup.yml +gobackup +build/ +dist/ +vendor/ +log/ diff --git a/DEVELOPMENT b/DEVELOPMENT new file mode 100644 index 0000000..df12e57 --- /dev/null +++ b/DEVELOPMENT @@ -0,0 +1,29 @@ +## Install + +- Install Go 1.16.x + +``` +$ brew install goreleaser +$ brew install dep +``` + +## Get Source + +``` +$ git clone https://github.com/4nkitd/gobackup.git +$ cd gobackup +$ go get +``` + +## Test + +``` +$ make test +``` + +## Release new version + +``` +$ git tag 0.7.3 +$ make release +``` diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..a1fa317 --- /dev/null +++ b/Makefile @@ -0,0 +1,7 @@ +test: + GO_ENV=test go test ./... +run: + go run main.go -- perform -m demo -c ./gobackup_test.yml +release: + @rm -Rf dist/ + @goreleaser --skip-validate diff --git a/README.md b/README.md new file mode 100644 index 0000000..52afb02 --- /dev/null +++ b/README.md @@ -0,0 +1,110 @@ +# GoBackup is a fullstack backup tool +design for web servers similar with [backup/backup](https://github.com/backup/backup), work with Crontab to backup automatically. + +## Features + +- No dependencies. +- Multiple Databases source support. +- Multiple Storage type support. +- Archive paths or files into a tar. + +## Current Support status + +### Databases + +- MySQL +- PostgreSQL +- Redis - `mode: sync/copy` +- MongoDB + +### Archive + +Use `tar` command to archive many file or path into a `.tar` file. + +### Compressor + +- Tgz - `.tar.gz` +- Uncompressed - `.tar` + +### Encryptor + +- OpenSSL - `aes-256-cbc` encrypt + +### Storages + +- Local +- FTP +- SCP - Upload via SSH copy +- [Amazon S3](https://aws.amazon.com/s3) + +## Install (macOS / Linux) + +```bash +$ curl -sSL https://git.io/gobackup | bash +``` + +after that, you will get `/usr/local/bin/gobackup` command. + +```bash +$ gobackup -h +NAME: + gobackup - Easy full stack backup operations on UNIX-like systems + +USAGE: + gobackup [global options] command [command options] [arguments...] + +VERSION: + 1.0.0 + +COMMANDS: + perform + help, h Shows a list of commands or help for one command + +GLOBAL OPTIONS: + --help, -h show help + --version, -v print the version +``` + +## Configuration + +GoBackup will seek config files in: + +- ~/.gobackup/gobackup.yml +- /etc/gobackup/gobackup.yml + +Example config: [gobackup_test.yml](https://github.com/4nkitd/gobackup/blob/master/gobackup_test.yml) + +```yml +# gobackup config example +# ----------------------- +models: + JobName: + compress_with: + type: tgz + store_with: + type: s3 + keep: 20 + bucket: gobackup-test + region: ap-south-1 + path: backups + access_key_id: Ohsgwk86h2ksas + secret_access_key: Ojsiw729wujhKdhwsIIOw9173 + databases: + DbName: + type: mysql + host: localhost + port: 3306 + database: test + username: root + password: 123456 +``` + +## Backup Run + +You may want run backup in scheduly, you need Crontab: + +```bash +gobackup perform >> ~/.gobackup/gobackup.log +``` +And after a day, you can check up the execute status by `~/.gobackup/gobackup.log`. + diff --git a/archive/archive.go b/archive/archive.go new file mode 100644 index 0000000..9695585 --- /dev/null +++ b/archive/archive.go @@ -0,0 +1,63 @@ +package archive + +import ( + "fmt" + "path" + "path/filepath" + + "github.com/4nkitd/gobackup/config" + "github.com/4nkitd/gobackup/helper" + "github.com/4nkitd/gobackup/logger" +) + +// Run archive +func Run(model config.ModelConfig) (err error) { + if model.Archive == nil { + return nil + } + + logger.Info("------------- Archives -------------") + + helper.MkdirP(model.DumpPath) + + includes := model.Archive.GetStringSlice("includes") + includes = cleanPaths(includes) + + excludes := model.Archive.GetStringSlice("excludes") + excludes = cleanPaths(excludes) + + if len(includes) == 0 { + return fmt.Errorf("archive.includes have no config") + } + logger.Info("=> includes", len(includes), "rules") + + opts := options(model.DumpPath, excludes, includes) + helper.Exec("tar", opts...) + + logger.Info("------------- Archives -------------\n") + + return nil +} + +func options(dumpPath string, excludes, includes []string) (opts []string) { + tarPath := path.Join(dumpPath, "archive.tar") + if helper.IsGnuTar { + opts = append(opts, "--ignore-failed-read") + } + opts = append(opts, "-cPf", tarPath) + + for _, exclude := range excludes { + opts = append(opts, "--exclude="+filepath.Clean(exclude)) + } + + opts = append(opts, includes...) + + return opts +} + +func cleanPaths(paths []string) (results []string) { + for _, p := range paths { + results = append(results, filepath.Clean(p)) + } + return +} diff --git a/archive/artchive_test.go b/archive/artchive_test.go new file mode 100644 index 0000000..61cdaec --- /dev/null +++ b/archive/artchive_test.go @@ -0,0 +1,42 @@ +package archive + +import ( + "strings" + "testing" + + "github.com/4nkitd/gobackup/config" + "github.com/4nkitd/gobackup/helper" + "github.com/stretchr/testify/assert" +) + +func TestRun(t *testing.T) { + // with nil Archive + model := config.ModelConfig{ + Archive: nil, + } + err := Run(model) + assert.NoError(t, err) +} + +func TestOptions(t *testing.T) { + includes := []string{ + "/foo/bar/dar", + "/bar/foo", + "/ddd", + } + + excludes := []string{ + "/hello/world", + "/cc/111", + } + + dumpPath := "~/work/dir" + + opts := options(dumpPath, excludes, includes) + cmd := strings.Join(opts, " ") + if helper.IsGnuTar { + assert.Equal(t, cmd, "--ignore-failed-read -cPf ~/work/dir/archive.tar --exclude=/hello/world --exclude=/cc/111 /foo/bar/dar /bar/foo /ddd") + } else { + assert.Equal(t, cmd, "-cPf ~/work/dir/archive.tar --exclude=/hello/world --exclude=/cc/111 /foo/bar/dar /bar/foo /ddd") + } +} diff --git a/compressor/base.go b/compressor/base.go new file mode 100644 index 0000000..f0f1562 --- /dev/null +++ b/compressor/base.go @@ -0,0 +1,65 @@ +package compressor + +import ( + "os" + "path" + "time" + + "github.com/4nkitd/gobackup/config" + "github.com/4nkitd/gobackup/logger" + "github.com/spf13/viper" +) + +// Base compressor +type Base struct { + name string + model config.ModelConfig + viper *viper.Viper +} + +// Context compressor +type Context interface { + perform() (archivePath string, err error) +} + +func (ctx *Base) archiveFilePath(ext string) string { + return path.Join(ctx.model.TempPath, time.Now().Format("2006.01.02.15.04.05")+ext) +} + +func newBase(model config.ModelConfig) (base Base) { + base = Base{ + name: model.Name, + model: model, + viper: model.CompressWith.Viper, + } + return +} + +// Run compressor +func Run(model config.ModelConfig) (archivePath string, err error) { + base := newBase(model) + + var ctx Context + switch model.CompressWith.Type { + case "tgz": + ctx = &Tgz{Base: base} + case "tar": + ctx = &Tar{Base: base} + default: + ctx = &Tar{} + } + + logger.Info("------------ Compressor -------------") + logger.Info("=> Compress | " + model.CompressWith.Type) + + // set workdir + os.Chdir(path.Join(model.DumpPath, "../")) + archivePath, err = ctx.perform() + if err != nil { + return + } + logger.Info("->", archivePath) + logger.Info("------------ Compressor -------------\n") + + return +} diff --git a/compressor/base_test.go b/compressor/base_test.go new file mode 100644 index 0000000..79073b6 --- /dev/null +++ b/compressor/base_test.go @@ -0,0 +1,41 @@ +package compressor + +import ( + "path" + "strings" + "testing" + "time" + + "github.com/4nkitd/gobackup/config" + "github.com/stretchr/testify/assert" +) + +type Monkey struct { + Base +} + +func (ctx Monkey) perform() (archivePath string, err error) { + result := "aaa" + return result, nil +} + +func TestBase_archiveFilePath(t *testing.T) { + base := Base{} + prefixPath := path.Join(base.model.TempPath, time.Now().Format("2006.01.02.15.04")) + assert.True(t, strings.HasPrefix(base.archiveFilePath(".tar"), prefixPath)) + assert.True(t, strings.HasSuffix(base.archiveFilePath(".tar"), ".tar")) +} + +func TestBaseInterface(t *testing.T) { + model := config.ModelConfig{ + Name: "TestMoneky", + } + base := newBase(model) + assert.Equal(t, base.name, model.Name) + assert.Equal(t, base.model, model) + + ctx := Monkey{Base: base} + result, err := ctx.perform() + assert.Equal(t, result, "aaa") + assert.Nil(t, err) +} diff --git a/compressor/tar.go b/compressor/tar.go new file mode 100644 index 0000000..24c84b9 --- /dev/null +++ b/compressor/tar.go @@ -0,0 +1,36 @@ +package compressor + +import ( + "github.com/4nkitd/gobackup/helper" +) + +// Tar noop compressor +// +// type: tar (store only) +type Tar struct { + Base +} + +func (ctx *Tar) perform() (archivePath string, err error) { + filePath := ctx.archiveFilePath(".tar") + + opts := ctx.options() + opts = append(opts, filePath) + opts = append(opts, ctx.name) + + _, err = helper.Exec("tar", opts...) + if err == nil { + archivePath = filePath + return + } + return +} + +func (ctx *Tar) options() (opts []string) { + if helper.IsGnuTar { + opts = append(opts, "--ignore-failed-read") + } + opts = append(opts, "-cf") + + return +} diff --git a/compressor/tar_test.go b/compressor/tar_test.go new file mode 100644 index 0000000..fefa0ea --- /dev/null +++ b/compressor/tar_test.go @@ -0,0 +1,20 @@ +package compressor + +import ( + "testing" + + "github.com/4nkitd/gobackup/helper" + "github.com/stretchr/testify/assert" +) + +func TestTar_options(t *testing.T) { + ctx := &Tar{} + opts := ctx.options() + if helper.IsGnuTar { + assert.Equal(t, opts[0], "--ignore-failed-read") + assert.Equal(t, opts[1], "-cf") + } else { + assert.Equal(t, opts[0], "-cf") + } + +} diff --git a/compressor/tgz.go b/compressor/tgz.go new file mode 100644 index 0000000..9e1a25e --- /dev/null +++ b/compressor/tgz.go @@ -0,0 +1,36 @@ +package compressor + +import ( + "github.com/4nkitd/gobackup/helper" +) + +// Tgz .tar.gz compressor +// +// type: tgz +type Tgz struct { + Base +} + +func (ctx *Tgz) perform() (archivePath string, err error) { + filePath := ctx.archiveFilePath(".tar.gz") + + opts := ctx.options() + opts = append(opts, filePath) + opts = append(opts, ctx.name) + + _, err = helper.Exec("tar", opts...) + if err == nil { + archivePath = filePath + return + } + return +} + +func (ctx *Tgz) options() (opts []string) { + if helper.IsGnuTar { + opts = append(opts, "--ignore-failed-read") + } + opts = append(opts, "-zcf") + + return +} diff --git a/compressor/tgz_test.go b/compressor/tgz_test.go new file mode 100644 index 0000000..01dc80c --- /dev/null +++ b/compressor/tgz_test.go @@ -0,0 +1,20 @@ +package compressor + +import ( + "testing" + + "github.com/4nkitd/gobackup/helper" + "github.com/stretchr/testify/assert" +) + +func TestTgz_options(t *testing.T) { + ctx := &Tgz{} + opts := ctx.options() + if helper.IsGnuTar { + assert.Equal(t, opts[0], "--ignore-failed-read") + assert.Equal(t, opts[1], "-zcf") + } else { + assert.Equal(t, opts[0], "-zcf") + } + +} diff --git a/config/config.go b/config/config.go new file mode 100644 index 0000000..88d1548 --- /dev/null +++ b/config/config.go @@ -0,0 +1,150 @@ +package config + +import ( + "fmt" + "os" + "path" + "time" + + "github.com/4nkitd/gobackup/logger" + "github.com/spf13/viper" +) + +var ( + // Exist Is config file exist + Exist bool + // Models configs + Models []ModelConfig + // HomeDir of user + HomeDir = os.Getenv("HOME") +) + +// ModelConfig for special case +type ModelConfig struct { + Name string + TempPath string + DumpPath string + CompressWith SubConfig + EncryptWith SubConfig + StoreWith SubConfig + Archive *viper.Viper + Databases []SubConfig + Storages []SubConfig + Viper *viper.Viper +} + +// SubConfig sub config info +type SubConfig struct { + Name string + Type string + Viper *viper.Viper +} + +// loadConfig from: +// - ./gobackup.yml +// - ~/.gobackup/gobackup.yml +// - /etc/gobackup/gobackup.yml +func Init(configFile string) { + viper.SetConfigType("yaml") + + // set config file directly + if len(configFile) > 0 { + viper.SetConfigFile(configFile) + } else { + viper.SetConfigName("gobackup") + + // ./gobackup.yml + viper.AddConfigPath(".") + // ~/.gobackup/gobackup.yml + viper.AddConfigPath("$HOME/.gobackup") // call multiple times to add many search paths + // /etc/gobackup/gobackup.yml + viper.AddConfigPath("/etc/gobackup/") // path to look for the config file in + } + + err := viper.ReadInConfig() + if err != nil { + logger.Error("Load gobackup config faild", err) + return + } + + Exist = true + Models = []ModelConfig{} + for key := range viper.GetStringMap("models") { + Models = append(Models, loadModel(key)) + } +} + +func loadModel(key string) (model ModelConfig) { + model.Name = key + model.TempPath = path.Join(os.TempDir(), "gobackup", fmt.Sprintf("%d", time.Now().UnixNano())) + model.DumpPath = path.Join(model.TempPath, key) + model.Viper = viper.Sub("models." + key) + + model.CompressWith = SubConfig{ + Type: model.Viper.GetString("compress_with.type"), + Viper: model.Viper.Sub("compress_with"), + } + + model.EncryptWith = SubConfig{ + Type: model.Viper.GetString("encrypt_with.type"), + Viper: model.Viper.Sub("encrypt_with"), + } + + model.StoreWith = SubConfig{ + Type: model.Viper.GetString("store_with.type"), + Viper: model.Viper.Sub("store_with"), + } + + model.Archive = model.Viper.Sub("archive") + + loadDatabasesConfig(&model) + loadStoragesConfig(&model) + + return +} + +func loadDatabasesConfig(model *ModelConfig) { + subViper := model.Viper.Sub("databases") + for key := range model.Viper.GetStringMap("databases") { + dbViper := subViper.Sub(key) + model.Databases = append(model.Databases, SubConfig{ + Name: key, + Type: dbViper.GetString("type"), + Viper: dbViper, + }) + } +} + +func loadStoragesConfig(model *ModelConfig) { + subViper := model.Viper.Sub("storages") + for key := range model.Viper.GetStringMap("storages") { + dbViper := subViper.Sub(key) + model.Storages = append(model.Storages, SubConfig{ + Name: key, + Type: dbViper.GetString("type"), + Viper: dbViper, + }) + } +} + +// GetModelByName get model by name +func GetModelByName(name string) (model *ModelConfig) { + for _, m := range Models { + if m.Name == name { + model = &m + return + } + } + return +} + +// GetDatabaseByName get database config by name +func (model *ModelConfig) GetDatabaseByName(name string) (subConfig *SubConfig) { + for _, m := range model.Databases { + if m.Name == name { + subConfig = &m + return + } + } + return +} diff --git a/config/config_test.go b/config/config_test.go new file mode 100644 index 0000000..dde62a7 --- /dev/null +++ b/config/config_test.go @@ -0,0 +1,72 @@ +package config + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func init() { + Init("../gobackup_test.yml") +} + +func TestModelsLength(t *testing.T) { + assert.Equal(t, Exist, true) + assert.Len(t, Models, 5) +} + +func TestModel(t *testing.T) { + model := GetModelByName("base_test") + + assert.Equal(t, model.Name, "base_test") + + // compress_with + assert.Equal(t, model.CompressWith.Type, "tgz") + assert.NotNil(t, model.CompressWith.Viper) + + // encrypt_with + assert.Equal(t, model.EncryptWith.Type, "openssl") + assert.NotNil(t, model.EncryptWith.Viper) + + // store_with + assert.Equal(t, model.StoreWith.Type, "local") + assert.Equal(t, model.StoreWith.Viper.GetString("path"), "/Users/jason/Downloads/backup1") + + // databases + assert.Len(t, model.Databases, 3) + + // mysql + db := model.GetDatabaseByName("dummy_test") + assert.Equal(t, db.Name, "dummy_test") + assert.Equal(t, db.Type, "mysql") + assert.Equal(t, db.Viper.GetString("host"), "localhost") + assert.Equal(t, db.Viper.GetString("port"), "3306") + assert.Equal(t, db.Viper.GetString("database"), "dummy_test") + assert.Equal(t, db.Viper.GetString("username"), "root") + assert.Equal(t, db.Viper.GetString("password"), "123456") + + // redis + db = model.GetDatabaseByName("redis1") + assert.Equal(t, db.Name, "redis1") + assert.Equal(t, db.Type, "redis") + assert.Equal(t, db.Viper.GetString("mode"), "sync") + assert.Equal(t, db.Viper.GetString("rdb_path"), "/var/db/redis/dump.rdb") + assert.Equal(t, db.Viper.GetBool("invoke_save"), true) + assert.Equal(t, db.Viper.GetString("password"), "456123") + + // redis + db = model.GetDatabaseByName("postgresql") + assert.Equal(t, db.Name, "postgresql") + assert.Equal(t, db.Type, "postgresql") + assert.Equal(t, db.Viper.GetString("host"), "localhost") + + // archive + includes := model.Archive.GetStringSlice("includes") + assert.Len(t, includes, 4) + assert.Contains(t, includes, "/home/ubuntu/.ssh/") + assert.Contains(t, includes, "/etc/nginx/nginx.conf") + + excludes := model.Archive.GetStringSlice("excludes") + assert.Len(t, excludes, 2) + assert.Contains(t, excludes, "/home/ubuntu/.ssh/known_hosts") +} diff --git a/database/base.go b/database/base.go new file mode 100644 index 0000000..652f83d --- /dev/null +++ b/database/base.go @@ -0,0 +1,85 @@ +package database + +import ( + "fmt" + "path" + + "github.com/4nkitd/gobackup/config" + "github.com/4nkitd/gobackup/helper" + "github.com/4nkitd/gobackup/logger" + "github.com/spf13/viper" +) + +// Base database +type Base struct { + model config.ModelConfig + dbConfig config.SubConfig + viper *viper.Viper + name string + dumpPath string +} + +// Context database interface +type Context interface { + perform() error +} + +func newBase(model config.ModelConfig, dbConfig config.SubConfig) (base Base) { + base = Base{ + model: model, + dbConfig: dbConfig, + viper: dbConfig.Viper, + name: dbConfig.Name, + } + base.dumpPath = path.Join(model.DumpPath, dbConfig.Type, base.name) + helper.MkdirP(base.dumpPath) + return +} + +// New - initialize Database +func runModel(model config.ModelConfig, dbConfig config.SubConfig) (err error) { + base := newBase(model, dbConfig) + var ctx Context + switch dbConfig.Type { + case "mysql": + ctx = &MySQL{Base: base} + case "redis": + ctx = &Redis{Base: base} + case "postgresql": + ctx = &PostgreSQL{Base: base} + case "mongodb": + ctx = &MongoDB{Base: base} + default: + logger.Warn(fmt.Errorf("model: %s databases.%s config `type: %s`, but is not implement", model.Name, dbConfig.Name, dbConfig.Type)) + return + } + + logger.Info("=> database |", dbConfig.Type, ":", base.name) + + // perform + err = ctx.perform() + if err != nil { + return err + } + logger.Info("") + + return +} + +// Run databases +func Run(model config.ModelConfig) error { + if len(model.Databases) == 0 { + return nil + } + + logger.Info("------------- Databases -------------") + for _, dbCfg := range model.Databases { + err := runModel(model, dbCfg) + if err != nil { + return err + } + } + logger.Info("------------- Databases -------------\n") + + return nil +} diff --git a/database/base_test.go b/database/base_test.go new file mode 100644 index 0000000..40511e4 --- /dev/null +++ b/database/base_test.go @@ -0,0 +1,58 @@ +package database + +import ( + "fmt" + "testing" + + "github.com/4nkitd/gobackup/config" + "github.com/stretchr/testify/assert" +) + +func init() { + config.Init("../gobackup_test.yml") +} + +type Monkey struct { + Base +} + +func (ctx Monkey) perform() error { + if ctx.model.Name != "TestMonkey" { + return fmt.Errorf("Error") + } + if ctx.dbConfig.Name != "mysql1" { + return fmt.Errorf("Error") + } + return nil +} + +func TestBaseInterface(t *testing.T) { + base := Base{ + model: config.ModelConfig{ + Name: "TestMonkey", + }, + dbConfig: config.SubConfig{ + Name: "mysql1", + }, + } + ctx := Monkey{Base: base} + err := ctx.perform() + assert.Nil(t, err) +} + +func TestBase_newBase(t *testing.T) { + model := config.ModelConfig{ + DumpPath: "/tmp/gobackup/test", + } + dbConfig := config.SubConfig{ + Type: "mysql", + Name: "mysql-master", + } + base := newBase(model, dbConfig) + + assert.Equal(t, base.model, model) + assert.Equal(t, base.dbConfig, dbConfig) + assert.Equal(t, base.viper, dbConfig.Viper) + assert.Equal(t, base.name, "mysql-master") + assert.Equal(t, base.dumpPath, "/tmp/gobackup/test/mysql/mysql-master") +} diff --git a/database/mongodb.go b/database/mongodb.go new file mode 100644 index 0000000..233165f --- /dev/null +++ b/database/mongodb.go @@ -0,0 +1,113 @@ +package database + +import ( + "fmt" + "strings" + + "github.com/4nkitd/gobackup/helper" + "github.com/4nkitd/gobackup/logger" +) + +// MongoDB database +// +// type: mongodb +// host: 127.0.0.1 +// port: 27017 +// database: +// username: +// password: +// authdb: +// oplog: false +type MongoDB struct { + Base + host string + port string + database string + username string + password string + authdb string + oplog bool +} + +var ( + mongodumpCli = "mongodump" +) + +func (ctx *MongoDB) perform() (err error) { + viper := ctx.viper + viper.SetDefault("oplog", false) + viper.SetDefault("host", "127.0.0.1") + viper.SetDefault("username", "root") + viper.SetDefault("port", 27017) + + ctx.host = viper.GetString("host") + ctx.port = viper.GetString("port") + ctx.database = viper.GetString("database") + ctx.username = viper.GetString("username") + ctx.password = viper.GetString("password") + ctx.oplog = viper.GetBool("oplog") + ctx.authdb = viper.GetString("authdb") + + err = ctx.dump() + if err != nil { + return err + } + return nil +} + +func (ctx *MongoDB) mongodump() string { + return mongodumpCli + " " + + ctx.nameOption() + " " + + ctx.credentialOptions() + " " + + ctx.connectivityOptions() + " " + + ctx.oplogOption() + " " + + "--out=" + ctx.dumpPath +} + +func (ctx *MongoDB) nameOption() string { + return "--db=" + ctx.database +} + +func (ctx *MongoDB) credentialOptions() string { + opts := []string{} + if len(ctx.username) > 0 { + opts = append(opts, "--username="+ctx.username) + } + if len(ctx.password) > 0 { + opts = append(opts, `--password=`+ctx.password) + } + if len(ctx.authdb) > 0 { + opts = append(opts, "--authenticationDatabase="+ctx.authdb) + } + return strings.Join(opts, " ") +} + +func (ctx *MongoDB) connectivityOptions() string { + opts := []string{} + if len(ctx.host) > 0 { + opts = append(opts, "--host="+ctx.host+"") + } + if len(ctx.port) > 0 { + opts = append(opts, "--port="+ctx.port+"") + } + + return strings.Join(opts, " ") +} + +func (ctx *MongoDB) oplogOption() string { + if ctx.oplog { + return "--oplog" + } + + return "" +} + +func (ctx *MongoDB) dump() error { + out, err := helper.Exec(ctx.mongodump()) + if err != nil { + return fmt.Errorf("-> Dump error: %s", err) + } + logger.Info(out) + logger.Info("dump path:", ctx.dumpPath) + return nil +} diff --git a/database/mongodb_test.go b/database/mongodb_test.go new file mode 100644 index 0000000..25e579b --- /dev/null +++ b/database/mongodb_test.go @@ -0,0 +1,60 @@ +package database + +import ( + // "github.com/spf13/viper" + "github.com/stretchr/testify/assert" + "testing" +) + +func TestMongoDB_credentialOptions(t *testing.T) { + ctx := &MongoDB{ + username: "foo", + password: "bar", + authdb: "sssbbb", + } + + assert.Equal(t, ctx.credentialOptions(), "--username=foo --password=bar --authenticationDatabase=sssbbb") +} + +func TestMongoDB_connectivityOptions(t *testing.T) { + ctx := &MongoDB{ + host: "10.11.12.13", + port: "12345", + } + assert.Equal(t, ctx.connectivityOptions(), "--host=10.11.12.13 --port=12345") + + ctx = &MongoDB{ + host: "10.11.12.13", + } + assert.Equal(t, ctx.connectivityOptions(), "--host=10.11.12.13") + + ctx = &MongoDB{ + port: "1122", + } + assert.Equal(t, ctx.connectivityOptions(), "--port=1122") +} + +func TestMongoDB_oplogOption(t *testing.T) { + ctx := &MongoDB{oplog: true} + assert.Equal(t, ctx.oplogOption(), "--oplog") + ctx.oplog = false + assert.Equal(t, ctx.oplogOption(), "") +} + +func TestMongoDB_mongodump(t *testing.T) { + base := Base{ + dumpPath: "/tmp/gobackup/test", + } + ctx := &MongoDB{ + Base: base, + host: "127.0.0.1", + port: "4567", + database: "hello", + username: "foo", + password: "bar", + authdb: "sssbbb", + oplog: true, + } + expect := "mongodump --db=hello --username=foo --password=bar --authenticationDatabase=sssbbb --host=127.0.0.1 --port=4567 --oplog --out=/tmp/gobackup/test" + assert.Equal(t, ctx.mongodump(), expect) +} diff --git a/database/mysql.go b/database/mysql.go new file mode 100644 index 0000000..c1ad5db --- /dev/null +++ b/database/mysql.go @@ -0,0 +1,88 @@ +package database + +import ( + "fmt" + "path" + "strings" + + "github.com/4nkitd/gobackup/helper" + "github.com/4nkitd/gobackup/logger" +) + +// MySQL database +// +// type: mysql +// host: 127.0.0.1 +// port: 3306 +// database: +// username: root +// password: +// additional_options: +type MySQL struct { + Base + host string + port string + database string + username string + password string + additionalOptions []string +} + +func (ctx *MySQL) perform() (err error) { + viper := ctx.viper + viper.SetDefault("host", "127.0.0.1") + viper.SetDefault("username", "root") + viper.SetDefault("port", 3306) + + ctx.host = viper.GetString("host") + ctx.port = viper.GetString("port") + ctx.database = viper.GetString("database") + ctx.username = viper.GetString("username") + ctx.password = viper.GetString("password") + addOpts := viper.GetString("additional_options") + if len(addOpts) > 0 { + ctx.additionalOptions = strings.Split(addOpts, " ") + } + + // mysqldump command + if len(ctx.database) == 0 { + return fmt.Errorf("mysql database config is required") + } + + err = ctx.dump() + return +} + +func (ctx *MySQL) dumpArgs() []string { + dumpArgs := []string{} + if len(ctx.host) > 0 { + dumpArgs = append(dumpArgs, "--host", ctx.host) + } + if len(ctx.port) > 0 { + dumpArgs = append(dumpArgs, "--port", ctx.port) + } + if len(ctx.username) > 0 { + dumpArgs = append(dumpArgs, "-u", ctx.username) + } + if len(ctx.password) > 0 { + dumpArgs = append(dumpArgs, `-p`+ctx.password) + } + if len(ctx.additionalOptions) > 0 { + dumpArgs = append(dumpArgs, ctx.additionalOptions...) + } + + dumpArgs = append(dumpArgs, ctx.database) + dumpFilePath := path.Join(ctx.dumpPath, ctx.database+".sql") + dumpArgs = append(dumpArgs, "--result-file="+dumpFilePath) + return dumpArgs +} + +func (ctx *MySQL) dump() error { + logger.Info("-> Dumping MySQL...") + _, err := helper.Exec("mysqldump", ctx.dumpArgs()...) + if err != nil { + return fmt.Errorf("-> Dump error: %s", err) + } + logger.Info("dump path:", ctx.dumpPath) + return nil +} diff --git a/database/mysql_test.go b/database/mysql_test.go new file mode 100644 index 0000000..0cfa2e1 --- /dev/null +++ b/database/mysql_test.go @@ -0,0 +1,93 @@ +package database + +import ( + "github.com/4nkitd/gobackup/config" + // "github.com/spf13/viper" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestMySQL_dumpArgs(t *testing.T) { + base := newBase( + config.ModelConfig{ + DumpPath: "/tmp/gobackup/test", + }, + config.SubConfig{ + Type: "mysql", + Name: "mysql1", + }, + ) + mysql := &MySQL{ + Base: base, + database: "dummy_test", + host: "127.0.0.2", + port: "6378", + password: "aaaa", + } + + dumpArgs := mysql.dumpArgs() + assert.Equal(t, dumpArgs, []string{ + "--host", + "127.0.0.2", + "--port", + "6378", + "-paaaa", + "dummy_test", + "--result-file=/tmp/gobackup/test/mysql/mysql1/dummy_test.sql", + }) +} + +func TestMySQL_dumpArgsWithAdditionalOptions(t *testing.T) { + base := newBase( + config.ModelConfig{ + DumpPath: "/tmp/gobackup/test", + }, + config.SubConfig{ + Type: "mysql", + Name: "mysql1", + }, + ) + mysql := &MySQL{ + Base: base, + database: "dummy_test", + host: "127.0.0.2", + port: "6378", + password: "*&^92'", + additionalOptions: []string{ + "--single-transaction", + "--quick", + }, + } + + dumpArgs := mysql.dumpArgs() + assert.Equal(t, dumpArgs, []string{ + "--host", + "127.0.0.2", + "--port", + "6378", + "-p*&^92'", + "--single-transaction", + "--quick", + "dummy_test", + "--result-file=/tmp/gobackup/test/mysql/mysql1/dummy_test.sql", + }) +} + +func TestMySQLPerform(t *testing.T) { + model := config.GetModelByName("base_test") + assert.NotNil(t, model) + + dbConfig := model.GetDatabaseByName("dummy_test") + assert.NotNil(t, dbConfig) + + base := newBase(*model, *dbConfig) + mysql := &MySQL{Base: base} + + mysql.perform() + assert.Equal(t, mysql.database, "dummy_test") + assert.Equal(t, mysql.host, "localhost") + assert.Equal(t, mysql.port, "3306") + assert.Equal(t, mysql.username, "root") + assert.Equal(t, mysql.password, "123456") +} diff --git a/database/postgresql.go b/database/postgresql.go new file mode 100644 index 0000000..12bfbad --- /dev/null +++ b/database/postgresql.go @@ -0,0 +1,83 @@ +package database + +import ( + "fmt" + "os" + "path" + "strings" + + "github.com/4nkitd/gobackup/helper" + "github.com/4nkitd/gobackup/logger" +) + +// PostgreSQL database +// +// type: postgresql +// host: localhost +// port: 5432 +// database: test +// username: +// password: +type PostgreSQL struct { + Base + host string + port string + database string + username string + password string + dumpCommand string +} + +func (ctx PostgreSQL) perform() (err error) { + viper := ctx.viper + viper.SetDefault("host", "localhost") + viper.SetDefault("port", 5432) + + ctx.host = viper.GetString("host") + ctx.port = viper.GetString("port") + ctx.database = viper.GetString("database") + ctx.username = viper.GetString("username") + ctx.password = viper.GetString("password") + + if err = ctx.prepare(); err != nil { + return + } + + err = ctx.dump() + return +} + +func (ctx *PostgreSQL) prepare() (err error) { + // mysqldump command + dumpArgs := []string{} + if len(ctx.database) == 0 { + return fmt.Errorf("PostgreSQL database config is required") + } + if len(ctx.host) > 0 { + dumpArgs = append(dumpArgs, "--host="+ctx.host) + } + if len(ctx.port) > 0 { + dumpArgs = append(dumpArgs, "--port="+ctx.port) + } + if len(ctx.username) > 0 { + dumpArgs = append(dumpArgs, "--username="+ctx.username) + } + + ctx.dumpCommand = "pg_dump " + strings.Join(dumpArgs, " ") + " " + ctx.database + + return nil +} + +func (ctx *PostgreSQL) dump() error { + dumpFilePath := path.Join(ctx.dumpPath, ctx.database+".sql") + logger.Info("-> Dumping PostgreSQL...") + if len(ctx.password) > 0 { + os.Setenv("PGPASSWORD", ctx.password) + } + _, err := helper.Exec(ctx.dumpCommand, "-f", dumpFilePath) + if err != nil { + return err + } + logger.Info("dump path:", dumpFilePath) + return nil +} diff --git a/database/redis.go b/database/redis.go new file mode 100644 index 0000000..fe607f0 --- /dev/null +++ b/database/redis.go @@ -0,0 +1,146 @@ +package database + +import ( + "fmt" + "path" + "regexp" + "strings" + + "github.com/4nkitd/gobackup/helper" + "github.com/4nkitd/gobackup/logger" +) + +type redisMode int + +const ( + redisModeSync redisMode = iota + redisModeCopy +) + +// Redis database +// +// type: redis +// mode: sync # or copy for use rdb_path +// invoke_save: true +// host: 192.168.1.2 +// port: 6379 +// password: +// rdb_path: /var/db/redis/dump.rdb +type Redis struct { + Base + host string + port string + password string + mode redisMode + invokeSave bool + // path of rdb file, example: /var/lib/redis/dump.rdb + rdbPath string +} + +var ( + redisCliCommand = "redis-cli" +) + +func (ctx *Redis) perform() (err error) { + viper := ctx.viper + viper.SetDefault("rdb_path", "/var/db/redis/dump.rdb") + viper.SetDefault("host", "127.0.0.1") + viper.SetDefault("port", "6379") + viper.SetDefault("invoke_save", true) + viper.SetDefault("mode", "copy") + + ctx.host = viper.GetString("host") + ctx.port = viper.GetString("port") + ctx.password = viper.GetString("password") + ctx.rdbPath = viper.GetString("rdb_path") + ctx.invokeSave = viper.GetBool("invoke_save") + + if viper.GetString("mode") == "sync" { + ctx.mode = redisModeSync + } else { + ctx.mode = redisModeCopy + + if !helper.IsExistsPath(ctx.rdbPath) { + return fmt.Errorf("Redis RDB file: %s does not exist", ctx.rdbPath) + } + } + + if err = ctx.prepare(); err != nil { + return + } + + logger.Info("-> Invoke save...") + if err = ctx.save(); err != nil { + return + } + + if ctx.mode == redisModeCopy { + err = ctx.copy() + } else { + err = ctx.sync() + } + if err != nil { + return + } + + return +} + +func (ctx *Redis) prepare() error { + // redis-cli command + args := []string{"redis-cli"} + if len(ctx.host) > 0 { + args = append(args, "-h "+ctx.host) + } + if len(ctx.port) > 0 { + args = append(args, "-p "+ctx.port) + } + if len(ctx.password) > 0 { + args = append(args, `-a `+ctx.password) + } + redisCliCommand = strings.Join(args, " ") + + return nil +} + +func (ctx *Redis) save() error { + if !ctx.invokeSave { + return nil + } + // FIXME: add retry + logger.Info("Perform redis-cli save...") + out, err := helper.Exec(redisCliCommand, "SAVE") + if err != nil { + return fmt.Errorf("redis-cli SAVE failed %s", err) + } + + if !regexp.MustCompile("OK$").MatchString(strings.TrimSpace(out)) { + return fmt.Errorf(`Failed to invoke the "SAVE" command Response was: %s`, out) + } + + return nil +} + +func (ctx *Redis) sync() error { + dumpFilePath := path.Join(ctx.dumpPath, "dump.rdb") + logger.Info("Syncing redis dump to", dumpFilePath) + _, err := helper.Exec(redisCliCommand, "--rdb", dumpFilePath) + if err != nil { + return fmt.Errorf("dump redis error: %s", err) + } + + if !helper.IsExistsPath(dumpFilePath) { + return fmt.Errorf("dump result file %s not found", dumpFilePath) + } + + return nil +} + +func (ctx *Redis) copy() error { + logger.Info("Copying redis dump to", ctx.dumpPath) + _, err := helper.Exec("cp", ctx.rdbPath, ctx.dumpPath) + if err != nil { + return fmt.Errorf("copy redis dump file error: %s", err) + } + return nil +} diff --git a/encryptor/base.go b/encryptor/base.go new file mode 100644 index 0000000..7e73074 --- /dev/null +++ b/encryptor/base.go @@ -0,0 +1,53 @@ +package encryptor + +import ( + "github.com/4nkitd/gobackup/config" + "github.com/4nkitd/gobackup/logger" + "github.com/spf13/viper" +) + +// Base encryptor +type Base struct { + model config.ModelConfig + viper *viper.Viper + archivePath string +} + +// Context encryptor interface +type Context interface { + perform() (encryptPath string, err error) +} + +func newBase(archivePath string, model config.ModelConfig) (base Base) { + base = Base{ + archivePath: archivePath, + model: model, + viper: model.EncryptWith.Viper, + } + return +} + +// Run compressor +func Run(archivePath string, model config.ModelConfig) (encryptPath string, err error) { + base := newBase(archivePath, model) + var ctx Context + switch model.EncryptWith.Type { + case "openssl": + ctx = &OpenSSL{Base: base} + default: + encryptPath = archivePath + return + } + + logger.Info("------------ Encryptor -------------") + + logger.Info("=> Encrypt | " + model.EncryptWith.Type) + encryptPath, err = ctx.perform() + if err != nil { + return + } + logger.Info("->", encryptPath) + logger.Info("------------ Encryptor -------------\n") + + return +} diff --git a/encryptor/open_ssl.go b/encryptor/open_ssl.go new file mode 100644 index 0000000..22f570b --- /dev/null +++ b/encryptor/open_ssl.go @@ -0,0 +1,53 @@ +package encryptor + +import ( + "fmt" + + "github.com/4nkitd/gobackup/helper" +) + +// OpenSSL encryptor for use openssl aes-256-cbc +// +// - base64: false +// - salt: true +// - password: +type OpenSSL struct { + Base + salt bool + base64 bool + password string +} + +func (ctx *OpenSSL) perform() (encryptPath string, err error) { + sslViper := ctx.viper + sslViper.SetDefault("salt", true) + sslViper.SetDefault("base64", false) + + ctx.salt = sslViper.GetBool("salt") + ctx.base64 = sslViper.GetBool("base64") + ctx.password = sslViper.GetString("password") + + if len(ctx.password) == 0 { + err = fmt.Errorf("password option is required") + return + } + + encryptPath = ctx.archivePath + ".enc" + + opts := ctx.options() + opts = append(opts, "-in", ctx.archivePath, "-out", encryptPath) + _, err = helper.Exec("openssl", opts...) + return +} + +func (ctx *OpenSSL) options() (opts []string) { + opts = append(opts, "aes-256-cbc") + if ctx.base64 { + opts = append(opts, "-base64") + } + if ctx.salt { + opts = append(opts, "-salt") + } + opts = append(opts, `-k`, ctx.password) + return opts +} diff --git a/encryptor/open_ssl_test.go b/encryptor/open_ssl_test.go new file mode 100644 index 0000000..e82b797 --- /dev/null +++ b/encryptor/open_ssl_test.go @@ -0,0 +1,26 @@ +package encryptor + +import ( + "github.com/stretchr/testify/assert" + "strings" + "testing" +) + +func TestOpenSSL_options(t *testing.T) { + ctx := &OpenSSL{ + password: "foo(872", + salt: false, + base64: true, + } + + opts := strings.Join(ctx.options(), " ") + assert.Equal(t, opts, "aes-256-cbc -base64 -k foo(872") + + ctx.salt = true + opts = strings.Join(ctx.options(), " ") + assert.Equal(t, opts, "aes-256-cbc -base64 -salt -k foo(872") + + ctx.base64 = false + opts = strings.Join(ctx.options(), " ") + assert.Equal(t, opts, "aes-256-cbc -salt -k foo(872") +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..522005d --- /dev/null +++ b/go.mod @@ -0,0 +1,30 @@ +module github.com/4nkitd/gobackup + +go 1.16 + +require ( + github.com/aliyun/aliyun-oss-go-sdk v2.1.8+incompatible + github.com/aws/aws-sdk-go v1.15.77 + github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f // indirect + github.com/bramvdbogaerde/go-scp v0.0.0-20170919175937-e1fc87afa325 + github.com/fatih/color v0.0.0-20170523135355-570b54cabe6b + github.com/fsnotify/fsnotify v1.4.9 // indirect + github.com/hashicorp/hcl v1.0.0 // indirect + github.com/magiconair/properties v1.8.5 // indirect + github.com/mattn/go-colorable v0.1.8 // indirect + github.com/mitchellh/mapstructure v1.4.1 // indirect + github.com/pelletier/go-toml v1.9.0 // indirect + github.com/satori/go.uuid v1.2.0 // indirect + github.com/secsy/goftp v0.0.0-20170729073433-503caa01c039 + github.com/spf13/afero v1.6.0 // indirect + github.com/spf13/cast v1.3.1 // indirect + github.com/spf13/jwalterweatherman v1.1.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect + github.com/spf13/viper v0.0.0-20170723055207-25b30aa063fc + github.com/stretchr/testify v1.7.0 + github.com/urfave/cli v1.22.5 + golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 + golang.org/x/net v0.0.0-20201110031124-69a78807bb2b // indirect + golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba // indirect + gopkg.in/yaml.v2 v2.2.8 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..295ec60 --- /dev/null +++ b/go.sum @@ -0,0 +1,89 @@ +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/aliyun/aliyun-oss-go-sdk v2.1.8+incompatible h1:hLUNPbx10wawWW7DeNExvTrlb90db3UnnNTFKHZEFhE= +github.com/aliyun/aliyun-oss-go-sdk v2.1.8+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8= +github.com/aws/aws-sdk-go v1.15.77 h1:qlut2MDI5mRKllPC6grO5n9M8UhPQg1TIA9cYAkC/gc= +github.com/aws/aws-sdk-go v1.15.77/go.mod h1:E3/ieXAlvM0XWO57iftYVDLLvQ824smPP3ATZkfNZeM= +github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f h1:ZNv7On9kyUzm7fvRZumSyy/IUiSC7AzL0I1jKKtwooA= +github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f/go.mod h1:AuiFmCCPBSrqvVMvuqFuk0qogytodnVFVSN5CeJB8Gc= +github.com/bramvdbogaerde/go-scp v0.0.0-20170919175937-e1fc87afa325 h1:0XaLkziw9H6fdpGICLFtfj1xs1azezsRgpKtpvHd7x8= +github.com/bramvdbogaerde/go-scp v0.0.0-20170919175937-e1fc87afa325/go.mod h1:aiQFnN5G0MivefWD+J4Em1a+CDyu/UBEmbNP5+8Gtd4= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/fatih/color v0.0.0-20170523135355-570b54cabe6b h1:qLH16nRUej+wMVo5JLUb7U0kYfnDZhLfrwUVTKhAkoA= +github.com/fatih/color v0.0.0-20170523135355-570b54cabe6b/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8 h1:12VvqtR6Aowv3l/EQUlocDHW2Cp4G9WJVH7uyH8QFJE= +github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= +github.com/magiconair/properties v1.8.5 h1:b6kJs+EmPFMYGkow9GiUyCyOvIwYetYJ3fSaWak/Gls= +github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= +github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8= +github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag= +github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/pelletier/go-toml v1.9.0 h1:NOd0BRdOKpPf0SxkL3HxSQOG7rNh+4kl6PHcBPFs7Q0= +github.com/pelletier/go-toml v1.9.0/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= +github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= +github.com/secsy/goftp v0.0.0-20170729073433-503caa01c039 h1:wV2Mw1I95SyKKg8X0748WGPPcXo++GZHn8x3OtH00r4= +github.com/secsy/goftp v0.0.0-20170729073433-503caa01c039/go.mod h1:MnkX001NG75g3p8bhFycnyIjeQoOjGL6CEIsdE/nKSY= +github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/spf13/afero v1.6.0 h1:xoax2sJ2DT8S8xA2paPFjDCScCNeWsg75VG0DLRreiY= +github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= +github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng= +github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= +github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v0.0.0-20170723055207-25b30aa063fc h1:fRl5Uvk/CZpx+8r7rQAGL3S8dJLLdP+Qd5em4mF7bzI= +github.com/spf13/viper v0.0.0-20170723055207-25b30aa063fc/go.mod h1:A8kyI5cUJhb8N+3pkfONlcEcZbueH6nhAm0Fq7SrnBM= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/urfave/cli v1.22.5 h1:lNq9sAHXK2qfdI8W+GRItjCEkI+2oR4d+MEHy1CKXoU= +github.com/urfave/cli v1.22.5/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b h1:uwuIcX0g4Yl1NC5XAz37xsr2lTtcqevgzYNVt49waME= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f h1:+Nyd8tzPX9R7BWHguqsrbFdRx3WQ/1ib8I44HXV5yTA= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba h1:O8mE0/t419eoIwhTFpKVkHiTs/Igowgfkj25AcZrtiE= +golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/gobackup_test.yml b/gobackup_test.yml new file mode 100644 index 0000000..818f6c6 --- /dev/null +++ b/gobackup_test.yml @@ -0,0 +1,104 @@ +# gobackup config example +# ----------------------- +# Put this file in follow place: +# ~/.gobackup/gobackup.yml or /etc/gobackup/gobackup.yml +models: + base_test: + compress_with: + type: tgz + encrypt_with: + type: openssl + password: 123456 + salt: false + openssl: true + store_with: + type: local + keep: 10 + path: /Users/jason/Downloads/backup1 + databases: + dummy_test: + type: mysql + host: localhost + port: 3306 + database: dummy_test + username: root + password: 123456 + redis1: + type: redis + mode: sync + rdb_path: /var/db/redis/dump.rdb + invoke_save: true + password: 456123 + postgresql: + type: postgresql + host: localhost + archive: + includes: + - /home/ubuntu/.ssh/ + - /etc/nginx/nginx.conf + - /etc/redis/redis.conf + - /etc/logrotate.d/ + excludes: + - /home/ubuntu/.ssh/known_hosts + - /etc/logrotate.d/syslog + normal_files: + store_with: + type: scp + keep: 10 + path: ~/backup + host: your-host.com + private_key: ~/.ssh/id_rsa + username: ubuntu + password: password + timeout: 300 + test_model: + compress_with: + type: tgz + store_with: + type: ftp + keep: 15 + path: /backup1/foo + host: your-host.com + port: 21 + timeout: 30 + username: user1 + password: pass1 + test_s3: + compress_with: + type: tgz + store_with: + type: s3 + keep: 20 + bucket: gobackup-test + region: ap-southeast-1 + path: backups + access_key_id: Ohsgwk86h2ks + secret_access_key: Ojsiw729wujhKdhwsIIOw9173 + demo: + compress_with: + type: tgz + encrypt_with: + type: openssl + password: 123456 + salt: false + openssl: true + store_with: + type: local + keep: 10 + path: /Users/jason/Downloads/backup1 + databases: + redis1: + type: redis + mode: sync + rdb_path: /var/db/redis/dump.rdb + invoke_save: true + password: 456123 + archive: + includes: + - /home/ubuntu/.ssh/ + - /etc/nginx/nginx.conf + - /etc/redis/redis.conf + - /etc/logrotate.d/ + excludes: + - /home/ubuntu/.ssh/known_hosts + - /etc/logrotate.d/syslog diff --git a/goreleaser.yml b/goreleaser.yml new file mode 100644 index 0000000..c4964c3 --- /dev/null +++ b/goreleaser.yml @@ -0,0 +1,13 @@ +builds: + - binary: gobackup + ldflags: -s -w -X main.version={{.Version}} + goos: + - darwin + - linux + - windows + goarch: + - amd64 + - arm64 +archives: + - name_template: "{{ .ProjectName }}-{{ .Os }}-{{ .Arch }}" + format: tar.gz diff --git a/helper/exec.go b/helper/exec.go new file mode 100644 index 0000000..9f2b3cf --- /dev/null +++ b/helper/exec.go @@ -0,0 +1,53 @@ +package helper + +import ( + "bytes" + "errors" + "fmt" + "os" + "os/exec" + "regexp" + "strings" + + "github.com/4nkitd/gobackup/logger" +) + +var ( + spaceRegexp = regexp.MustCompile("[\\s]+") +) + +// Exec cli commands +func Exec(command string, args ...string) (output string, err error) { + commands := spaceRegexp.Split(command, -1) + command = commands[0] + commandArgs := []string{} + if len(commands) > 1 { + commandArgs = commands[1:] + } + if len(args) > 0 { + commandArgs = append(commandArgs, args...) + } + + fullCommand, err := exec.LookPath(command) + if err != nil { + return "", fmt.Errorf("%s cannot be found", command) + } + + cmd := exec.Command(fullCommand, commandArgs...) + cmd.Env = os.Environ() + + var stdErr bytes.Buffer + cmd.Stderr = &stdErr + + // logger.Debug(fullCommand, " ", strings.Join(commandArgs, " ")) + + out, err := cmd.Output() + if err != nil { + logger.Debug(fullCommand, " ", strings.Join(commandArgs, " ")) + err = errors.New(stdErr.String()) + return + } + + output = strings.Trim(string(out), "\n") + return +} diff --git a/helper/exec_test.go b/helper/exec_test.go new file mode 100644 index 0000000..4c4cd1a --- /dev/null +++ b/helper/exec_test.go @@ -0,0 +1,29 @@ +package helper + +import ( + "github.com/stretchr/testify/assert" + "testing" +) + +func TestExec(t *testing.T) { + res, err := Exec("head", "-n1", "./exec_test.go") + assert.Nil(t, err) + assert.Equal(t, res, "package helper") + + res, err = Exec("head -n1 ./exec_test.go") + assert.Nil(t, err) + assert.Equal(t, res, "package helper") + + res, err = Exec("head -n1 ./exec_test.go") + assert.Nil(t, err) + assert.Equal(t, res, "package helper") + + res, err = Exec("head -n1", "./exec_test.go") + assert.Nil(t, err) + assert.Equal(t, res, "package helper") + + res, err = Exec("not-found-command", "foo") + assert.NotNil(t, err) + assert.Equal(t, err.Error(), "not-found-command cannot be found") + assert.Empty(t, res) +} diff --git a/helper/filepath.go b/helper/filepath.go new file mode 100644 index 0000000..9d81994 --- /dev/null +++ b/helper/filepath.go @@ -0,0 +1,35 @@ +package helper + +import ( + "os" + "path" +) + +// IsExistsPath check path exist +func IsExistsPath(p string) bool { + _, err := os.Stat(p) + if err != nil { + return os.IsExist(err) + } + return true +} + +// MkdirP like mkdir -p +func MkdirP(dirPath string) { + if _, err := os.Stat(dirPath); os.IsNotExist(err) { + os.MkdirAll(dirPath, 0777) + } +} + +// ExplandHome ~/foo -> /home/jason/foo +func ExplandHome(filePath string) string { + if len(filePath) < 2 { + return filePath + } + + if filePath[:2] != "~/" { + return filePath + } + + return path.Join(os.Getenv("HOME"), filePath[2:]) +} diff --git a/helper/filepath_test.go b/helper/filepath_test.go new file mode 100644 index 0000000..00d75c7 --- /dev/null +++ b/helper/filepath_test.go @@ -0,0 +1,44 @@ +package helper + +import ( + "github.com/stretchr/testify/assert" + "os" + "path" + "testing" +) + +func TestIsExistsPath(t *testing.T) { + exist := IsExistsPath("foo/bar") + assert.False(t, exist) + + exist = IsExistsPath("./filepath_test.go") + assert.True(t, exist) +} + +func TestMkdirP(t *testing.T) { + dest := path.Join(os.TempDir(), "test-mkdir-p") + exist := IsExistsPath(dest) + assert.False(t, exist) + + MkdirP(dest) + defer os.Remove(dest) + exist = IsExistsPath(dest) + assert.True(t, exist) +} + +func TestExplandHome(t *testing.T) { + newPath := ExplandHome("") + assert.Equal(t, newPath, "") + + newPath = ExplandHome("/home/jason/111") + assert.Equal(t, newPath, "/home/jason/111") + + newPath = ExplandHome("~") + assert.Equal(t, newPath, "~") + + newPath = ExplandHome("~/") + assert.NotEqual(t, newPath[:2], "~/") + + newPath = ExplandHome("~/foo/bar/dar") + assert.Equal(t, newPath, path.Join(os.Getenv("HOME"), "/foo/bar/dar")) +} diff --git a/helper/utils.go b/helper/utils.go new file mode 100644 index 0000000..326fb41 --- /dev/null +++ b/helper/utils.go @@ -0,0 +1,29 @@ +package helper + +import ( + "strings" +) + +var ( + // IsGnuTar show tar type + IsGnuTar = false +) + +func init() { + checkIsGnuTar() +} + +func checkIsGnuTar() { + out, _ := Exec("tar", "--version") + IsGnuTar = strings.Contains(out, "GNU") +} + +// CleanHost clean host url ftp://foo.bar.com -> foo.bar.com +func CleanHost(host string) string { + // ftp://ftp.your-host.com -> ftp.your-host.com + if strings.Contains(host, "://") { + return strings.Split(host, "://")[1] + } + + return host +} diff --git a/helper/utils_test.go b/helper/utils_test.go new file mode 100644 index 0000000..efb80ce --- /dev/null +++ b/helper/utils_test.go @@ -0,0 +1,23 @@ +package helper + +import ( + "runtime" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestUtils_init(t *testing.T) { + if runtime.GOOS == "linux" { + assert.Equal(t, IsGnuTar, true) + } else { + assert.Equal(t, IsGnuTar, false) + } +} + +func TestCleanHost(t *testing.T) { + assert.Equal(t, "foo.bar.com", CleanHost("foo.bar.com")) + assert.Equal(t, "foo.bar.com", CleanHost("ftp://foo.bar.com")) + assert.Equal(t, "foo.bar.com", CleanHost("http://foo.bar.com")) + assert.Equal(t, "", CleanHost("http://")) +} diff --git a/install b/install new file mode 100644 index 0000000..e28c3ff --- /dev/null +++ b/install @@ -0,0 +1,17 @@ +version='v1.0.1' +if [[ `uname` == 'Darwin' ]]; then + platform='darwin' +else + platform='linux' +fi +curl -sSLo gobackup.tar.gz https://github.com/4nkitd/gobackup/releases/download/$version/gobackup-$platform-amd64.tar.gz +tar zxf gobackup.tar.gz + +if [[ `whoami` == 'root' ]]; then + mv gobackup /usr/local/bin/gobackup +else + sudo mv gobackup /usr/local/bin/gobackup +fi +mkdir -p ~/.gobackup && touch ~/.gobackup/gobackup.yml +rm gobackup.tar.gz + diff --git a/logger/logger.go b/logger/logger.go new file mode 100644 index 0000000..088e6ec --- /dev/null +++ b/logger/logger.go @@ -0,0 +1,54 @@ +package logger + +import ( + "fmt" + "github.com/fatih/color" + "log" + "os" +) + +var ( + logFlag = log.Ldate | log.Ltime + myLog = log.New(os.Stdout, "", logFlag) +) + +func init() { + isTest := os.Getenv("GO_ENV") == "test" + if isTest { + os.MkdirAll("../log", 0777) + logfile, _ := os.OpenFile("../log/test.log", os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666) + myLog = log.New(logfile, "", logFlag) + } +} + +// Print log +func Print(v ...interface{}) { + myLog.Print(v...) +} + +// Println log +func Println(v ...interface{}) { + myLog.Println(v...) +} + +// Debug log +func Debug(v ...interface{}) { + myLog.Println("[debug]", fmt.Sprint(v...)) +} + +// Info log +func Info(v ...interface{}) { + myLog.Println(v...) +} + +// Warn log +func Warn(v ...interface{}) { + c := color.YellowString(fmt.Sprint(v...)) + myLog.Println(c) +} + +// Error log +func Error(v ...interface{}) { + c := color.RedString(fmt.Sprint(v...)) + myLog.Println(c) +} diff --git a/main.go b/main.go new file mode 100644 index 0000000..59bedec --- /dev/null +++ b/main.go @@ -0,0 +1,78 @@ +package main + +import ( + "os" + + "github.com/4nkitd/gobackup/config" + "github.com/4nkitd/gobackup/model" + "github.com/urfave/cli" +) + +const ( + usage = "Easy full stack backup operations on UNIX-like systems" +) + +var ( + modelName = "" + configFile = "" + version = "master" +) + +func main() { + app := cli.NewApp() + app.Version = version + app.Name = "gobackup" + app.Usage = usage + + app.Commands = []cli.Command{ + { + Name: "perform", + Flags: []cli.Flag{ + cli.StringFlag{ + Name: "model, m", + Usage: "Model name that you want execute", + Destination: &modelName, + }, + cli.StringFlag{ + Name: "config, c", + Usage: "Special a config file", + Destination: &configFile, + }, + }, + Action: func(c *cli.Context) error { + config.Init(configFile) + + if len(modelName) == 0 { + performAll() + } else { + performOne(modelName) + } + + return nil + }, + }, + } + + app.Run(os.Args) +} + +func performAll() { + for _, modelConfig := range config.Models { + m := model.Model{ + Config: modelConfig, + } + m.Perform() + } +} + +func performOne(modelName string) { + for _, modelConfig := range config.Models { + if modelConfig.Name == modelName { + m := model.Model{ + Config: modelConfig, + } + m.Perform() + return + } + } +} diff --git a/model/model.go b/model/model.go new file mode 100644 index 0000000..8ee4379 --- /dev/null +++ b/model/model.go @@ -0,0 +1,75 @@ +package model + +import ( + "os" + + "github.com/4nkitd/gobackup/archive" + "github.com/4nkitd/gobackup/compressor" + "github.com/4nkitd/gobackup/config" + "github.com/4nkitd/gobackup/database" + "github.com/4nkitd/gobackup/encryptor" + "github.com/4nkitd/gobackup/logger" + "github.com/4nkitd/gobackup/storage" +) + +// Model class +type Model struct { + Config config.ModelConfig +} + +// Perform model +func (ctx Model) Perform() { + logger.Info("======== " + ctx.Config.Name + " ========") + logger.Info("WorkDir:", ctx.Config.DumpPath+"\n") + + defer func() { + if r := recover(); r != nil { + ctx.cleanup() + } + + ctx.cleanup() + }() + + err := database.Run(ctx.Config) + if err != nil { + logger.Error(err) + return + } + + if ctx.Config.Archive != nil { + err = archive.Run(ctx.Config) + if err != nil { + logger.Error(err) + return + } + } + + archivePath, err := compressor.Run(ctx.Config) + if err != nil { + logger.Error(err) + return + } + + archivePath, err = encryptor.Run(archivePath, ctx.Config) + if err != nil { + logger.Error(err) + return + } + + err = storage.Run(ctx.Config, archivePath) + if err != nil { + logger.Error(err) + return + } + +} + +// Cleanup model temp files +func (ctx Model) cleanup() { + logger.Info("Cleanup temp: " + ctx.Config.TempPath + "/\n") + err := os.RemoveAll(ctx.Config.TempPath) + if err != nil { + logger.Error("Cleanup temp dir "+ctx.Config.TempPath+" error:", err) + } + logger.Info("======= End " + ctx.Config.Name + " =======\n\n") +} diff --git a/storage/base.go b/storage/base.go new file mode 100644 index 0000000..1fcd623 --- /dev/null +++ b/storage/base.go @@ -0,0 +1,80 @@ +package storage + +import ( + "fmt" + "path/filepath" + + "github.com/4nkitd/gobackup/config" + "github.com/4nkitd/gobackup/logger" + "github.com/spf13/viper" +) + +// Base storage +type Base struct { + model config.ModelConfig + archivePath string + viper *viper.Viper + keep int +} + +// Context storage interface +type Context interface { + open() error + close() + upload(fileKey string) error + delete(fileKey string) error +} + +func newBase(model config.ModelConfig, archivePath string) (base Base) { + base = Base{ + model: model, + archivePath: archivePath, + viper: model.StoreWith.Viper, + } + + if base.viper != nil { + base.keep = base.viper.GetInt("keep") + } + + return +} + +// Run storage +func Run(model config.ModelConfig, archivePath string) (err error) { + logger.Info("------------- Storage --------------") + newFileKey := filepath.Base(archivePath) + base := newBase(model, archivePath) + var ctx Context + switch model.StoreWith.Type { + case "local": + ctx = &Local{Base: base} + case "ftp": + ctx = &FTP{Base: base} + case "scp": + ctx = &SCP{Base: base} + case "s3": + ctx = &S3{Base: base} + case "oss": + ctx = &OSS{Base: base} + default: + return fmt.Errorf("[%s] storage type has not implement", model.StoreWith.Type) + } + + logger.Info("=> Storage | " + model.StoreWith.Type) + err = ctx.open() + if err != nil { + return err + } + defer ctx.close() + + err = ctx.upload(newFileKey) + if err != nil { + return err + } + + cycler := Cycler{} + cycler.run(model.Name, newFileKey, base.keep, ctx.delete) + + logger.Info("------------- Storage --------------\n") + return nil +} diff --git a/storage/base_test.go b/storage/base_test.go new file mode 100644 index 0000000..e61f932 --- /dev/null +++ b/storage/base_test.go @@ -0,0 +1,19 @@ +package storage + +import ( + "testing" + + "github.com/4nkitd/gobackup/config" + "github.com/stretchr/testify/assert" +) + +func TestBase_newBase(t *testing.T) { + model := config.ModelConfig{} + archivePath := "/tmp/gobackup/test-storeage/foo.zip" + base := newBase(model, archivePath) + + assert.Equal(t, base.archivePath, archivePath) + assert.Equal(t, base.model, model) + assert.Equal(t, base.viper, model.Viper) + assert.Equal(t, base.keep, 0) +} diff --git a/storage/cycler.go b/storage/cycler.go new file mode 100644 index 0000000..f10b28c --- /dev/null +++ b/storage/cycler.go @@ -0,0 +1,109 @@ +package storage + +import ( + "encoding/json" + "io/ioutil" + "os" + "path" + "time" + + "github.com/4nkitd/gobackup/config" + "github.com/4nkitd/gobackup/helper" + "github.com/4nkitd/gobackup/logger" +) + +type PackageList []Package + +type Package struct { + FileKey string `json:"file_key"` + CreatedAt time.Time `json:"created_at"` +} + +var ( + cyclerPath = path.Join(config.HomeDir, ".gobackup/cycler") +) + +type Cycler struct { + packages PackageList + isLoaded bool +} + +func (c *Cycler) add(fileKey string) { + c.packages = append(c.packages, Package{ + FileKey: fileKey, + CreatedAt: time.Now(), + }) +} + +func (c *Cycler) shiftByKeep(keep int) (first *Package) { + total := len(c.packages) + if total <= keep { + return nil + } + + first, c.packages = &c.packages[0], c.packages[1:] + return +} + +func (c *Cycler) run(model string, fileKey string, keep int, deletePackage func(fileKey string) error) { + cyclerFileName := path.Join(cyclerPath, model+".json") + + c.load(cyclerFileName) + c.add(fileKey) + defer c.save(cyclerFileName) + + if keep == 0 { + return + } + + for { + pkg := c.shiftByKeep(keep) + if pkg == nil { + break + } + + err := deletePackage(pkg.FileKey) + if err != nil { + logger.Warn("remove failed: ", err) + } + } +} + +func (c *Cycler) load(cyclerFileName string) { + helper.MkdirP(cyclerPath) + + // write example JSON if not exist + if !helper.IsExistsPath(cyclerFileName) { + ioutil.WriteFile(cyclerFileName, []byte("[{}]"), os.ModePerm) + } + + f, err := ioutil.ReadFile(cyclerFileName) + if err != nil { + logger.Error("Load cycler.json failed:", err) + return + } + err = json.Unmarshal(f, &c.packages) + if err != nil { + logger.Error("Unmarshal cycler.json failed:", err) + } + c.isLoaded = true +} + +func (c *Cycler) save(cyclerFileName string) { + if !c.isLoaded { + logger.Warn("Skip save cycler.json because it not loaded") + return + } + + data, err := json.Marshal(&c.packages) + if err != nil { + logger.Error("Marshal packages to cycler.json failed: ", err) + return + } + + err = ioutil.WriteFile(cyclerFileName, data, os.ModePerm) + if err != nil { + logger.Error("Save cycler.json failed: ", err) + return + } +} diff --git a/storage/cycler_test.go b/storage/cycler_test.go new file mode 100644 index 0000000..d2d53fd --- /dev/null +++ b/storage/cycler_test.go @@ -0,0 +1,44 @@ +package storage + +import ( + "github.com/stretchr/testify/assert" + "testing" + "time" +) + +func TestCycler_add(t *testing.T) { + cycler := Cycler{} + cycler.add("foo") + cycler.add("bar") + + assert.Equal(t, len(cycler.packages), 2) +} + +func TestCycler_shiftByKeep(t *testing.T) { + cycler := Cycler{ + packages: PackageList{ + Package{ + FileKey: "p1", + CreatedAt: time.Now(), + }, + Package{ + FileKey: "p2", + CreatedAt: time.Now(), + }, + }, + } + cycler.add("p3") + cycler.add("p4") + cycler.add("p5") + cycler.add("p6") + + pkg := cycler.shiftByKeep(2) + assert.Equal(t, len(cycler.packages), 5) + assert.Equal(t, pkg.FileKey, "p1") + pkg = cycler.shiftByKeep(2) + assert.Equal(t, len(cycler.packages), 4) + assert.Equal(t, pkg.FileKey, "p2") + pkg = cycler.shiftByKeep(4) + assert.Equal(t, len(cycler.packages), 4) + assert.Nil(t, pkg) +} diff --git a/storage/ftp.go b/storage/ftp.go new file mode 100644 index 0000000..b1fc7ae --- /dev/null +++ b/storage/ftp.go @@ -0,0 +1,91 @@ +package storage + +import ( + "os" + "path" + + "github.com/4nkitd/gobackup/helper" + + // "crypto/tls" + "time" + + "github.com/4nkitd/gobackup/logger" + "github.com/secsy/goftp" +) + +// FTP storage +// +// type: ftp +// path: /backups +// host: ftp.your-host.com +// port: 21 +// timeout: 30 +// username: +// password: +type FTP struct { + Base + path string + host string + port string + username string + password string + + client *goftp.Client +} + +func (ctx *FTP) open() (err error) { + ctx.viper.SetDefault("port", "21") + ctx.viper.SetDefault("timeout", 300) + + ctx.host = helper.CleanHost(ctx.viper.GetString("host")) + ctx.port = ctx.viper.GetString("port") + ctx.path = ctx.viper.GetString("path") + ctx.username = ctx.viper.GetString("username") + ctx.password = ctx.viper.GetString("password") + + ftpConfig := goftp.Config{ + User: ctx.viper.GetString("username"), + Password: ctx.viper.GetString("password"), + Timeout: ctx.viper.GetDuration("timeout") * time.Second, + } + ctx.client, err = goftp.DialConfig(ftpConfig, ctx.host+":"+ctx.port) + if err != nil { + return err + } + return +} + +func (ctx *FTP) close() { + ctx.client.Close() +} + +func (ctx *FTP) upload(fileKey string) (err error) { + logger.Info("-> Uploading...") + _, err = ctx.client.Stat(ctx.path) + if os.IsNotExist(err) { + if _, err := ctx.client.Mkdir(ctx.path); err != nil { + return err + } + } + + file, err := os.Open(ctx.archivePath) + if err != nil { + return err + } + defer file.Close() + + remotePath := path.Join(ctx.path, fileKey) + err = ctx.client.Store(remotePath, file) + if err != nil { + return err + } + + logger.Info("Store successed") + return nil +} + +func (ctx *FTP) delete(fileKey string) (err error) { + remotePath := path.Join(ctx.path, fileKey) + err = ctx.client.Delete(remotePath) + return +} diff --git a/storage/local.go b/storage/local.go new file mode 100644 index 0000000..ecd7e32 --- /dev/null +++ b/storage/local.go @@ -0,0 +1,39 @@ +package storage + +import ( + "path" + + "github.com/4nkitd/gobackup/helper" + "github.com/4nkitd/gobackup/logger" +) + +// Local storage +// +// type: local +// path: /data/backups +type Local struct { + Base + destPath string +} + +func (ctx *Local) open() (err error) { + ctx.destPath = ctx.model.StoreWith.Viper.GetString("path") + helper.MkdirP(ctx.destPath) + return +} + +func (ctx *Local) close() {} + +func (ctx *Local) upload(fileKey string) (err error) { + _, err = helper.Exec("cp", ctx.archivePath, ctx.destPath) + if err != nil { + return err + } + logger.Info("Store successed", ctx.destPath) + return nil +} + +func (ctx *Local) delete(fileKey string) (err error) { + _, err = helper.Exec("rm", path.Join(ctx.destPath, fileKey)) + return +} diff --git a/storage/oss.go b/storage/oss.go new file mode 100644 index 0000000..ee46c53 --- /dev/null +++ b/storage/oss.go @@ -0,0 +1,102 @@ +package storage + +import ( + "path" + + "github.com/4nkitd/gobackup/logger" + "github.com/aliyun/aliyun-oss-go-sdk/oss" +) + +// OSS - Aliyun OSS storage +// +// type: oss +// bucket: gobackup-test +// endpoint: oss-cn-beijing.aliyuncs.com +// path: / +// access_key_id: your-access-key-id +// access_key_secret: your-access-key-secret +// max_retries: 5 +// timeout: 300 +// threads: 1 (1 .. 100) +type OSS struct { + Base + endpoint string + bucket string + accessKeyID string + accessKeySecret string + path string + maxRetries int + timeout int + client *oss.Bucket + threads int +} + +var ( + // 1 Mb one part + ossPartSize int64 = 1024 * 1024 +) + +func (ctx *OSS) open() (err error) { + ctx.viper.SetDefault("endpoint", "oss-cn-beijing.aliyuncs.com") + ctx.viper.SetDefault("max_retries", 3) + ctx.viper.SetDefault("path", "/") + ctx.viper.SetDefault("timeout", 300) + ctx.viper.SetDefault("threads", 1) + + ctx.endpoint = ctx.viper.GetString("endpoint") + ctx.bucket = ctx.viper.GetString("bucket") + ctx.accessKeyID = ctx.viper.GetString("access_key_id") + ctx.accessKeySecret = ctx.viper.GetString("access_key_secret") + ctx.path = ctx.viper.GetString("path") + ctx.maxRetries = ctx.viper.GetInt("max_retries") + ctx.timeout = ctx.viper.GetInt("timeout") + ctx.threads = ctx.viper.GetInt("threads") + + // limit thread in 1..100 + if ctx.threads < 1 { + ctx.threads = 1 + } + if ctx.threads > 100 { + ctx.threads = 100 + } + + logger.Info("endpoint:", ctx.endpoint) + logger.Info("bucket:", ctx.bucket) + + ossClient, err := oss.New(ctx.endpoint, ctx.accessKeyID, ctx.accessKeySecret) + if err != nil { + return err + } + ossClient.Config.Timeout = uint(ctx.timeout) + ossClient.Config.RetryTimes = uint(ctx.maxRetries) + + ctx.client, err = ossClient.Bucket(ctx.bucket) + if err != nil { + return err + } + + return +} + +func (ctx *OSS) close() { +} + +func (ctx *OSS) upload(fileKey string) (err error) { + remotePath := path.Join(ctx.path, fileKey) + + logger.Info("-> Uploading OSS...") + err = ctx.client.UploadFile(remotePath, ctx.archivePath, ossPartSize, oss.Routines(ctx.threads)) + + if err != nil { + return err + } + logger.Info("Success") + + return nil +} + +func (ctx *OSS) delete(fileKey string) (err error) { + remotePath := path.Join(ctx.path, fileKey) + err = ctx.client.DeleteObject(remotePath) + return +} diff --git a/storage/s3.go b/storage/s3.go new file mode 100644 index 0000000..78b03b6 --- /dev/null +++ b/storage/s3.go @@ -0,0 +1,92 @@ +package storage + +import ( + "fmt" + "os" + "path" + + "github.com/4nkitd/gobackup/logger" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/aws/aws-sdk-go/service/s3/s3manager" +) + +// S3 - Amazon S3 storage +// +// type: s3 +// bucket: gobackup-test +// region: us-east-1 +// path: backups +// access_key_id: your-access-key-id +// secret_access_key: your-secret-access-key +// max_retries: 5 +// timeout: 300 +type S3 struct { + Base + bucket string + path string + client *s3manager.Uploader +} + +func (ctx *S3) open() (err error) { + ctx.viper.SetDefault("region", "us-east-1") + cfg := aws.NewConfig() + endpoint := ctx.viper.GetString("endpoint") + if len(endpoint) > 0 { + cfg.Endpoint = aws.String(endpoint) + cfg.S3ForcePathStyle = aws.Bool(true) + } + cfg.Credentials = credentials.NewStaticCredentials( + ctx.viper.GetString("access_key_id"), + ctx.viper.GetString("secret_access_key"), + ctx.viper.GetString("token"), + ) + cfg.Region = aws.String(ctx.viper.GetString("region")) + cfg.MaxRetries = aws.Int(ctx.viper.GetInt("max_retries")) + + ctx.bucket = ctx.viper.GetString("bucket") + ctx.path = ctx.viper.GetString("path") + + sess := session.Must(session.NewSession(cfg)) + ctx.client = s3manager.NewUploader(sess) + + return +} + +func (ctx *S3) close() {} + +func (ctx *S3) upload(fileKey string) (err error) { + f, err := os.Open(ctx.archivePath) + if err != nil { + return fmt.Errorf("failed to open file %q, %v", ctx.archivePath, err) + } + + remotePath := path.Join(ctx.path, fileKey) + + input := &s3manager.UploadInput{ + Bucket: aws.String(ctx.bucket), + Key: aws.String(remotePath), + Body: f, + } + + logger.Info("-> S3 Uploading...") + result, err := ctx.client.Upload(input) + if err != nil { + return fmt.Errorf("failed to upload file, %v", err) + } + + logger.Info("=>", result.Location) + return nil +} + +func (ctx *S3) delete(fileKey string) (err error) { + remotePath := path.Join(ctx.path, fileKey) + input := &s3.DeleteObjectInput{ + Bucket: aws.String(ctx.bucket), + Key: aws.String(remotePath), + } + _, err = ctx.client.S3.DeleteObject(input) + return +} diff --git a/storage/scp.go b/storage/scp.go new file mode 100644 index 0000000..bfe8fb2 --- /dev/null +++ b/storage/scp.go @@ -0,0 +1,113 @@ +package storage + +import ( + "os" + "path" + "time" + + "github.com/4nkitd/gobackup/helper" + "golang.org/x/crypto/ssh" + + // "crypto/tls" + "github.com/4nkitd/gobackup/logger" + "github.com/bramvdbogaerde/go-scp" + "github.com/bramvdbogaerde/go-scp/auth" +) + +// SCP storage +// +// type: scp +// host: 192.168.1.2 +// port: 22 +// username: root +// password: +// timeout: 300 +// private_key: ~/.ssh/id_rsa +type SCP struct { + Base + path string + host string + port string + privateKey string + username string + password string + client scp.Client +} + +func (ctx *SCP) open() (err error) { + ctx.viper.SetDefault("port", "22") + ctx.viper.SetDefault("timeout", 300) + ctx.viper.SetDefault("private_key", "~/.ssh/id_rsa") + + ctx.host = ctx.viper.GetString("host") + ctx.port = ctx.viper.GetString("port") + ctx.path = ctx.viper.GetString("path") + ctx.username = ctx.viper.GetString("username") + ctx.password = ctx.viper.GetString("password") + ctx.privateKey = helper.ExplandHome(ctx.viper.GetString("private_key")) + var clientConfig ssh.ClientConfig + logger.Info("PrivateKey", ctx.privateKey) + clientConfig, err = auth.PrivateKey( + ctx.username, + ctx.privateKey, + ssh.InsecureIgnoreHostKey(), + ) + if err != nil { + logger.Warn(err) + logger.Info("PrivateKey fail, Try User@Host with Password") + clientConfig = ssh.ClientConfig{ + User: ctx.username, + HostKeyCallback: ssh.InsecureIgnoreHostKey(), + } + } + clientConfig.Timeout = ctx.viper.GetDuration("timeout") * time.Second + if len(ctx.password) > 0 { + clientConfig.Auth = append(clientConfig.Auth, ssh.Password(ctx.password)) + } + + ctx.client = scp.NewClient(ctx.host+":"+ctx.port, &clientConfig) + + err = ctx.client.Connect() + if err != nil { + return err + } + defer ctx.client.Session.Close() + ctx.client.Session.Run("mkdir -p " + ctx.path) + return +} + +func (ctx *SCP) close() {} + +func (ctx *SCP) upload(fileKey string) (err error) { + err = ctx.client.Connect() + if err != nil { + return err + } + defer ctx.client.Session.Close() + + file, err := os.Open(ctx.archivePath) + if err != nil { + return err + } + defer file.Close() + + remotePath := path.Join(ctx.path, fileKey) + logger.Info("-> scp", remotePath) + ctx.client.CopyFromFile(*file, remotePath, "0655") + + logger.Info("Store successed") + return nil +} + +func (ctx *SCP) delete(fileKey string) (err error) { + err = ctx.client.Connect() + if err != nil { + return + } + defer ctx.client.Session.Close() + + remotePath := path.Join(ctx.path, fileKey) + logger.Info("-> remove", remotePath) + err = ctx.client.Session.Run("rm " + remotePath) + return +}