diff --git a/.github/workflows/tag.yaml b/.github/workflows/tag.yaml index 888867f..22c2e87 100644 --- a/.github/workflows/tag.yaml +++ b/.github/workflows/tag.yaml @@ -39,7 +39,7 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Build - run: "make dockerfile assets_build build GO_VERSION=${{ env.GO_VERSION }} POSTGRES_VERSION=${{ matrix.POSTGRES_VERSION }}" + run: "make build GO_VERSION=${{ env.GO_VERSION }} POSTGRES_VERSION=${{ matrix.POSTGRES_VERSION }}" - name: Build docker image and push release image run: "make ci_rename_release_binary ci_release_snapshot ci_release POSTGRES_VERSION=${{ matrix.POSTGRES_VERSION }}" diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 62d5169..52b6165 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -41,7 +41,7 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Build - run: "make dockerfile assets_build build ci_check_embedded_binaries GO_VERSION=${{ env.GO_VERSION }} POSTGRES_VERSION=${{ matrix.POSTGRES_VERSION }}" + run: "make build GO_VERSION=${{ env.GO_VERSION }} POSTGRES_VERSION=${{ matrix.POSTGRES_VERSION }}" - name: Test run: "make test POSTGRES_VERSION=${{ matrix.POSTGRES_VERSION }}" diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..f979dbb --- /dev/null +++ b/Dockerfile @@ -0,0 +1,26 @@ +ARG POSTGRES_VERSION +ARG GO_VERSION + +# ============================================= +# Build application && embed PostgreSQL in it +# ============================================= +FROM golang:${GO_VERSION} as builder + +ADD . /workspace +RUN apt-get update && apt-get install make -y + +RUN cd /workspace && \ + make build + + +# ============================================================ +# Create target image basing on a specific PostgreSQL version +# ============================================================ +FROM postgres:${POSTGRES_VERSION} + +RUN apt-get update && apt-get install -y gpg + +COPY --from=builder /workspace/.build/pgbr /usr/bin/pgbr +RUN chmod +x /usr/bin/pgbr + +ENTRYPOINT ["/usr/bin/pgbr"] diff --git a/Makefile b/Makefile index 8176a0c..a7a31f6 100644 --- a/Makefile +++ b/Makefile @@ -1,33 +1,15 @@ include test.mk include ci.mk -PG_DATA=assets/.build/data - -copy_libs_and_executables: - mkdir -p ${PG_DATA} - cp -p /usr/lib/postgresql/${POSTGRES_VERSION}/bin/psql ${PG_DATA}/psql - cp -p /usr/lib/postgresql/${POSTGRES_VERSION}/bin/pg_dumpall ${PG_DATA}/pg_dumpall - cp -p /usr/lib/postgresql/${POSTGRES_VERSION}/bin/pg_dump ${PG_DATA}/pg_dump - cp -p /usr/lib/postgresql/${POSTGRES_VERSION}/bin/pg_restore ${PG_DATA}/pg_restore - - ./hack/get-binary-with-libs.py /usr/lib/postgresql/${POSTGRES_VERSION}/bin/psql ./${PG_DATA} - ./hack/get-binary-with-libs.py /usr/lib/postgresql/${POSTGRES_VERSION}/bin/pg_dump ./${PG_DATA} - ./hack/get-binary-with-libs.py /usr/lib/postgresql/${POSTGRES_VERSION}/bin/pg_dumpall ./${PG_DATA} - ./hack/get-binary-with-libs.py /usr/lib/postgresql/${POSTGRES_VERSION}/bin/pg_restore ./${PG_DATA} - clean: rm -rf .build/* -assets_build: ## Builds PostgreSQL assets using docker - docker build . -f build.Dockerfile --build-arg POSTGRES_VERSION=${POSTGRES_VERSION} --build-arg GO_VERSION=${GO_VERSION} -t build - @docker rm -f builder 2>/dev/null - docker create --name builder build - docker cp builder:/workspace/.build ./ - docker cp builder:/workspace/assets ./ - build: CGO_ENABLED=0 GO111MODULE=on go build -tags nomemcopy -o ./.build/pgbr chmod +x ./.build/pgbr test: + export PGBR_USE_CONTAINER=true; \ + export PGBR_CONTAINER_IMAGE=bitnami/postgresql; \ + export POSTGRES_VERSION=${POSTGRES_VERSION}; \ go test -v ./... diff --git a/README.md b/README.md index d1238da..36b54a1 100644 --- a/README.md +++ b/README.md @@ -4,22 +4,15 @@ pgbr PostgreSQL simple backup & restore helper tool created for usage with Backup Repository, but can be used also standalone. **Features:** -- `psql`, `pg_dump`, `pg_dumpall`, `pg_restore` packaged **in a single binary**, available as subcommands e.g. `pgbr psql -- -h 127.0.0.1 -d riotkit -c "SELECT 1"` -- Minimum system requirements, no extra binaries or libraries and **even no PostgreSQL client is required, just this one binary** - Opinionated backup & restore commands basing on PostgreSQL built-in commands **Requirements:** -- [patchelf](https://github.com/NixOS/patchelf) -- Linux +- Linux (x86_64/amd64 architecture) +- PostgreSQL in desired version Conception ---------- -**Single-binary** - -`pgbr` binary has compiled PostgreSQL tools, including libc (musl/glibc) and dynamic libraries, everything is unpacked into temporary directory, then `patchelf` patches -interpreter to match `ld-linux` or `ld-musl` at selected path and `pgbr` is invoking a process with extra `LD_LIBRARY_PATH` environment set. - **Sensible defaults** Backup & Restore should be simple and fault-tolerant, that's why this tool is automating basic things like disconnecting clients, or connecting @@ -53,82 +46,6 @@ Procedure: cat dump.gz | ./.build/pgbr db restore --password riotkit --user riotkit --connection-database=postgres ``` -psql & pg_dump & pg_dumpall & pg_restore ----------------------------------------- - -- psql: `pgbr psql -- ...` -- pg_dump: `pgbr pg_dump -- ...` -- pg_dumpall: `pgbr pg_dumpall -- ...` -- pg_restore: `pgbr pg_restore -- ...` - -```bash -pgbr psql -- --help - -#psql is the PostgreSQL interactive terminal. -# -#Usage: -# psql [OPTION]... [DBNAME [USERNAME]] -# -#General options: -# -c, --command=COMMAND run only single command (SQL or internal) and exit -# -d, --dbname=DBNAME database name to connect to (default: "damian") -# -f, --file=FILENAME execute commands from file, then exit -# -l, --list list available databases, then exit -# -v, --set=, --variable=NAME=VALUE -# set psql variable NAME to VALUE -# (e.g., -v ON_ERROR_STOP=1) -# -V, --version output version information, then exit -# -X, --no-psqlrc do not read startup file (~/.psqlrc) -# -1 ("one"), --single-transaction -# execute as a single transaction (if non-interactive) -# -?, --help[=options] show this help, then exit -# --help=commands list backslash commands, then exit -# --help=variables list special variables, then exit -# -#Input and output options: -# -a, --echo-all echo all input from script -# -b, --echo-errors echo failed commands -# -e, --echo-queries echo commands sent to server -# -E, --echo-hidden display queries that internal commands generate -# -L, --log-file=FILENAME send session log to file -# -n, --no-readline disable enhanced command line editing (readline) -# -o, --output=FILENAME send query results to file (or |pipe) -# -q, --quiet run quietly (no messages, only query output) -# -s, --single-step single-step mode (confirm each query) -# -S, --single-line single-line mode (end of line terminates SQL command) -# -#Output format options: -# -A, --no-align unaligned table output mode -# --csv CSV (Comma-Separated Values) table output mode -# -F, --field-separator=STRING -# field separator for unaligned output (default: "|") -# -H, --html HTML table output mode -# -P, --pset=VAR[=ARG] set printing option VAR to ARG (see \pset command) -# -R, --record-separator=STRING -# record separator for unaligned output (default: newline) -# -t, --tuples-only print rows only -# -T, --table-attr=TEXT set HTML table tag attributes (e.g., width, border) -# -x, --expanded turn on expanded table output -# -z, --field-separator-zero -# set field separator for unaligned output to zero byte -# -0, --record-separator-zero -# set record separator for unaligned output to zero byte -# -#Connection options: -# -h, --host=HOSTNAME database server host or socket directory (default: "local socket") -# -p, --port=PORT database server port (default: "5432") -# -U, --username=USERNAME database user name (default: "damian") -# -w, --no-password never prompt for password -# -W, --password force password prompt (should happen automatically) -# -#For more information, type "\?" (for internal commands) or "\help" (for SQL -#commands) from within psql, or consult the psql section in the PostgreSQL -#documentation. -# -#Report bugs to . -#PostgreSQL home page: -``` - Passing extra arguments ----------------------- @@ -139,3 +56,18 @@ Both `pgbr db backup` and `pgbr db restore` are supporting UNIX-like parameters ```bash pgbr db backup --password riotkit --user riotkit -- --role=my-role > dump.gz ``` + +Using tooling from docker image +------------------------------- + +`pgbr` can rely on your host OS or use `docker run` to trigger `psql`, `pg_dump`, `pg_dumpall` or `pg_restore`. + +In order to use a docker image that will provide client tools you can set those environment variables: + +```bash +export PGBR_USE_CONTAINER=true +export POSTGRES_VERSION=15 + +# optional: image name +export PGBR_CONTAINER_IMAGE=my-registry/image-name +``` diff --git a/assets/.build/data/.gitkeep b/assets/.build/data/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/assets/extraction.go b/assets/extraction.go deleted file mode 100644 index bf212d8..0000000 --- a/assets/extraction.go +++ /dev/null @@ -1,122 +0,0 @@ -package assets - -import ( - "fmt" - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - "os" - "os/exec" - "path" - "path/filepath" -) - -// ExtractAllFromMemory unpacks all stored libraries and binaries into single directory -func ExtractAllFromMemory(targetDir string) (bool, error) { - var hasAtLeastOneError bool - - if err := os.RemoveAll(targetDir); err != nil { - return false, errors.Wrapf(err, "cannot delete temporary directory at path: '%s'", targetDir) - } - - if err := os.MkdirAll(targetDir, 0755); err != nil { - return false, errors.Wrapf(err, "cannot create directory '%s'", targetDir) - } - - dir, readErr := Res.ReadDir(".build/data") - if readErr != nil { - return false, errors.Wrap(readErr, "cannot read directory from the go:embed") - } - - for _, asset := range dir { - assetName := asset.Name() - if asset.IsDir() { - continue - } - - logrus.Debugf("Extracting '%s' into '%s'", assetName, targetDir) - data, err := Res.ReadFile(".build/data/" + assetName) - - if err != nil { - logrus.Error(errors.Wrap(err, "cannot unpack file from single-binary")) - hasAtLeastOneError = true - } - - subdir := "" - baseName := path.Base(assetName) - - if baseName == "pg_dumpall" || baseName == "pg_restore" || baseName == "psql" || baseName == "pg_dump" { - subdir = "bin/" - _ = os.Mkdir(targetDir+"/"+subdir, 0755) - } - - err = os.WriteFile(targetDir+"/"+subdir+baseName, data, 0755) - if err != nil { - logrus.Error(errors.Wrap(err, "cannot unpack file from single-binary")) - hasAtLeastOneError = true - } - } - - return !hasAtLeastOneError, nil -} - -func PatchBinaries(targetDir string) error { - // todo: if patchelf is accessible, else return nil - - absDir, absErr := filepath.Abs(targetDir) - if absErr != nil { - return errors.Wrapf(absErr, "Cannot find absolute path for '%s'", targetDir) - } - - interpreterPath := findInterpreterPath(absDir) - logrus.Debugf("Interpreter path: '%s'", interpreterPath) - - if interpreterPath == "" { - return errors.New("cannot find ld-linux or ld-musl") - } - - binDir := absDir + "/bin" - _ = os.MkdirAll(binDir, 0755) - - files, err := os.ReadDir(binDir) - if err != nil { - return errors.Wrapf(err, "Cannot list directory '%s' to patch binaries", targetDir) - } - - for _, file := range files { - logrus.Debugf("Processing '%s' with patchelf", binDir+"/"+file.Name()) - - cmd := fmt.Sprintf("patchelf --set-interpreter %s %s", interpreterPath, binDir+"/"+file.Name()) - c := exec.Command("/bin/sh", "-c", cmd) - c.Stdout = os.Stdout - c.Stderr = os.Stderr - c.Stdin = os.Stdin - c.Env = os.Environ() - if waitErr := c.Run(); waitErr != nil { - return errors.Wrapf(waitErr, "patchelf failed to set ld-linux path: %s", cmd) - } - } - - return nil -} - -func findInterpreterPath(libPath string) string { - possibleMatches := [][]string{ - // musl (Alpine Linux) - glob(libPath + "/ld-musl-*"), - // libc (all others) - glob(libPath + "/ld-linux*"), - } - - for _, matches := range possibleMatches { - if len(matches) > 0 { - return matches[0] - } - } - - return "" -} - -func glob(pattern string) []string { - matches, _ := filepath.Glob(pattern) - return matches -} diff --git a/assets/unpacking.go b/assets/unpacking.go deleted file mode 100644 index 6dae056..0000000 --- a/assets/unpacking.go +++ /dev/null @@ -1,31 +0,0 @@ -package assets - -import ( - "embed" - "log" - "os" -) - -var ( - //go:embed .build/data - Res embed.FS -) - -func UnpackOrExit() string { - tempDir := "/tmp/br-pgbr" - if os.Getenv("BR_TEMP_DIR") != "" { - tempDir = os.Getenv("BR_TEMP_DIR") - } - - // prepare binaries - if unpacked, err := ExtractAllFromMemory(tempDir); err != nil || !unpacked { - if err == nil && !unpacked { - log.Fatalf("Cannot unpack binaries and libraries to '%s'", tempDir) - } - log.Fatal(err) - } - if err := PatchBinaries(tempDir); err != nil { - log.Fatal(err) - } - return tempDir -} diff --git a/build.Dockerfile b/build.Dockerfile deleted file mode 100644 index cbba4e6..0000000 --- a/build.Dockerfile +++ /dev/null @@ -1,29 +0,0 @@ -ARG POSTGRES_VERSION -ARG GO_VERSION - -# ======================================== -# Collect specific version of PostgreSQL -# ======================================== -FROM postgres:${POSTGRES_VERSION} as postgres - -ARG POSTGRES_VERSION=${POSTGRES_VERSION} - -RUN apt-get update && apt-get install python3 make bash util-linux gcc-multilib file patchelf -y - -ADD . /workspace -WORKDIR /workspace - -# collect binaries and libraries in .build/data directory -RUN cd /workspace && make copy_libs_and_executables POSTGRES_VERSION=${POSTGRES_VERSION} - - -# ============================================= -# Build application && embed PostgreSQL in it -# ============================================= -FROM golang:${GO_VERSION} as builder - -COPY --from=postgres /workspace /workspace -RUN apt-get update && apt-get install make -y - -RUN cd /workspace && \ - make build diff --git a/ci.mk b/ci.mk index bb30f07..bef860c 100644 --- a/ci.mk +++ b/ci.mk @@ -1,20 +1,8 @@ GO_VERSION=1.19 -POSTGRES_VERSION=15.0 - -ci_check_embedded_binaries: - ./.build/pgbr pg_dump -- --version - ./.build/pgbr pg_dumpall -- --version - ./.build/pgbr pg_restore -- --version - ./.build/pgbr psql -- --version - -dockerfile: - mkdir -p .build - cat build.Dockerfile > .build/Dockerfile - echo "" >> .build/Dockerfile - cat release.Dockerfile >> .build/Dockerfile +POSTGRES_VERSION=15 ci_release_snapshot: - docker build . -f .build/Dockerfile --build-arg POSTGRES_VERSION=${POSTGRES_VERSION} --build-arg GO_VERSION=${GO_VERSION} -t ghcr.io/riotkit-org/pgbr:latest-pg${POSTGRES_VERSION} + docker build . --build-arg POSTGRES_VERSION=${POSTGRES_VERSION} --build-arg GO_VERSION=${GO_VERSION} -t ghcr.io/riotkit-org/pgbr:latest-pg${POSTGRES_VERSION} docker push ghcr.io/riotkit-org/pgbr:latest-pg${POSTGRES_VERSION} ci_release: diff --git a/cmd/db/backup.go b/cmd/db/backup.go index a4385d9..050d54d 100644 --- a/cmd/db/backup.go +++ b/cmd/db/backup.go @@ -1,15 +1,19 @@ package db import ( + "bytes" "fmt" "github.com/riotkit-org/br-pg-simple-backup/cmd/base" - "github.com/riotkit-org/br-pg-simple-backup/cmd/wrapper" + "github.com/riotkit-org/br-pg-simple-backup/cmd/runner" "github.com/spf13/cobra" ) // NewBackupCommand creates the new command -func NewBackupCommand(libDir string) *cobra.Command { - app := &BackupCommand{} +func NewBackupCommand(captureOutput bool, stdinBuffer *bytes.Buffer) (*cobra.Command, *BackupCommand) { + app := &BackupCommand{ + CaptureOutput: captureOutput, + StdinBuffer: stdinBuffer, + } var basicOpts base.BasicOptions command := &cobra.Command{ @@ -19,7 +23,9 @@ func NewBackupCommand(libDir string) *cobra.Command { RunE: func(command *cobra.Command, args []string) error { app.ExtraArgs = command.Flags().Args() base.PreCommandRun(command, &basicOpts) - return app.Run(libDir) + out, err := app.Run() + app.Output = out + return err }, } @@ -32,7 +38,7 @@ func NewBackupCommand(libDir string) *cobra.Command { command.Flags().StringVarP(&app.InitialDbName, "connection-database", "D", "postgres", "Any, even empty database name to connect to initially") base.PopulateFlags(command, &basicOpts) - return command + return command, app } type BackupCommand struct { @@ -44,17 +50,20 @@ type BackupCommand struct { InitialDbName string CompressionLevel int ExtraArgs []string + CaptureOutput bool + + // used for testing + Output []byte + StdinBuffer *bytes.Buffer } // Run Executes the command and outputs a stream to the stdout -func (bc *BackupCommand) Run(libDir string) error { +func (bc *BackupCommand) Run() ([]byte, error) { dumpArgs := []string{ "--clean", "--host", bc.Hostname, fmt.Sprintf("--port=%v", bc.Port), "--username", bc.Username, - "--format=c", // custom format for pg_restore - fmt.Sprintf("--compress=%v", bc.CompressionLevel), } envVars := []string{ "PGPASSWORD=" + bc.Password, @@ -65,11 +74,16 @@ func (bc *BackupCommand) Run(libDir string) error { // difference between pg_dump and pg_dumpall if !bc.allDatabases() { // pg_dump - dumpArgs = append(dumpArgs, "--create", "--blobs", bc.Database) + dumpArgs = append(dumpArgs, + "--create", + "--blobs", bc.Database, + "--format=c", // custom format for pg_restore + fmt.Sprintf("--compress=%v", bc.CompressionLevel), + ) binName = "pg_dump" } else { // pg_dumpall - dumpArgs = append(dumpArgs, "--superuser="+bc.Username, "--dbname="+bc.InitialDbName) + dumpArgs = append(dumpArgs, "--superuser="+bc.Username, "--database="+bc.InitialDbName) binName = "pg_dumpall" } @@ -78,7 +92,7 @@ func (bc *BackupCommand) Run(libDir string) error { dumpArgs = append(dumpArgs, bc.ExtraArgs...) } - return wrapper.RunWrappedPGCommand(libDir, binName, dumpArgs, envVars) + return runner.Run(binName, dumpArgs, envVars, bc.CaptureOutput, bc.StdinBuffer) } func (bc *BackupCommand) allDatabases() bool { diff --git a/cmd/db/db_test.go b/cmd/db/db_test.go index cf3ecfc..4c2076f 100644 --- a/cmd/db/db_test.go +++ b/cmd/db/db_test.go @@ -1,9 +1,10 @@ package db_test import ( + "bytes" "context" - "github.com/riotkit-org/br-pg-simple-backup/assets" "github.com/riotkit-org/br-pg-simple-backup/cmd/db" + "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/testcontainers/testcontainers-go" "github.com/testcontainers/testcontainers-go/wait" @@ -13,28 +14,89 @@ import ( "testing" ) -func TestBackup(t *testing.T) { +func setupContainer() (testcontainers.Container, string) { container, err := createContainer() if err != nil { log.Fatal(err) } - defer container.Terminate(context.Background()) endpoint, _ := container.Endpoint(context.Background(), "") port := strings.Split(endpoint, ":")[1] - path := assets.UnpackOrExit() - cmd := db.NewBackupCommand(path) - cmd.SetArgs([]string{ - "--host=127.0.0.1", - "--port=" + port, - "--user=anarchism", - "--password=syndicalism", + return container, port +} + +func TestBackupAndRestoreSingleDatabase(t *testing.T) { + container, _ := setupContainer() + defer container.Terminate(context.Background()) + ip, _ := container.ContainerIP(context.TODO()) + + logrus.SetLevel(logrus.DebugLevel) + + // Backup first + backupCmd, backupApp := db.NewBackupCommand(true, &bytes.Buffer{}) + backupCmd.SetArgs([]string{ + "--host=" + ip, + "--port=5432", + "--user=postgres", + "--password=postgres", + "--db-name=riotkit", + }) + + bErr := backupCmd.Execute() + assert.Nil(t, bErr, "Expected that the command will not return error: "+string(backupApp.Output)) + out := backupApp.Output + + restoreStdinBuff := &bytes.Buffer{} + restoreStdinBuff.Write(out) + + // Then restore + restoreCmd, restoreApp := db.NewRestoreCommand(true, restoreStdinBuff) + restoreCmd.SetArgs([]string{ + "--host=" + ip, + "--port=5432", + "--user=postgres", + "--password=postgres", "--db-name=riotkit", + }) + err := backupCmd.Execute() + assert.Nil(t, err, "Expected that the restore will succeed. Output: "+string(restoreApp.Output)) +} + +func TestBackupAndRestoreAllDatabases(t *testing.T) { + container, _ := setupContainer() + defer container.Terminate(context.Background()) + ip, _ := container.ContainerIP(context.TODO()) + logrus.SetLevel(logrus.DebugLevel) + + // Backup first + backupCmd, backupApp := db.NewBackupCommand(true, &bytes.Buffer{}) + backupCmd.SetArgs([]string{ + "--host=" + ip, + "--port=5432", + "--user=postgres", + "--password=postgres", // not using --db-name, in effect dumpall will be used }) - assert.Nil(t, cmd.Execute(), "Expected that the command will not return error") + + bErr := backupCmd.Execute() + assert.Nil(t, bErr, "Expected that the command will not return error: "+string(backupApp.Output)) + out := backupApp.Output + + restoreStdinBuff := &bytes.Buffer{} + restoreStdinBuff.Write(out) + + // Then restore + restoreCmd, restoreApp := db.NewRestoreCommand(true, restoreStdinBuff) + restoreCmd.SetArgs([]string{ + "--host=" + ip, + "--port=5432", + "--user=postgres", + "--password=postgres", + }) + err := backupCmd.Execute() + assert.Nil(t, err, "Expected that the restore will succeed. Output: "+string(restoreApp.Output)) } func createContainer() (testcontainers.Container, error) { @@ -45,9 +107,10 @@ func createContainer() (testcontainers.Container, error) { WaitingFor: wait.ForLog("database system is ready to accept connections"), AutoRemove: true, Env: map[string]string{ - "POSTGRESQL_DATABASE": "riotkit", - "POSTGRESQL_USERNAME": "anarchism", - "POSTGRESQL_PASSWORD": "syndicalism", + "POSTGRESQL_DATABASE": "riotkit", + "POSTGRESQL_USERNAME": "anarchism", + "POSTGRESQL_PASSWORD": "syndicalism", + "POSTGRESQL_POSTGRES_PASSWORD": "postgres", }, } pg, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ diff --git a/cmd/db/main.go b/cmd/db/main.go index 9d57583..f3dde8e 100644 --- a/cmd/db/main.go +++ b/cmd/db/main.go @@ -1,11 +1,12 @@ package db import ( + "bytes" "github.com/spf13/cobra" ) // NewDbCommand creates the new command -func NewDbCommand(libDir string) *cobra.Command { +func NewDbCommand() *cobra.Command { command := &cobra.Command{ Use: "db", Short: "Operations on database level using dumps", @@ -14,8 +15,11 @@ func NewDbCommand(libDir string) *cobra.Command { }, } - command.AddCommand(NewBackupCommand(libDir)) - command.AddCommand(NewRestoreCommand(libDir)) + backupCmd, _ := NewBackupCommand(false, &bytes.Buffer{}) + command.AddCommand(backupCmd) + + restoreCmd, _ := NewRestoreCommand(false, &bytes.Buffer{}) + command.AddCommand(restoreCmd) return command } diff --git a/cmd/db/restore.go b/cmd/db/restore.go index 7da743b..6919279 100644 --- a/cmd/db/restore.go +++ b/cmd/db/restore.go @@ -1,12 +1,13 @@ package db import ( + "bytes" "context" "fmt" pgx "github.com/jackc/pgx/v4" "github.com/pkg/errors" "github.com/riotkit-org/br-pg-simple-backup/cmd/base" - "github.com/riotkit-org/br-pg-simple-backup/cmd/wrapper" + "github.com/riotkit-org/br-pg-simple-backup/cmd/runner" "github.com/sirupsen/logrus" "github.com/spf13/cobra" "strings" @@ -15,8 +16,11 @@ import ( const TechDatabaseName = "br_empty_conn_db" // NewRestoreCommand creates the new command -func NewRestoreCommand(libDir string) *cobra.Command { - app := &RestoreCommand{} +func NewRestoreCommand(captureOutput bool, stdinBuffer *bytes.Buffer) (*cobra.Command, *RestoreCommand) { + app := &RestoreCommand{ + CaptureOutput: captureOutput, + StdinBuffer: stdinBuffer, + } var basicOpts base.BasicOptions command := &cobra.Command{ @@ -26,7 +30,7 @@ func NewRestoreCommand(libDir string) *cobra.Command { RunE: func(command *cobra.Command, args []string) error { app.ExtraArgs = command.Flags().Args() base.PreCommandRun(command, &basicOpts) - return app.Run(libDir) + return app.Run() }, } @@ -38,7 +42,7 @@ func NewRestoreCommand(libDir string) *cobra.Command { command.Flags().StringVarP(&app.InitialDbName, "connection-database", "D", "postgres", "Any, even empty database name to connect to initially") base.PopulateFlags(command, &basicOpts) - return command + return command, app } type RestoreCommand struct { @@ -49,10 +53,16 @@ type RestoreCommand struct { Password string ExtraArgs []string DatabaseName string + CaptureOutput bool + Buffer *bytes.Buffer + + // for testing purposes + Output []byte + StdinBuffer *bytes.Buffer } // Run Executes the command and outputs a stream to the stdout -func (bc *RestoreCommand) Run(libDir string) error { +func (bc *RestoreCommand) Run() error { // 0) Prepare a database we will be connecting to, instead of connecting to target database if err := bc.createTechnicalDatabase(); err != nil { return err @@ -78,15 +88,26 @@ func (bc *RestoreCommand) Run(libDir string) error { envVars := []string{ "PGPASSWORD=" + bc.Password, } - if restoreErr := wrapper.RunWrappedPGCommand(libDir, "pg_restore", bc.buildRestoreArgs(), envVars); restoreErr != nil { - return errors.Wrap(restoreErr, "Cannot restore backup") + + if bc.DatabaseName != "" { + // for single database we run pg_restore + _, restoreErr := runner.Run("pg_restore", bc.buildPGRestoreArgs(), envVars, bc.CaptureOutput, bc.StdinBuffer) + if restoreErr != nil { + return errors.Wrap(restoreErr, "Cannot restore backup using pg_restore") + } + } else { + // for multiple databases we run psql + _, restoreErr := runner.Run("psql", bc.buildPSQLArgs(), envVars, bc.CaptureOutput, bc.StdinBuffer) + if restoreErr != nil { + return errors.Wrap(restoreErr, "Cannot restore backup using psql") + } } logrus.Info("Database restored.") return nil } -func (bc *RestoreCommand) buildRestoreArgs() []string { +func (bc *RestoreCommand) buildPGRestoreArgs() []string { restoreArgs := []string{ "--clean", "--create", @@ -107,6 +128,20 @@ func (bc *RestoreCommand) buildRestoreArgs() []string { return restoreArgs } +func (bc *RestoreCommand) buildPSQLArgs() []string { + psqlArgs := []string{ + "--host", bc.Hostname, + "--port", fmt.Sprintf("%v", bc.Port), + "--username", bc.Username, + "--dbname=" + bc.InitialDbName, + "--no-readline", + } + if len(bc.ExtraArgs) > 0 { + psqlArgs = append(psqlArgs, bc.ExtraArgs...) + } + return psqlArgs +} + // createTechnicalDatabase Creates an additional database to which we will be connecting, which will be excluded from the backup & restore process func (bc *RestoreCommand) createTechnicalDatabase() error { client := bc.createClient() diff --git a/cmd/root.go b/cmd/root.go index 29e7ce3..73ea472 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -2,24 +2,19 @@ package cmd import ( "github.com/riotkit-org/br-pg-simple-backup/cmd/db" - "github.com/riotkit-org/br-pg-simple-backup/cmd/wrapper" "github.com/spf13/cobra" ) // Main creates the new command -func Main(libDir string) *cobra.Command { +func Main() *cobra.Command { cmd := &cobra.Command{ Use: "pgbr", - Short: "PostgreSQL backup & restore wrapper for Backup Repository. Works also as a standalone, single-binary backup make & restore utility", + Short: "PostgreSQL backup & restore runner for Backup Repository (and not only)", RunE: func(cmd *cobra.Command, args []string) error { return cmd.Help() }, } - cmd.AddCommand(wrapper.NewCmdPostgresWrapper(libDir, "psql", "psql")) - cmd.AddCommand(wrapper.NewCmdPostgresWrapper(libDir, "pg_dump", "pg_dump")) - cmd.AddCommand(wrapper.NewCmdPostgresWrapper(libDir, "pg_dumpall", "pg_dumpall")) - cmd.AddCommand(wrapper.NewCmdPostgresWrapper(libDir, "pg_restore", "pg_restore")) - cmd.AddCommand(db.NewDbCommand(libDir)) + cmd.AddCommand(db.NewDbCommand()) return cmd } diff --git a/cmd/runner/main.go b/cmd/runner/main.go new file mode 100644 index 0000000..f1e5a0b --- /dev/null +++ b/cmd/runner/main.go @@ -0,0 +1,59 @@ +package runner + +import ( + "bytes" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + "os" + "os/exec" +) + +func Run(binName string, execArgs []string, envVars []string, captureOutput bool, inputStdin *bytes.Buffer) ([]byte, error) { + logrus.Debugf("Running '%s' %v", binName, execArgs) + var c *exec.Cmd + + if os.Getenv("PGBR_USE_CONTAINER") != "" && os.Getenv("POSTGRES_VERSION") != "" { + containerImage := "bitnami/postgresql" + // allow to override the image + if os.Getenv("PGBR_CONTAINER_IMAGE") != "" { + containerImage = os.Getenv("PGBR_CONTAINER_IMAGE") + } + containerImage += ":" + os.Getenv("POSTGRES_VERSION") + + args := []string{"run"} + + // apply env variables as "-e" docker run commandline switches + for _, envVar := range envVars { + args = append(args, "-e", envVar) + } + + args = append(args, "-i", "--entrypoint", binName, "--rm", containerImage) + args = append(args, execArgs...) + + c = exec.Command("docker", args...) + } else { + c = exec.Command(binName, execArgs...) + env := os.Environ() + env = append(env, envVars...) + c.Env = env + } + + // allow to optionally capture output into a buffer + if captureOutput { + c.Stdin = inputStdin + out, waitErr := c.CombinedOutput() + if waitErr != nil { + return out, errors.Wrapf(waitErr, "error invoking '%s'", binName) + } + return out, nil + } + + c.Stdout = os.Stdout + c.Stderr = os.Stderr + c.Stdin = os.Stdin + + if waitErr := c.Run(); waitErr != nil { + return []byte{}, errors.Wrapf(waitErr, "error invoking '%s'", binName) + } + return []byte{}, nil +} diff --git a/cmd/wrapper/main.go b/cmd/wrapper/main.go deleted file mode 100644 index 6df0fe4..0000000 --- a/cmd/wrapper/main.go +++ /dev/null @@ -1,50 +0,0 @@ -package wrapper - -import ( - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - "github.com/spf13/cobra" - "os" - "os/exec" -) - -// NewCmdPostgresWrapper creates the new command -func NewCmdPostgresWrapper(libDir string, binName string, cmdName string) *cobra.Command { - app := &Wrapper{} - - command := &cobra.Command{ - Use: cmdName, - SilenceUsage: true, - Short: binName + " wrapper, pass " + binName + " parameters after '--'", - RunE: func(command *cobra.Command, args []string) error { - return app.Run(libDir, binName, command.Flags().Args()) - }, - } - return command -} - -type Wrapper struct{} - -func (dw *Wrapper) Run(libDir string, binName string, execArgs []string) error { - return RunWrappedPGCommand(libDir, binName, execArgs, []string{}) -} - -func RunWrappedPGCommand(libDir string, binName string, execArgs []string, envVars []string) error { - logrus.Debugf("Running '%s' %v", binName, execArgs) - - fullPath := libDir + "/bin/" + binName - c := exec.Command(fullPath, execArgs...) - - env := os.Environ() - env = append(env, "LD_LIBRARY_PATH="+libDir) - env = append(env, envVars...) - - c.Stdout = os.Stdout - c.Stderr = os.Stderr - c.Stdin = os.Stdin - c.Env = env - if waitErr := c.Run(); waitErr != nil { - return errors.Wrapf(waitErr, "error invoking '%s' (%s), LD_LIBRARY_PATH=%s", binName, fullPath, libDir) - } - return nil -} diff --git a/hack/get-binary-with-libs.py b/hack/get-binary-with-libs.py deleted file mode 100755 index 48a53f5..0000000 --- a/hack/get-binary-with-libs.py +++ /dev/null @@ -1,72 +0,0 @@ -#!/usr/bin/env python3 -import os -import subprocess -import re -import sys - -ALREADY_COPIED = [] - - -def copy_dependencies(bin_name: str, target_dir: str): - if os.path.isfile(bin_name): - bin_path = bin_name - else: - bin_path = subprocess.check_output(['whereis', bin_name]).decode('utf-8').split(' ')[1].strip() - subprocess.check_call(["file", bin_path]) - - if os.path.islink(bin_path): - bin_path = os.path.realpath(bin_path) - - try: - os.mkdir(target_dir) - except FileExistsError: - pass - - print(f' >> Copying dependencies for {bin_name}') - copy_dependencies_for_path(bin_path, target_dir, depth=0) - - -def copy_dependencies_for_path(bin_path: str, target_dir: str, depth: int = 0): - if bin_path in ALREADY_COPIED: - return - - ldd = subprocess.check_output(['ldd', bin_path]).decode('utf-8').split("\n") - - print(ldd) - - for line in ldd: - is_static_ld = "=>" not in line and "ld-" in line - - if not line.strip() or ("=>" not in line and not is_static_ld): - continue - - if is_static_ld: - parsed = re.findall('\s*([\/A-Za-z\-_.0-9]+)\ ', line) - orig_name = os.path.basename(parsed[0]) - else: - try: - parsed = re.findall('=>\s*([\/A-Za-z\-_.0-9]+)\ ', line) - orig_name = re.findall('\s*([A-Za-z\-_.0-9]+)\s*=>', line) - except Exception as e: - print(">> Line caused error: ", line) - print(" Exception: ", e) - raise - - orig_name = orig_name[0] - - if parsed[0] == "ldd": - # ['\tldd (0x7f5a18c9c000)', '\tlibc.musl-x86_64.so.1 => ldd (0x7f5a18c9c000)', ''] - continue - - real_path = subprocess.check_output(['readlink', '-f', parsed[0]]).decode('utf-8').strip() - print((" " * depth * 2) + f">> Copying {real_path} -> {orig_name}") - subprocess.check_call(['cp', real_path, target_dir + "/" + orig_name]) - - if not is_static_ld: - copy_dependencies_for_path(real_path, target_dir, depth + 1) - - ALREADY_COPIED.append(bin_path) - - -if __name__ == '__main__': - copy_dependencies(sys.argv[1], sys.argv[2]) diff --git a/main.go b/main.go index a85e94f..a47a49f 100644 --- a/main.go +++ b/main.go @@ -1,14 +1,12 @@ package main import ( - "github.com/riotkit-org/br-pg-simple-backup/assets" "github.com/riotkit-org/br-pg-simple-backup/cmd" "os" ) func main() { - tempDir := assets.UnpackOrExit() - command := cmd.Main(tempDir) + command := cmd.Main() args := os.Args if args != nil { diff --git a/release.Dockerfile b/release.Dockerfile deleted file mode 100644 index bf5baf3..0000000 --- a/release.Dockerfile +++ /dev/null @@ -1,11 +0,0 @@ -# Intentionally we do not use Alpine Linux there, to ensure binaries compatibility with at least Debian-like systems. We test on CI on Ubuntu and produce Debian images. - -# ===================== -# Create target image -# ===================== -FROM debian:bullseye-slim - -RUN apt-get update && apt-get install -y patchelf gpg - -COPY --from=builder /workspace/.build/pgbr /usr/bin/pgbr -ENTRYPOINT ["/usr/bin/pgbr"]