Merge pull request #19 from arenadata/1.24.0-sync

ADBDEV-2506 1.24.0 sync

Makefile

@@ -22,21 +22,20 @@ SUBDIRS_ALL=$(SUBDIRS_HAS_UNIT) integration/ end_to_end/
 GOLANG_LINTER=$(GOPATH)/bin/golangci-lint
 GINKGO=$(GOPATH)/bin/ginkgo
 GOIMPORTS=$(GOPATH)/bin/goimports
-GO_ENV=GO111MODULE=on # ensures the project still compiles in $GOPATH/src using golang versions 1.12 and below
-GO_BUILD=$(GO_ENV) go build -mod=readonly
+GO_BUILD=go build -mod=readonly
 DEBUG=-gcflags=all="-N -l"
 
 CUSTOM_BACKUP_DIR ?= "/tmp"
 helper_path ?= $(BIN_DIR)/$(HELPER)
 
 depend :
-		$(GO_ENV) go mod download
+	go mod download
 
-$(GINKGO) :
-		$(GO_ENV) go install github.com/onsi/ginkgo/ginkgo
+$(GINKGO) : # v1.14.0 is compatible with centos6 default gcc version
+	go install github.com/onsi/ginkgo/ginkgo@v1.14.0
 
 $(GOIMPORTS) :
-		$(GO_ENV) go install golang.org/x/tools/cmd/goimports
+	go install golang.org/x/tools/cmd/goimports@latest
 
 format : $(GOIMPORTS)
 		@goimports -w $(shell find . -type f -name '*.go' -not -path "./vendor/*")
@@ -52,21 +51,21 @@ lint : $(GOLANG_LINTER)
 		golangci-lint run --tests=false
 
 unit : $(GINKGO)
-		$(GO_ENV) ginkgo $(GINKGO_FLAGS) $(SUBDIRS_HAS_UNIT) 2>&1
+	ginkgo $(GINKGO_FLAGS) $(SUBDIRS_HAS_UNIT) 2>&1
 
 unit_all_gpdb_versions : $(GINKGO)
-		TEST_GPDB_VERSION=4.3.999 $(GO_ENV) ginkgo $(GINKGO_FLAGS) $(SUBDIRS_HAS_UNIT) 2>&1
-		TEST_GPDB_VERSION=5.999.0 $(GO_ENV) ginkgo $(GINKGO_FLAGS) $(SUBDIRS_HAS_UNIT) 2>&1
-		TEST_GPDB_VERSION=6.999.0 $(GO_ENV) ginkgo $(GINKGO_FLAGS) $(SUBDIRS_HAS_UNIT) 2>&1
-		TEST_GPDB_VERSION=7.0.0 $(GO_ENV) ginkgo $(GINKGO_FLAGS) $(SUBDIRS_HAS_UNIT) 2>&1 # GPDB master
+		TEST_GPDB_VERSION=4.3.999 ginkgo $(GINKGO_FLAGS) $(SUBDIRS_HAS_UNIT) 2>&1
+		TEST_GPDB_VERSION=5.999.0 ginkgo $(GINKGO_FLAGS) $(SUBDIRS_HAS_UNIT) 2>&1
+		TEST_GPDB_VERSION=6.999.0 ginkgo $(GINKGO_FLAGS) $(SUBDIRS_HAS_UNIT) 2>&1
+		TEST_GPDB_VERSION=7.0.0 ginkgo $(GINKGO_FLAGS) $(SUBDIRS_HAS_UNIT) 2>&1 # GPDB master
 
 integration : $(GINKGO)
-		$(GO_ENV) ginkgo $(GINKGO_FLAGS) integration 2>&1
+	ginkgo $(GINKGO_FLAGS) integration 2>&1
 
 test : build unit integration
 
 end_to_end : $(GINKGO)
-		$(GO_ENV) ginkgo $(GINKGO_FLAGS) -slowSpecThreshold=10 end_to_end -- --custom_backup_dir $(CUSTOM_BACKUP_DIR) 2>&1
+	ginkgo $(GINKGO_FLAGS) -slowSpecThreshold=10 end_to_end -- --custom_backup_dir $(CUSTOM_BACKUP_DIR) 2>&1
 
 coverage :
 		@./show_coverage.sh
@@ -108,6 +107,10 @@ clean :
 		rm -f $(BIN_DIR)/$(BACKUP) $(BACKUP) $(BIN_DIR)/$(RESTORE) $(RESTORE) $(BIN_DIR)/$(HELPER) $(HELPER)
 		# Test artifacts
 		rm -rf /tmp/go-build* /tmp/gexec_artifacts* /tmp/ginkgo*
+		docker stop s3-minio # stop minio before removing its data directories
+		docker rm s3-minio
+		rm -rf /tmp/minio
+		rm -f /tmp/minio_config.yaml
 		# Code coverage files
 		rm -rf /tmp/cover* /tmp/unit*
 		go clean -i -r -x -testcache -modcache
@@ -126,3 +129,12 @@ info-report:
 	@echo "Info and verbose messaging:"
 	@echo ""
 	@ag "gplog.Info|gplog.Verbose" --ignore "*_test*"
+
+test-s3-local: build install
+	${PWD}/plugins/generate_minio_config.sh
+	mkdir -p /tmp/minio/gpbackup-s3-test
+	docker run -d --name s3-minio -p 9000:9000 -p 9001:9001 -v /tmp/minio:/data/minio quay.io/minio/minio server /data/minio --console-address ":9001"
+	sleep 2 # Wait for minio server to start up
+	${PWD}/plugins/plugin_test.sh $(BIN_DIR)/gpbackup_s3_plugin /tmp/minio_config.yaml
+	docker stop s3-minio
+	docker rm s3-minio

README.md

@@ -27,7 +27,7 @@ The `build` target will put the `gpbackup` and `gprestore` binaries in `$HOME/go
 
 This will also attempt to copy `gpbackup_helper` to the greenplum segments (retrieving hostnames from `gp_segment_configuration`). Pay attention to the output as it will indicate whether this operation was successful.
 
-`make build_linux` and `make build_mac` are for cross compiling between macOS and Linux
+`make build_linux` is for cross compiling on macOS, and the target is Linux.
 
 `make install` will scp the `gpbackup_helper` binary (used with -single-data-file flag) to all hosts
 

arenadata/run_gpbackup_tests.bash

@@ -14,6 +14,6 @@ source /usr/local/greenplum-db-devel/greenplum_path.sh;
 source ~/gpdb_src/gpAux/gpdemo/gpdemo-env.sh;
 gpconfig -c shared_preload_libraries -v dummy_seclabel;
 gpstop -ar;
-wget https://golang.org/dl/go1.14.15.linux-amd64.tar.gz;
-tar -C ~/ -xzf go1.14.15.linux-amd64.tar.gz;
+wget https://golang.org/dl/go1.17.6.linux-amd64.tar.gz;
+tar -C ~/ -xzf go1.17.6.linux-amd64.tar.gz;
 PATH=$PATH:~/go/bin GOPATH=~/go make depend build install test end_to_end -C go/src/github.com/greenplum-db/gpbackup/"

backup/backup.go

@@ -1,6 +1,7 @@
 package backup
 
 import (
+	"encoding/json"
 	"errors"
 	"fmt"
 	"os"
@@ -522,3 +523,77 @@ func CreateInitialSegmentPipes(oidList []string, c *cluster.Cluster, connectionP
 	}
 	return maxPipes
 }
+
+type TableLocks struct {
+	Oid         uint32
+	Database    string
+	Relation    string
+	Mode        string
+	Application string
+	Granted     string
+	User        string
+	Pid         string
+}
+
+func getTableLocks(table Table) []TableLocks {
+	conn := dbconn.NewDBConnFromEnvironment(MustGetFlagString(options.DBNAME))
+	conn.MustConnect(1)
+	var query string
+	defer conn.Close()
+	if conn.Version.Before("6") {
+		query = fmt.Sprintf(`
+		SELECT c.oid as oid,
+		coalesce(a.datname, '') as database,
+		n.nspname || '.' || l.relation as relation,
+		l.mode,
+		l.GRANTED as granted,
+		coalesce(a.application_name, '') as application,
+		coalesce(a.usename, '') as user,
+		a.procpid as pid
+		FROM pg_stat_activity a
+		JOIN pg_locks l ON l.pid = a.procpid
+		JOIN pg_class c on c.oid = l.relation
+		JOIN pg_namespace n on n.oid=c.relnamespace
+		WHERE (a.datname = '%s' OR a.datname IS NULL)
+		AND NOT a.procpid = pg_backend_pid()
+		AND relation = '%s'::regclass
+		AND mode = 'AccessExclusiveLock'
+		ORDER BY a.query_start;
+		`, conn.DBName, table.FQN())
+	} else {
+		query = fmt.Sprintf(`
+		SELECT c.oid as oid,
+		coalesce(a.datname, '') as database,
+		n.nspname || '.' || l.relation relation,
+		l.mode,
+		l.GRANTED as granted,
+		coalesce(a.application_name, '') as application,
+		coalesce(a.usename, '') as user,
+		a.pid
+		FROM pg_stat_activity a
+		JOIN pg_locks l ON l.pid = a.pid
+		JOIN pg_class c on c.oid = l.relation
+		JOIN pg_namespace n on n.oid=c.relnamespace
+		WHERE (a.datname = '%s' OR a.datname IS NULL)
+		AND NOT a.pid = pg_backend_pid()
+		AND relation = '%s'::regclass
+		AND mode = 'AccessExclusiveLock'
+		ORDER BY a.query_start;
+		`, conn.DBName, table.FQN())
+	}
+
+	locksResults := make([]TableLocks, 0)
+	err := conn.Select(&locksResults, query)
+	if err != nil {
+		gplog.FatalOnError(err)
+	}
+
+	return locksResults
+}
+
+func logTableLocks(table Table, whichConn int) {
+	locks := getTableLocks(table)
+	jsonData, _ := json.Marshal(&locks)
+	gplog.Warn("Worker %d could not acquire AccessShareLock for table %s. Terminating worker and deferring table to main worker thread.",	whichConn,table.FQN())
+	gplog.Warn("Locks held on table %s: %s", table.FQN(), jsonData)
+}

backup/data.go

@@ -172,9 +172,8 @@ func backupDataForAllTablesCopyQueue(tables []Table) []map[uint32]int64 {
 						// the following WARN message is nicely outputted.
 						fmt.Printf("\n")
 					}
-					gplog.Warn("Worker %d could not acquire AccessShareLock for table %s. Terminating worker and deferring table to main worker thread.",
-						whichConn, table.FQN())
-
+					// Log locks held on the table
+					logTableLocks(table, whichConn)
 					oidMap.Store(table.Oid, Deferred)
 					// Rollback transaction since it's in an aborted state
 					connectionPool.MustRollback(whichConn)
@@ -300,9 +299,8 @@ func backupDataForAllTables(tables []Table) []map[uint32]int64 {
 							// the following WARN message is nicely outputted.
 							fmt.Printf("\n")
 						}
-						gplog.Warn("Worker %d could not acquire AccessShareLock for table %s. Terminating worker and deferring table to main worker thread.",
-							whichConn, table.FQN())
-
+						// Log locks held on the table
+						logTableLocks(table, whichConn)
 						// Defer table to main worker thread
 						deferredTablesMutex.Lock()
 						deferredTables = append(deferredTables, table)
@@ -395,6 +393,5 @@ func LockTableNoWait(dataTable Table, connNum int) error {
 	if err != nil {
 		return err
 	}
-
 	return nil
 }