Merge pull request #26 from arenadata/1.28.0-sync

ADBDEV-3822 1.28.0 sync

CONTRIBUTING.md

@@ -2,7 +2,7 @@
 
 We warmly welcome and greatly appreciate contributions from the
 community. By participating you agree to the [code of
-conduct](https://github.com/greenplum-db/gpbackup/blob/master/CODE-OF-CONDUCT.md).
+conduct](https://github.com/greenplum-db/gpbackup/blob/main/CODE-OF-CONDUCT.md).
 Overall, we follow GPDB's comprehensive contribution policy. Please
 refer to it [here](https://github.com/greenplum-db/gpdb#contributing)
 for details.
@@ -22,7 +22,7 @@ for details.
 * Try and follow similar coding styles as found throughout the code
   base.
 * Make commits as logical units for ease of reviewing.
-* Rebase with master often to stay in sync with upstream.
+* Rebase with main often to stay in sync with upstream.
 * Add new tests to cover your code. We use
   [Ginkgo](http://onsi.github.io/ginkgo/) and
   [Gomega](https://onsi.github.io/gomega/) for testing.
@@ -44,9 +44,9 @@ git commit --fixup <commit SHA>
   -- or --
 git commit --squash <commit SHA>
 ```
-* Once approved, before merging into master squash your fixups with:
+* Once approved, before merging into main squash your fixups with:
 ```
-git rebase -i --autosquash origin/master
+git rebase -i --autosquash origin/main
 git push --force-with-lease $USER <my-feature-branch>
 ```
 

Makefile

@@ -27,15 +27,25 @@ DEBUG=-gcflags=all="-N -l"
 CUSTOM_BACKUP_DIR ?= "/tmp"
 helper_path ?= $(BIN_DIR)/$(HELPER)
 
+# Prefer gpsync as the newer utility, fall back to gpscp if not present (older installs)
+ifeq (, $(shell which gpsync))
+COPYUTIL=gpscp
+else
+COPYUTIL=gpsync
+endif
+
 depend :
 	go mod download
 
-$(GINKGO) : # v1.14.0 is compatible with centos6 default gcc version
-	go install github.com/onsi/ginkgo/v2/ginkgo@v2.2.0
+$(GINKGO) :
+	go install github.com/onsi/ginkgo/v2/ginkgo@v2.8.4
 
 $(GOIMPORTS) :
 	go install golang.org/x/tools/cmd/goimports@latest
 
+$(GOSQLITE) :
+	go install github.com/mattn/go-sqlite3
+
 format : $(GOIMPORTS)
 		@goimports -w $(shell find . -type f -name '*.go' -not -path "./vendor/*")
 
@@ -56,28 +66,28 @@ unit_all_gpdb_versions : $(GINKGO)
 		TEST_GPDB_VERSION=4.3.999 ginkgo $(GINKGO_FLAGS) $(SUBDIRS_HAS_UNIT) 2>&1
 		TEST_GPDB_VERSION=5.999.0 ginkgo $(GINKGO_FLAGS) $(SUBDIRS_HAS_UNIT) 2>&1
 		TEST_GPDB_VERSION=6.999.0 ginkgo $(GINKGO_FLAGS) $(SUBDIRS_HAS_UNIT) 2>&1
-		TEST_GPDB_VERSION=7.999.0 ginkgo $(GINKGO_FLAGS) $(SUBDIRS_HAS_UNIT) 2>&1 # GPDB master
+		TEST_GPDB_VERSION=7.999.0 ginkgo $(GINKGO_FLAGS) $(SUBDIRS_HAS_UNIT) 2>&1 # GPDB main
 
 integration : $(GINKGO)
 	ginkgo $(GINKGO_FLAGS) integration 2>&1
 
 test : build unit integration
 
 end_to_end : $(GINKGO)
-	ginkgo $(GINKGO_FLAGS) --slow-spec-threshold=10s end_to_end -- --custom_backup_dir $(CUSTOM_BACKUP_DIR) 2>&1
+	ginkgo $(GINKGO_FLAGS) --timeout=3h --poll-progress-after=0s end_to_end -- --custom_backup_dir $(CUSTOM_BACKUP_DIR) 2>&1
 
 coverage :
 		@./show_coverage.sh
 
-build :
-		$(GO_BUILD) -tags '$(BACKUP)' -o $(BIN_DIR)/$(BACKUP) -ldflags "-X $(BACKUP_VERSION_STR)"
-		$(GO_BUILD) -tags '$(RESTORE)' -o $(BIN_DIR)/$(RESTORE) -ldflags "-X $(RESTORE_VERSION_STR)"
-		$(GO_BUILD) -tags '$(HELPER)' -o $(BIN_DIR)/$(HELPER) -ldflags "-X $(HELPER_VERSION_STR)"
+build : $(GOSQLITE)
+		CGO_ENABLED=1 $(GO_BUILD) -tags '$(BACKUP)' -o $(BIN_DIR)/$(BACKUP) --ldflags '-X $(BACKUP_VERSION_STR)'
+		CGO_ENABLED=1 $(GO_BUILD) -tags '$(RESTORE)' -o $(BIN_DIR)/$(RESTORE) --ldflags '-X $(RESTORE_VERSION_STR)'
+		CGO_ENABLED=1 $(GO_BUILD) -tags '$(HELPER)' -o $(BIN_DIR)/$(HELPER) --ldflags '-X $(HELPER_VERSION_STR)'
 
 debug :
-		$(GO_BUILD) -tags '$(BACKUP)' -o $(BIN_DIR)/$(BACKUP) -ldflags "-X $(BACKUP_VERSION_STR)" $(DEBUG)
-		$(GO_BUILD) -tags '$(RESTORE)' -o $(BIN_DIR)/$(RESTORE) -ldflags "-X $(RESTORE_VERSION_STR)" $(DEBUG)
-		$(GO_BUILD) -tags '$(HELPER)' -o $(BIN_DIR)/$(HELPER) -ldflags "-X $(HELPER_VERSION_STR)" $(DEBUG)
+		CGO_ENABLED=1 $(GO_BUILD) -tags '$(BACKUP)' -o $(BIN_DIR)/$(BACKUP) -ldflags "-X $(BACKUP_VERSION_STR)" $(DEBUG)
+		CGO_ENABLED=1 $(GO_BUILD) -tags '$(RESTORE)' -o $(BIN_DIR)/$(RESTORE) -ldflags "-X $(RESTORE_VERSION_STR)" $(DEBUG)
+		CGO_ENABLED=1 $(GO_BUILD) -tags '$(HELPER)' -o $(BIN_DIR)/$(HELPER) -ldflags "-X $(HELPER_VERSION_STR)" $(DEBUG)
 
 build_linux :
 		env GOOS=linux GOARCH=amd64 $(GO_BUILD) -tags '$(BACKUP)' -o $(BACKUP) -ldflags "-X $(BACKUP_VERSION_STR)"
@@ -88,7 +98,7 @@ install :
 		cp $(BIN_DIR)/$(BACKUP) $(BIN_DIR)/$(RESTORE) $(GPHOME)/bin
 		@psql -X -t -d template1 -c 'select distinct hostname from gp_segment_configuration where content != -1' > /tmp/seg_hosts 2>/dev/null; \
 		if [ $$? -eq 0 ]; then \
-			gpscp -f /tmp/seg_hosts $(helper_path) =:$(GPHOME)/bin/$(HELPER); \
+			$(COPYUTIL) -f /tmp/seg_hosts $(helper_path) =:$(GPHOME)/bin/$(HELPER); \
 			if [ $$? -eq 0 ]; then \
 				echo 'Successfully copied gpbackup_helper to $(GPHOME) on all segments'; \
 			else \

README.md

@@ -5,6 +5,7 @@
 ## Pre-Requisites
 
 The project requires the Go Programming language version 1.11 or higher. Follow the directions [here](https://golang.org/doc/) for installation, usage and configuration instructions.
+The project also has a dependency on `sqlite3`. This is installed by default on many platforms, but you must install it on your system if it is not present.
 
 ## Downloading
 

arenadata/run_gpbackup_tests.bash

@@ -9,11 +9,11 @@ make -C gpdb_src/contrib/dummy_seclabel/ install
 gpdb_src/concourse/scripts/setup_gpadmin_user.bash
 make_cluster
 
+wget https://golang.org/dl/go1.20.5.linux-amd64.tar.gz -O - | tar -C /opt -xz;
+
 su - gpadmin -c "
 source /usr/local/greenplum-db-devel/greenplum_path.sh;
 source ~/gpdb_src/gpAux/gpdemo/gpdemo-env.sh;
 gpconfig -c shared_preload_libraries -v dummy_seclabel;
 gpstop -ar;
-wget https://golang.org/dl/go1.17.6.linux-amd64.tar.gz;
-tar -C ~/ -xzf go1.17.6.linux-amd64.tar.gz;
-PATH=$PATH:~/go/bin GOPATH=~/go make depend build install test end_to_end -C go/src/github.com/greenplum-db/gpbackup/"
+PATH=$PATH:/opt/go/bin:~/go/bin GOPATH=~/go make depend build install test end_to_end -C /home/gpadmin/go/src/github.com/greenplum-db/gpbackup"

backup/backup.go

@@ -382,22 +382,39 @@ func DoTeardown() {
 		if statErr != nil { // Even if this isn't os.IsNotExist, don't try to write a report file in case of further errors
 			return
 		}
-		historyFilename := globalFPInfo.GetBackupHistoryFilePath()
+		historyDBName := globalFPInfo.GetBackupHistoryDatabasePath()
+		historyFileLegacyName := globalFPInfo.GetBackupHistoryFilePath()
 		reportFilename := globalFPInfo.GetBackupReportFilePath()
 		configFilename := globalFPInfo.GetConfigFilePath()
 
 		time.Sleep(time.Second) // We sleep for 1 second to ensure multiple backups do not start within the same second.
 
+		// Check if legacy history file is still present, log warning if so. Only log if we're planning to use history db.
+		var err error
+		if _, err = os.Stat(historyFileLegacyName); err == nil && !MustGetFlagBool(options.NO_HISTORY) {
+			gplog.Warn("Legacy gpbackup_history file %s is still present. Please run 'gpbackup_manager migrate-history' to add entries from that file to the history database.", historyFileLegacyName)
+		}
+
 		if backupReport != nil {
 			if !backupFailed {
 				backupReport.BackupConfig.Status = history.BackupStatusSucceed
 			}
 			backupReport.ConstructBackupParamsString()
 			backupReport.BackupConfig.SegmentCount = len(globalCluster.ContentIDs) - 1
-			err := history.WriteBackupHistory(historyFilename, &backupReport.BackupConfig)
-			if err != nil {
-				gplog.Error(fmt.Sprintf("%v", err))
+
+			if !MustGetFlagBool(options.NO_HISTORY) {
+				historyDB, err := history.InitializeHistoryDatabase(historyDBName)
+				if err != nil {
+					gplog.Error(fmt.Sprintf("%v", err))
+				} else {
+					err = history.StoreBackupHistory(historyDB, &backupReport.BackupConfig)
+					historyDB.Close()
+					if err != nil {
+						gplog.Error(fmt.Sprintf("%v", err))
+					}
+				}
 			}
+
 			history.WriteConfigFile(&backupReport.BackupConfig, configFilename)
 			if backupReport.BackupConfig.EndTime == "" {
 				backupReport.BackupConfig.EndTime = history.CurrentTimestamp()
@@ -447,7 +464,7 @@ func DoCleanup(backupFailed bool) {
 			// If the terminate query is sent via a connection with an active COPY command, and the COPY's pipe is cleaned up, the COPY query will hang.
 			// This results in the DoCleanup function passed to the signal handler to never return, blocking the os.Exit call
 			if wasTerminated {
-				// It is possible for the COPY command to become orphaned if an agent process is killed
+				// It is possible for the COPY command to become orphaned if an agent process is stopped
 				utils.TerminateHangingCopySessions(connectionPool, globalFPInfo, fmt.Sprintf("gpbackup_%s", globalFPInfo.Timestamp))
 			}
 			if backupFailed {

backup/data.go

@@ -51,7 +51,7 @@ func AddTableDataEntriesToTOC(tables []Table, rowsCopiedMaps []map[uint32]int64)
 				}
 			}
 			attributes := ConstructTableAttributesList(table.ColumnDefs)
-			globalTOC.AddMasterDataEntry(table.Schema, table.Name, table.Oid, attributes, rowsCopied, table.PartitionLevelInfo.RootName, table.DistPolicy)
+			globalTOC.AddCoordinatorDataEntry(table.Schema, table.Name, table.Oid, attributes, rowsCopied, table.PartitionLevelInfo.RootName, table.DistPolicy)
 		}
 	}
 }
@@ -144,7 +144,7 @@ func backupDataForAllTables(tables []Table) []map[uint32]int64 {
 	rowsCopiedMaps := make([]map[uint32]int64, connectionPool.NumConns)
 	/*
 	 * We break when an interrupt is received and rely on
-	 * TerminateHangingCopySessions to kill any COPY statements
+	 * TerminateHangingCopySessions to halt any COPY statements
 	 * in progress if they don't finish on their own.
 	 */
 	tasks := make(chan Table, len(tables))
@@ -337,7 +337,9 @@ func GetBackupDataSet(tables []Table) ([]Table, int64) {
 // the lock, the call will fail instead of block. Return the failure for handling.
 func LockTableNoWait(dataTable Table, connNum int) error {
 	var lockMode string
-	if connectionPool.Version.AtLeast("6.21.0") {
+	if connectionPool.Version.AtLeast("7") {
+		lockMode = `IN ACCESS SHARE MODE NOWAIT COORDINATOR ONLY`
+	} else if connectionPool.Version.AtLeast("6.21.0") {
 		lockMode = `IN ACCESS SHARE MODE NOWAIT MASTER ONLY`
 	} else {
 		lockMode = `IN ACCESS SHARE MODE NOWAIT`

backup/data_test.go

@@ -54,7 +54,7 @@ var _ = Describe("backup/data tests", func() {
 		It("adds an entry for a regular table to the TOC", func() {
 			tables := []backup.Table{table}
 			backup.AddTableDataEntriesToTOC(tables, rowsCopiedMaps)
-			expectedDataEntries := []toc.MasterDataEntry{{Schema: "public", Name: "table", Oid: 1, AttributeString: "(a)"}}
+			expectedDataEntries := []toc.CoordinatorDataEntry{{Schema: "public", Name: "table", Oid: 1, AttributeString: "(a)"}}
 			Expect(tocfile.DataEntries).To(Equal(expectedDataEntries))
 		})
 		It("does not add an entry for an external table to the TOC", func() {

backup/incremental.go

@@ -1,10 +1,11 @@
 package backup
 
 import (
+	"fmt"
 	"path"
 
 	"github.com/greenplum-db/gp-common-go-libs/gplog"
-	"github.com/greenplum-db/gp-common-go-libs/iohelper"
+	"github.com/greenplum-db/gp-common-go-libs/operating"
 	"github.com/greenplum-db/gpbackup/history"
 	"github.com/greenplum-db/gpbackup/options"
 	"github.com/greenplum-db/gpbackup/toc"
@@ -43,13 +44,12 @@ func GetTargetBackupTimestamp() string {
 
 func GetLatestMatchingBackupTimestamp() string {
 	latestTimestamp := ""
-	var contents *history.History
 	var latestMatchingBackupHistoryEntry *history.BackupConfig
-	var err error
-	if iohelper.FileExistsAndIsReadable(globalFPInfo.GetBackupHistoryFilePath()) {
-		contents, _, err = history.NewHistory(globalFPInfo.GetBackupHistoryFilePath())
-		gplog.FatalOnError(err)
-		latestMatchingBackupHistoryEntry = GetLatestMatchingBackupConfig(contents, &backupReport.BackupConfig)
+
+	historyDBPath := globalFPInfo.GetBackupHistoryDatabasePath()
+	_, err := operating.System.Stat(historyDBPath)
+	if err == nil {
+		latestMatchingBackupHistoryEntry = GetLatestMatchingBackupConfig(historyDBPath, &backupReport.BackupConfig)
 	}
 
 	if latestMatchingBackupHistoryEntry == nil {
@@ -62,10 +62,53 @@ func GetLatestMatchingBackupTimestamp() string {
 	return latestTimestamp
 }
 
-func GetLatestMatchingBackupConfig(history *history.History, currentBackupConfig *history.BackupConfig) *history.BackupConfig {
-	for _, backupConfig := range history.BackupConfigs {
-		if matchesIncrementalFlags(&backupConfig, currentBackupConfig) && !backupConfig.Failed() {
-			return &backupConfig
+func GetLatestMatchingBackupConfig(historyDBPath string, currentBackupConfig *history.BackupConfig) *history.BackupConfig {
+	// get list of timestamps for backups that match filterable flags, most recent first, then
+	// iterate through them querying and checking one at a time. this is necessary due to the
+	// impracticality of checking the include and exclude sets directly in a query
+
+	historyDB, _ := history.InitializeHistoryDatabase(historyDBPath)
+
+	whereClause := fmt.Sprintf(`backup_dir = '%s' AND database_name = '%s' AND leaf_partition_data = %v
+		AND plugin = '%s' AND single_data_file = %v AND compressed = %v AND date_deleted = ''`,
+		MustGetFlagString(options.BACKUP_DIR),
+		currentBackupConfig.DatabaseName,
+		MustGetFlagBool(options.LEAF_PARTITION_DATA),
+		currentBackupConfig.Plugin,
+		MustGetFlagBool(options.SINGLE_DATA_FILE),
+		currentBackupConfig.Compressed)
+
+	getBackupTimetampsQuery := fmt.Sprintf(`
+		SELECT timestamp
+		FROM backups
+		WHERE %s
+		ORDER BY timestamp DESC`, whereClause)
+	timestampRows, err := historyDB.Query(getBackupTimetampsQuery)
+	if err != nil {
+		gplog.Error(err.Error())
+		return nil
+	}
+	defer timestampRows.Close()
+
+	timestamps := make([]string, 0)
+	for timestampRows.Next() {
+		var timestamp string
+		err = timestampRows.Scan(&timestamp)
+		if err != nil {
+			gplog.Error(err.Error())
+			return nil
+		}
+		timestamps = append(timestamps, timestamp)
+	}
+
+	for _, ts := range timestamps {
+		backupConfig, err := history.GetBackupConfig(ts, historyDB)
+		if err != nil {
+			gplog.Error(err.Error())
+			return nil
+		}
+		if !backupConfig.Failed() && matchesIncrementalFlags(backupConfig, currentBackupConfig) {
+			return backupConfig
 		}
 	}