Merge branch '1.29.1-conflicts' into 1.29.1-sync

arenadata/Dockerfile

@@ -0,0 +1,3 @@
+FROM hub.adsw.io/library/gpdb6_regress:latest
+
+COPY . /home/gpadmin/go/src/github.com/greenplum-db/gpbackup

arenadata/README.md

@@ -0,0 +1,6 @@
+## How to run tests
+
+```bash
+docker build -t gpbackup:test -f arenadata/Dockerfile .
+docker run --rm -it --sysctl 'kernel.sem=500 1024000 200 4096' gpbackup:test bash -c "ssh-keygen -A && /usr/sbin/sshd && bash /home/gpadmin/go/src/github.com/greenplum-db/gpbackup/arenadata/run_gpbackup_tests.bash"
+```

arenadata/arenadata_helper.go

@@ -0,0 +1,42 @@
+package arenadata
+
+import (
+	"regexp"
+	"strconv"
+
+	"github.com/greenplum-db/gp-common-go-libs/gplog"
+	"github.com/pkg/errors"
+)
+
+var (
+	adPattern = regexp.MustCompile(`_arenadata(\d+)`)
+)
+
+func EnsureAdVersionCompatibility(backupVersion string, restoreVersion string) {
+	var (
+		adBackup, adRestore int
+		err                 error
+	)
+	if strVersion := adPattern.FindAllStringSubmatch(backupVersion, -1); len(strVersion) > 0 {
+		adBackup, err = strconv.Atoi(strVersion[0][1])
+		gplog.FatalOnError(err)
+	} else {
+		gplog.Fatal(errors.Errorf("Invalid arenadata version format for gpbackup: %s", backupVersion), "")
+	}
+	if strVersion := adPattern.FindAllStringSubmatch(restoreVersion, -1); len(strVersion) > 0 {
+		adRestore, err = strconv.Atoi(strVersion[0][1])
+		gplog.FatalOnError(err)
+	} else {
+		gplog.Fatal(errors.Errorf("Invalid arenadata version format for gprestore: %s", restoreVersion), "")
+	}
+	if adRestore < adBackup {
+		gplog.Fatal(errors.Errorf("gprestore arenadata%d cannot restore a backup taken with gpbackup arenadata%d; please use gprestore arenadata%d or later.",
+			adRestore, adBackup, adBackup), "")
+	}
+}
+
+// fullVersion: gpbackup version + '_' + arenadata release + ('+' + gpbackup build)
+// example: 1.20.4_arenadata2+dev.1.g768b7e0 -> 1.20.4+dev.1.g768b7e0
+func GetOriginalVersion(fullVersion string) string {
+	return adPattern.ReplaceAllString(fullVersion, "")
+}

arenadata/run_gpbackup_tests.bash

@@ -0,0 +1,19 @@
+#!/bin/bash -l
+
+set -eox pipefail
+
+source gpdb_src/concourse/scripts/common.bash
+install_and_configure_gpdb
+make -C gpdb_src/src/test/regress/
+make -C gpdb_src/contrib/dummy_seclabel/ install
+gpdb_src/concourse/scripts/setup_gpadmin_user.bash
+make_cluster
+
+wget https://golang.org/dl/go1.20.5.linux-amd64.tar.gz -O - | tar -C /opt -xz;
+
+su - gpadmin -c "
+source /usr/local/greenplum-db-devel/greenplum_path.sh;
+source ~/gpdb_src/gpAux/gpdemo/gpdemo-env.sh;
+gpconfig -c shared_preload_libraries -v dummy_seclabel;
+gpstop -ar;
+PATH=$PATH:/opt/go/bin:~/go/bin GOPATH=~/go make depend build install test end_to_end -C /home/gpadmin/go/src/github.com/greenplum-db/gpbackup"

backup/backup.go

@@ -242,10 +242,10 @@ func backupPredata(metadataFile *utils.FileWithByteCount, tables []Table, tableO
 	objects := make([]Sortable, 0)
 	metadataMap := make(MetadataMap)
 
+	objects = append(objects, convertToSortableSlice(tables)...)
 	if !tableOnly {
 		functions, funcInfoMap = retrieveFunctions(&objects, metadataMap)
 	}
-	objects = append(objects, convertToSortableSlice(tables)...)
 	relationMetadata := GetMetadataForObjectType(connectionPool, TYPE_RELATION)
 	addToMetadataMap(relationMetadata, metadataMap)
 

backup/dependencies.go

@@ -2,6 +2,7 @@ package backup
 
 import (
 	"fmt"
+	"sort"
 	"strings"
 
 	"github.com/greenplum-db/gp-common-go-libs/dbconn"
@@ -87,6 +88,24 @@ type Sortable interface {
 	GetUniqueID() UniqueID
 }
 
+type Sortables []Sortable
+
+func (s Sortables) Len() int {
+	return len(s)
+}
+func (s Sortables) Swap(i, j int) {
+	s[i], s[j] = s[j], s[i]
+}
+func (s Sortables) Less(i, j int) bool {
+	return s[i].GetUniqueID().Oid < s[j].GetUniqueID().Oid
+}
+
+func SortByOid(sortables []Sortable) []Sortable {
+	s := Sortables(sortables)
+	sort.Sort(s)
+	return []Sortable(s)
+}
+
 func TopologicalSort(slice []Sortable, dependencies DependencyMap) []Sortable {
 	inDegrees := make(map[UniqueID]int)
 	dependencyIndexes := make(map[UniqueID]int)

backup/queries_relations.go

@@ -98,17 +98,29 @@ func getUserTableRelations(connectionPool *dbconn.DBConn) []Relation {
 	}
 
 	query := fmt.Sprintf(`
-	SELECT n.oid AS schemaoid,
-		c.oid AS oid,
-		quote_ident(n.nspname) AS schema,
-		quote_ident(c.relname) AS name
-	FROM pg_class c
+	SELECT schemaoid, oid, schema, name FROM (
+		SELECT n.oid AS schemaoid,
+			c.oid AS oid,
+			quote_ident(n.nspname) AS schema,
+			quote_ident(c.relname) AS name,
+			coalesce(prt.pages, c.relpages) AS pages
+		FROM pg_class c
 		JOIN pg_namespace n ON c.relnamespace = n.oid
-	WHERE %s
-		%s
-		AND relkind IN (%s)
-		AND %s
-		ORDER BY c.oid`,
+		LEFT JOIN (
+			SELECT
+				p.parrelid,
+				sum(pc.relpages) AS pages
+			FROM pg_partition_rule AS pr
+			JOIN pg_partition AS p ON pr.paroid = p.oid
+			JOIN pg_class AS pc ON pr.parchildrelid = pc.oid
+			GROUP BY p.parrelid
+		) AS prt ON prt.parrelid = c.oid
+		WHERE %s
+			%s
+			AND relkind IN (%s)
+			AND %s
+	) res
+	ORDER BY pages DESC, oid`,
 		relationAndSchemaFilterClause(), childPartitionFilter, relkindFilter, ExtensionFilterClause("c"))
 
 	results := make([]Relation, 0)
@@ -128,15 +140,27 @@ func getUserTableRelationsWithIncludeFiltering(connectionPool *dbconn.DBConn, in
 	includeOids := getOidsFromRelationList(connectionPool, includedRelationsQuoted)
 	oidStr := strings.Join(includeOids, ", ")
 	query := fmt.Sprintf(`
-	SELECT n.oid AS schemaoid,
-		c.oid AS oid,
-		quote_ident(n.nspname) AS schema,
-		quote_ident(c.relname) AS name
-	FROM pg_class c
+	SELECT schemaoid, oid, schema, name FROM (
+		SELECT n.oid AS schemaoid,
+			c.oid AS oid,
+			quote_ident(n.nspname) AS schema,
+			quote_ident(c.relname) AS name,
+			coalesce(prt.pages, c.relpages) AS pages
+		FROM pg_class c
 		JOIN pg_namespace n ON c.relnamespace = n.oid
-	WHERE c.oid IN (%s)
+		LEFT JOIN (
+			SELECT
+				p.parrelid,
+				sum(pc.relpages) AS pages
+			FROM pg_partition_rule AS pr
+			JOIN pg_partition AS p ON pr.paroid = p.oid
+			JOIN pg_class AS pc ON pr.parchildrelid = pc.oid
+			GROUP BY p.parrelid
+		) AS prt ON prt.parrelid = c.oid
+		WHERE c.oid IN (%s)
 		AND relkind IN (%s)
-	ORDER BY c.oid`, oidStr, relkindFilter)
+	) res
+	ORDER BY pages DESC, oid`, oidStr, relkindFilter)
 
 	results := make([]Relation, 0)
 	err := connectionPool.Select(&results, query)

backup/wrappers.go

@@ -631,6 +631,7 @@ func backupDependentObjects(metadataFile *utils.FileWithByteCount, tables []Tabl
 
 	gplog.Verbose("Writing CREATE statements for dependent objects to metadata file")
 
+	sortables = SortByOid(sortables)
 	backupSet := createBackupSet(sortables)
 	relevantDeps := GetDependencies(connectionPool, backupSet, tables)
 	if connectionPool.Version.Is("4") && !tableOnly {

end_to_end/end_to_end_suite_test.go

@@ -1673,6 +1673,31 @@ LANGUAGE plpgsql NO SQL;`)
 
 				assertArtifactsCleaned(restoreConn, timestamp)
 			})
+			It("runs gpbackup and gprestore to backup functions depending on table row's type", func() {
+				skipIfOldBackupVersionBefore("1.19.0")
+
+				testhelper.AssertQueryRuns(backupConn, "CREATE TABLE table_provides_type (n int);")
+				defer testhelper.AssertQueryRuns(backupConn, "DROP TABLE table_provides_type;")
+
+				testhelper.AssertQueryRuns(backupConn, "INSERT INTO table_provides_type values (1);")
+				testhelper.AssertQueryRuns(backupConn, "CREATE OR REPLACE FUNCTION func_depends_on_row_type(arg table_provides_type[]) RETURNS void AS $$ BEGIN; SELECT NULL; END; $$ LANGUAGE SQL;")
+
+				defer testhelper.AssertQueryRuns(backupConn, "DROP FUNCTION func_depends_on_row_type(arg table_provides_type[]);")
+
+				timestamp := gpbackup(gpbackupPath, backupHelperPath)
+				gprestore(gprestorePath, restoreHelperPath, timestamp,
+					"--redirect-db", "restoredb")
+
+				assertRelationsCreated(restoreConn, TOTAL_RELATIONS+1) // for 1 new table
+				assertDataRestored(restoreConn, schema2TupleCounts)
+				assertDataRestored(restoreConn, map[string]int{
+					"public.foo":                 40000,
+					"public.holds":               50000,
+					"public.sales":               13,
+					"public.table_provides_type": 1})
+
+				assertArtifactsCleaned(restoreConn, timestamp)
+			})
 			It("Can restore xml with xmloption set to document", func() {
 				testutils.SkipIfBefore6(backupConn)
 				// Set up the XML table that contains XML content

end_to_end/locks_test.go

@@ -200,8 +200,8 @@ var _ = Describe("Deadlock handling", func() {
 		// Concurrently wait for gpbackup to block on the trigger metadata dump section. Once we
 		// see gpbackup blocked, request AccessExclusiveLock (to imitate a TRUNCATE or VACUUM
 		// FULL) on all the test tables.
-		dataTables := []string{`public."FOObar"`, "public.foo", "public.holds", "public.sales", "public.bigtable",
-			"schema2.ao1", "schema2.ao2", "schema2.foo2", "schema2.foo3", "schema2.returns"}
+		dataTables := []string{"public.bigtable", "public.holds", "public.foo", "schema2.foo3", "schema2.ao1",
+			"schema2.ao2", `public."FOObar"`, "public.sales", "schema2.foo2", "schema2.returns"}
 		for _, dataTable := range dataTables {
 			go func(dataTable string) {
 				accessExclusiveLockConn := testutils.SetupTestDbConn("testdb")
@@ -260,7 +260,7 @@ var _ = Describe("Deadlock handling", func() {
 		// Check that 10 deadlock traps were placed during the test
 		Expect(accessExclBlockedLockCount).To(Equal(10))
 		// No non-main worker should have been able to run COPY due to deadlock detection
-		for i := 1; i < 2; i++ {
+		for i := 1; i <= 2; i++ {
 			expectedLockString := fmt.Sprintf("[DEBUG]:-Worker %d: LOCK TABLE ", i)
 			Expect(stdout).To(ContainSubstring(expectedLockString))
 
@@ -270,7 +270,7 @@ var _ = Describe("Deadlock handling", func() {
 			unexpectedCopyString := fmt.Sprintf("[DEBUG]:-Worker %d: COPY ", i)
 			Expect(stdout).ToNot(ContainSubstring(unexpectedCopyString))
 
-			expectedLockString = fmt.Sprintf(`Locks held on table %s`, dataTables[i])
+			expectedLockString = fmt.Sprintf(`Locks held on table %s`, dataTables[i-1])
 			Expect(stdout).To(ContainSubstring(expectedLockString))
 
 			Expect(stdout).To(ContainSubstring(`"Mode":"AccessExclusiveLock"`))

helper/helper.go

@@ -8,7 +8,6 @@ import (
 	"os"
 	"os/signal"
 	"path/filepath"
-	"sort"
 	"strconv"
 	"strings"
 	"sync"
@@ -210,7 +209,6 @@ func getOidListFromFile(oidFileName string) ([]int, error) {
 		num, _ := strconv.Atoi(oid)
 		oidList[i] = num
 	}
-	sort.Ints(oidList)
 	return oidList, nil
 }