Make it possible to backup/restore extension configuration tables (#75)

This patch adds support for extension configuration tables to gpbackup. The
extension script can specify tables that need to be backuped/restored using the
standard function pg_catalog.pg_extension_config_dump. As a result, gpbackup
will include the data of the specified tables (but not their definition) in the
backup. If the second argument to the pg_catalog.pg_extension_config_dump
function is an empty string, then gpbackup will dump the entire contents of the
table. A non-empty string in the second argument of the
pg_catalog.pg_extension_config_dump function means a WHERE clause that filters
out the data to be dumped. GPDB versions earlier than 7 do not support the
IGNORE EXTERNAL PARTITIONS option for the COPY ( query ) TO command, so this
option is omitted if an additional filter is present.

(cherry picked from commit e75a86a)

backup/data.go

@@ -89,12 +89,24 @@ func CopyTableOut(connectionPool *dbconn.DBConn, table Table, destinationToWrite
 		// process column names to exclude generated columns from data copy out
 		columnNames = ConstructTableAttributesList(table.ColumnDefs)
 	}
+	tableName := table.FQN()
+	ignoreExternalPartitions := " IGNORE EXTERNAL PARTITIONS"
+	if table.ExtensionTableConfig != nil && *table.ExtensionTableConfig != "" {
+		if columnNames == "" {
+			columnNames = ConstructTableAttributesList(table.ColumnDefs)
+		}
+		tableName = fmt.Sprintf("(SELECT %s FROM %s %s)", columnNames[1:len(columnNames)-1], tableName, *table.ExtensionTableConfig)
+		columnNames = ""
+		if connectionPool.Version.Before("7") {
+			ignoreExternalPartitions = ""
+		}
+	}
 
 	workerInfo := ""
 	if gplog.GetVerbosity() >= gplog.LOGVERBOSE {
 		workerInfo = fmt.Sprintf("Worker %d: ", connNum)
 	}
-	query := fmt.Sprintf("COPY %s%s TO %s WITH CSV DELIMITER '%s' ON SEGMENT IGNORE EXTERNAL PARTITIONS;", table.FQN(), columnNames, copyCommand, tableDelim)
+	query := fmt.Sprintf("COPY %s%s TO %s WITH CSV DELIMITER '%s' ON SEGMENT%s;", tableName, columnNames, copyCommand, tableDelim, ignoreExternalPartitions)
 	if connectionPool.Version.AtLeast("7") {
 		utils.LogProgress(`%sExecuting "%s" on coordinator`, workerInfo, query)
 	} else {

backup/predata_relations.go

@@ -47,15 +47,27 @@ func SplitTablesByPartitionType(tables []Table, includeList []options.Relation)
 					// and does not need the suffix added
 					table.Name = AppendExtPartSuffix(table.Name)
 				}
-				metadataTables = append(metadataTables, table)
+				if table.ExtensionTableConfig == nil {
+					// If the table is not a configuration table for the extension,
+					// then add it to the metadata.
+					metadataTables = append(metadataTables, table)
+				}
 			}
 			partType := table.PartitionLevelInfo.Level
 			if connectionPool.Version.AtLeast("7") {
 				// In GPDB 7+, we need to dump out the leaf partition DDL along with their
 				// ALTER TABLE ATTACH PARTITION commands to construct the partition table
-				metadataTables = append(metadataTables, table)
+				if table.ExtensionTableConfig == nil {
+					// If the table is not a configuration table for the extension,
+					// then add it to the metadata.
+					metadataTables = append(metadataTables, table)
+				}
 			} else if partType != "l" && partType != "i" {
-				metadataTables = append(metadataTables, table)
+				if table.ExtensionTableConfig == nil {
+					// If the table is not a configuration table for the extension,
+					// then add it to the metadata.
+					metadataTables = append(metadataTables, table)
+				}
 			}
 			if MustGetFlagBool(options.LEAF_PARTITION_DATA) {
 				if partType != "p" && partType != "i" {
@@ -94,7 +106,11 @@ func SplitTablesByPartitionType(tables []Table, includeList []options.Relation)
 				table.Name = AppendExtPartSuffix(table.Name)
 			}
 
-			metadataTables = append(metadataTables, table)
+			if table.ExtensionTableConfig == nil {
+				// If the table is not a configuration table for the extension,
+				// then add it to the metadata.
+				metadataTables = append(metadataTables, table)
+			}
 			// In GPDB 7+, we need to filter out leaf and intermediate subroot partitions
 			// since the COPY will be called on the top-most root partition. It just so
 			// happens that those particular partition types will always have an

backup/queries_relations.go

@@ -220,6 +220,25 @@ func GetForeignTableRelations(connectionPool *dbconn.DBConn) []Relation {
 	return results
 }
 
+func GetExtensionTableRelations(connectionPool *dbconn.DBConn) []Relation {
+	query := fmt.Sprintf(`
+	SELECT n.oid AS schemaoid,
+		c.oid,
+		quote_ident(n.nspname) AS schema,
+		quote_ident(c.relname) AS name
+	FROM pg_class c
+		JOIN pg_namespace n ON c.relnamespace = n.oid
+	WHERE %s
+		AND relkind IN ('r', 'p')
+		AND c.oid IN (select unnest(extconfig) from pg_catalog.pg_extension)`,
+		relationAndSchemaFilterClause())
+
+	results := make([]Relation, 0)
+	err := connectionPool.Select(&results, query)
+	gplog.FatalOnError(err)
+	return results
+}
+
 type Sequence struct {
 	Relation
 	OwningTableOid          string

backup/queries_table_defs.go

@@ -83,6 +83,7 @@ type TableDefinition struct {
 	PartitionKeyDef         string
 	AttachPartitionInfo     AttachPartitionInfo
 	ForceRowSecurity        bool
+	ExtensionTableConfig    *string
 }
 
 /*
@@ -110,6 +111,7 @@ func ConstructDefinitionsForTables(connectionPool *dbconn.DBConn, tableRelations
 	partitionKeyDefs := GetPartitionKeyDefs(connectionPool)
 	attachPartitionInfo := GetAttachPartitionInfo(connectionPool)
 	forceRowSecurity := GetForceRowSecurity(connectionPool)
+	extensionTableConfs := GetExtensionTableConfigs(connectionPool)
 
 	gplog.Verbose("Constructing table definition map")
 	for _, tableRel := range tableRelations {
@@ -134,6 +136,7 @@ func ConstructDefinitionsForTables(connectionPool *dbconn.DBConn, tableRelations
 			PartitionKeyDef:         partitionKeyDefs[oid],
 			AttachPartitionInfo:     attachPartitionInfo[oid],
 			ForceRowSecurity:        forceRowSecurity[oid],
+			ExtensionTableConfig:    extensionTableConfs[oid],
 		}
 		if tableDef.Inherits == nil {
 			tableDef.Inherits = []string{}
@@ -841,3 +844,21 @@ func selectAsOidToStringMap(connectionPool *dbconn.DBConn, query string) map[uin
 	}
 	return resultMap
 }
+
+type ExtensionTableConfig struct {
+	Oid       uint32
+	Condition string
+}
+
+func GetExtensionTableConfigs(connectionPool *dbconn.DBConn) map[uint32]*string {
+	gplog.Verbose("Retrieving extension table information")
+	results := make([]ExtensionTableConfig, 0)
+	err := connectionPool.Select(&results, "SELECT unnest(extconfig) oid, unnest(extcondition) condition FROM pg_catalog.pg_extension")
+	gplog.FatalOnError(err)
+	resultMap := make(map[uint32]*string)
+	for _, result := range results {
+		condition := strings.Clone(result.Condition)
+		resultMap[result.Oid] = &condition
+	}
+	return resultMap
+}

backup/wrappers.go

@@ -212,6 +212,7 @@ func RetrieveAndProcessTables() ([]Table, []Table, []Table) {
 	if connectionPool.Version.AtLeast("6") {
 		tableRelations = append(tableRelations, GetForeignTableRelations(connectionPool)...)
 	}
+	tableRelations = append(tableRelations, GetExtensionTableRelations(connectionPool)...)
 
 	allTables := ConstructDefinitionsForTables(connectionPool, tableRelations)
 

end_to_end/end_to_end_suite_test.go

@@ -1183,6 +1183,8 @@ var _ = Describe("backup and restore end to end tests", func() {
 				CREATE EXTENSION test_ext2;
 				CREATE EXTENSION test_ext4;
 				CREATE EXTENSION test_ext1;
+				INSERT INTO test1 SELECT i FROM generate_series(1, 100) i;
+				INSERT INTO test2 SELECT i, i % 2 != 0 FROM generate_series(1, 100) i;
 			`)
 			defer testhelper.AssertQueryRuns(backupConn, `
 				DROP EXTENSION test_ext1;
@@ -1192,12 +1194,15 @@ var _ = Describe("backup and restore end to end tests", func() {
 				DROP EXTENSION test_ext3;
 			`)
 
-			output := gpbackup(gpbackupPath, backupHelperPath,
-				"--metadata-only")
+			output := gpbackup(gpbackupPath, backupHelperPath)
 			timestamp := getBackupTimestamp(string(output))
 			gprestore(gprestorePath, restoreHelperPath, timestamp,
 				"--redirect-db", "restoredb")
 
+			assertDataRestored(restoreConn, map[string]int{
+				"public.test1": 100,
+				"public.test2": 50,
+			})
 			assertArtifactsCleaned(timestamp)
 		})
 	})

end_to_end/resources/test_ext1--1.0.sql

@@ -1,3 +1,8 @@
 /* src/test/modules/test_extensions/test_ext1--1.0.sql */
 -- complain if script is sourced in psql, rather than via CREATE EXTENSION
 \echo Use "CREATE EXTENSION test_ext1" to load this file. \quit
+
+CREATE TABLE test1 (i int) DISTRIBUTED BY (i);
+CREATE TABLE test2 (i int, b bool) DISTRIBUTED BY (i);
+SELECT pg_catalog.pg_extension_config_dump('test1', '');
+SELECT pg_catalog.pg_extension_config_dump('test2', 'WHERE b');