ADBDEV-5541: Backup statistics by one row

backup/queries_statistics.go

@@ -53,7 +53,7 @@ type AttributeStatistic struct {
 	Values5      pq.StringArray `db:"stavalues5"`
 }
 
-func GetAttributeStatistics(connectionPool *dbconn.DBConn, tables []Table) map[uint32][]AttributeStatistic {
+func GetAttributeStatistics(connectionPool *dbconn.DBConn, tables []Table, processRow func(attStat *AttributeStatistic)) {
 	inheritClause := ""
 	statSlotClause := ""
 	if connectionPool.Version.AtLeast("6") {
@@ -119,14 +119,15 @@ func GetAttributeStatistics(connectionPool *dbconn.DBConn, tables []Table) map[u
 		inheritClause, statSlotClause, statCollationClause,
 		SchemaFilterClause("n"), utils.SliceToQuotedString(tablenames))
 
-	results := make([]AttributeStatistic, 0)
-	err := connectionPool.Select(&results, query)
+	rows, err := connectionPool.Query(query)
 	gplog.FatalOnError(err)
-	stats := make(map[uint32][]AttributeStatistic)
-	for _, stat := range results {
-		stats[stat.Oid] = append(stats[stat.Oid], stat)
+	for rows.Next() {
+		var attStat AttributeStatistic
+		err = rows.StructScan(&attStat)
+		gplog.FatalOnError(err)
+		processRow(&attStat)
 	}
-	return stats
+	gplog.FatalOnError(rows.Err())
 }
 
 type TupleStatistic struct {
@@ -137,7 +138,7 @@ type TupleStatistic struct {
 	RelTuples float64
 }
 
-func GetTupleStatistics(connectionPool *dbconn.DBConn, tables []Table) map[uint32]TupleStatistic {
+func GetTupleStatistics(connectionPool *dbconn.DBConn, tables []Table, processRow func(tupleStat *TupleStatistic)) {
 	tablenames := make([]string, 0)
 	for _, table := range tables {
 		tablenames = append(tablenames, table.FQN())
@@ -155,12 +156,13 @@ func GetTupleStatistics(connectionPool *dbconn.DBConn, tables []Table) map[uint3
 	ORDER BY n.nspname, c.relname`,
 		SchemaFilterClause("n"), utils.SliceToQuotedString(tablenames))
 
-	results := make([]TupleStatistic, 0)
-	err := connectionPool.Select(&results, query)
+	rows, err := connectionPool.Query(query)
 	gplog.FatalOnError(err)
-	stats := make(map[uint32]TupleStatistic)
-	for _, stat := range results {
-		stats[stat.Oid] = stat
+	for rows.Next() {
+		var tupleStat TupleStatistic
+		err = rows.StructScan(&tupleStat)
+		gplog.FatalOnError(err)
+		processRow(&tupleStat)
 	}
-	return stats
+	gplog.FatalOnError(rows.Err())
 }

backup/statistics.go

@@ -14,16 +14,15 @@ import (
 	"github.com/lib/pq"
 )
 
-func PrintStatisticsStatements(statisticsFile *utils.FileWithByteCount, tocfile *toc.TOC, tables []Table, attStats map[uint32][]AttributeStatistic, tupleStats map[uint32]TupleStatistic) {
-	for _, table := range tables {
-		tupleQuery := GenerateTupleStatisticsQuery(table, tupleStats[table.Oid])
-		printStatisticsStatementForTable(statisticsFile, tocfile, table, tupleQuery)
-		for _, attStat := range attStats[table.Oid] {
-			attributeQueries := GenerateAttributeStatisticsQueries(table, attStat)
-			for _, attrQuery := range attributeQueries {
-				printStatisticsStatementForTable(statisticsFile, tocfile, table, attrQuery)
-			}
-		}
+func PrintTupleStatisticsStatementForTable(statisticsFile *utils.FileWithByteCount, tocfile *toc.TOC, table Table, tupleStat TupleStatistic) {
+	tupleQuery := GenerateTupleStatisticsQuery(table, tupleStat)
+	printStatisticsStatementForTable(statisticsFile, tocfile, table, tupleQuery)
+}
+
+func PrintAttributeStatisticsStatementForTable(statisticsFile *utils.FileWithByteCount, tocfile *toc.TOC, table Table, attStat AttributeStatistic) {
+	attributeQueries := GenerateAttributeStatisticsQueries(table, attStat)
+	for _, attrQuery := range attributeQueries {
+		printStatisticsStatementForTable(statisticsFile, tocfile, table, attrQuery)
 	}
 }
 

backup/statistics_test.go

@@ -6,12 +6,22 @@ import (
 	"github.com/greenplum-db/gpbackup/backup"
 	"github.com/greenplum-db/gpbackup/testutils"
 	"github.com/greenplum-db/gpbackup/toc"
+	"github.com/greenplum-db/gpbackup/utils"
 	"github.com/lib/pq"
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 )
 
+func PrintStatisticsStatements(statisticsFile *utils.FileWithByteCount, tocfile *toc.TOC, tables []backup.Table, attStats map[uint32][]backup.AttributeStatistic, tupleStats map[uint32]backup.TupleStatistic) {
+	for _, table := range tables {
+		backup.PrintTupleStatisticsStatementForTable(statisticsFile, tocfile, table, tupleStats[table.Oid])
+		for _, attStat := range attStats[table.Oid] {
+			backup.PrintAttributeStatisticsStatementForTable(statisticsFile, tocfile, table, attStat)
+		}
+	}
+}
+
 var _ = Describe("backup/statistics tests", func() {
 	getStatInsertReplace := func(smallint int, oid int) (string, string, string, string, string) {
 		insertReplace1, insertReplace2, insertReplace3, insertReplace4, insertReplace5 := "", "", "", "", ""
@@ -67,7 +77,7 @@ var _ = Describe("backup/statistics tests", func() {
 				456: attStat2,
 			}
 
-			backup.PrintStatisticsStatements(backupfile, tocfile, tables, attStats, tupleStats)
+			PrintStatisticsStatements(backupfile, tocfile, tables, attStats, tupleStats)
 			testutils.ExpectEntry(tocfile.StatisticsEntries, 0, "testschema", "", "testtable1", toc.OBJ_STATISTICS)
 			testutils.ExpectEntry(tocfile.StatisticsEntries, 1, "testschema", "", "testtable2", toc.OBJ_STATISTICS)
 			testutils.ExpectEntry(tocfile.StatisticsEntries, 2, "testschema", "", "testtable2", toc.OBJ_STATISTICS)

backup/wrappers.go

@@ -772,11 +772,17 @@ func backupExtendedStatistic(metadataFile *utils.FileWithByteCount) {
  */
 
 func backupTableStatistics(statisticsFile *utils.FileWithByteCount, tables []Table) {
-	attStats := GetAttributeStatistics(connectionPool, tables)
-	tupleStats := GetTupleStatistics(connectionPool, tables)
-
 	backupSessionGUC(statisticsFile)
-	PrintStatisticsStatements(statisticsFile, globalTOC, tables, attStats, tupleStats)
+	tablesMap := make(map[uint32]Table)
+	for _, table := range tables {
+		tablesMap[table.Oid] = table
+	}
+	GetTupleStatistics(connectionPool, tables, func(tupleStat *TupleStatistic) {
+		PrintTupleStatisticsStatementForTable(statisticsFile, globalTOC, tablesMap[tupleStat.Oid], *tupleStat)
+	})
+	GetAttributeStatistics(connectionPool, tables, func(attStat *AttributeStatistic) {
+		PrintAttributeStatisticsStatementForTable(statisticsFile, globalTOC, tablesMap[attStat.Oid], *attStat)
+	})
 }
 
 func backupIncrementalMetadata() {

integration/statistics_create_test.go

@@ -5,11 +5,22 @@ import (
 	"github.com/greenplum-db/gp-common-go-libs/testhelper"
 	"github.com/greenplum-db/gpbackup/backup"
 	"github.com/greenplum-db/gpbackup/testutils"
+	"github.com/greenplum-db/gpbackup/toc"
+	"github.com/greenplum-db/gpbackup/utils"
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 )
 
+func PrintStatisticsStatements(statisticsFile *utils.FileWithByteCount, tocfile *toc.TOC, tables []backup.Table, attStats map[uint32][]backup.AttributeStatistic, tupleStats map[uint32]backup.TupleStatistic) {
+	for _, table := range tables {
+		backup.PrintTupleStatisticsStatementForTable(statisticsFile, tocfile, table, tupleStats[table.Oid])
+		for _, attStat := range attStats[table.Oid] {
+			backup.PrintAttributeStatisticsStatementForTable(statisticsFile, tocfile, table, attStat)
+		}
+	}
+}
+
 var _ = Describe("backup integration tests", func() {
 	BeforeEach(func() {
 		tocfile, backupfile = testutils.InitializeTestTOC(buffer, "predata")
@@ -30,22 +41,34 @@ var _ = Describe("backup integration tests", func() {
 			oldTableOid := testutils.OidFromObjectName(connectionPool, "public", "foo", backup.TYPE_RELATION)
 			tables[0].Oid = oldTableOid
 
-			beforeAttStats := backup.GetAttributeStatistics(connectionPool, tables)
-			beforeTupleStats := backup.GetTupleStatistics(connectionPool, tables)
+			beforeAttStats := make(map[uint32][]backup.AttributeStatistic)
+			backup.GetAttributeStatistics(connectionPool, tables, func(attStat *backup.AttributeStatistic) {
+				beforeAttStats[attStat.Oid] = append(beforeAttStats[attStat.Oid], *attStat)
+			})
+			beforeTupleStats := make(map[uint32]backup.TupleStatistic)
+			backup.GetTupleStatistics(connectionPool, tables, func(tupleStat *backup.TupleStatistic) {
+				beforeTupleStats[tupleStat.Oid] = *tupleStat
+			})
 			beforeTupleStat := beforeTupleStats[oldTableOid]
 
 			// Drop and recreate the table to clear the statistics
 			testhelper.AssertQueryRuns(connectionPool, "DROP TABLE public.foo")
 			testhelper.AssertQueryRuns(connectionPool, "CREATE TABLE public.foo(i int, j text, k bool)")
 
 			// Reload the retrieved statistics into the new table
-			backup.PrintStatisticsStatements(backupfile, tocfile, tables, beforeAttStats, beforeTupleStats)
+			PrintStatisticsStatements(backupfile, tocfile, tables, beforeAttStats, beforeTupleStats)
 			testhelper.AssertQueryRuns(connectionPool, buffer.String())
 
 			newTableOid := testutils.OidFromObjectName(connectionPool, "public", "foo", backup.TYPE_RELATION)
 			tables[0].Oid = newTableOid
-			afterAttStats := backup.GetAttributeStatistics(connectionPool, tables)
-			afterTupleStats := backup.GetTupleStatistics(connectionPool, tables)
+			afterAttStats := make(map[uint32][]backup.AttributeStatistic)
+			backup.GetAttributeStatistics(connectionPool, tables, func(attStat *backup.AttributeStatistic) {
+				afterAttStats[attStat.Oid] = append(afterAttStats[attStat.Oid], *attStat)
+			})
+			afterTupleStats := make(map[uint32]backup.TupleStatistic)
+			backup.GetTupleStatistics(connectionPool, tables, func(tupleStat *backup.TupleStatistic) {
+				afterTupleStats[tupleStat.Oid] = *tupleStat
+			})
 			afterTupleStat := afterTupleStats[newTableOid]
 
 			oldAtts := beforeAttStats[oldTableOid]
@@ -74,22 +97,34 @@ var _ = Describe("backup integration tests", func() {
 			oldTableOid := testutils.OidFromObjectName(connectionPool, "public", "foo''\"''''bar", backup.TYPE_RELATION)
 			tables[0].Oid = oldTableOid
 
-			beforeAttStats := backup.GetAttributeStatistics(connectionPool, tables)
-			beforeTupleStats := backup.GetTupleStatistics(connectionPool, tables)
+			beforeAttStats := make(map[uint32][]backup.AttributeStatistic)
+			backup.GetAttributeStatistics(connectionPool, tables, func(attStat *backup.AttributeStatistic) {
+				beforeAttStats[attStat.Oid] = append(beforeAttStats[attStat.Oid], *attStat)
+			})
+			beforeTupleStats := make(map[uint32]backup.TupleStatistic)
+			backup.GetTupleStatistics(connectionPool, tables, func(tupleStat *backup.TupleStatistic) {
+				beforeTupleStats[tupleStat.Oid] = *tupleStat
+			})
 			beforeTupleStat := beforeTupleStats[oldTableOid]
 
 			// Drop and recreate the table to clear the statistics
 			testhelper.AssertQueryRuns(connectionPool, "DROP TABLE public.\"foo'\"\"''bar\"")
 			testhelper.AssertQueryRuns(connectionPool, "CREATE TABLE public.\"foo'\"\"''bar\"(i int, j text, k bool)")
 
 			// Reload the retrieved statistics into the new table
-			backup.PrintStatisticsStatements(backupfile, tocfile, tables, beforeAttStats, beforeTupleStats)
+			PrintStatisticsStatements(backupfile, tocfile, tables, beforeAttStats, beforeTupleStats)
 			testhelper.AssertQueryRuns(connectionPool, buffer.String())
 
 			newTableOid := testutils.OidFromObjectName(connectionPool, "public", "foo''\"''''bar", backup.TYPE_RELATION)
 			tables[0].Oid = newTableOid
-			afterAttStats := backup.GetAttributeStatistics(connectionPool, tables)
-			afterTupleStats := backup.GetTupleStatistics(connectionPool, tables)
+			afterAttStats := make(map[uint32][]backup.AttributeStatistic)
+			backup.GetAttributeStatistics(connectionPool, tables, func(attStat *backup.AttributeStatistic) {
+				afterAttStats[attStat.Oid] = append(afterAttStats[attStat.Oid], *attStat)
+			})
+			afterTupleStats := make(map[uint32]backup.TupleStatistic)
+			backup.GetTupleStatistics(connectionPool, tables, func(tupleStat *backup.TupleStatistic) {
+				afterTupleStats[tupleStat.Oid] = *tupleStat
+			})
 			afterTupleStat := afterTupleStats[newTableOid]
 
 			oldAtts := beforeAttStats[oldTableOid]

integration/statistics_queries_test.go

@@ -31,7 +31,10 @@ var _ = Describe("backup integration tests", func() {
 	})
 	Describe("GetAttributeStatistics", func() {
 		It("returns attribute statistics for a table", func() {
-			attStats := backup.GetAttributeStatistics(connectionPool, tables)
+			attStats := make(map[uint32][]backup.AttributeStatistic)
+			backup.GetAttributeStatistics(connectionPool, tables, func(attStat *backup.AttributeStatistic) {
+				attStats[attStat.Oid] = append(attStats[attStat.Oid], *attStat)
+			})
 			Expect(attStats).To(HaveLen(1))
 			Expect(attStats[tableOid]).To(HaveLen(3))
 			tableAttStatsI := attStats[tableOid][0]
@@ -83,7 +86,10 @@ var _ = Describe("backup integration tests", func() {
 	})
 	Describe("GetTupleStatistics", func() {
 		It("returns tuple statistics for a table", func() {
-			tupleStats := backup.GetTupleStatistics(connectionPool, tables)
+			tupleStats := make(map[uint32]backup.TupleStatistic)
+			backup.GetTupleStatistics(connectionPool, tables, func(tupleStat *backup.TupleStatistic) {
+				tupleStats[tupleStat.Oid] = *tupleStat
+			})
 			Expect(tupleStats).To(HaveLen(1))
 			tableTupleStats := tupleStats[tableOid]