Skip to content

Commit

Permalink
Fix backup statistics for leaf partitions (#42)
Browse files Browse the repository at this point in the history 
gpbackup does not backup statistics for leaf partitions with
leaf-partition-data option

This patch solves the problem by backuping statistics for all specified
tables, including leaf partitions, and not just root partitions.

The test for this patch is based on the test for the 85384fa.
And, as stated in that patch, it removes the unnecessary
condition when comparing the number of statistics.

Cherry-picked-from: 0ce7382
(cherry picked from commit 6b5fe21)
(cherry picked from commit 2de6423)
  • Loading branch information
@RekGRpth
RekGRpth committed Aug 1, 2024
1 parent e82255d commit 33acd29
Show file tree
Hide file tree
Showing 4 changed files with 19 additions and 29 deletions.
4 changes: 2 additions & 2 deletions backup/backup.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -128,7 +128,7 @@ func DoBackup() {
}

gplog.Info("Gathering table state information")
metadataTables, dataTables := RetrieveAndProcessTables()
metadataTables, dataTables, allTables := RetrieveAndProcessTables()
dataTables, numExtOrForeignTables := GetBackupDataSet(dataTables)
if len(dataTables) == 0 && !backupReport.MetadataOnly {
gplog.Warn("No tables in backup set contain data. Performing metadata-only backup instead.")
Expand Down Expand Up @@ -195,7 +195,7 @@ func DoBackup() {

printDataBackupWarnings(numExtOrForeignTables)
if MustGetFlagBool(options.WITH_STATS) {
backupStatistics(metadataTables)
backupStatistics(allTables)
}

globalTOC.WriteToFileAndMakeReadOnly(globalFPInfo.GetTOCFilePath())
Expand Down
8 changes: 4 additions & 4 deletions backup/wrappers.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -203,7 +203,7 @@ func createBackupDirectoriesOnAllHosts() {
* Metadata retrieval wrapper functions
*/

func RetrieveAndProcessTables() ([]Table, []Table) {
func RetrieveAndProcessTables() ([]Table, []Table, []Table) {
includedRelations := GetIncludedUserTableRelations(connectionPool, IncludedRelationFqns)
tableRelations := ConvertRelationsOptionsToBackup(includedRelations)

Expand All @@ -213,12 +213,12 @@ func RetrieveAndProcessTables() ([]Table, []Table) {
tableRelations = append(tableRelations, GetForeignTableRelations(connectionPool)...)
}

tables := ConstructDefinitionsForTables(connectionPool, tableRelations)
allTables := ConstructDefinitionsForTables(connectionPool, tableRelations)

metadataTables, dataTables := SplitTablesByPartitionType(tables, IncludedRelationFqns)
metadataTables, dataTables := SplitTablesByPartitionType(allTables, IncludedRelationFqns)
objectCounts["Tables"] = len(metadataTables)

return metadataTables, dataTables
return metadataTables, dataTables, allTables
}

func retrieveFunctions(sortables *[]Sortable, metadataMap MetadataMap) ([]Function, map[uint32]FunctionInfo) {
Expand Down
16 changes: 3 additions & 13 deletions end_to_end/end_to_end_suite_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1605,13 +1605,6 @@ var _ = Describe("backup and restore end to end tests", func() {
skipIfOldBackupVersionBefore("1.18.0")

testhelper.AssertQueryRuns(backupConn, `
CREATE TABLE et (
id character varying(13),
flg smallint,
dttm timestamp without time zone,
src character varying(80)
) WITH (appendonly='true', orientation='row', compresstype=zstd, compresslevel='3') DISTRIBUTED BY (id);
CREATE TABLE pt (
id character varying(13),
flg smallint,
Expand All @@ -1622,19 +1615,16 @@ var _ = Describe("backup and restore end to end tests", func() {
);
INSERT INTO pt(id, flg, dttm, src) VALUES (1, 1, now(), 'val');
INSERT INTO et(id, flg, dttm, src) VALUES (2, 2, now(), 'val');
ANALYZE pt;
ANALYZE et;
ANALYZE ROOTPARTITION pt;
ALTER TABLE pt EXCHANGE PARTITION src_mdm WITH TABLE et;
`)

defer testhelper.AssertQueryRuns(backupConn,
`DROP TABLE et CASCADE; DROP TABLE pt CASCADE;`)
`DROP TABLE pt CASCADE;`)
outputBkp := gpbackup(gpbackupPath, backupHelperPath,
"--with-stats",
"--leaf-partition-data",
"--backup-dir", backupDir, "--single-backup-dir")
timestamp := getBackupTimestamp(string(outputBkp))

Expand All @@ -1654,7 +1644,7 @@ var _ = Describe("backup and restore end to end tests", func() {
assertPGClassStatsRestored(backupConn, restoreConn, publicSchemaTupleCounts)
assertPGClassStatsRestored(backupConn, restoreConn, schema2TupleCounts)

statsQuery := fmt.Sprintf(`SELECT count(*) AS string FROM pg_statistic st left join pg_class cl on st.starelid = cl.oid left join pg_namespace nm on cl.relnamespace = nm.oid where cl.relname != 'pt_1_prt_src_mdm' AND %s;`, backup.SchemaFilterClause("nm"))
statsQuery := fmt.Sprintf(`SELECT count(*) AS string FROM pg_statistic st left join pg_class cl on st.starelid = cl.oid left join pg_namespace nm on cl.relnamespace = nm.oid where %s;`, backup.SchemaFilterClause("nm"))
backupStatisticCount := dbconn.MustSelectString(backupConn, statsQuery)
restoredStatisticsCount := dbconn.MustSelectString(restoreConn, statsQuery)

Expand Down
20 changes: 10 additions & 10 deletions integration/wrappers_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ var _ = Describe("Wrappers Integration", func() {
connectionPool.MustBegin(0)
defer connectionPool.MustCommit(0)

_, dataTables := backup.RetrieveAndProcessTables()
_, dataTables, _ := backup.RetrieveAndProcessTables()
Expect(len(dataTables)).To(Equal(2))
Expect(dataTables[0].Name).To(Equal("foo"))
Expect(dataTables[1].Name).To(Equal(`"BAR"`))
Expand Down Expand Up @@ -67,7 +67,7 @@ var _ = Describe("Wrappers Integration", func() {
Expect(err).To(Not(HaveOccurred()))
backup.ValidateAndProcessFilterLists(subject)

_, dataTables := backup.RetrieveAndProcessTables()
_, dataTables, _ := backup.RetrieveAndProcessTables()
Expect(len(dataTables)).To(Equal(3))
Expect(dataTables[0].Name).To(Equal("thousands"))
Expect(dataTables[1].Name).To(Equal("ten"))
Expand All @@ -87,7 +87,7 @@ var _ = Describe("Wrappers Integration", func() {
Expect(err).To(Not(HaveOccurred()))
backup.ValidateAndProcessFilterLists(subject)

_, dataTables := backup.RetrieveAndProcessTables()
_, dataTables, _ := backup.RetrieveAndProcessTables()
Expect(len(dataTables)).To(Equal(2))
Expect(dataTables[0].Name).To(Equal("ten"))
Expect(dataTables[1].Name).To(Equal("empty"))
Expand All @@ -105,7 +105,7 @@ var _ = Describe("Wrappers Integration", func() {
Expect(err).To(Not(HaveOccurred()))
backup.ValidateAndProcessFilterLists(subject)

_, dataTables := backup.RetrieveAndProcessTables()
_, dataTables, _ := backup.RetrieveAndProcessTables()
Expect(len(dataTables)).To(Equal(2))
Expect(dataTables[0].Name).To(Equal("ten"))
Expect(dataTables[1].Name).To(Equal("empty"))
Expand Down Expand Up @@ -142,7 +142,7 @@ var _ = Describe("Wrappers Integration", func() {
Expect(err).To(Not(HaveOccurred()))
backup.ValidateAndProcessFilterLists(subject)

_, dataTables := backup.RetrieveAndProcessTables()
_, dataTables, _ := backup.RetrieveAndProcessTables()
Expect(len(dataTables)).To(Equal(2))
Expect(dataTables[0].Name).To(Equal("thousands"))
Expect(dataTables[1].Name).To(Equal("empty"))
Expand All @@ -160,7 +160,7 @@ var _ = Describe("Wrappers Integration", func() {
Expect(err).To(Not(HaveOccurred()))
backup.ValidateAndProcessFilterLists(subject)

_, dataTables := backup.RetrieveAndProcessTables()
_, dataTables, _ := backup.RetrieveAndProcessTables()
Expect(len(dataTables)).To(Equal(2))
Expect(dataTables[0].Name).To(Equal("thousands"))
Expect(dataTables[1].Name).To(Equal("empty"))
Expand Down Expand Up @@ -201,7 +201,7 @@ var _ = Describe("Wrappers Integration", func() {
err = backup.ExpandIncludesForPartitions(connectionPool, subject, includeOids, backupCmdFlags)
Expect(err).ShouldNot(HaveOccurred())

_, dataTables := backup.RetrieveAndProcessTables()
_, dataTables, _ := backup.RetrieveAndProcessTables()
Expect(len(dataTables)).To(Equal(2))
Expect(dataTables[0].Name).To(Equal("partition_table"))
Expect(dataTables[1].Name).To(Equal("ten"))
Expand All @@ -221,7 +221,7 @@ var _ = Describe("Wrappers Integration", func() {
err = backup.ExpandIncludesForPartitions(connectionPool, subject, includeOids, backupCmdFlags)
Expect(err).ShouldNot(HaveOccurred())

_, dataTables := backup.RetrieveAndProcessTables()
_, dataTables, _ := backup.RetrieveAndProcessTables()

Expect(len(dataTables)).To(Equal(3))
Expect(dataTables[0].Name).To(Equal("partition_table_1_prt_girls"))
Expand Down Expand Up @@ -256,7 +256,7 @@ var _ = Describe("Wrappers Integration", func() {
Expect(err).To(Not(HaveOccurred()))
backup.ValidateAndProcessFilterLists(subject)

_, dataTables := backup.RetrieveAndProcessTables()
_, dataTables, _ := backup.RetrieveAndProcessTables()
Expect(len(dataTables)).To(Equal(3))

Expect(dataTables[0].Name).To(Equal("thousands"))
Expand All @@ -279,7 +279,7 @@ var _ = Describe("Wrappers Integration", func() {
Expect(err).To(Not(HaveOccurred()))
backup.ValidateAndProcessFilterLists(subject)

_, dataTables := backup.RetrieveAndProcessTables()
_, dataTables, _ := backup.RetrieveAndProcessTables()
Expect(len(dataTables)).To(Equal(3))

Expect(dataTables[0].Name).To(Equal("thousands"))
Expand Down

0 comments on commit 33acd29

Please sign in to comment.