Skip to content

Commit

Permalink
Revert "Fix backup statistics for leaf partitions (#42)"
Browse files Browse the repository at this point in the history 
This reverts commit 2de6423 to solve conflicts
with 12bca9e.
  • Loading branch information
@RekGRpth
RekGRpth committed Jul 16, 2024
1 parent 82fd612 commit 723484e
Show file tree
Hide file tree
Showing 4 changed files with 29 additions and 19 deletions.
4 changes: 2 additions & 2 deletions backup/backup.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -128,7 +128,7 @@ func DoBackup() {
}

gplog.Info("Gathering table state information")
metadataTables, dataTables, allTables := RetrieveAndProcessTables()
metadataTables, dataTables := RetrieveAndProcessTables()
dataTables, numExtOrForeignTables := GetBackupDataSet(dataTables)
if len(dataTables) == 0 && !backupReport.MetadataOnly {
gplog.Warn("No tables in backup set contain data. Performing metadata-only backup instead.")
Expand Down Expand Up @@ -195,7 +195,7 @@ func DoBackup() {

printDataBackupWarnings(numExtOrForeignTables)
if MustGetFlagBool(options.WITH_STATS) {
backupStatistics(allTables)
backupStatistics(metadataTables)
}

globalTOC.WriteToFileAndMakeReadOnly(globalFPInfo.GetTOCFilePath())
Expand Down
8 changes: 4 additions & 4 deletions backup/wrappers.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -205,7 +205,7 @@ func createBackupDirectoriesOnAllHosts() {
* Metadata retrieval wrapper functions
*/

func RetrieveAndProcessTables() ([]Table, []Table, []Table) {
func RetrieveAndProcessTables() ([]Table, []Table) {
includedRelations := GetIncludedUserTableRelations(connectionPool, IncludedRelationFqns)
tableRelations := ConvertRelationsOptionsToBackup(includedRelations)

Expand All @@ -215,12 +215,12 @@ func RetrieveAndProcessTables() ([]Table, []Table, []Table) {
tableRelations = append(tableRelations, GetForeignTableRelations(connectionPool)...)
}

allTables := ConstructDefinitionsForTables(connectionPool, tableRelations)
tables := ConstructDefinitionsForTables(connectionPool, tableRelations)

metadataTables, dataTables := SplitTablesByPartitionType(allTables, IncludedRelationFqns)
metadataTables, dataTables := SplitTablesByPartitionType(tables, IncludedRelationFqns)
objectCounts["Tables"] = len(metadataTables)

return metadataTables, dataTables, allTables
return metadataTables, dataTables
}

func retrieveFunctions(sortables *[]Sortable, metadataMap MetadataMap) ([]Function, map[uint32]FunctionInfo) {
Expand Down
16 changes: 13 additions & 3 deletions end_to_end/end_to_end_suite_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1527,6 +1527,13 @@ var _ = Describe("backup and restore end to end tests", func() {
skipIfOldBackupVersionBefore("1.18.0")

testhelper.AssertQueryRuns(backupConn, `
CREATE TABLE et (
id character varying(13),
flg smallint,
dttm timestamp without time zone,
src character varying(80)
) WITH (appendonly='true', orientation='row', compresstype=zstd, compresslevel='3') DISTRIBUTED BY (id);
CREATE TABLE pt (
id character varying(13),
flg smallint,
Expand All @@ -1537,16 +1544,19 @@ var _ = Describe("backup and restore end to end tests", func() {
);
INSERT INTO pt(id, flg, dttm, src) VALUES (1, 1, now(), 'val');
INSERT INTO et(id, flg, dttm, src) VALUES (2, 2, now(), 'val');
ANALYZE pt;
ANALYZE et;
ANALYZE ROOTPARTITION pt;
ALTER TABLE pt EXCHANGE PARTITION src_mdm WITH TABLE et;
`)

defer testhelper.AssertQueryRuns(backupConn,
`DROP TABLE pt CASCADE;`)
`DROP TABLE et CASCADE; DROP TABLE pt CASCADE;`)
timestamp := gpbackup(gpbackupPath, backupHelperPath,
"--with-stats",
"--leaf-partition-data",
"--backup-dir", backupDir, "--single-backup-dir")
files, err := path.Glob(path.Join(backupDir, "backups/*",
timestamp, "*statistics.sql"))
Expand All @@ -1564,7 +1574,7 @@ var _ = Describe("backup and restore end to end tests", func() {
assertPGClassStatsRestored(backupConn, restoreConn, publicSchemaTupleCounts)
assertPGClassStatsRestored(backupConn, restoreConn, schema2TupleCounts)

statsQuery := fmt.Sprintf(`SELECT count(*) AS string FROM pg_statistic st left join pg_class cl on st.starelid = cl.oid left join pg_namespace nm on cl.relnamespace = nm.oid where %s;`, backup.SchemaFilterClause("nm"))
statsQuery := fmt.Sprintf(`SELECT count(*) AS string FROM pg_statistic st left join pg_class cl on st.starelid = cl.oid left join pg_namespace nm on cl.relnamespace = nm.oid where cl.relname != 'pt_1_prt_src_mdm' AND %s;`, backup.SchemaFilterClause("nm"))
backupStatisticCount := dbconn.MustSelectString(backupConn, statsQuery)
restoredStatisticsCount := dbconn.MustSelectString(restoreConn, statsQuery)

Expand Down
20 changes: 10 additions & 10 deletions integration/wrappers_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ var _ = Describe("Wrappers Integration", func() {
connectionPool.MustBegin(0)
defer connectionPool.MustCommit(0)

_, dataTables, _ := backup.RetrieveAndProcessTables()
_, dataTables := backup.RetrieveAndProcessTables()
Expect(len(dataTables)).To(Equal(2))
Expect(dataTables[0].Name).To(Equal("foo"))
Expect(dataTables[1].Name).To(Equal(`"BAR"`))
Expand All @@ -63,7 +63,7 @@ var _ = Describe("Wrappers Integration", func() {
rootCmd := &cobra.Command{}
backup.DoInit(rootCmd) // initialize the ObjectCount

_, dataTables, _ := backup.RetrieveAndProcessTables()
_, dataTables := backup.RetrieveAndProcessTables()
Expect(len(dataTables)).To(Equal(3))
Expect(dataTables[0].Name).To(Equal("thousands"))
Expect(dataTables[1].Name).To(Equal("ten"))
Expand All @@ -76,7 +76,7 @@ var _ = Describe("Wrappers Integration", func() {
backup.DoInit(rootCmd) // initialize the ObjectCount
rootCmd.Execute()

_, dataTables, _ := backup.RetrieveAndProcessTables()
_, dataTables := backup.RetrieveAndProcessTables()
Expect(len(dataTables)).To(Equal(2))
Expect(dataTables[0].Name).To(Equal("ten"))
Expect(dataTables[1].Name).To(Equal("empty"))
Expand All @@ -88,7 +88,7 @@ var _ = Describe("Wrappers Integration", func() {
backup.DoInit(rootCmd) // initialize the ObjectCount
rootCmd.Execute()

_, dataTables, _ := backup.RetrieveAndProcessTables()
_, dataTables := backup.RetrieveAndProcessTables()
Expect(len(dataTables)).To(Equal(2))
Expect(dataTables[0].Name).To(Equal("ten"))
Expect(dataTables[1].Name).To(Equal("empty"))
Expand Down Expand Up @@ -119,7 +119,7 @@ var _ = Describe("Wrappers Integration", func() {
backup.DoInit(rootCmd) // initialize the ObjectCount
rootCmd.Execute()

_, dataTables, _ := backup.RetrieveAndProcessTables()
_, dataTables := backup.RetrieveAndProcessTables()
Expect(len(dataTables)).To(Equal(2))
Expect(dataTables[0].Name).To(Equal("thousands"))
Expect(dataTables[1].Name).To(Equal("empty"))
Expand All @@ -131,7 +131,7 @@ var _ = Describe("Wrappers Integration", func() {
backup.DoInit(rootCmd) // initialize the ObjectCount
rootCmd.Execute()

_, dataTables, _ := backup.RetrieveAndProcessTables()
_, dataTables := backup.RetrieveAndProcessTables()
Expect(len(dataTables)).To(Equal(2))
Expect(dataTables[0].Name).To(Equal("thousands"))
Expect(dataTables[1].Name).To(Equal("empty"))
Expand Down Expand Up @@ -168,7 +168,7 @@ var _ = Describe("Wrappers Integration", func() {
err := opts.ExpandIncludesForPartitions(connectionPool, rootCmd.Flags())
Expect(err).ShouldNot(HaveOccurred())

_, dataTables, _ := backup.RetrieveAndProcessTables()
_, dataTables := backup.RetrieveAndProcessTables()
Expect(len(dataTables)).To(Equal(2))
Expect(dataTables[0].Name).To(Equal("partition_table"))
Expect(dataTables[1].Name).To(Equal("ten"))
Expand All @@ -184,7 +184,7 @@ var _ = Describe("Wrappers Integration", func() {
err := opts.ExpandIncludesForPartitions(connectionPool, rootCmd.Flags())
Expect(err).ShouldNot(HaveOccurred())

_, dataTables, _ := backup.RetrieveAndProcessTables()
_, dataTables := backup.RetrieveAndProcessTables()

Expect(len(dataTables)).To(Equal(3))
Expect(dataTables[0].Name).To(Equal("partition_table_1_prt_girls"))
Expand Down Expand Up @@ -215,7 +215,7 @@ var _ = Describe("Wrappers Integration", func() {
rootCmd := &cobra.Command{}
backup.DoInit(rootCmd) // initialize the ObjectCount

_, dataTables, _ := backup.RetrieveAndProcessTables()
_, dataTables := backup.RetrieveAndProcessTables()
Expect(len(dataTables)).To(Equal(3))

Expect(dataTables[0].Name).To(Equal("thousands"))
Expand All @@ -234,7 +234,7 @@ var _ = Describe("Wrappers Integration", func() {
rootCmd := &cobra.Command{}
backup.DoInit(rootCmd) // initialize the ObjectCount

_, dataTables, _ := backup.RetrieveAndProcessTables()
_, dataTables := backup.RetrieveAndProcessTables()
Expect(len(dataTables)).To(Equal(3))

Expect(dataTables[0].Name).To(Equal("thousands"))
Expand Down

0 comments on commit 723484e

Please sign in to comment.