From 1dd79be9bfd704c11e1d4312f4f1880e1966b94d Mon Sep 17 00:00:00 2001 From: Georgy Shelkovy Date: Thu, 10 Oct 2024 09:58:41 +0500 Subject: [PATCH 1/8] Refactor diskquota local_table_stats_map --- src/gp_activetable.c | 33 ++++++++++++--------------------- src/gp_activetable.h | 6 ++++++ src/quotamodel.c | 17 +++++++---------- 3 files changed, 25 insertions(+), 31 deletions(-) diff --git a/src/gp_activetable.c b/src/gp_activetable.c index adbd69ef..3e320ab6 100644 --- a/src/gp_activetable.c +++ b/src/gp_activetable.c @@ -374,8 +374,8 @@ gp_fetch_active_tables(bool is_init) Assert(Gp_role == GP_ROLE_DISPATCH); memset(&ctl, 0, sizeof(ctl)); - ctl.keysize = sizeof(TableEntryKey); - ctl.entrysize = sizeof(DiskQuotaActiveTableEntry); + ctl.keysize = sizeof(Oid); + ctl.entrysize = sizeof(ActiveTableEntryCombined) + SEGCOUNT * sizeof(Size); ctl.hcxt = CurrentMemoryContext; local_table_stats_map = diskquota_hash_create("local active table map with relfilenode info", 1024, &ctl, @@ -949,8 +949,7 @@ load_table_size(HTAB *local_table_stats_map) TupleDesc tupdesc; int i; bool found; - TableEntryKey key; - DiskQuotaActiveTableEntry *quota_entry; + ActiveTableEntryCombined *quota_entry; SPIPlanPtr plan; Portal portal; char *sql = "select tableid, size, segid from diskquota.table_size"; @@ -1017,13 +1016,10 @@ load_table_size(HTAB *local_table_stats_map) dat = SPI_getbinval(tup, tupdesc, 3, &isnull); if (isnull) continue; segid = DatumGetInt16(dat); - key.reloid = reloid; - key.segid = segid; - quota_entry = (DiskQuotaActiveTableEntry *)hash_search(local_table_stats_map, &key, HASH_ENTER, &found); + quota_entry = (ActiveTableEntryCombined *)hash_search(local_table_stats_map, &reloid, HASH_ENTER, &found); quota_entry->reloid = reloid; - quota_entry->tablesize = size; - quota_entry->segid = segid; + quota_entry->tablesize[segid + 1] = size; } SPI_freetuptable(SPI_tuptable); SPI_cursor_fetch(portal, true, 10000); @@ -1168,8 +1164,7 @@ pull_active_table_size_from_seg(HTAB *local_table_stats_map, char *active_oid_ar bool found; Oid reloid; int segId; - TableEntryKey key; - DiskQuotaActiveTableEntry *entry; + ActiveTableEntryCombined *entry; PGresult *pgresult = cdb_pgresults.pg_results[i]; @@ -1184,39 +1179,35 @@ pull_active_table_size_from_seg(HTAB *local_table_stats_map, char *active_oid_ar { reloid = atooid(PQgetvalue(pgresult, j, 0)); tableSize = (Size)atoll(PQgetvalue(pgresult, j, 1)); - key.reloid = reloid; /* for diskquota extension version is 1.0, pgresult doesn't contain segid */ if (PQnfields(pgresult) == 3) { /* get the segid, tablesize for each table */ segId = atoi(PQgetvalue(pgresult, j, 2)); - key.segid = segId; - entry = (DiskQuotaActiveTableEntry *)hash_search(local_table_stats_map, &key, HASH_ENTER, &found); + entry = (ActiveTableEntryCombined *)hash_search(local_table_stats_map, &reloid, HASH_ENTER, &found); if (!found) { /* receive table size info from the first segment */ entry->reloid = reloid; - entry->segid = segId; } - entry->tablesize = tableSize; + entry->tablesize[segId + 1] = tableSize; } /* when segid is -1, the tablesize is the sum of tablesize of master and all segments */ - key.segid = -1; - entry = (DiskQuotaActiveTableEntry *)hash_search(local_table_stats_map, &key, HASH_ENTER, &found); + segId = -1; + entry = (ActiveTableEntryCombined *)hash_search(local_table_stats_map, &reloid, HASH_ENTER, &found); if (!found) { /* receive table size info from the first segment */ entry->reloid = reloid; - entry->tablesize = tableSize; - entry->segid = -1; + entry->tablesize[segId + 1] = tableSize; } else { /* sum table size from all the segments */ - entry->tablesize = entry->tablesize + tableSize; + entry->tablesize[segId + 1] = entry->tablesize[segId + 1] + tableSize; } } } diff --git a/src/gp_activetable.h b/src/gp_activetable.h index 1ff10d60..0bac9d55 100644 --- a/src/gp_activetable.h +++ b/src/gp_activetable.h @@ -37,6 +37,12 @@ typedef struct DiskQuotaActiveTableEntry Size tablesize; } DiskQuotaActiveTableEntry; +typedef struct ActiveTableEntryCombined +{ + Oid reloid; + Size tablesize[]; +} ActiveTableEntryCombined; + extern HTAB *gp_fetch_active_tables(bool force); extern void init_active_table_hook(void); extern void init_shm_worker_active_tables(void); diff --git a/src/quotamodel.c b/src/quotamodel.c index f558bc66..4b6cba01 100644 --- a/src/quotamodel.c +++ b/src/quotamodel.c @@ -918,9 +918,8 @@ calculate_table_disk_usage(bool is_init, HTAB *local_active_table_stat_map) TableSizeEntry *tsentry = NULL; Oid relOid; HASH_SEQ_STATUS iter; - DiskQuotaActiveTableEntry *active_table_entry; + ActiveTableEntryCombined *active_table_entry; TableSizeEntryKey key; - TableEntryKey active_table_key; List *oidlist; ListCell *l; int delete_entries_num = 0; @@ -1042,10 +1041,8 @@ calculate_table_disk_usage(bool is_init, HTAB *local_active_table_stat_map) /* mark tsentry is_exist */ if (tsentry) set_table_size_entry_flag(tsentry, TABLE_EXIST); - active_table_key.reloid = relOid; - active_table_key.segid = cur_segid; - active_table_entry = (DiskQuotaActiveTableEntry *)hash_search( - local_active_table_stat_map, &active_table_key, HASH_FIND, &active_tbl_found); + active_table_entry = (ActiveTableEntryCombined *)hash_search( + local_active_table_stat_map, &relOid, HASH_FIND, &active_tbl_found); /* skip to recalculate the tables which are not in active list */ if (active_tbl_found) @@ -1055,15 +1052,15 @@ calculate_table_disk_usage(bool is_init, HTAB *local_active_table_stat_map) /* pretend process as utility mode, and append the table size on master */ Gp_role = GP_ROLE_UTILITY; - active_table_entry->tablesize += calculate_table_size(relOid); + active_table_entry->tablesize[cur_segid + 1] += calculate_table_size(relOid); Gp_role = GP_ROLE_DISPATCH; } /* firstly calculate the updated total size of a table */ - updated_total_size = active_table_entry->tablesize - TableSizeEntryGetSize(tsentry, cur_segid); + updated_total_size = active_table_entry->tablesize[cur_segid + 1] - TableSizeEntryGetSize(tsentry, cur_segid); /* update the table_size entry */ - TableSizeEntrySetSize(tsentry, cur_segid, active_table_entry->tablesize); + TableSizeEntrySetSize(tsentry, cur_segid, active_table_entry->tablesize[cur_segid + 1]); TableSizeEntrySetFlushFlag(tsentry, cur_segid); /* update the disk usage, there may be entries in the map whose keys are InvlidOid as the tsentry does @@ -1347,7 +1344,7 @@ dispatch_rejectmap(HTAB *local_active_table_stat_map) { HASH_SEQ_STATUS hash_seq; GlobalRejectMapEntry *rejectmap_entry; - DiskQuotaActiveTableEntry *active_table_entry; + ActiveTableEntryCombined *active_table_entry; int num_entries, count = 0; CdbPgResults cdb_pgresults = {NULL, 0}; StringInfoData rows; From 4f444f67221fe7fb2893634bfac112e7ab6c14b1 Mon Sep 17 00:00:00 2001 From: Georgy Shelkovy Date: Thu, 10 Oct 2024 10:40:06 +0500 Subject: [PATCH 2/8] fix --- src/gp_activetable.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/gp_activetable.h b/src/gp_activetable.h index 0bac9d55..4ea51048 100644 --- a/src/gp_activetable.h +++ b/src/gp_activetable.h @@ -40,7 +40,7 @@ typedef struct DiskQuotaActiveTableEntry typedef struct ActiveTableEntryCombined { Oid reloid; - Size tablesize[]; + Size tablesize[1]; /* variable length array */ } ActiveTableEntryCombined; extern HTAB *gp_fetch_active_tables(bool force); From ae19d343fca06beb651cc14c95a8287731fffb3a Mon Sep 17 00:00:00 2001 From: Georgy Shelkovy Date: Thu, 10 Oct 2024 10:54:41 +0500 Subject: [PATCH 3/8] format --- src/gp_activetable.c | 40 ++++++++++++++++++------------------- src/quotamodel.c | 47 ++++++++++++++++++++++---------------------- 2 files changed, 44 insertions(+), 43 deletions(-) diff --git a/src/gp_activetable.c b/src/gp_activetable.c index 3e320ab6..1ef423f0 100644 --- a/src/gp_activetable.c +++ b/src/gp_activetable.c @@ -946,13 +946,13 @@ get_active_tables_oid(void) static void load_table_size(HTAB *local_table_stats_map) { - TupleDesc tupdesc; - int i; - bool found; - ActiveTableEntryCombined *quota_entry; - SPIPlanPtr plan; - Portal portal; - char *sql = "select tableid, size, segid from diskquota.table_size"; + TupleDesc tupdesc; + int i; + bool found; + ActiveTableEntryCombined *quota_entry; + SPIPlanPtr plan; + Portal portal; + char *sql = "select tableid, size, segid from diskquota.table_size"; if ((plan = SPI_prepare(sql, 0, NULL)) == NULL) ereport(ERROR, (errmsg("[diskquota] SPI_prepare(\"%s\") failed", sql))); @@ -1015,10 +1015,10 @@ load_table_size(HTAB *local_table_stats_map) size = DatumGetInt64(dat); dat = SPI_getbinval(tup, tupdesc, 3, &isnull); if (isnull) continue; - segid = DatumGetInt16(dat); + segid = DatumGetInt16(dat); quota_entry = (ActiveTableEntryCombined *)hash_search(local_table_stats_map, &reloid, HASH_ENTER, &found); - quota_entry->reloid = reloid; + quota_entry->reloid = reloid; quota_entry->tablesize[segid + 1] = size; } SPI_freetuptable(SPI_tuptable); @@ -1160,11 +1160,11 @@ pull_active_table_size_from_seg(HTAB *local_table_stats_map, char *active_oid_ar /* sum table size from each segment into local_table_stats_map */ for (i = 0; i < cdb_pgresults.numResults; i++) { - Size tableSize; - bool found; - Oid reloid; - int segId; - ActiveTableEntryCombined *entry; + Size tableSize; + bool found; + Oid reloid; + int segId; + ActiveTableEntryCombined *entry; PGresult *pgresult = cdb_pgresults.pg_results[i]; @@ -1177,14 +1177,14 @@ pull_active_table_size_from_seg(HTAB *local_table_stats_map, char *active_oid_ar for (j = 0; j < PQntuples(pgresult); j++) { - reloid = atooid(PQgetvalue(pgresult, j, 0)); - tableSize = (Size)atoll(PQgetvalue(pgresult, j, 1)); + reloid = atooid(PQgetvalue(pgresult, j, 0)); + tableSize = (Size)atoll(PQgetvalue(pgresult, j, 1)); /* for diskquota extension version is 1.0, pgresult doesn't contain segid */ if (PQnfields(pgresult) == 3) { /* get the segid, tablesize for each table */ - segId = atoi(PQgetvalue(pgresult, j, 2)); - entry = (ActiveTableEntryCombined *)hash_search(local_table_stats_map, &reloid, HASH_ENTER, &found); + segId = atoi(PQgetvalue(pgresult, j, 2)); + entry = (ActiveTableEntryCombined *)hash_search(local_table_stats_map, &reloid, HASH_ENTER, &found); if (!found) { @@ -1196,12 +1196,12 @@ pull_active_table_size_from_seg(HTAB *local_table_stats_map, char *active_oid_ar /* when segid is -1, the tablesize is the sum of tablesize of master and all segments */ segId = -1; - entry = (ActiveTableEntryCombined *)hash_search(local_table_stats_map, &reloid, HASH_ENTER, &found); + entry = (ActiveTableEntryCombined *)hash_search(local_table_stats_map, &reloid, HASH_ENTER, &found); if (!found) { /* receive table size info from the first segment */ - entry->reloid = reloid; + entry->reloid = reloid; entry->tablesize[segId + 1] = tableSize; } else diff --git a/src/quotamodel.c b/src/quotamodel.c index 4b6cba01..08244c66 100644 --- a/src/quotamodel.c +++ b/src/quotamodel.c @@ -912,18 +912,18 @@ merge_uncommitted_table_to_oidlist(List *oidlist) static void calculate_table_disk_usage(bool is_init, HTAB *local_active_table_stat_map) { - bool table_size_map_found; - bool active_tbl_found; - int64 updated_total_size; - TableSizeEntry *tsentry = NULL; - Oid relOid; - HASH_SEQ_STATUS iter; - ActiveTableEntryCombined *active_table_entry; - TableSizeEntryKey key; - List *oidlist; - ListCell *l; - int delete_entries_num = 0; - StringInfoData delete_statement; + bool table_size_map_found; + bool active_tbl_found; + int64 updated_total_size; + TableSizeEntry *tsentry = NULL; + Oid relOid; + HASH_SEQ_STATUS iter; + ActiveTableEntryCombined *active_table_entry; + TableSizeEntryKey key; + List *oidlist; + ListCell *l; + int delete_entries_num = 0; + StringInfoData delete_statement; initStringInfo(&delete_statement); @@ -1041,8 +1041,8 @@ calculate_table_disk_usage(bool is_init, HTAB *local_active_table_stat_map) /* mark tsentry is_exist */ if (tsentry) set_table_size_entry_flag(tsentry, TABLE_EXIST); - active_table_entry = (ActiveTableEntryCombined *)hash_search( - local_active_table_stat_map, &relOid, HASH_FIND, &active_tbl_found); + active_table_entry = (ActiveTableEntryCombined *)hash_search(local_active_table_stat_map, &relOid, + HASH_FIND, &active_tbl_found); /* skip to recalculate the tables which are not in active list */ if (active_tbl_found) @@ -1057,7 +1057,8 @@ calculate_table_disk_usage(bool is_init, HTAB *local_active_table_stat_map) Gp_role = GP_ROLE_DISPATCH; } /* firstly calculate the updated total size of a table */ - updated_total_size = active_table_entry->tablesize[cur_segid + 1] - TableSizeEntryGetSize(tsentry, cur_segid); + updated_total_size = + active_table_entry->tablesize[cur_segid + 1] - TableSizeEntryGetSize(tsentry, cur_segid); /* update the table_size entry */ TableSizeEntrySetSize(tsentry, cur_segid, active_table_entry->tablesize[cur_segid + 1]); @@ -1342,14 +1343,14 @@ flush_local_reject_map(void) static void dispatch_rejectmap(HTAB *local_active_table_stat_map) { - HASH_SEQ_STATUS hash_seq; - GlobalRejectMapEntry *rejectmap_entry; - ActiveTableEntryCombined *active_table_entry; - int num_entries, count = 0; - CdbPgResults cdb_pgresults = {NULL, 0}; - StringInfoData rows; - StringInfoData active_oids; - StringInfoData sql; + HASH_SEQ_STATUS hash_seq; + GlobalRejectMapEntry *rejectmap_entry; + ActiveTableEntryCombined *active_table_entry; + int num_entries, count = 0; + CdbPgResults cdb_pgresults = {NULL, 0}; + StringInfoData rows; + StringInfoData active_oids; + StringInfoData sql; initStringInfo(&rows); initStringInfo(&active_oids); From c39833734b38517d71c94a0146c5481f5203d0e2 Mon Sep 17 00:00:00 2001 From: Georgy Shelkovy Date: Thu, 10 Oct 2024 12:09:57 +0500 Subject: [PATCH 4/8] optimize --- src/gp_activetable.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/gp_activetable.c b/src/gp_activetable.c index 1ef423f0..cf27e612 100644 --- a/src/gp_activetable.c +++ b/src/gp_activetable.c @@ -1207,7 +1207,7 @@ pull_active_table_size_from_seg(HTAB *local_table_stats_map, char *active_oid_ar else { /* sum table size from all the segments */ - entry->tablesize[segId + 1] = entry->tablesize[segId + 1] + tableSize; + entry->tablesize[segId + 1] += tableSize; } } } From e70ff2265592374e6014c9f205d5bad1aa95a945 Mon Sep 17 00:00:00 2001 From: Georgy Shelkovy Date: Thu, 10 Oct 2024 12:12:04 +0500 Subject: [PATCH 5/8] fix --- src/gp_activetable.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/gp_activetable.c b/src/gp_activetable.c index cf27e612..4224ee2e 100644 --- a/src/gp_activetable.c +++ b/src/gp_activetable.c @@ -375,7 +375,7 @@ gp_fetch_active_tables(bool is_init) memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); - ctl.entrysize = sizeof(ActiveTableEntryCombined) + SEGCOUNT * sizeof(Size); + ctl.entrysize = offsetof(ActiveTableEntryCombined, tablesize) + (SEGCOUNT + 1) * sizeof(Size); ctl.hcxt = CurrentMemoryContext; local_table_stats_map = diskquota_hash_create("local active table map with relfilenode info", 1024, &ctl, From 7e6c0c3b5e2109d2e4c9c01b4ab4369086fff162 Mon Sep 17 00:00:00 2001 From: Georgy Shelkovy Date: Thu, 10 Oct 2024 12:56:07 +0500 Subject: [PATCH 6/8] fix --- src/gp_activetable.c | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/src/gp_activetable.c b/src/gp_activetable.c index 4224ee2e..98ce6dba 100644 --- a/src/gp_activetable.c +++ b/src/gp_activetable.c @@ -1179,29 +1179,23 @@ pull_active_table_size_from_seg(HTAB *local_table_stats_map, char *active_oid_ar { reloid = atooid(PQgetvalue(pgresult, j, 0)); tableSize = (Size)atoll(PQgetvalue(pgresult, j, 1)); + entry = (ActiveTableEntryCombined *)hash_search(local_table_stats_map, &reloid, HASH_ENTER, &found); + /* for diskquota extension version is 1.0, pgresult doesn't contain segid */ if (PQnfields(pgresult) == 3) { /* get the segid, tablesize for each table */ segId = atoi(PQgetvalue(pgresult, j, 2)); - entry = (ActiveTableEntryCombined *)hash_search(local_table_stats_map, &reloid, HASH_ENTER, &found); - - if (!found) - { - /* receive table size info from the first segment */ - entry->reloid = reloid; - } entry->tablesize[segId + 1] = tableSize; } /* when segid is -1, the tablesize is the sum of tablesize of master and all segments */ segId = -1; - entry = (ActiveTableEntryCombined *)hash_search(local_table_stats_map, &reloid, HASH_ENTER, &found); + // entry = (ActiveTableEntryCombined *)hash_search(local_table_stats_map, &reloid, HASH_ENTER, &found); if (!found) { /* receive table size info from the first segment */ - entry->reloid = reloid; entry->tablesize[segId + 1] = tableSize; } else From 81e9f795d3bec562de7b260637947c7cdc6e6fe3 Mon Sep 17 00:00:00 2001 From: Georgy Shelkovy Date: Thu, 10 Oct 2024 12:57:10 +0500 Subject: [PATCH 7/8] rm --- src/gp_activetable.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/gp_activetable.c b/src/gp_activetable.c index 98ce6dba..40e5d6a9 100644 --- a/src/gp_activetable.c +++ b/src/gp_activetable.c @@ -1191,7 +1191,6 @@ pull_active_table_size_from_seg(HTAB *local_table_stats_map, char *active_oid_ar /* when segid is -1, the tablesize is the sum of tablesize of master and all segments */ segId = -1; - // entry = (ActiveTableEntryCombined *)hash_search(local_table_stats_map, &reloid, HASH_ENTER, &found); if (!found) { From 2351a5a7495863cf273c3cf13f8ebab684994596 Mon Sep 17 00:00:00 2001 From: Georgy Shelkovy Date: Thu, 10 Oct 2024 12:57:58 +0500 Subject: [PATCH 8/8] format --- src/gp_activetable.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/gp_activetable.c b/src/gp_activetable.c index 40e5d6a9..0888c4de 100644 --- a/src/gp_activetable.c +++ b/src/gp_activetable.c @@ -1185,7 +1185,7 @@ pull_active_table_size_from_seg(HTAB *local_table_stats_map, char *active_oid_ar if (PQnfields(pgresult) == 3) { /* get the segid, tablesize for each table */ - segId = atoi(PQgetvalue(pgresult, j, 2)); + segId = atoi(PQgetvalue(pgresult, j, 2)); entry->tablesize[segId + 1] = tableSize; }