Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

ADBDEV-6520: Refactor strings into arguments when deleting/inserting into a table #41

Open
wants to merge 75 commits into
base: gpdb
Choose a base branch
from
Open
Changes from 74 commits
Commits
Show all changes
75 commits
Select commit Hold shift + click to select a range
1701482
Use short-lived SPI contexts
RekGRpth Oct 28, 2024
ce6fc81
warn
RekGRpth Oct 28, 2024
7fabd15
format
RekGRpth Oct 28, 2024
d628f7c
transaction
RekGRpth Oct 28, 2024
cded6ba
fix
RekGRpth Oct 28, 2024
f5fd4ce
optimize
RekGRpth Oct 28, 2024
865f9f8
context
RekGRpth Oct 28, 2024
ff7ab07
inc
RekGRpth Oct 28, 2024
7aa4972
Refactor strings to arguments in diskquota
RekGRpth Oct 28, 2024
983283c
snapshot
RekGRpth Oct 28, 2024
851e619
Merge branch 'ADBDEV-6577' into ADBDEV-6520
RekGRpth Oct 28, 2024
ca7bcda
optimize
RekGRpth Oct 28, 2024
65174b9
optimize
RekGRpth Oct 28, 2024
71e72a5
optimize
RekGRpth Oct 28, 2024
cf88580
optimize
RekGRpth Oct 28, 2024
0a83f76
fix
RekGRpth Oct 28, 2024
90a6577
rename
RekGRpth Oct 29, 2024
3ff7a3a
rename
RekGRpth Oct 29, 2024
7067668
Merge branch 'ADBDEV-6577' into ADBDEV-6520
RekGRpth Oct 29, 2024
469ed54
rm
RekGRpth Oct 29, 2024
4e81a71
use enum and bit mask
RekGRpth Oct 30, 2024
25684b4
fix
RekGRpth Oct 30, 2024
0b49c70
formet
RekGRpth Oct 30, 2024
8652e1a
fix
RekGRpth Oct 30, 2024
e41ea95
optimize
RekGRpth Oct 30, 2024
f64447a
opmimize
RekGRpth Oct 30, 2024
85e6997
Merge branch 'ADBDEV-6577' into ADBDEV-6520
RekGRpth Oct 30, 2024
e26a435
format
RekGRpth Oct 30, 2024
f2e66f2
format
RekGRpth Oct 30, 2024
3897da2
upper
RekGRpth Oct 30, 2024
ea50d56
optimize
RekGRpth Oct 30, 2024
a33012b
fix
RekGRpth Oct 30, 2024
8ae3517
fix
RekGRpth Oct 30, 2024
ad3a95f
fix
RekGRpth Oct 30, 2024
0069950
use context and error
RekGRpth Oct 30, 2024
e40f6a2
Merge branch 'ADBDEV-6577' into ADBDEV-6520
RekGRpth Oct 30, 2024
f9fa83a
format
RekGRpth Oct 30, 2024
9356184
move connect closer to execute
RekGRpth Oct 30, 2024
56f770e
additional move
RekGRpth Oct 30, 2024
8db6ada
rm
RekGRpth Oct 30, 2024
77212b6
mv
RekGRpth Oct 30, 2024
bb37067
mv
RekGRpth Oct 30, 2024
02020c1
partial revert
RekGRpth Oct 30, 2024
1aaf144
mv
RekGRpth Oct 30, 2024
cc262d3
mv
RekGRpth Oct 30, 2024
5cde9fa
partial revert
RekGRpth Oct 30, 2024
7d135ec
simplify
RekGRpth Oct 30, 2024
909f31c
fix
RekGRpth Oct 30, 2024
f05e5c6
simplify
RekGRpth Oct 30, 2024
5691fee
revert
RekGRpth Oct 30, 2024
40e2ff4
fix
RekGRpth Oct 30, 2024
b2a7df4
nl
RekGRpth Oct 30, 2024
acf32c2
Merge branch 'ADBDEV-6577' into ADBDEV-6520
RekGRpth Oct 30, 2024
3add2f3
format
RekGRpth Oct 30, 2024
f0bd18e
simplify
RekGRpth Oct 31, 2024
b73cb13
unuify
RekGRpth Oct 31, 2024
eb44b0f
optimize
RekGRpth Oct 31, 2024
a945451
Merge branch 'ADBDEV-6577' into ADBDEV-6520
RekGRpth Oct 31, 2024
555a8d1
format
RekGRpth Oct 31, 2024
5550449
fix
RekGRpth Oct 31, 2024
5b9e622
Merge branch 'ADBDEV-6577' into ADBDEV-6520
RekGRpth Oct 31, 2024
0bae1e1
simplify
RekGRpth Oct 31, 2024
5efc75c
Merge branch 'ADBDEV-6577' into ADBDEV-6520
RekGRpth Oct 31, 2024
dbfa9ab
rename
RekGRpth Oct 31, 2024
b3b8175
rename
RekGRpth Oct 31, 2024
9e126e2
rename
RekGRpth Oct 31, 2024
88224e0
mv
RekGRpth Oct 31, 2024
e818c97
mv
RekGRpth Oct 31, 2024
11602fd
mv
RekGRpth Oct 31, 2024
63becb0
Merge branch 'ADBDEV-6577' into ADBDEV-6520
RekGRpth Oct 31, 2024
4423e43
format
RekGRpth Oct 31, 2024
3777151
remove redundant code
RekGRpth Oct 31, 2024
180545c
Merge branch 'ADBDEV-6577' into ADBDEV-6520
RekGRpth Oct 31, 2024
c61fe34
Merge branch 'gpdb' into ADBDEV-6520
RekGRpth Nov 1, 2024
c153f30
add proto
RekGRpth Nov 7, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
140 changes: 70 additions & 70 deletions src/quotamodel.c
Original file line number Diff line number Diff line change
Expand Up @@ -239,7 +239,7 @@ static bool get_table_size_entry_flag(TableSizeEntry *entry, TableSizeEntryFlag
static void reset_table_size_entry_flag(TableSizeEntry *entry, TableSizeEntryFlag flag);
static void set_table_size_entry_flag(TableSizeEntry *entry, TableSizeEntryFlag flag);

static void delete_from_table_size_map(char *str);
static void delete_from_table_size_map(ArrayBuildState *tableids, ArrayBuildState *segids);
hilltracer marked this conversation as resolved.
Show resolved Hide resolved

/* add a new entry quota or update the old entry quota */
static void
Expand Down Expand Up @@ -909,10 +909,7 @@ calculate_table_disk_usage(bool is_init, HTAB *local_active_table_stat_map)
TableSizeEntryKey key;
List *oidlist;
ListCell *l;
int delete_entries_num = 0;
StringInfoData delete_statement;

initStringInfo(&delete_statement);
ArrayBuildState *tableids = NULL, *segids = NULL;

/*
* unset is_exist flag for tsentry in table_size_map this is used to
Expand Down Expand Up @@ -968,15 +965,15 @@ calculate_table_disk_usage(bool is_init, HTAB *local_active_table_stat_map)

for (int i = -1; i < SEGCOUNT; i++)
{
appendStringInfo(&delete_statement, "%s(%u,%d)", (delete_entries_num == 0) ? " " : ", ", relOid, i);

delete_entries_num++;
tableids =
accumArrayResult(tableids, ObjectIdGetDatum(relOid), false, OIDOID, CurrentMemoryContext);
segids = accumArrayResult(segids, Int16GetDatum(i), false, INT2OID, CurrentMemoryContext);

if (delete_entries_num > SQL_MAX_VALUES_NUMBER)
if (tableids->nelems > SQL_MAX_VALUES_NUMBER)
{
delete_from_table_size_map(delete_statement.data);
resetStringInfo(&delete_statement);
delete_entries_num = 0;
delete_from_table_size_map(tableids, segids);
tableids = NULL;
segids = NULL;
}
}

Expand Down Expand Up @@ -1106,9 +1103,8 @@ calculate_table_disk_usage(bool is_init, HTAB *local_active_table_stat_map)
}
}

if (delete_entries_num) delete_from_table_size_map(delete_statement.data);
if (tableids) delete_from_table_size_map(tableids, segids);
hilltracer marked this conversation as resolved.
Show resolved Hide resolved

pfree(delete_statement.data);
list_free(oidlist);

/*
Expand Down Expand Up @@ -1137,39 +1133,47 @@ calculate_table_disk_usage(bool is_init, HTAB *local_active_table_stat_map)
}

static void
delete_from_table_size_map(char *str)
delete_from_table_size_map(ArrayBuildState *tableids, ArrayBuildState *segids)
{
StringInfoData delete_statement;

initStringInfo(&delete_statement);
appendStringInfo(&delete_statement,
"WITH deleted_table AS ( VALUES %s ) "
"delete from diskquota.table_size "
"where (tableid, segid) in ( SELECT * FROM deleted_table );",
str);
bool connected_in_this_function = SPI_connect_if_not_yet();
int ret = SPI_execute(delete_statement.data, false, 0);
Datum tableid = makeArrayResult(tableids, CurrentMemoryContext);
Datum segid = makeArrayResult(segids, CurrentMemoryContext);
bool connected_in_this_function = SPI_connect_if_not_yet();
int ret = SPI_execute_with_args(
"delete from diskquota.table_size where (tableid, segid) in (select * from unnest($1, $2))", 2,
(Oid[]){OIDARRAYOID, INT2ARRAYOID}, (Datum[]){tableid, segid}, NULL, false, 0);
if (ret != SPI_OK_DELETE)
ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR),
errmsg("[diskquota] delete_from_table_size_map SPI_execute failed: error code %d", ret)));
SPI_finish_if(connected_in_this_function);
pfree(delete_statement.data);
pfree(DatumGetPointer(tableid));
pfree(DatumGetPointer(segid));
}

static void
insert_into_table_size_map(char *str)
update_table_size_map(ArrayBuildState *tableids, ArrayBuildState *sizes, ArrayBuildState *segids)
{
StringInfoData insert_statement;

initStringInfo(&insert_statement);
appendStringInfo(&insert_statement, "insert into diskquota.table_size values %s;", str);
bool connected_in_this_function = SPI_connect_if_not_yet();
int ret = SPI_execute(insert_statement.data, false, 0);
Datum tableid = makeArrayResult(tableids, CurrentMemoryContext);
Datum size = makeArrayResult(sizes, CurrentMemoryContext);
Datum segid = makeArrayResult(segids, CurrentMemoryContext);
bool connected_in_this_function = SPI_connect_if_not_yet();
int ret = SPI_execute_with_args(
"delete from diskquota.table_size where (tableid, segid) in (select * from unnest($1, $2))", 2,
(Oid[]){OIDARRAYOID, INT2ARRAYOID}, (Datum[]){tableid, segid}, NULL, false, 0);
if (ret != SPI_OK_DELETE)
ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR),
errmsg("[diskquota] delete_from_table_size_map SPI_execute failed: error code %d", ret)));
SPI_finish_if(connected_in_this_function);
connected_in_this_function = SPI_connect_if_not_yet();
ret = SPI_execute_with_args("insert into diskquota.table_size select * from unnest($1, $2, $3)", 3,
(Oid[]){OIDARRAYOID, INT8ARRAYOID, INT2ARRAYOID}, (Datum[]){tableid, size, segid}, NULL,
false, 0);
if (ret != SPI_OK_INSERT)
ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR),
errmsg("[diskquota] insert_into_table_size_map SPI_execute failed: error code %d", ret)));
SPI_finish_if(connected_in_this_function);
pfree(insert_statement.data);
pfree(DatumGetPointer(tableid));
pfree(DatumGetPointer(size));
pfree(DatumGetPointer(segid));
}

/*
Expand All @@ -1183,20 +1187,24 @@ flush_to_table_size(void)
{
HASH_SEQ_STATUS iter;
TableSizeEntry *tsentry = NULL;
StringInfoData delete_statement;
StringInfoData insert_statement;
int delete_entries_num = 0;
int insert_entries_num = 0;
struct
{
ArrayBuildState *tableids;
ArrayBuildState *segids;
} delete = {0};
struct
{
ArrayBuildState *tableids;
ArrayBuildState *sizes;
ArrayBuildState *segids;
} update = {0};

/* TODO: Add flush_size_interval to avoid flushing size info in every loop */

/* Disable ORCA since it does not support non-scalar subqueries. */
bool old_optimizer = optimizer;
optimizer = false;

initStringInfo(&insert_statement);
initStringInfo(&delete_statement);

hash_seq_init(&iter, table_size_map);
while ((tsentry = hash_seq_search(&iter)) != NULL)
{
Expand All @@ -1207,37 +1215,32 @@ flush_to_table_size(void)
/* delete dropped table from both table_size_map and table table_size */
if (!get_table_size_entry_flag(tsentry, TABLE_EXIST))
{
appendStringInfo(&delete_statement, "%s(%u,%d)", (delete_entries_num == 0) ? " " : ", ",
tsentry->key.reloid, i);
delete_entries_num++;
if (delete_entries_num > SQL_MAX_VALUES_NUMBER)
delete.tableids = accumArrayResult(delete.tableids, ObjectIdGetDatum(tsentry->key.reloid), false,
OIDOID, CurrentMemoryContext);
delete.segids = accumArrayResult(delete.segids, Int16GetDatum(i), false, INT2OID, CurrentMemoryContext);

if (delete.tableids->nelems > SQL_MAX_VALUES_NUMBER)
{
delete_from_table_size_map(delete_statement.data);
resetStringInfo(&delete_statement);
delete_entries_num = 0;
delete_from_table_size_map(delete.tableids, delete.segids);
delete.tableids = NULL;
delete.segids = NULL;
}
}
/* update the table size by delete+insert in table table_size */
else if (TableSizeEntryGetFlushFlag(tsentry, i))
{
appendStringInfo(&delete_statement, "%s(%u,%d)", (delete_entries_num == 0) ? " " : ", ",
tsentry->key.reloid, i);
appendStringInfo(&insert_statement, "%s(%u,%ld,%d)", (insert_entries_num == 0) ? " " : ", ",
tsentry->key.reloid, TableSizeEntryGetSize(tsentry, i), i);
delete_entries_num++;
insert_entries_num++;

if (delete_entries_num > SQL_MAX_VALUES_NUMBER)
{
delete_from_table_size_map(delete_statement.data);
resetStringInfo(&delete_statement);
delete_entries_num = 0;
}
if (insert_entries_num > SQL_MAX_VALUES_NUMBER)
update.tableids = accumArrayResult(update.tableids, ObjectIdGetDatum(tsentry->key.reloid), false,
OIDOID, CurrentMemoryContext);
update.sizes = accumArrayResult(update.sizes, Int64GetDatum(TableSizeEntryGetSize(tsentry, i)), false,
INT8OID, CurrentMemoryContext);
update.segids = accumArrayResult(update.segids, Int16GetDatum(i), false, INT2OID, CurrentMemoryContext);

if (update.tableids->nelems > SQL_MAX_VALUES_NUMBER)
{
insert_into_table_size_map(insert_statement.data);
resetStringInfo(&insert_statement);
insert_entries_num = 0;
update_table_size_map(update.tableids, update.sizes, update.segids);
update.tableids = NULL;
update.sizes = NULL;
update.segids = NULL;
}

TableSizeEntryResetFlushFlag(tsentry, i);
Expand All @@ -1249,13 +1252,10 @@ flush_to_table_size(void)
}
}

if (delete_entries_num) delete_from_table_size_map(delete_statement.data);
if (insert_entries_num) insert_into_table_size_map(insert_statement.data);
if (delete.tableids) delete_from_table_size_map(delete.tableids, delete.segids);
if (update.tableids) update_table_size_map(update.tableids, update.sizes, update.segids);

hilltracer marked this conversation as resolved.
Show resolved Hide resolved
optimizer = old_optimizer;

pfree(delete_statement.data);
pfree(insert_statement.data);
}

/*
Expand Down
Loading