Merge pull request #3 from arenadata/1.0.3-sync

ADBDEV-1244 1.0.3 sync

concourse/scripts/test_diskquota.sh

@@ -5,6 +5,8 @@ set -exo pipefail
 CWDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 TOP_DIR=${CWDIR}/../../../
 GPDB_CONCOURSE_DIR=${TOP_DIR}/gpdb_src/concourse/scripts
+CUT_NUMBER=6
+
 source "${GPDB_CONCOURSE_DIR}/common.bash"
 function test(){	
 	chown -R gpadmin:gpadmin ${TOP_DIR};
@@ -22,7 +24,7 @@ function test(){
 		trap "[ -s regression.diffs ] && grep -v GP_IGNORE regression.diffs" EXIT
 		make installcheck
 		[ -s regression.diffs ] && grep -v GP_IGNORE regression.diffs && exit 1
-		ps -ef | grep postgres| grep qddir| cut -d ' ' -f 6 | xargs kill -9
+		ps -ef | grep postgres| grep qddir| cut -d ' ' -f ${CUT_NUMBER} | xargs kill -9
 		export PGPORT=6001
 		echo "export PGPROT=\$PGPORT" >> /usr/local/greenplum-db-devel/greenplum_path.sh
 		source /usr/local/greenplum-db-devel/greenplum_path.sh
@@ -51,6 +53,9 @@ function _main() {
 
 	time make_cluster
 	time install_diskquota
+	if [ "${DISKQUOTA_OS}" == "rhel7" ]; then
+		CUT_NUMBER=5
+	fi
 
 	time test
 }

concourse/tasks/test_diskquota.yml

@@ -10,3 +10,4 @@ inputs:
 run:
   path: diskquota_src/concourse/scripts/test_diskquota.sh
 params:
+  DISKQUOTA_OS:

diskquota--1.0.sql

@@ -9,6 +9,7 @@ CREATE SCHEMA diskquota;
 CREATE TABLE diskquota.quota_config (targetOid oid, quotatype int, quotalimitMB int8, PRIMARY KEY(targetOid, quotatype));
 
 SELECT pg_catalog.pg_extension_config_dump('diskquota.quota_config', '');
+SELECT gp_segment_id, pg_catalog.pg_extension_config_dump('diskquota.quota_config', '') from gp_dist_random('gp_id');
 
 CREATE FUNCTION diskquota.set_schema_quota(text, text)
 RETURNS void STRICT
@@ -20,6 +21,11 @@ RETURNS void STRICT
 AS 'MODULE_PATHNAME'
 LANGUAGE C;
 
+CREATE FUNCTION diskquota.update_diskquota_db_list(oid, int4)
+RETURNS void STRICT
+AS 'MODULE_PATHNAME'
+LANGUAGE C;
+
 CREATE TABLE diskquota.table_size (tableid oid, size bigint, PRIMARY KEY(tableid));
 
 CREATE TABLE diskquota.state (state int, PRIMARY KEY(state));

diskquota.c

@@ -54,9 +54,6 @@
 #include "diskquota.h"
 PG_MODULE_MAGIC;
 
-/* max number of monitored database with diskquota enabled */
-#define MAX_NUM_MONITORED_DB 10
-
 #define DISKQUOTA_DB	"diskquota"
 #define DISKQUOTA_APPLICATION_NAME  "gp_reserved_gpdiskquota"
 
@@ -541,7 +538,10 @@ create_monitor_db_table(void)
 	bool		ret = true;
 
 	sql = "create schema if not exists diskquota_namespace;"
-		"create table if not exists diskquota_namespace.database_list(dbid oid not null unique);";
+		"create table if not exists diskquota_namespace.database_list(dbid oid not null unique);"
+		"create schema if not exists diskquota;"
+		"create or replace function diskquota.update_diskquota_db_list(oid, int4) returns void "
+		"strict as '$libdir/diskquota' language C;";
 
 	StartTransactionCommand();
 
@@ -640,6 +640,7 @@ start_workers_from_dblist(void)
 			ereport(LOG, (errmsg("[diskquota launcher] database(oid:%u) in table database_list is not a valid database", dbid)));
 			continue;
 		}
+		elog(WARNING, "start workers");
 		if (!start_worker_by_dboid(dbid))
 			ereport(ERROR, (errmsg("[diskquota launcher] start worker process of database(oid:%u) failed", dbid)));
 		num++;
@@ -807,14 +808,15 @@ on_add_db(Oid dbid, MessageResult * code)
 		*code = ERR_START_WORKER;
 		ereport(ERROR, (errmsg("[diskquota launcher] failed to start worker - dbid=%u", dbid)));
 	}
+
 }
 
 /*
  * Handle message: drop extension diskquota
  * do our best to:
  * 1. kill the associated worker process
  * 2. delete dbid from diskquota_namespace.database_list
- * 3. invalidate black-map entries from shared memory
+ * 3. invalidate black-map entries and monitoring_dbid_cache from shared memory
  */
 static void
 on_del_db(Oid dbid, MessageResult * code)
@@ -887,6 +889,16 @@ del_dbid_from_database_list(Oid dbid)
 	{
 		ereport(ERROR, (errmsg("[diskquota launcher] SPI_execute sql:'%s', errno:%d", str.data, errno)));
 	}
+	pfree(str.data);
+
+	/* clean the dbid from shared memory*/
+	initStringInfo(&str);
+	appendStringInfo(&str, "select gp_segment_id, diskquota.update_diskquota_db_list(%u, 1)"
+			" from gp_dist_random('gp_id');", dbid);
+	ret = SPI_execute(str.data, true, 0);
+	if (ret != SPI_OK_SELECT)
+		ereport(ERROR, (errmsg("[diskquota launcher] SPI_execute sql:'%s', errno:%d", str.data, errno)));
+	pfree(str.data);
 }
 
 /*
@@ -929,7 +941,7 @@ terminate_all_workers(void)
 
 	/*
 	 * terminate the worker processes. since launcher will exit immediately,
-	 * we skip to clear the disk_quota_worker_map
+	 * we skip to clear the disk_quota_worker_map and monitoring_dbid_cache
 	 */
 	while ((hash_entry = hash_seq_search(&iter)) != NULL)
 	{

diskquota.h

@@ -3,6 +3,9 @@
 
 #include "storage/lwlock.h"
 
+/* max number of monitored database with diskquota enabled */
+#define MAX_NUM_MONITORED_DB 10
+
 typedef enum
 {
 	NAMESPACE_QUOTA,
@@ -21,12 +24,14 @@ typedef enum
 	DISKQUOTA_READY_STATE
 }			DiskQuotaState;
 
+#define DiskQuotaLocksItemNumber (5)
 struct DiskQuotaLocks
 {
 	LWLock	   *active_table_lock;
 	LWLock	   *black_map_lock;
 	LWLock	   *extension_ddl_message_lock;
 	LWLock	   *extension_ddl_lock; /* ensure create diskquota extension serially */
+	LWLock	   *monitoring_dbid_cache_lock;
 };
 typedef struct DiskQuotaLocks DiskQuotaLocks;
 

diskquota_schedule

@@ -1,13 +1,15 @@
 test: init
 test: prepare
-test: test_table_size
+# disable this tese due to GPDB behavior change
+# test: test_table_size
 test: test_fast_disk_check
+#test: test_insert_after_drop
 test: test_role test_schema test_drop_table test_column test_copy test_update test_toast test_truncate test_reschema test_temp_role test_rename test_delete_quota test_mistake
 test: test_truncate
 test: test_delete_quota
 test: test_partition
 test: test_vacuum
 test: test_primary_failure
 test: test_extension
+test: test_manytable
 test: clean
-test: test_insert_after_drop

diskquota_schedule_int

@@ -1,10 +1,11 @@
 test: init
 test: prepare
-test: test_table_size
+#test: test_table_size
 test: test_fast_disk_check
 test: test_role test_schema test_drop_table test_column test_copy test_update test_toast test_truncate test_reschema test_temp_role test_rename test_delete_quota test_mistake
 test: test_truncate
 test: test_delete_quota
 test: test_partition
+test: test_manytable
 test: clean
-test: test_insert_after_drop
+#test: test_insert_after_drop

diskquota_utility.c

@@ -39,14 +39,18 @@
 #include "utils/memutils.h"
 #include "utils/numeric.h"
 
+#include <cdb/cdbvars.h>
+
 #include "diskquota.h"
+#include "gp_activetable.h"
 
 /* disk quota helper function */
 
 PG_FUNCTION_INFO_V1(init_table_size_table);
 PG_FUNCTION_INFO_V1(diskquota_start_worker);
 PG_FUNCTION_INFO_V1(set_schema_quota);
 PG_FUNCTION_INFO_V1(set_role_quota);
+PG_FUNCTION_INFO_V1(update_diskquota_db_list);
 
 /* timeout count to wait response from launcher process, in 1/10 sec */
 #define WAIT_TIME_COUNT  1200
@@ -635,3 +639,51 @@ get_size_in_mb(char *str)
 
 	return result;
 }
+
+/*
+ * Function to update the db list on each segment
+ */
+Datum
+update_diskquota_db_list(PG_FUNCTION_ARGS)
+{
+	Oid	dbid = PG_GETARG_OID(0);
+	int	mode = PG_GETARG_INT32(1);
+	bool	found = false;
+
+	if (!superuser())
+	{
+		ereport(ERROR,
+				(errcode(ERRCODE_INSUFFICIENT_PRIVILEGE),
+				 errmsg("must be superuser to update db list")));
+	}
+
+	/* add/remove the dbid to monitoring database cache to filter out table not under
+	* monitoring in hook functions
+	*/
+
+	LWLockAcquire(diskquota_locks.monitoring_dbid_cache_lock, LW_EXCLUSIVE);
+	if (mode == 0)
+	{	
+		Oid *entry = NULL;
+		entry = hash_search(monitoring_dbid_cache, &dbid, HASH_ENTER, &found);
+		elog(WARNING, "add dbid %u into SHM", dbid);
+		if (!found && entry == NULL)
+		{
+			ereport(WARNING,
+				(errmsg("can't alloc memory on dbid cache, there ary too many databases to monitor")));
+		}
+	}
+	else if (mode == 1)
+	{
+		hash_search(monitoring_dbid_cache, &dbid, HASH_REMOVE, &found);
+		if (!found)
+		{
+			ereport(WARNING,
+				(errmsg("cannot remove the database from db list, dbid not found")));
+		}
+	}
+	LWLockRelease(diskquota_locks.monitoring_dbid_cache_lock);
+
+	PG_RETURN_VOID();
+
+}

expected/test_manytable.out

@@ -0,0 +1,24 @@
+-- start_ignore
+-- test case manytable change cluster level config, can not run in parallel.
+\! gpconfig -c diskquota.max_active_tables -v 2 > /dev/null
+-- end_ignore
+\! echo $?
+0
+CREATE DATABASE test_manytable01;
+CREATE DATABASE test_manytable02;
+\c test_manytable01
+CREATE TABLE a01(i int) DISTRIBUTED BY (i);
+CREATE TABLE a02(i int) DISTRIBUTED BY (i);
+CREATE TABLE a03(i int) DISTRIBUTED BY (i);
+INSERT INTO a01 values(generate_series(0, 500));
+INSERT INTO a02 values(generate_series(0, 500));
+INSERT INTO a03 values(generate_series(0, 500));
+\c test_manytable02
+CREATE TABLE b01(i int) DISTRIBUTED BY (i);
+INSERT INTO b01 values(generate_series(0, 500));
+\c postgres
+DROP DATABASE test_manytable01;
+DROP DATABASE test_manytable02;
+-- start_ignore
+\! gpconfig -c diskquota.max_active_tables -v 1024 > /dev/null
+-- end_ignore

expected/test_table_size.out

@@ -12,7 +12,7 @@ select pg_sleep(2);
 create table buffer(oid oid, relname name, size bigint);
 NOTICE:  Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'oid' as the Greenplum Database data distribution key for this table.
 HINT:  The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew.
-with size as ( select oid,relname,pg_total_relation_size(oid) from pg_class) insert into buffer select size.oid, size.relname, size.pg_total_relation_size  from size, diskquota.table_size as dt where dt.tableid = size.oid and relname = 'a';
+insert into buffer select oid, relname, pg_total_relation_size(oid) from pg_class, diskquota.table_size as dt where dt.size = oid and relname = 'a';
 insert into buffer select oid, relname, sum(pg_total_relation_size(oid)) from gp_dist_random('pg_class') where oid > 16384 and (relkind='r' or relkind='m') and relname = 'a' group by oid, relname;
 select sum(buffer.size) = diskquota.table_size.size from buffer, diskquota.table_size where buffer.oid = diskquota.table_size.tableid group by diskquota.table_size.size;
  ?column? 

gp_activetable.c

@@ -51,6 +51,7 @@ typedef struct DiskQuotaSetOFCache
 }			DiskQuotaSetOFCache;
 
 HTAB	   *active_tables_map = NULL;
+HTAB	   *monitoring_dbid_cache = NULL;
 
 /* active table hooks which detect the disk file size change. */
 static file_create_hook_type prev_file_create_hook = NULL;
@@ -161,14 +162,26 @@ report_active_table_helper(const RelFileNodeBackend *relFileNode)
 	DiskQuotaActiveTableFileEntry *entry;
 	DiskQuotaActiveTableFileEntry item;
 	bool		found = false;
+	Oid dbid = relFileNode->node.dbNode;
 
 
 	/* We do not collect the active table in either master or mirror segments  */
 	if (IS_QUERY_DISPATCHER() || IsRoleMirror())
 	{
 		return;
 	}
-
+
+	/* do not collect active table info when the database is not under monitoring.
+	 * this operation is read-only and does not require absolutely exact.
+	 * read the cache with out shared lock */
+	hash_search(monitoring_dbid_cache, &dbid, HASH_FIND, &found);
+
+	if (!found)
+	{
+		return;
+	}
+	found = false;
+
 	MemSet(&item, 0, sizeof(DiskQuotaActiveTableFileEntry));
 	item.dbid = relFileNode->node.dbNode;
 	item.relfilenode = relFileNode->node.relNode;

gp_activetable.h

@@ -24,6 +24,7 @@ extern void init_shm_worker_active_tables(void);
 extern void init_lock_active_tables(void);
 
 extern HTAB *active_tables_map;
+extern HTAB *monitoring_dbid_cache;
 
 #define atooid(x)  ((Oid) strtoul((x), NULL, 10))