Skip to content

Commit

Permalink
enhan(check project): WiP
Browse files Browse the repository at this point in the history
  • Loading branch information
@xtorret
xtorret committed Dec 27, 2024
1 parent c450ece commit eacc4eb
Show file tree
Hide file tree
Showing 9 changed files with 630 additions and 906 deletions.
1 change: 0 additions & 1 deletion updates/40/40000/utils/ddl.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -186,7 +186,6 @@ END; $$;

SELECT gw_fct_admin_manage_fields($${"data":{"action":"ADD","table":"sys_fprocess", "column":"except_level", "dataType":"integer"}}$$);
SELECT gw_fct_admin_manage_fields($${"data":{"action":"ADD","table":"sys_fprocess", "column":"except_msg", "dataType":"text"}}$$);
SELECT gw_fct_admin_manage_fields($${"data":{"action":"ADD","table":"sys_fprocess", "column":"except_msg_feature", "dataType":"text"}}$$);
SELECT gw_fct_admin_manage_fields($${"data":{"action":"ADD","table":"sys_fprocess", "column":"fprocess_name", "dataType":"text"}}$$);
SELECT gw_fct_admin_manage_fields($${"data":{"action":"ADD","table":"sys_fprocess", "column":"query_text", "dataType":"text"}}$$);
SELECT gw_fct_admin_manage_fields($${"data":{"action":"ADD","table":"sys_fprocess", "column":"info_msg", "dataType":"text"}}$$);
Expand Down
12 changes: 12 additions & 0 deletions updates/40/40000/utils/dml.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -873,3 +873,15 @@ ALTER TABLE IF EXISTS config_style ADD CONSTRAINT idval_chk UNIQUE (idval);

-- 20/12/2024
UPDATE sys_param_user SET id='plan_psector_current' WHERE id='plan_psector_vdefault';

-- 26/12/2024
INSERT INTO sys_function (id, function_name, project_type, function_type, input_params, return_type, descript, sys_role, sample_query, "source")
VALUES(3362, 'gw_fct_create_checktables', 'utils', 'function', 'json', 'json', 'Create temporal tables for check process.', 'role_basic', NULL, 'core');

INSERT INTO sys_function (id, function_name, project_type, function_type, input_params, return_type, descript, sys_role, sample_query, "source")
VALUES(3364, 'gw_fct_setcheckdatabase', 'utils', 'function', 'json', 'json', 'Check database exceptions.', 'role_basic', NULL, 'core');

INSERT INTO sys_function (id, function_name, project_type, function_type, input_params, return_type, descript, sys_role, sample_query, "source")
VALUES(3366, 'gw_fct_create_logreturn', 'utils', 'function', 'json', 'json', 'Create log return for check functions.', 'role_basic', NULL, 'core');

update sys_fprocess set query_text = replace(query_text,'v_prefix_', 't_');
241 changes: 103 additions & 138 deletions utils/fct/gw_fct_check_fprocess.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@ select gw_fct_check_fprocess($${"data":{"parameters":{"functionFid": '||v_fid||'
DECLARE
v_function_fid integer;
v_check_fid integer;
v_prefix_table text;
--
v_rec record;
v_rec_anl record;
Expand All @@ -23,165 +22,131 @@ v_sql text;
v_text_aux text;
v_exc_msg text;
v_iscount boolean;
v_graphclass text;
v_version text;
v_process text;

BEGIN

-- get input params
v_function_fid := (((p_data ->>'data')::json->>'parameters')::json->>'functionFid')::integer;
v_check_fid := (((p_data ->> 'data')::json->>'parameters')::json->> 'checkFid')::integer;
v_prefix_table := (((p_data ->> 'data')::json->>'parameters')::json->> 'prefixTable')::text;
v_graphclass := (((p_data ->> 'data')::json->>'parameters')::json->> 'graphClass')::text;
v_process := (((p_data ->> 'data')::json->>'parameters')::json->> 'process');

SELECT giswater INTO v_version FROM sys_version order by id desc limit 1;


-- get fprocess data
select * into v_rec from sys_fprocess where fid = v_check_fid;

-- only for graphanalytics_check_data
if lower(v_graphclass) = 'all' then

execute 'SELECT gw_fct_check_fprocess($${"data":{"parameters":{"functionFid": 125, "checkFid":"'||v_check_fid||'", "prefixTable": "",
"graphClass":"DMA"}}}$$)';

execute 'SELECT gw_fct_check_fprocess($${"data":{"parameters":{"functionFid": 125, "checkFid":"'||v_check_fid||'", "prefixTable": "",
"graphClass":"DQA"}}}$$)';

execute 'SELECT gw_fct_check_fprocess($${"data":{"parameters":{"functionFid": 125, "checkFid":"'||v_check_fid||'", "prefixTable": "",
"graphClass":"SECTOR"}}}$$)';

execute 'SELECT gw_fct_check_fprocess($${"data":{"parameters":{"functionFid": 125, "checkFid":"'||v_check_fid||'", "prefixTable": "",
"graphClass":"PRESSZONE"}}}$$)';

end if;


if lower(v_graphclass) = 'all' then

return '{"status": "accepted"}'::json;

end if;

if v_rec.query_text ilike '%v_graphclass%' then

v_rec.query_text = replace(v_rec.query_text, 'v_graphclass', lower(v_graphclass));
v_rec.info_msg = replace(v_rec.info_msg, 'v_graphclass', lower(v_graphclass));

end if;


-- replace flag word with target table
if v_rec.query_text ilike '%v_prefix_%' then
v_rec.query_text = replace(v_rec.query_text, 'v_prefix_', v_prefix_table);
v_rec.info_msg = replace(v_rec.info_msg, 'v_prefix_', v_prefix_table);
end if;


-- manage query count
if v_rec.query_text ilike '%string_agg%' and v_rec.fid <> 317 then

execute 'with mec as ('||v_rec.query_text||'),
b as (select unnest(string_to_array("string_agg", ''; '')) as "string_agg" from mec)
select count(*) from b'
into v_count;

else

execute 'select count(*) from ('||v_rec.query_text||')a'
into v_count;

end if;


-- get text variables according to singular/plural values
v_exc_msg = v_rec.except_msg;

if v_count = 1 then

v_text_aux = 'There is ';

v_exc_msg =
concat(
substring(split_part(v_rec.except_msg, ' ', 1) FROM 1 FOR length(split_part(v_rec.except_msg, ' ', 1)) - 1),
' ',
substring(v_rec.except_msg FROM length(split_part(v_rec.except_msg, ' ', 1)) + 2)
);

elsif v_count > 1 then

v_text_aux = 'There are ';


end if;


-- manage result (audit_check_data)
IF v_count > 0 and v_rec.except_level > 1 then

INSERT INTO temp_audit_check_data (fid, criticity, result_id, error_message, fcount)
values (v_function_fid, v_rec.except_level, v_check_fid, concat(
case when v_rec.except_level = 2 then 'ERROR-' when v_rec.except_level = 3 then 'WARNING-' end ,
v_check_fid, ': ', concat(v_text_aux, v_count, ' ', v_exc_msg)), 9999);

ELSE

INSERT INTO temp_audit_check_data (fid, criticity, result_id, error_message, fcount)
values (v_function_fid, 3, v_check_fid, concat('INFO: ', v_rec.info_msg), 7777);

END IF;


-- manage result (anl_tables)
if v_rec.query_text ilike '%the_geom%' then

-- geom_type from result
v_sql = 'select a.*, '||quote_literal(v_exc_msg)||' from ('||v_rec.query_text||') a';

execute 'select distinct st_geometrytype(the_geom) from ('||v_sql||')a limit 1' into v_geom_type;


if v_geom_type = 'ST_LineString' then

execute '
insert into anl_arc (arc_id, arccat_id, expl_id, fid, the_geom, descript)
select arc_id, arccat_id, expl_id, '||v_check_fid||', the_geom, '||quote_literal(v_exc_msg)||' from ('||v_sql||')b';

elsif v_geom_type = 'ST_Point' then

if v_rec.query_text ilike 'SELECT node_id%' then
IF v_process = 'check' THEN

-- manage query count
if v_rec.query_text ilike '%string_agg%' and v_rec.fid <> 317 then
execute 'with mec as ('||v_rec.query_text||'),
b as (select unnest(string_to_array("string_agg", ''; '')) as "string_agg" from mec)
select count(*) from b'
into v_count;
else
execute 'select count(*) from ('||v_rec.query_text||')a'
into v_count;
end if;

-- get text variables according to singular/plural values
v_exc_msg = v_rec.except_msg;
if v_count = 1 then
v_text_aux = 'There is ';
v_exc_msg =
concat(
substring(split_part(v_rec.except_msg, ' ', 1) FROM 1 FOR length(split_part(v_rec.except_msg, ' ', 1)) - 1),
' ',
substring(v_rec.except_msg FROM length(split_part(v_rec.except_msg, ' ', 1)) + 2)
);
elsif v_count > 1 then
v_text_aux = 'There are ';
end if;

-- manage result (audit_check_data)
IF v_count > 0 and v_rec.except_level > 1 then

INSERT INTO t_audit_check_data (fid, criticity, result_id, error_message, fcount)
values (v_function_fid, v_rec.except_level, v_check_fid, concat(
case when v_rec.except_level = 2 then 'ERROR-' when v_rec.except_level = 3 then 'WARNING-' end ,
v_check_fid, ': ', concat(v_text_aux, v_count, ' ', v_exc_msg)), 9999);
ELSE
INSERT INTO t_audit_check_data (fid, criticity, result_id, error_message, fcount)
values (v_function_fid, 3, v_check_fid, concat('INFO: ', v_rec.info_msg), 7777);

END IF;

-- manage result (anl_tables)
if v_rec.query_text ilike '%the_geom%' then

v_sql = 'select a.*, '||quote_literal(v_exc_msg)||' from ('||v_rec.query_text||') a';
execute 'select distinct st_geometrytype(the_geom) from ('||v_sql||')a limit 1' into v_geom_type;

if v_geom_type = 'ST_LineString' then
execute '
insert into anl_node (node_id, nodecat_id, expl_id, fid, the_geom, descript)
select node_id, nodecat_id, expl_id, '||v_check_fid||', the_geom, '||quote_literal(v_exc_msg)||' from ('||v_sql||')b';

elsif v_rec.query_text ilike '% SELECT connec_id%' then
insert into t_anl_arc (arc_id, arccat_id, expl_id, fid, the_geom, descript)
select arc_id, arccat_id, expl_id, '||v_check_fid||', the_geom, '||quote_literal(v_exc_msg)||' from ('||v_sql||')b';
elsif v_geom_type = 'ST_Point' then

if v_rec.query_text ilike 'SELECT node_id%' then
execute '
insert into t_anl_node (node_id, nodecat_id, expl_id, fid, the_geom, descript)
select node_id, nodecat_id, expl_id, '||v_check_fid||', the_geom, '||quote_literal(v_exc_msg)||' from ('||v_sql||')b';

elsif v_rec.query_text ilike '% SELECT connec_id%' then
execute '
insert into t_anl_connec (connec_id, conneccat_id, expl_id, fid, the_geom, descript)
select connec_id, conneccat_id, expl_id, '||v_check_fid||', the_geom, '||quote_literal(v_exc_msg)||' from ('||v_sql||')b';
end if;

elsif v_geom_type = 'ST_MultiPolygon' then
execute '
insert into anl_connec (connec_id, conneccat_id, expl_id, fid, the_geom, descript)
select connec_id, conneccat_id, expl_id, '||v_check_fid||', the_geom, '||quote_literal(v_exc_msg)||' from ('||v_sql||')b';

insert into t_anl_polygon (pol_id, pol_type, expl_id, fid, the_geom, descript)
select pol_id, pol_type, expl_id, '||v_check_fid||', the_geom, '||quote_literal(v_exc_msg)||' from ('||v_sql||')b';
end if;
end if;

ELSIF v_process = 'finish' THEN

elsif v_geom_type = 'ST_MultiPolygon' then

execute '
insert into anl_polygon (pol_id, pol_type, expl_id, fid, the_geom, descript)
select pol_id, pol_type, expl_id, '||v_check_fid||', the_geom, '||quote_literal(v_exc_msg)||' from ('||v_sql||')b';
if v_rec.query_text ilike '%the_geom%' then

v_sql = 'select a.*, '||quote_literal(v_exc_msg)||' from ('||v_rec.query_text||') a';
execute 'select distinct st_geometrytype(the_geom) from ('||v_sql||')a limit 1' into v_geom_type;

if v_geom_type = 'ST_LineString' then
EXECUTE 'DELETE FROM anl_arc WHERE fid = '||v_rec.fid||' AND cur_user = current_user';
EXECUTE 'insert into anl_arc (arc_id, arccat_id, expl_id, fid, the_geom, descript, cur_user)
SELECT arc_id, arccat_id, expl_id, fid, the_geom, descript, current_user FROM t_anl_arc WHERE fid = v_rec.fid';

elsif v_geom_type = 'ST_Point' then
if v_rec.query_text ilike 'SELECT node_id%' then
EXECUTE 'DELETE FROM anl_node WHERE fid = '||v_rec.fid||' AND cur_user = current_user';
EXECUTE 'insert into anl_node (node_id, nodecat_id, expl_id, fid, the_geom, descript, cur_user)
SELECT node_id, nodecat_id, expl_id, fid, the_geom, descript, current_user FROM t_anl_node WHERE fid = v_rec.fid';

elsif v_rec.query_text ilike '% SELECT connec_id%' then
EXECUTE 'DELETE FROM anl_connec WHERE fid = '||v_rec.fid||' AND cur_user = current_user';
EXECUTE 'insert into anl_arc (connec_id, conneccat_id, expl_id, fid, the_geom, descript, cur_user)
SELECT connec_id, conneccat_id, expl_id, fid, the_geom, descript, current_user FROM t_anl_connec WHERE fid = v_rec.fid';

elsif v_rec.query_text ilike '% SELECT gully_id%' then
EXECUTE 'DELETE FROM anl_gully WHERE fid = '||v_rec.fid||' AND cur_user = current_user';
EXECUTE 'insert into anl_gully (gully_id, gullycat_id, expl_id, fid, the_geom, descript, cur_user)
SELECT arc_id, gullycat_id, expl_id, fid, the_geom, descript, current_user FROM t_anl_gully WHERE fid = v_rec.fid';
end if;

elsif v_geom_type = 'ST_MultiPolygon' then
EXECUTE 'DELETE FROM anl_polygon WHERE fid = '||v_rec.fid||' AND cur_user = current_user';
EXECUTE 'insert into anl_polygon (pol_id, pol_type expl_id, fid, the_geom, descript, cur_user)
SELECT pol_id, pol_type, expl_id, fid, the_geom, descript, current_user FROM t_anl_polygon WHERE fid = v_rec.fid';
end if;
end if;

end if;

END IF;

-- Return
RETURN gw_fct_json_create_return(('{"status":"Accepted", "message":{"level":1, "text":"Data quality analysis done succesfully"}, "version":"'||v_version||'"'||
',"body":{"form":{}'||
',"data":{ "info":""}}'||
'}')::json, 2776, null, null, null);
RETURN '{"status":"ok"}';

END;
$BODY$
LANGUAGE plpgsql VOLATILE
COST 100;
COST 100;
Loading

0 comments on commit eacc4eb

Please sign in to comment.