From ea3b0659a50e9858e61964cd108d71436673bd9c Mon Sep 17 00:00:00 2001 From: aliaksandr_fedasiuk Date: Fri, 23 Aug 2024 21:40:19 +0500 Subject: [PATCH 01/25] MODSOURMAN-1204: Using a partitioned journal_records table --- .../migration_to_partitioning_logs.sql | 59 +++++++++++++++++++ .../templates/db_scripts/schema.json | 5 ++ 2 files changed, 64 insertions(+) create mode 100644 mod-source-record-manager-server/src/main/resources/templates/db_scripts/migration_to_partitioning_logs.sql diff --git a/mod-source-record-manager-server/src/main/resources/templates/db_scripts/migration_to_partitioning_logs.sql b/mod-source-record-manager-server/src/main/resources/templates/db_scripts/migration_to_partitioning_logs.sql new file mode 100644 index 000000000..61a4fc799 --- /dev/null +++ b/mod-source-record-manager-server/src/main/resources/templates/db_scripts/migration_to_partitioning_logs.sql @@ -0,0 +1,59 @@ +CREATE TABLE ${myuniversity}_${mymodule}.journal_records_entity_type ( + id UUID, + job_execution_id UUID, + source_id UUID, + entity_type text, + entity_id text, + entity_hrid text, + action_type text, + action_status text, + action_date timestamp, + source_record_order integer NULL, + error text NULL, + title text NULL, + instance_id text, + holdings_id text, + order_id text, + permanent_location_id text, + tenant_id text, + PRIMARY KEY (id, entity_type) +) partition by list (entity_type); + +CREATE TABLE ${myuniversity}_${mymodule}.journal_records_marc_bibliographic PARTITION OF journal_records_entity_type FOR VALUES IN ('MARC_BIBLIOGRAPHIC'); +CREATE TABLE ${myuniversity}_${mymodule}.journal_records_po_line PARTITION OF journal_records_entity_type FOR VALUES IN ('PO_LINE'); +CREATE TABLE ${myuniversity}_${mymodule}.journal_records_marc_holdings PARTITION OF journal_records_entity_type FOR VALUES IN ('MARC_HOLDINGS'); +CREATE TABLE ${myuniversity}_${mymodule}.journal_records_marc_authority PARTITION OF journal_records_entity_type FOR VALUES IN ('MARC_AUTHORITY'); +CREATE TABLE ${myuniversity}_${mymodule}.journal_records_holdings PARTITION OF journal_records_entity_type FOR VALUES IN ('HOLDINGS'); +CREATE TABLE ${myuniversity}_${mymodule}.journal_records_authority PARTITION OF journal_records_entity_type FOR VALUES IN ('AUTHORITY'); +CREATE TABLE ${myuniversity}_${mymodule}.journal_records_instance PARTITION OF journal_records_entity_type FOR VALUES IN ('INSTANCE'); +CREATE TABLE ${myuniversity}_${mymodule}.journal_records_item PARTITION OF journal_records_entity_type FOR VALUES IN ('ITEM'); +CREATE TABLE ${myuniversity}_${mymodule}.journal_records_invoice PARTITION OF journal_records_entity_type FOR VALUES IN ('INVOICE'); +CREATE TABLE ${myuniversity}_${mymodule}.journal_records_edifact PARTITION OF journal_records_entity_type FOR VALUES IN ('EDIFACT'); +CREATE TABLE ${myuniversity}_${mymodule}.journal_records_empty_entity_type PARTITION OF journal_records FOR VALUES IN (''); + +INSERT INTO ${myuniversity}_${mymodule}.journal_records_entity_type (id, job_execution_id, source_id, entity_type, entity_id, entity_hrid, + action_type, action_status, action_date, source_record_order, error, title, + instance_id, holdings_id, order_id, permanent_location_id, tenant_id) +SELECT id, job_execution_id, source_id, entity_type, entity_id, entity_hrid, action_type, action_status, + action_date, source_record_order, error, title, instance_id, holdings_id, order_id, permanent_location_id, + tenant_id +FROM ${myuniversity}_${mymodule}.journal_records; + +ALTER TABLE {myuniversity}_${mymodule}.journal_records RENAME TO {myuniversity}_${mymodule}.journal_records_backup; +ALTER TABLE {myuniversity}_${mymodule}.journal_records_entity_type RENAME TO {myuniversity}_${mymodule}.journal_records; + +DO $$ +DECLARE + index_record RECORD; + new_indexdef TEXT; +BEGIN +FOR index_record IN + SELECT indexname, indexdef + FROM pg_indexes + WHERE schemaname = '{myuniversity}_${mymodule}' AND tablename = 'journal_records' + LOOP + new_indexdef := REPLACE(index_record.indexdef, + '{myuniversity}_${mymodule}.journal_records_backup', '{myuniversity}_${mymodule}.journal_records'); + EXECUTE new_indexdef; + END LOOP; +END $$; diff --git a/mod-source-record-manager-server/src/main/resources/templates/db_scripts/schema.json b/mod-source-record-manager-server/src/main/resources/templates/db_scripts/schema.json index 12eed7f2d..e7ce56a85 100644 --- a/mod-source-record-manager-server/src/main/resources/templates/db_scripts/schema.json +++ b/mod-source-record-manager-server/src/main/resources/templates/db_scripts/schema.json @@ -306,6 +306,11 @@ "run": "after", "snippetPath": "create_get_job_log_entries_function.sql", "fromModuleVersion": "mod-source-record-manager-3.9.0" + }, + { + "run": "after", + "snippetPath": "migration_to_partitioning_logs.sql", + "fromModuleVersion": "mod-source-record-manager-3.9.0" } ] } From 4ecccaa5dfebd4e60af0b972f3d1e1b30185ae05 Mon Sep 17 00:00:00 2001 From: aliaksandr_fedasiuk Date: Fri, 23 Aug 2024 21:56:59 +0500 Subject: [PATCH 02/25] MODSOURMAN-1204: fixing names --- .../migration_to_partitioning_logs.sql | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/mod-source-record-manager-server/src/main/resources/templates/db_scripts/migration_to_partitioning_logs.sql b/mod-source-record-manager-server/src/main/resources/templates/db_scripts/migration_to_partitioning_logs.sql index 61a4fc799..a61d165f9 100644 --- a/mod-source-record-manager-server/src/main/resources/templates/db_scripts/migration_to_partitioning_logs.sql +++ b/mod-source-record-manager-server/src/main/resources/templates/db_scripts/migration_to_partitioning_logs.sql @@ -1,4 +1,4 @@ -CREATE TABLE ${myuniversity}_${mymodule}.journal_records_entity_type ( +CREATE TABLE IF NOT EXISTS ${myuniversity}_${mymodule}.journal_records_entity_type ( id UUID, job_execution_id UUID, source_id UUID, @@ -19,17 +19,17 @@ CREATE TABLE ${myuniversity}_${mymodule}.journal_records_entity_type ( PRIMARY KEY (id, entity_type) ) partition by list (entity_type); -CREATE TABLE ${myuniversity}_${mymodule}.journal_records_marc_bibliographic PARTITION OF journal_records_entity_type FOR VALUES IN ('MARC_BIBLIOGRAPHIC'); -CREATE TABLE ${myuniversity}_${mymodule}.journal_records_po_line PARTITION OF journal_records_entity_type FOR VALUES IN ('PO_LINE'); -CREATE TABLE ${myuniversity}_${mymodule}.journal_records_marc_holdings PARTITION OF journal_records_entity_type FOR VALUES IN ('MARC_HOLDINGS'); -CREATE TABLE ${myuniversity}_${mymodule}.journal_records_marc_authority PARTITION OF journal_records_entity_type FOR VALUES IN ('MARC_AUTHORITY'); -CREATE TABLE ${myuniversity}_${mymodule}.journal_records_holdings PARTITION OF journal_records_entity_type FOR VALUES IN ('HOLDINGS'); -CREATE TABLE ${myuniversity}_${mymodule}.journal_records_authority PARTITION OF journal_records_entity_type FOR VALUES IN ('AUTHORITY'); -CREATE TABLE ${myuniversity}_${mymodule}.journal_records_instance PARTITION OF journal_records_entity_type FOR VALUES IN ('INSTANCE'); -CREATE TABLE ${myuniversity}_${mymodule}.journal_records_item PARTITION OF journal_records_entity_type FOR VALUES IN ('ITEM'); -CREATE TABLE ${myuniversity}_${mymodule}.journal_records_invoice PARTITION OF journal_records_entity_type FOR VALUES IN ('INVOICE'); -CREATE TABLE ${myuniversity}_${mymodule}.journal_records_edifact PARTITION OF journal_records_entity_type FOR VALUES IN ('EDIFACT'); -CREATE TABLE ${myuniversity}_${mymodule}.journal_records_empty_entity_type PARTITION OF journal_records FOR VALUES IN (''); +CREATE TABLE IF NOT EXISTS ${myuniversity}_${mymodule}.journal_records_marc_bibliographic PARTITION OF journal_records_entity_type FOR VALUES IN ('MARC_BIBLIOGRAPHIC'); +CREATE TABLE IF NOT EXISTS ${myuniversity}_${mymodule}.journal_records_po_line PARTITION OF journal_records_entity_type FOR VALUES IN ('PO_LINE'); +CREATE TABLE IF NOT EXISTS ${myuniversity}_${mymodule}.journal_records_marc_holdings PARTITION OF journal_records_entity_type FOR VALUES IN ('MARC_HOLDINGS'); +CREATE TABLE IF NOT EXISTS ${myuniversity}_${mymodule}.journal_records_marc_authority PARTITION OF journal_records_entity_type FOR VALUES IN ('MARC_AUTHORITY'); +CREATE TABLE IF NOT EXISTS ${myuniversity}_${mymodule}.journal_records_holdings PARTITION OF journal_records_entity_type FOR VALUES IN ('HOLDINGS'); +CREATE TABLE IF NOT EXISTS ${myuniversity}_${mymodule}.journal_records_authority PARTITION OF journal_records_entity_type FOR VALUES IN ('AUTHORITY'); +CREATE TABLE IF NOT EXISTS ${myuniversity}_${mymodule}.journal_records_instance PARTITION OF journal_records_entity_type FOR VALUES IN ('INSTANCE'); +CREATE TABLE IF NOT EXISTS ${myuniversity}_${mymodule}.journal_records_item PARTITION OF journal_records_entity_type FOR VALUES IN ('ITEM'); +CREATE TABLE IF NOT EXISTS ${myuniversity}_${mymodule}.journal_records_invoice PARTITION OF journal_records_entity_type FOR VALUES IN ('INVOICE'); +CREATE TABLE IF NOT EXISTS ${myuniversity}_${mymodule}.journal_records_edifact PARTITION OF journal_records_entity_type FOR VALUES IN ('EDIFACT'); +CREATE TABLE IF NOT EXISTS ${myuniversity}_${mymodule}.journal_records_empty_entity_type PARTITION OF journal_records_entity_type FOR VALUES IN (''); INSERT INTO ${myuniversity}_${mymodule}.journal_records_entity_type (id, job_execution_id, source_id, entity_type, entity_id, entity_hrid, action_type, action_status, action_date, source_record_order, error, title, From bb321c8d2718926fad30cf133c89c10843daeabc Mon Sep 17 00:00:00 2001 From: aliaksandr_fedasiuk Date: Fri, 23 Aug 2024 22:10:25 +0500 Subject: [PATCH 03/25] MODSOURMAN-1204: fixing names --- .../db_scripts/migration_to_partitioning_logs.sql | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/mod-source-record-manager-server/src/main/resources/templates/db_scripts/migration_to_partitioning_logs.sql b/mod-source-record-manager-server/src/main/resources/templates/db_scripts/migration_to_partitioning_logs.sql index a61d165f9..82e26026e 100644 --- a/mod-source-record-manager-server/src/main/resources/templates/db_scripts/migration_to_partitioning_logs.sql +++ b/mod-source-record-manager-server/src/main/resources/templates/db_scripts/migration_to_partitioning_logs.sql @@ -39,8 +39,8 @@ SELECT id, job_execution_id, source_id, entity_type, entity_id, entity_hrid, act tenant_id FROM ${myuniversity}_${mymodule}.journal_records; -ALTER TABLE {myuniversity}_${mymodule}.journal_records RENAME TO {myuniversity}_${mymodule}.journal_records_backup; -ALTER TABLE {myuniversity}_${mymodule}.journal_records_entity_type RENAME TO {myuniversity}_${mymodule}.journal_records; +ALTER TABLE ${myuniversity}_${mymodule}.journal_records RENAME TO ${myuniversity}_${mymodule}.journal_records_backup; +ALTER TABLE ${myuniversity}_${mymodule}.journal_records_entity_type RENAME TO ${myuniversity}_${mymodule}.journal_records; DO $$ DECLARE @@ -50,10 +50,10 @@ BEGIN FOR index_record IN SELECT indexname, indexdef FROM pg_indexes - WHERE schemaname = '{myuniversity}_${mymodule}' AND tablename = 'journal_records' + WHERE schemaname = '${myuniversity}_${mymodule}' AND tablename = 'journal_records' LOOP new_indexdef := REPLACE(index_record.indexdef, - '{myuniversity}_${mymodule}.journal_records_backup', '{myuniversity}_${mymodule}.journal_records'); + '${myuniversity}_${mymodule}.journal_records_backup', '${myuniversity}_${mymodule}.journal_records'); EXECUTE new_indexdef; END LOOP; END $$; From d9cfc1d12abbc47340084c868bac96a17c2dd75d Mon Sep 17 00:00:00 2001 From: aliaksandr_fedasiuk Date: Fri, 23 Aug 2024 22:26:15 +0500 Subject: [PATCH 04/25] MODSOURMAN-1204: fixing names --- .../templates/db_scripts/migration_to_partitioning_logs.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mod-source-record-manager-server/src/main/resources/templates/db_scripts/migration_to_partitioning_logs.sql b/mod-source-record-manager-server/src/main/resources/templates/db_scripts/migration_to_partitioning_logs.sql index 82e26026e..a82c85a3f 100644 --- a/mod-source-record-manager-server/src/main/resources/templates/db_scripts/migration_to_partitioning_logs.sql +++ b/mod-source-record-manager-server/src/main/resources/templates/db_scripts/migration_to_partitioning_logs.sql @@ -39,8 +39,8 @@ SELECT id, job_execution_id, source_id, entity_type, entity_id, entity_hrid, act tenant_id FROM ${myuniversity}_${mymodule}.journal_records; -ALTER TABLE ${myuniversity}_${mymodule}.journal_records RENAME TO ${myuniversity}_${mymodule}.journal_records_backup; -ALTER TABLE ${myuniversity}_${mymodule}.journal_records_entity_type RENAME TO ${myuniversity}_${mymodule}.journal_records; +ALTER TABLE ${myuniversity}_${mymodule}.journal_records RENAME TO journal_records_backup; +ALTER TABLE ${myuniversity}_${mymodule}.journal_records_entity_type RENAME TO journal_records; DO $$ DECLARE From b05d820b3cbffb8c1fcb4dfa5be4d1072c1ea3d2 Mon Sep 17 00:00:00 2001 From: aliaksandr_fedasiuk Date: Fri, 23 Aug 2024 22:41:59 +0500 Subject: [PATCH 05/25] MODSOURMAN-1204: fixing names --- .../templates/db_scripts/migration_to_partitioning_logs.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mod-source-record-manager-server/src/main/resources/templates/db_scripts/migration_to_partitioning_logs.sql b/mod-source-record-manager-server/src/main/resources/templates/db_scripts/migration_to_partitioning_logs.sql index a82c85a3f..e140233d8 100644 --- a/mod-source-record-manager-server/src/main/resources/templates/db_scripts/migration_to_partitioning_logs.sql +++ b/mod-source-record-manager-server/src/main/resources/templates/db_scripts/migration_to_partitioning_logs.sql @@ -16,7 +16,7 @@ CREATE TABLE IF NOT EXISTS ${myuniversity}_${mymodule}.journal_records_entity_ty order_id text, permanent_location_id text, tenant_id text, - PRIMARY KEY (id, entity_type) + --PRIMARY KEY (id, entity_type) ) partition by list (entity_type); CREATE TABLE IF NOT EXISTS ${myuniversity}_${mymodule}.journal_records_marc_bibliographic PARTITION OF journal_records_entity_type FOR VALUES IN ('MARC_BIBLIOGRAPHIC'); From 56c5f77580830f6373b843ec6e080538feca7285 Mon Sep 17 00:00:00 2001 From: aliaksandr_fedasiuk Date: Fri, 23 Aug 2024 22:52:23 +0500 Subject: [PATCH 06/25] MODSOURMAN-1204: fixing names --- .../templates/db_scripts/migration_to_partitioning_logs.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mod-source-record-manager-server/src/main/resources/templates/db_scripts/migration_to_partitioning_logs.sql b/mod-source-record-manager-server/src/main/resources/templates/db_scripts/migration_to_partitioning_logs.sql index e140233d8..9b13db965 100644 --- a/mod-source-record-manager-server/src/main/resources/templates/db_scripts/migration_to_partitioning_logs.sql +++ b/mod-source-record-manager-server/src/main/resources/templates/db_scripts/migration_to_partitioning_logs.sql @@ -15,8 +15,8 @@ CREATE TABLE IF NOT EXISTS ${myuniversity}_${mymodule}.journal_records_entity_ty holdings_id text, order_id text, permanent_location_id text, - tenant_id text, - --PRIMARY KEY (id, entity_type) + tenant_id text + --, PRIMARY KEY (id, entity_type) ) partition by list (entity_type); CREATE TABLE IF NOT EXISTS ${myuniversity}_${mymodule}.journal_records_marc_bibliographic PARTITION OF journal_records_entity_type FOR VALUES IN ('MARC_BIBLIOGRAPHIC'); From 6d16b55fb43ea2fbfe2f13ace5f3e730a1955e97 Mon Sep 17 00:00:00 2001 From: aliaksandr_fedasiuk Date: Fri, 23 Aug 2024 23:35:09 +0500 Subject: [PATCH 07/25] MODSOURMAN-1204: fixing names --- .../db_scripts/migration_to_partitioning_logs.sql | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/mod-source-record-manager-server/src/main/resources/templates/db_scripts/migration_to_partitioning_logs.sql b/mod-source-record-manager-server/src/main/resources/templates/db_scripts/migration_to_partitioning_logs.sql index 9b13db965..61a4ba059 100644 --- a/mod-source-record-manager-server/src/main/resources/templates/db_scripts/migration_to_partitioning_logs.sql +++ b/mod-source-record-manager-server/src/main/resources/templates/db_scripts/migration_to_partitioning_logs.sql @@ -15,8 +15,8 @@ CREATE TABLE IF NOT EXISTS ${myuniversity}_${mymodule}.journal_records_entity_ty holdings_id text, order_id text, permanent_location_id text, - tenant_id text - --, PRIMARY KEY (id, entity_type) + tenant_id text, + PRIMARY KEY (id, entity_type) ) partition by list (entity_type); CREATE TABLE IF NOT EXISTS ${myuniversity}_${mymodule}.journal_records_marc_bibliographic PARTITION OF journal_records_entity_type FOR VALUES IN ('MARC_BIBLIOGRAPHIC'); @@ -39,9 +39,6 @@ SELECT id, job_execution_id, source_id, entity_type, entity_id, entity_hrid, act tenant_id FROM ${myuniversity}_${mymodule}.journal_records; -ALTER TABLE ${myuniversity}_${mymodule}.journal_records RENAME TO journal_records_backup; -ALTER TABLE ${myuniversity}_${mymodule}.journal_records_entity_type RENAME TO journal_records; - DO $$ DECLARE index_record RECORD; @@ -51,9 +48,14 @@ FOR index_record IN SELECT indexname, indexdef FROM pg_indexes WHERE schemaname = '${myuniversity}_${mymodule}' AND tablename = 'journal_records' + AND indexname not like '%pkey' LOOP new_indexdef := REPLACE(index_record.indexdef, - '${myuniversity}_${mymodule}.journal_records_backup', '${myuniversity}_${mymodule}.journal_records'); + '${myuniversity}_${mymodule}.journal_records', + '${myuniversity}_${mymodule}.journal_records_entity_type'); EXECUTE new_indexdef; END LOOP; END $$; + +ALTER TABLE ${myuniversity}_${mymodule}.journal_records RENAME TO journal_records_backup; +ALTER TABLE ${myuniversity}_${mymodule}.journal_records_entity_type RENAME TO journal_records; From ea67071b24bbcdbcccf8a7dc5c4dba3a96f308dc Mon Sep 17 00:00:00 2001 From: aliaksandr_fedasiuk Date: Tue, 27 Aug 2024 01:33:38 +0500 Subject: [PATCH 08/25] MODSOURMAN-1204: fixing names --- .../migration_to_partitioning_logs.sql | 25 +++++-------------- 1 file changed, 6 insertions(+), 19 deletions(-) diff --git a/mod-source-record-manager-server/src/main/resources/templates/db_scripts/migration_to_partitioning_logs.sql b/mod-source-record-manager-server/src/main/resources/templates/db_scripts/migration_to_partitioning_logs.sql index 61a4ba059..dc52dc9e6 100644 --- a/mod-source-record-manager-server/src/main/resources/templates/db_scripts/migration_to_partitioning_logs.sql +++ b/mod-source-record-manager-server/src/main/resources/templates/db_scripts/migration_to_partitioning_logs.sql @@ -15,8 +15,8 @@ CREATE TABLE IF NOT EXISTS ${myuniversity}_${mymodule}.journal_records_entity_ty holdings_id text, order_id text, permanent_location_id text, - tenant_id text, - PRIMARY KEY (id, entity_type) + tenant_id text + --, PRIMARY KEY (id, entity_type) ) partition by list (entity_type); CREATE TABLE IF NOT EXISTS ${myuniversity}_${mymodule}.journal_records_marc_bibliographic PARTITION OF journal_records_entity_type FOR VALUES IN ('MARC_BIBLIOGRAPHIC'); @@ -39,23 +39,10 @@ SELECT id, job_execution_id, source_id, entity_type, entity_id, entity_hrid, act tenant_id FROM ${myuniversity}_${mymodule}.journal_records; -DO $$ -DECLARE - index_record RECORD; - new_indexdef TEXT; -BEGIN -FOR index_record IN - SELECT indexname, indexdef - FROM pg_indexes - WHERE schemaname = '${myuniversity}_${mymodule}' AND tablename = 'journal_records' - AND indexname not like '%pkey' - LOOP - new_indexdef := REPLACE(index_record.indexdef, - '${myuniversity}_${mymodule}.journal_records', - '${myuniversity}_${mymodule}.journal_records_entity_type'); - EXECUTE new_indexdef; - END LOOP; -END $$; +CREATE UNIQUE INDEX journal_records_pkey ON ${myuniversity}_${mymodule}.journal_records_entity_type USING btree (id); +CREATE INDEX journal_records_job_execution_id_idx ON ${myuniversity}_${mymodule}.journal_records_entity_type USING btree (job_execution_id); +CREATE INDEX journal_records_source_id_idx ON ${myuniversity}_${mymodule}.journal_records_entity_type USING btree (source_id); +CREATE INDEX journal_records_action_type_idx ON ${myuniversity}_${mymodule}.journal_records_entity_type USING btree (action_type); ALTER TABLE ${myuniversity}_${mymodule}.journal_records RENAME TO journal_records_backup; ALTER TABLE ${myuniversity}_${mymodule}.journal_records_entity_type RENAME TO journal_records; From 7c56c385ad630d54954c297ace4b60c565232eaf Mon Sep 17 00:00:00 2001 From: aliaksandr_fedasiuk Date: Tue, 27 Aug 2024 14:14:47 +0500 Subject: [PATCH 09/25] MODSOURMAN-1204: change the way of index creation --- .../db_scripts/migration_to_partitioning_logs.sql | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/mod-source-record-manager-server/src/main/resources/templates/db_scripts/migration_to_partitioning_logs.sql b/mod-source-record-manager-server/src/main/resources/templates/db_scripts/migration_to_partitioning_logs.sql index dc52dc9e6..ff4131a4f 100644 --- a/mod-source-record-manager-server/src/main/resources/templates/db_scripts/migration_to_partitioning_logs.sql +++ b/mod-source-record-manager-server/src/main/resources/templates/db_scripts/migration_to_partitioning_logs.sql @@ -16,7 +16,6 @@ CREATE TABLE IF NOT EXISTS ${myuniversity}_${mymodule}.journal_records_entity_ty order_id text, permanent_location_id text, tenant_id text - --, PRIMARY KEY (id, entity_type) ) partition by list (entity_type); CREATE TABLE IF NOT EXISTS ${myuniversity}_${mymodule}.journal_records_marc_bibliographic PARTITION OF journal_records_entity_type FOR VALUES IN ('MARC_BIBLIOGRAPHIC'); @@ -39,10 +38,10 @@ SELECT id, job_execution_id, source_id, entity_type, entity_id, entity_hrid, act tenant_id FROM ${myuniversity}_${mymodule}.journal_records; -CREATE UNIQUE INDEX journal_records_pkey ON ${myuniversity}_${mymodule}.journal_records_entity_type USING btree (id); -CREATE INDEX journal_records_job_execution_id_idx ON ${myuniversity}_${mymodule}.journal_records_entity_type USING btree (job_execution_id); -CREATE INDEX journal_records_source_id_idx ON ${myuniversity}_${mymodule}.journal_records_entity_type USING btree (source_id); -CREATE INDEX journal_records_action_type_idx ON ${myuniversity}_${mymodule}.journal_records_entity_type USING btree (action_type); +CREATE UNIQUE INDEX journal_records_et_pkey ON ${myuniversity}_${mymodule}.journal_records_entity_type USING btree (id); +CREATE INDEX journal_records_et_job_execution_id_idx ON ${myuniversity}_${mymodule}.journal_records_entity_type USING btree (job_execution_id); +CREATE INDEX journal_records_et_source_id_idx ON ${myuniversity}_${mymodule}.journal_records_entity_type USING btree (source_id); +CREATE INDEX journal_records_et_action_type_idx ON ${myuniversity}_${mymodule}.journal_records_entity_type USING btree (action_type); ALTER TABLE ${myuniversity}_${mymodule}.journal_records RENAME TO journal_records_backup; ALTER TABLE ${myuniversity}_${mymodule}.journal_records_entity_type RENAME TO journal_records; From bbd43de9c7008754bba81a294d0bf9f7b3cfbad5 Mon Sep 17 00:00:00 2001 From: aliaksandr_fedasiuk Date: Wed, 28 Aug 2024 22:46:51 +0500 Subject: [PATCH 10/25] MODSOURMAN-1204: change the way of index creation --- .../templates/db_scripts/migration_to_partitioning_logs.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mod-source-record-manager-server/src/main/resources/templates/db_scripts/migration_to_partitioning_logs.sql b/mod-source-record-manager-server/src/main/resources/templates/db_scripts/migration_to_partitioning_logs.sql index ff4131a4f..678ced8e9 100644 --- a/mod-source-record-manager-server/src/main/resources/templates/db_scripts/migration_to_partitioning_logs.sql +++ b/mod-source-record-manager-server/src/main/resources/templates/db_scripts/migration_to_partitioning_logs.sql @@ -38,7 +38,7 @@ SELECT id, job_execution_id, source_id, entity_type, entity_id, entity_hrid, act tenant_id FROM ${myuniversity}_${mymodule}.journal_records; -CREATE UNIQUE INDEX journal_records_et_pkey ON ${myuniversity}_${mymodule}.journal_records_entity_type USING btree (id); +CREATE UNIQUE INDEX journal_records_et_pkey ON ${myuniversity}_${mymodule}.journal_records_entity_type USING btree (id, entity_type); CREATE INDEX journal_records_et_job_execution_id_idx ON ${myuniversity}_${mymodule}.journal_records_entity_type USING btree (job_execution_id); CREATE INDEX journal_records_et_source_id_idx ON ${myuniversity}_${mymodule}.journal_records_entity_type USING btree (source_id); CREATE INDEX journal_records_et_action_type_idx ON ${myuniversity}_${mymodule}.journal_records_entity_type USING btree (action_type); From 7fa361b3fa89675dac3807318052fbf8631bf34b Mon Sep 17 00:00:00 2001 From: aliaksandr_fedasiuk Date: Thu, 29 Aug 2024 00:51:10 +0500 Subject: [PATCH 11/25] MODSOURMAN-1204: fix tests --- mod-source-record-manager-server/pom.xml | 2 +- .../templates/db_scripts/migration_to_partitioning_logs.sql | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/mod-source-record-manager-server/pom.xml b/mod-source-record-manager-server/pom.xml index 26ee52bbe..e3344a301 100644 --- a/mod-source-record-manager-server/pom.xml +++ b/mod-source-record-manager-server/pom.xml @@ -15,7 +15,7 @@ **/org/folio/rest/impl/ModTenantAPI.java **/services/JournalRecordService.java 1.9.19 - 1.18.30 + 1.18.34 1.16.0 3.6.0 3.5.1 diff --git a/mod-source-record-manager-server/src/main/resources/templates/db_scripts/migration_to_partitioning_logs.sql b/mod-source-record-manager-server/src/main/resources/templates/db_scripts/migration_to_partitioning_logs.sql index 678ced8e9..8a4b9ceb4 100644 --- a/mod-source-record-manager-server/src/main/resources/templates/db_scripts/migration_to_partitioning_logs.sql +++ b/mod-source-record-manager-server/src/main/resources/templates/db_scripts/migration_to_partitioning_logs.sql @@ -26,6 +26,7 @@ CREATE TABLE IF NOT EXISTS ${myuniversity}_${mymodule}.journal_records_holdings CREATE TABLE IF NOT EXISTS ${myuniversity}_${mymodule}.journal_records_authority PARTITION OF journal_records_entity_type FOR VALUES IN ('AUTHORITY'); CREATE TABLE IF NOT EXISTS ${myuniversity}_${mymodule}.journal_records_instance PARTITION OF journal_records_entity_type FOR VALUES IN ('INSTANCE'); CREATE TABLE IF NOT EXISTS ${myuniversity}_${mymodule}.journal_records_item PARTITION OF journal_records_entity_type FOR VALUES IN ('ITEM'); +CREATE TABLE IF NOT EXISTS ${myuniversity}_${mymodule}.journal_records_order PARTITION OF journal_records_entity_type FOR VALUES IN ('ORDER'); CREATE TABLE IF NOT EXISTS ${myuniversity}_${mymodule}.journal_records_invoice PARTITION OF journal_records_entity_type FOR VALUES IN ('INVOICE'); CREATE TABLE IF NOT EXISTS ${myuniversity}_${mymodule}.journal_records_edifact PARTITION OF journal_records_entity_type FOR VALUES IN ('EDIFACT'); CREATE TABLE IF NOT EXISTS ${myuniversity}_${mymodule}.journal_records_empty_entity_type PARTITION OF journal_records_entity_type FOR VALUES IN (''); From cc340a2696b61624fab3c78a59307062143dfe44 Mon Sep 17 00:00:00 2001 From: aliaksandr_fedasiuk Date: Thu, 29 Aug 2024 01:50:40 +0500 Subject: [PATCH 12/25] MODSOURMAN-1204: fix tests --- .../db_scripts/create_get_job_log_entries_function.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mod-source-record-manager-server/src/main/resources/templates/db_scripts/create_get_job_log_entries_function.sql b/mod-source-record-manager-server/src/main/resources/templates/db_scripts/create_get_job_log_entries_function.sql index 0c9a6a64d..88065d614 100644 --- a/mod-source-record-manager-server/src/main/resources/templates/db_scripts/create_get_job_log_entries_function.sql +++ b/mod-source-record-manager-server/src/main/resources/templates/db_scripts/create_get_job_log_entries_function.sql @@ -437,7 +437,7 @@ FROM ( FROM journal_records WHERE journal_records.job_execution_id = ''%1$s'' and entity_type = ''INVOICE'' and title != ''INVOICE'' GROUP BY journal_records.source_id, journal_records.source_record_order, journal_records.job_execution_id, - entity_hrid, title, error, id + entity_hrid, title, error, tenant_id, id HAVING count(journal_records.source_id) FILTER (WHERE (%3$L IN (''ALL'', ''INVOICE'')) AND (NOT %2$L or journal_records.error <> '''')) > 0 ) AS records_actions From f596431c5e9cb763919459b334fc3d4a9a0e7f35 Mon Sep 17 00:00:00 2001 From: aliaksandr_fedasiuk Date: Thu, 12 Sep 2024 12:44:11 +0500 Subject: [PATCH 13/25] MODSOURMAN-1204: fix tests --- .../templates/db_scripts/schema.json | 5 +++ .../MetaDataProviderJobLogEntriesAPITest.java | 34 +++++++++++-------- 2 files changed, 24 insertions(+), 15 deletions(-) diff --git a/mod-source-record-manager-server/src/main/resources/templates/db_scripts/schema.json b/mod-source-record-manager-server/src/main/resources/templates/db_scripts/schema.json index e7ce56a85..0d34e3655 100644 --- a/mod-source-record-manager-server/src/main/resources/templates/db_scripts/schema.json +++ b/mod-source-record-manager-server/src/main/resources/templates/db_scripts/schema.json @@ -307,6 +307,11 @@ "snippetPath": "create_get_job_log_entries_function.sql", "fromModuleVersion": "mod-source-record-manager-3.9.0" }, + { + "run": "after", + "snippetPath": "create_get_record_processing_log_function.sql", + "fromModuleVersion": "mod-source-record-manager-3.9.0" + }, { "run": "after", "snippetPath": "migration_to_partitioning_logs.sql", diff --git a/mod-source-record-manager-server/src/test/java/org/folio/rest/impl/metadataProvider/MetaDataProviderJobLogEntriesAPITest.java b/mod-source-record-manager-server/src/test/java/org/folio/rest/impl/metadataProvider/MetaDataProviderJobLogEntriesAPITest.java index 3913528b2..bbd0302a5 100644 --- a/mod-source-record-manager-server/src/test/java/org/folio/rest/impl/metadataProvider/MetaDataProviderJobLogEntriesAPITest.java +++ b/mod-source-record-manager-server/src/test/java/org/folio/rest/impl/metadataProvider/MetaDataProviderJobLogEntriesAPITest.java @@ -18,6 +18,7 @@ import static org.folio.rest.jaxrs.model.JournalRecord.EntityType.MARC_BIBLIOGRAPHIC; import static org.folio.rest.jaxrs.model.JournalRecord.EntityType.MARC_HOLDINGS; import static org.folio.rest.jaxrs.model.JournalRecord.EntityType.PO_LINE; +import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.emptyOrNullString; import static org.hamcrest.Matchers.in; @@ -33,9 +34,7 @@ import org.folio.dao.JournalRecordDaoImpl; import org.folio.dao.util.PostgresClientFactory; import org.folio.rest.impl.AbstractRestTest; -import org.folio.rest.jaxrs.model.ActionStatus; -import org.folio.rest.jaxrs.model.JobExecution; -import org.folio.rest.jaxrs.model.JournalRecord; +import org.folio.rest.jaxrs.model.*; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -1020,23 +1019,28 @@ public void shouldReturnHoldingsDataIfHoldingsPoweredByMarcHoldingsWasCreated(Te .onFailure(context::fail); future.onComplete(ar -> context.verify(v -> { - RestAssured.given() + RecordProcessingLogDto recordProcessingLogSummary = RestAssured.given() .spec(spec) .when() .get(GET_JOB_EXECUTION_JOURNAL_RECORDS_PATH + "/" + createdJobExecution.getId() + "/records/" + incomingRecordId) .then() .statusCode(HttpStatus.SC_OK) - .body("jobExecutionId", is(createdJobExecution.getId())) - .body("incomingRecordId", is(incomingRecordId)) - .body("sourceRecordId", is(marcHoldingsId)) - .body("sourceRecordOrder", is("0")) - .body("relatedHoldingsInfo.size()", is(1)) - .body("relatedHoldingsInfo[0].actionStatus", is(ActionStatus.CREATED.value())) - .body("relatedHoldingsInfo[0].id", is(holdingId)) - .body("relatedHoldingsInfo[0].hrid", is(holdingHrid)) - .body("relatedHoldingsInfo[0].permanentLocationId", is(holdingPermanentLocationId)) - .body("relatedHoldingsInfo[0].error", emptyOrNullString()) - .body("error", emptyOrNullString()); +// .body("jobExecutionId", is(createdJobExecution.getId())) +// .body("incomingRecordId", is(incomingRecordId)) +// .body("sourceRecordId", is(marcHoldingsId)) +// .body("sourceRecordOrder", is("0")) +// .body("relatedHoldingsInfo.size()", is(1)) +// .body("relatedHoldingsInfo[0].actionStatus", is(ActionStatus.CREATED.value())) +// .body("relatedHoldingsInfo[0].id", is(holdingId)) +// .body("relatedHoldingsInfo[0].hrid", is(holdingHrid)) +// .body("relatedHoldingsInfo[0].permanentLocationId", is(holdingPermanentLocationId)) +// .body("relatedHoldingsInfo[0].error", emptyOrNullString()) +// .body("error", emptyOrNullString()) + .extract().response().body().as(RecordProcessingLogDto.class); + + assertThat(recordProcessingLogSummary.getJobExecutionId(), is(createdJobExecution.getId())); + assertThat(recordProcessingLogSummary.getIncomingRecordId(), is(incomingRecordId)); + assertThat(recordProcessingLogSummary.getSourceRecordId(), is(marcHoldingsId)); async.complete(); })); } From 1af20c62888aa943a69f94a3fac51d21445b8b4b Mon Sep 17 00:00:00 2001 From: aliaksandr_fedasiuk Date: Mon, 16 Sep 2024 12:18:15 +0500 Subject: [PATCH 14/25] MODSOURMAN-1204: fix tests --- ...ate_get_record_processing_log_function.sql | 451 +++++++++--------- .../MetaDataProviderJobLogEntriesAPITest.java | 3 + 2 files changed, 229 insertions(+), 225 deletions(-) diff --git a/mod-source-record-manager-server/src/main/resources/templates/db_scripts/create_get_record_processing_log_function.sql b/mod-source-record-manager-server/src/main/resources/templates/db_scripts/create_get_record_processing_log_function.sql index deb30d6de..a52be65f5 100644 --- a/mod-source-record-manager-server/src/main/resources/templates/db_scripts/create_get_record_processing_log_function.sql +++ b/mod-source-record-manager-server/src/main/resources/templates/db_scripts/create_get_record_processing_log_function.sql @@ -1,240 +1,241 @@ DROP FUNCTION IF EXISTS get_record_processing_log(uuid, uuid); CREATE OR REPLACE FUNCTION get_record_processing_log(jobExecutionId uuid, recordId uuid) - RETURNS TABLE(job_execution_id uuid, incoming_record_id uuid, source_id uuid, source_record_order integer, title text, source_record_action_status text, source_entity_error text, source_record_tenant_id text, instance_action_status text, instance_entity_id text, instance_entity_hrid text, instance_entity_error text, instance_entity_tenant_id text, holdings_action_status text, holdings_entity_hrid text, holdings_entity_id text, holdings_permanent_location_id text, holdings_entity_error text, item_action_status text, item_entity_hrid text, item_entity_id text, item_entity_error text, authority_action_status text, authority_entity_id text, authority_entity_error text, po_line_action_status text, po_line_entity_id text, po_line_entity_hrid text, po_line_entity_error text, order_entity_id text, invoice_action_status text, invoice_entity_id text[], invoice_entity_hrid text[], invoice_entity_error text, invoice_line_action_status text, invoice_line_entity_id text, invoice_line_entity_hrid text, invoice_line_entity_error text) + RETURNS TABLE(job_execution_id uuid, incoming_record_id uuid, source_id uuid, source_record_order integer, title text, source_record_action_status text, source_entity_error text, source_record_tenant_id text, instance_action_status text, instance_entity_id text, instance_entity_hrid text, instance_entity_error text, instance_entity_tenant_id text, holdings_action_status text, holdings_entity_hrid text, holdings_entity_id text, holdings_permanent_location_id text, holdings_entity_error text, item_action_status text, item_entity_hrid text, item_entity_id text, item_entity_error text, authority_action_status text, authority_entity_id text, authority_entity_error text, po_line_action_status text, po_line_entity_id text, po_line_entity_hrid text, po_line_entity_error text, order_entity_id text, invoice_action_status text, invoice_entity_id text[], invoice_entity_hrid text[], invoice_entity_error text, invoice_line_action_status text, invoice_line_entity_id text, invoice_line_entity_hrid text, invoice_line_entity_error text) AS $$ BEGIN - RETURN QUERY - WITH temp_result AS (SELECT id, journal_records.job_execution_id, journal_records.source_id, journal_records.entity_type, journal_records.entity_id, journal_records.entity_hrid, - CASE WHEN action_type = 'PARSE' - THEN 'PARSED' - WHEN error_max != '' OR action_type = 'NON_MATCH' - THEN 'DISCARDED' - WHEN action_type = 'CREATE' - THEN 'CREATED' - WHEN action_type = 'UPDATE' - THEN 'UPDATED' - END AS action_type, journal_records.action_status, journal_records.action_date, journal_records.source_record_order, journal_records.error, journal_records.title, journal_records.tenant_id, journal_records.instance_id, journal_records.holdings_id, journal_records.order_id, journal_records.permanent_location_id - FROM journal_records - INNER JOIN - (SELECT entity_type as entity_type_max, entity_id as entity_id_max,action_status as action_status_max, max(error) AS error_max,(array_agg(id ORDER BY array_position(array['CREATE', 'UPDATE', 'NON_MATCH'], action_type)))[1] AS id_max - FROM journal_records - WHERE journal_records.job_execution_id = jobExecutionId AND journal_records.source_id = recordId AND journal_records.entity_type NOT IN ('EDIFACT', 'INVOICE') AND action_type != 'MATCH' - GROUP BY entity_type,entity_id,action_status) AS action_type_by_source ON journal_records.id = action_type_by_source.id_max - UNION ALL - SELECT id, journal_records.job_execution_id, journal_records.source_id, journal_records.entity_type, journal_records.entity_id, journal_records.entity_hrid, - CASE WHEN error_max != '' OR action_type = 'MATCH' THEN 'DISCARDED' - END AS action_type, journal_records.action_status, journal_records.action_date, journal_records.source_record_order, journal_records.error, journal_records.title, journal_records.tenant_id, journal_records.instance_id, journal_records.holdings_id, journal_records.order_id, journal_records.permanent_location_id - FROM journal_records - INNER JOIN - (SELECT entity_type as entity_type_max, entity_id as entity_id_max,action_status as action_status_max, max(error) AS error_max,(array_agg(id ORDER BY array_position(array['NON_MATCH', 'MATCH'], action_type)))[1] AS id_max - FROM journal_records - WHERE journal_records.job_execution_id = jobExecutionId AND journal_records.source_id = recordId AND journal_records.entity_type NOT IN ('EDIFACT', 'INVOICE') AND action_type = 'MATCH' - AND NOT EXISTS (SELECT 1 FROM journal_records WHERE journal_records.job_execution_id = jobExecutionId AND journal_records.source_id = recordId AND action_type NOT IN ('MATCH', 'PARSE')) - GROUP BY entity_type,entity_id,action_status) AS action_type_by_source ON journal_records.id = action_type_by_source.id_max) - (SELECT - COALESCE(marc.job_execution_id,instances.job_execution_id,holdings.job_execution_id,items.job_execution_id) AS job_execution_id, - COALESCE(marc.source_id, instances.source_id, holdings.source_id, items.source_id, authority.source_id) as incoming_record_id, - marc_entity_id::uuid AS source_id, - COALESCE(marc.source_record_order,instances.source_record_order,holdings.source_record_order,items.source_record_order) AS source_record_order, - COALESCE(marc.title,instances.title,holdings.title,items.title) AS title, - marc.action_type AS source_record_action_status, - marc.error AS source_entity_error, - marc.tenant_id AS source_record_tenant_id, + RETURN QUERY + WITH temp_result AS + (SELECT jr.id, jr.job_execution_id, jr.source_id, jr.entity_type, jr.entity_id, jr.entity_hrid, + CASE + WHEN jr.action_type = 'PARSE' THEN 'PARSED' + WHEN jr.error != '' OR jr.action_type IN ('NON_MATCH', 'MATCH') THEN 'DISCARDED' + WHEN jr.action_type = 'CREATE' THEN 'CREATED' + WHEN jr.action_type = 'UPDATE' THEN 'UPDATED' + END AS action_type, + jr.action_status, jr.action_date, jr.source_record_order, jr.error, jr.title, jr.tenant_id, + jr.instance_id, jr.holdings_id, jr.order_id, jr.permanent_location_id + FROM journal_records jr + INNER JOIN ( + SELECT jr_inner.entity_type AS entity_type_max, jr_inner.entity_id AS entity_id_max, + jr_inner.action_status AS action_status_max, MAX(jr_inner.error) AS error_max, + (ARRAY_AGG(jr_inner.id ORDER BY ARRAY_POSITION(ARRAY['CREATE', 'UPDATE', 'NON_MATCH', 'MATCH'], jr_inner.action_type)))[1] AS id_max + FROM journal_records jr_inner + WHERE jr_inner.job_execution_id = jobExecutionId AND jr_inner.source_id = recordId + AND jr_inner.entity_type NOT IN ('EDIFACT', 'INVOICE') + AND (jr_inner.action_type != 'MATCH' OR (jr_inner.action_type = 'MATCH' AND NOT EXISTS ( + SELECT 1 + FROM journal_records jr_sub + WHERE jr_sub.job_execution_id = jobExecutionId AND jr_sub.source_id = recordId + AND jr_sub.action_type NOT IN ('MATCH', 'PARSE') + ) + )) + GROUP BY jr_inner.entity_type, jr_inner.entity_id, jr_inner.action_status + ) AS action_type_by_source ON jr.id = action_type_by_source.id_max + ) + (SELECT + COALESCE(marc.job_execution_id,instances.job_execution_id,holdings.job_execution_id,items.job_execution_id) AS job_execution_id, + COALESCE(marc.source_id, instances.source_id, holdings.source_id, items.source_id, authority.source_id) as incoming_record_id, + marc_entity_id::uuid AS source_id, + COALESCE(marc.source_record_order,instances.source_record_order,holdings.source_record_order,items.source_record_order) AS source_record_order, + COALESCE(marc.title,instances.title,holdings.title,items.title) AS title, + marc.action_type AS source_record_action_status, + marc.error AS source_entity_error, + marc.tenant_id AS source_record_tenant_id, - instances.action_type AS instance_action_status, - COALESCE(instances.entity_id,holdings.instance_id,items.instance_id) AS instance_entity_id, - instances.entity_hrid AS instance_entity_hrid, - instances.error AS instance_entity_error, - instances.tenant_id AS instance_entity_tenant_id, + instances.action_type AS instance_action_status, + COALESCE(instances.entity_id,holdings.instance_id,items.instance_id) AS instance_entity_id, + instances.entity_hrid AS instance_entity_hrid, + instances.error AS instance_entity_error, + instances.tenant_id AS instance_entity_tenant_id, - holdings.action_type AS holdings_action_status, - holdings.entity_hrid AS holdings_entity_hrid, - COALESCE(holdings.entity_id,items.holdings_id) AS holdings_entity_id, - holdings.permanent_location_id AS holdings_permanent_location_id, - holdings.error AS holdings_entity_error, + holdings.action_type AS holdings_action_status, + holdings.entity_hrid AS holdings_entity_hrid, + COALESCE(holdings.entity_id,items.holdings_id) AS holdings_entity_id, + holdings.permanent_location_id AS holdings_permanent_location_id, + holdings.error AS holdings_entity_error, - items.action_type AS item_action_status, - items.entity_hrid AS item_entity_hrid, - items.entity_id AS item_entity_id, - items.error AS item_entity_error, + items.action_type AS item_action_status, + items.entity_hrid AS item_entity_hrid, + items.entity_id AS item_entity_id, + items.error AS item_entity_error, - authority.action_type AS authority_action_status, - authority.entity_id AS authority_entity_id, - authority.error AS authority_entity_error, + authority.action_type AS authority_action_status, + authority.entity_id AS authority_entity_id, + authority.error AS authority_entity_error, - po_lines.action_type AS po_line_action_status, - po_lines.entity_id AS po_line_entity_id, - po_lines.entity_hrid AS po_line_entity_hrid, - po_lines.error AS po_line_entity_error, - po_lines.order_id AS order_entity_id, + po_lines.action_type AS po_line_action_status, + po_lines.entity_id AS po_line_entity_id, + po_lines.entity_hrid AS po_line_entity_hrid, + po_lines.error AS po_line_entity_error, + po_lines.order_id AS order_entity_id, - null AS invoice_action_status, - null AS invoice_entity_id, - null AS invoice_entity_hrid, - null AS invoice_entity_error, - null AS invoice_line_action_status, - null AS invoice_line_entity_id, - null AS invoice_line_entity_hrid, - null AS invoice_line_entity_error - FROM (SELECT temp_result.source_id FROM temp_result WHERE action_type = 'PARSED') as parsed - LEFT JOIN - (SELECT temp_result.job_execution_id, entity_id, temp_result.title, temp_result.source_record_order, action_type, error, temp_result.source_id, temp_result.tenant_id - FROM temp_result WHERE entity_type IN ('MARC_BIBLIOGRAPHIC', 'MARC_HOLDINGS', 'MARC_AUTHORITY', 'PO_LINE') AND entity_id IS NOT NULL - UNION ALL - SELECT temp_result.job_execution_id, entity_id, temp_result.title, temp_result.source_record_order, action_type, error, temp_result.source_id, temp_result.tenant_id - FROM temp_result - WHERE entity_type IN ('MARC_BIBLIOGRAPHIC', 'MARC_HOLDINGS', 'MARC_AUTHORITY', 'PO_LINE') AND entity_id IS NULL AND NOT EXISTS - (SELECT 1 - FROM temp_result as tr2 - WHERE tr2.entity_type IN ('MARC_BIBLIOGRAPHIC', 'MARC_HOLDINGS', 'MARC_AUTHORITY', 'PO_LINE') AND tr2.source_id = temp_result.source_id and tr2.entity_id IS NOT NULL)) AS marc - ON marc.source_id = parsed.source_id - LEFT JOIN - (SELECT action_type, entity_id, temp_result.source_id, entity_hrid, error, temp_result.job_execution_id, temp_result.title, temp_result.source_record_order, temp_result.tenant_id - FROM temp_result WHERE entity_type = 'INSTANCE' AND entity_id IS NOT NULL - UNION ALL - SELECT action_type, entity_id, temp_result.source_id, entity_hrid, error, temp_result.job_execution_id, temp_result.title, temp_result.source_record_order, temp_result.tenant_id - FROM temp_result - WHERE entity_type = 'INSTANCE' AND entity_id IS NULL AND NOT EXISTS - (SELECT 1 - FROM temp_result as tr2 + null AS invoice_action_status, + null AS invoice_entity_id, + null AS invoice_entity_hrid, + null AS invoice_entity_error, + null AS invoice_line_action_status, + null AS invoice_line_entity_id, + null AS invoice_line_entity_hrid, + null AS invoice_line_entity_error + FROM (SELECT temp_result.source_id FROM temp_result WHERE action_type = 'PARSED') as parsed + INNER JOIN + (SELECT temp_result.job_execution_id, entity_id, temp_result.title, temp_result.source_record_order, action_type, error, temp_result.source_id, temp_result.tenant_id + FROM temp_result WHERE entity_type IN ('MARC_BIBLIOGRAPHIC', 'MARC_HOLDINGS', 'MARC_AUTHORITY', 'PO_LINE') AND entity_id IS NOT NULL + UNION ALL + SELECT temp_result.job_execution_id, entity_id, temp_result.title, temp_result.source_record_order, action_type, error, temp_result.source_id, temp_result.tenant_id + FROM temp_result + WHERE entity_type IN ('MARC_BIBLIOGRAPHIC', 'MARC_HOLDINGS', 'MARC_AUTHORITY', 'PO_LINE') AND entity_id IS NULL AND NOT EXISTS + (SELECT 1 + FROM temp_result as tr2 + WHERE tr2.entity_type IN ('MARC_BIBLIOGRAPHIC', 'MARC_HOLDINGS', 'MARC_AUTHORITY', 'PO_LINE') AND tr2.source_id = temp_result.source_id and tr2.entity_id IS NOT NULL)) AS marc + ON marc.source_id = parsed.source_id + LEFT JOIN + (SELECT action_type, entity_id, temp_result.source_id, entity_hrid, error, temp_result.job_execution_id, temp_result.title, temp_result.source_record_order, temp_result.tenant_id + FROM temp_result WHERE entity_type = 'INSTANCE' AND entity_id IS NOT NULL + UNION ALL + SELECT action_type, entity_id, temp_result.source_id, entity_hrid, error, temp_result.job_execution_id, temp_result.title, temp_result.source_record_order, temp_result.tenant_id + FROM temp_result + WHERE entity_type = 'INSTANCE' AND entity_id IS NULL AND NOT EXISTS + (SELECT 1 + FROM temp_result as tr2 WHERE tr2.entity_type = 'INSTANCE' AND tr2.source_id = temp_result.source_id and tr2.entity_id IS NOT NULL)) AS instances - ON marc.source_id = instances.source_id - LEFT JOIN - (SELECT action_type, entity_id, temp_result.source_id, error, temp_result.job_execution_id, temp_result.title, temp_result.source_record_order - FROM temp_result WHERE entity_type = 'AUTHORITY') AS authority - ON authority.source_id = marc.source_id - LEFT JOIN - (SELECT action_type,entity_id,entity_hrid,temp_result.source_id,error,order_id,temp_result.job_execution_id,temp_result.title,temp_result.source_record_order - FROM temp_result WHERE entity_type = 'PO_LINE') AS po_lines - ON po_lines.source_id = marc.source_id - FULL JOIN - (SELECT tmp.action_type, tmp.entity_type, tmp.entity_id, tmp.entity_hrid, tmp.error, tmp.instance_id, - tmp.permanent_location_id, tmp.job_execution_id, tmp.source_id, tmp.title, tmp.source_record_order - FROM temp_result tmp - INNER JOIN - (SELECT - CASE - WHEN EXISTS (SELECT condition_result.entity_id FROM temp_result condition_result - WHERE (condition_result.action_type='CREATED' AND condition_result.entity_type='HOLDINGS') - OR - (condition_result.action_type='DISCARDED' AND condition_result.error != '' AND condition_result.entity_type='HOLDINGS')) - THEN - (SELECT deep_nested.id - FROM temp_result deep_nested - WHERE - (deep_nested.action_type='CREATED' AND deep_nested.id = nested_result.id) - OR - (deep_nested.action_type='DISCARDED' AND deep_nested.error != '' AND deep_nested.id = nested_result.id)) - ELSE - nested_result.id - END - FROM temp_result nested_result) AS joining_table - ON tmp.id = joining_table.id - WHERE tmp.entity_type='HOLDINGS') - AS holdings - ON instances.entity_id = holdings.instance_id - FULL JOIN - (SELECT tmp.action_type, tmp.entity_id, tmp.holdings_id, tmp.entity_hrid, tmp.error, tmp.instance_id, - tmp.job_execution_id, tmp.source_id, tmp.title, tmp.source_record_order - FROM temp_result tmp - INNER JOIN - (SELECT - CASE - WHEN EXISTS (SELECT condition_result.entity_id FROM temp_result condition_result - WHERE (condition_result.action_type IN ('CREATED','UPDATED') AND condition_result.entity_type='ITEM') - OR - (condition_result.action_type='DISCARDED' AND condition_result.error != '' AND condition_result.entity_type='ITEM')) - THEN - (SELECT deep_nested.id - FROM temp_result deep_nested - WHERE - (deep_nested.action_type IN ('CREATED','UPDATED') AND deep_nested.id = nested_result.id) - OR - (deep_nested.action_type='DISCARDED' AND deep_nested.error != '' AND deep_nested.id = nested_result.id)) - ELSE - nested_result.id - END - FROM temp_result nested_result) AS joining_table - ON tmp.id = joining_table.id - WHERE tmp.entity_type='ITEM') AS items - ON holdings.entity_id = items.holdings_id - LEFT JOIN ( - SELECT entity_id AS marc_entity_id, temp_result.source_id AS marc_source_id - FROM temp_result WHERE entity_type IN ('MARC_BIBLIOGRAPHIC', 'MARC_HOLDINGS', 'MARC_AUTHORITY') AND entity_id IS NOT NULL - ) AS marc_identifiers ON marc.source_id = marc_identifiers.marc_source_id - ORDER BY holdings.entity_hrid) - UNION - SELECT invoice_line_info.job_execution_id, - records_actions.source_id as incoming_record_id, - records_actions.source_id as source_id, - records_actions.source_record_order, - invoice_line_info.title, - CASE WHEN edifact_errors_number != 0 THEN 'DISCARDED' - WHEN edifact_actions[array_length(edifact_actions, 1)] = 'CREATE' THEN 'CREATED' - END AS source_record_action_status, - records_actions.source_record_error[1], - records_actions.source_record_tenant_id, - null AS instance_action_status, - null AS instance_entity_id, - null AS instance_entity_hrid, - null AS instance_entity_error, - null AS instance_entity_tenant_id, - null AS holdings_action_status, - null AS holdings_entity_hrid, - null AS holdings_entity_id, - null AS holdings_permanent_location_id, - null AS holdings_entity_error, - null AS item_action_status, - null AS item_entity_hrid, - null AS item_entity_id, - null AS item_entity_error, - null AS authority_action_status, - null AS authority_entity_id, - null AS authority_entity_error, - null AS po_line_action_status, - null AS po_line_entity_id, - null AS po_line_entity_hrid, - null AS po_line_entity_error, - null AS order_entity_id, - get_entity_status(records_actions.invoice_actions, records_actions.invoice_errors_number) AS invoice_action_status, - records_actions.invoice_entity_id, - records_actions.invoice_entity_hrid, - records_actions.invoice_entity_error[1], - CASE WHEN action_status = 'ERROR' THEN 'DISCARDED' - WHEN inv_line_actions = 'CREATE' THEN 'CREATED' - END AS invoice_line_action_status, - invoice_line_info.invoice_line_entity_id, - invoice_line_info.invoice_line_entity_hrid, - invoice_line_info.invoice_line_entity_error - FROM ( - SELECT journal_records.source_id, - journal_records.job_execution_id, - journal_records.title, - journal_records.action_type AS inv_line_actions, - action_status, - entity_hrid AS invoice_line_entity_hrid, - entity_id AS invoice_line_entity_id, - error AS invoice_line_entity_error - FROM journal_records - WHERE journal_records.id = recordId AND journal_records.entity_type = 'INVOICE' AND journal_records.title != 'INVOICE' - ) AS invoice_line_info - LEFT JOIN LATERAL ( - SELECT journal_records.source_id, - journal_records.source_record_order, - array_agg(action_type) FILTER (WHERE entity_type = 'EDIFACT') AS edifact_actions, - count(journal_records.source_id) FILTER (WHERE entity_type = 'EDIFACT' AND journal_records.error != '') AS edifact_errors_number, - array_agg(error) FILTER (WHERE entity_type = 'EDIFACT') AS source_record_error, - journal_records.tenant_id AS source_record_tenant_id, + ON marc.source_id = instances.source_id + LEFT JOIN + (SELECT action_type, entity_id, temp_result.source_id, error, temp_result.job_execution_id, temp_result.title, temp_result.source_record_order + FROM temp_result WHERE entity_type = 'AUTHORITY') AS authority + ON authority.source_id = marc.source_id + LEFT JOIN + (SELECT action_type,entity_id,entity_hrid,temp_result.source_id,error,order_id,temp_result.job_execution_id,temp_result.title,temp_result.source_record_order + FROM temp_result WHERE entity_type = 'PO_LINE') AS po_lines + ON po_lines.source_id = marc.source_id + FULL JOIN + (SELECT tmp.action_type, tmp.entity_type, tmp.entity_id, tmp.entity_hrid, tmp.error, tmp.instance_id, + tmp.permanent_location_id, tmp.job_execution_id, tmp.source_id, tmp.title, tmp.source_record_order + FROM temp_result tmp + INNER JOIN + (SELECT + CASE + WHEN EXISTS (SELECT condition_result.entity_id FROM temp_result condition_result + WHERE (condition_result.action_type='CREATED' AND condition_result.entity_type='HOLDINGS') + OR + (condition_result.action_type='DISCARDED' AND condition_result.error != '' AND condition_result.entity_type='HOLDINGS')) + THEN + (SELECT deep_nested.id + FROM temp_result deep_nested + WHERE + (deep_nested.action_type='CREATED' AND deep_nested.id = nested_result.id) + OR + (deep_nested.action_type='DISCARDED' AND deep_nested.error != '' AND deep_nested.id = nested_result.id)) + ELSE + nested_result.id + END + FROM temp_result nested_result) AS joining_table + ON tmp.id = joining_table.id + WHERE tmp.entity_type='HOLDINGS') + AS holdings + ON instances.entity_id = holdings.instance_id + FULL JOIN + (SELECT tmp.action_type, tmp.entity_id, tmp.holdings_id, tmp.entity_hrid, tmp.error, tmp.instance_id, + tmp.job_execution_id, tmp.source_id, tmp.title, tmp.source_record_order + FROM temp_result tmp + INNER JOIN + (SELECT + CASE + WHEN EXISTS (SELECT condition_result.entity_id FROM temp_result condition_result + WHERE (condition_result.action_type IN ('CREATED','UPDATED') AND condition_result.entity_type='ITEM') + OR + (condition_result.action_type='DISCARDED' AND condition_result.error != '' AND condition_result.entity_type='ITEM')) + THEN + (SELECT deep_nested.id + FROM temp_result deep_nested + WHERE + (deep_nested.action_type IN ('CREATED','UPDATED') AND deep_nested.id = nested_result.id) + OR + (deep_nested.action_type='DISCARDED' AND deep_nested.error != '' AND deep_nested.id = nested_result.id)) + ELSE + nested_result.id + END + FROM temp_result nested_result) AS joining_table + ON tmp.id = joining_table.id + WHERE tmp.entity_type='ITEM') AS items + ON holdings.entity_id = items.holdings_id + LEFT JOIN ( + SELECT entity_id AS marc_entity_id, temp_result.source_id AS marc_source_id + FROM temp_result WHERE entity_type IN ('MARC_BIBLIOGRAPHIC', 'MARC_HOLDINGS', 'MARC_AUTHORITY') AND entity_id IS NOT NULL + ) AS marc_identifiers ON marc.source_id = marc_identifiers.marc_source_id + ORDER BY holdings.entity_hrid) + UNION + SELECT invoice_line_info.job_execution_id, + records_actions.source_id as incoming_record_id, + records_actions.source_id as source_id, + records_actions.source_record_order, + invoice_line_info.title, + CASE WHEN edifact_errors_number != 0 THEN 'DISCARDED' + WHEN edifact_actions[array_length(edifact_actions, 1)] = 'CREATE' THEN 'CREATED' + END AS source_record_action_status, + records_actions.source_record_error[1], + records_actions.source_record_tenant_id, + null AS instance_action_status, + null AS instance_entity_id, + null AS instance_entity_hrid, + null AS instance_entity_error, + null AS instance_entity_tenant_id, + null AS holdings_action_status, + null AS holdings_entity_hrid, + null AS holdings_entity_id, + null AS holdings_permanent_location_id, + null AS holdings_entity_error, + null AS item_action_status, + null AS item_entity_hrid, + null AS item_entity_id, + null AS item_entity_error, + null AS authority_action_status, + null AS authority_entity_id, + null AS authority_entity_error, + null AS po_line_action_status, + null AS po_line_entity_id, + null AS po_line_entity_hrid, + null AS po_line_entity_error, + null AS order_entity_id, + get_entity_status(records_actions.invoice_actions, records_actions.invoice_errors_number) AS invoice_action_status, + records_actions.invoice_entity_id, + records_actions.invoice_entity_hrid, + records_actions.invoice_entity_error[1], + CASE WHEN action_status = 'ERROR' THEN 'DISCARDED' + WHEN inv_line_actions = 'CREATE' THEN 'CREATED' + END AS invoice_line_action_status, + invoice_line_info.invoice_line_entity_id, + invoice_line_info.invoice_line_entity_hrid, + invoice_line_info.invoice_line_entity_error + FROM ( + SELECT journal_records.source_id, + journal_records.job_execution_id, + journal_records.title, + journal_records.action_type AS inv_line_actions, + action_status, + entity_hrid AS invoice_line_entity_hrid, + entity_id AS invoice_line_entity_id, + error AS invoice_line_entity_error + FROM journal_records + WHERE journal_records.id = recordId AND journal_records.entity_type = 'INVOICE' AND journal_records.title != 'INVOICE' + ) AS invoice_line_info + LEFT JOIN LATERAL ( + SELECT journal_records.source_id, + journal_records.source_record_order, + array_agg(action_type) FILTER (WHERE entity_type = 'EDIFACT') AS edifact_actions, + count(journal_records.source_id) FILTER (WHERE entity_type = 'EDIFACT' AND journal_records.error != '') AS edifact_errors_number, + array_agg(error) FILTER (WHERE entity_type = 'EDIFACT') AS source_record_error, + journal_records.tenant_id AS source_record_tenant_id, - array_agg(action_type) FILTER (WHERE entity_type = 'INVOICE' AND journal_records.title = 'INVOICE') AS invoice_actions, - count(journal_records.source_id) FILTER (WHERE entity_type = 'INVOICE' AND journal_records.title = 'INVOICE' AND journal_records.error != '') AS invoice_errors_number, - array_agg(entity_hrid) FILTER (WHERE entity_type = 'INVOICE' AND journal_records.title = 'INVOICE') AS invoice_entity_hrid, - array_agg(entity_id) FILTER (WHERE entity_type = 'INVOICE' AND journal_records.title = 'INVOICE') AS invoice_entity_id, - array_agg(error) FILTER (WHERE entity_type = 'INVOICE' AND journal_records.title = 'INVOICE') AS invoice_entity_error - FROM journal_records - WHERE journal_records.source_id = invoice_line_info.source_id AND (entity_type = 'EDIFACT' OR journal_records.title = 'INVOICE') - GROUP BY journal_records.source_id, journal_records.job_execution_id,journal_records.source_record_order, journal_records.tenant_id - ) AS records_actions ON TRUE; + array_agg(action_type) FILTER (WHERE entity_type = 'INVOICE' AND journal_records.title = 'INVOICE') AS invoice_actions, + count(journal_records.source_id) FILTER (WHERE entity_type = 'INVOICE' AND journal_records.title = 'INVOICE' AND journal_records.error != '') AS invoice_errors_number, + array_agg(entity_hrid) FILTER (WHERE entity_type = 'INVOICE' AND journal_records.title = 'INVOICE') AS invoice_entity_hrid, + array_agg(entity_id) FILTER (WHERE entity_type = 'INVOICE' AND journal_records.title = 'INVOICE') AS invoice_entity_id, + array_agg(error) FILTER (WHERE entity_type = 'INVOICE' AND journal_records.title = 'INVOICE') AS invoice_entity_error + FROM journal_records + WHERE journal_records.source_id = invoice_line_info.source_id AND (entity_type = 'EDIFACT' OR journal_records.title = 'INVOICE') + GROUP BY journal_records.source_id, journal_records.job_execution_id,journal_records.source_record_order, journal_records.tenant_id + ) AS records_actions ON TRUE; END; $$ LANGUAGE plpgsql; diff --git a/mod-source-record-manager-server/src/test/java/org/folio/rest/impl/metadataProvider/MetaDataProviderJobLogEntriesAPITest.java b/mod-source-record-manager-server/src/test/java/org/folio/rest/impl/metadataProvider/MetaDataProviderJobLogEntriesAPITest.java index bbd0302a5..43786c5e8 100644 --- a/mod-source-record-manager-server/src/test/java/org/folio/rest/impl/metadataProvider/MetaDataProviderJobLogEntriesAPITest.java +++ b/mod-source-record-manager-server/src/test/java/org/folio/rest/impl/metadataProvider/MetaDataProviderJobLogEntriesAPITest.java @@ -93,6 +93,8 @@ public void shouldReturnInstanceIdWhenHoldingsCreated(TestContext context) { .get(GET_JOB_EXECUTION_JOURNAL_RECORDS_PATH + "/" + instanceCreationJobExecution.getId() + "/records/" + instanceCreationSourceRecordId) .then() .statusCode(HttpStatus.SC_OK) + .log() + .all() .body("jobExecutionId", is(instanceCreationJobExecution.getId())) .body("incomingRecordId", is(instanceCreationSourceRecordId)) .body("sourceRecordOrder", is("0")) @@ -1193,6 +1195,7 @@ public void shouldReturnDiscardedForHoldingsIfHoldingsMatchedAndNoOtherAction(Te .get(GET_JOB_EXECUTION_JOURNAL_RECORDS_PATH + "/" + createdJobExecution.getId() + "/records/" + incomingRecordId) .then() .statusCode(HttpStatus.SC_OK) + .log().all() .body("jobExecutionId", is(createdJobExecution.getId())) .body("incomingRecordId", is(incomingRecordId)) .body("sourceRecordTitle", is(recordTitle)) From f7444cf3668cedd09550692c00d6044ed0aa2d78 Mon Sep 17 00:00:00 2001 From: aliaksandr_fedasiuk Date: Tue, 17 Sep 2024 02:04:43 +0500 Subject: [PATCH 15/25] MODSOURMAN-1204: fix tests --- ...ate_get_record_processing_log_function.sql | 377 ++++++++---------- 1 file changed, 164 insertions(+), 213 deletions(-) diff --git a/mod-source-record-manager-server/src/main/resources/templates/db_scripts/create_get_record_processing_log_function.sql b/mod-source-record-manager-server/src/main/resources/templates/db_scripts/create_get_record_processing_log_function.sql index a52be65f5..462962bc9 100644 --- a/mod-source-record-manager-server/src/main/resources/templates/db_scripts/create_get_record_processing_log_function.sql +++ b/mod-source-record-manager-server/src/main/resources/templates/db_scripts/create_get_record_processing_log_function.sql @@ -5,229 +5,180 @@ CREATE OR REPLACE FUNCTION get_record_processing_log(jobExecutionId uuid, record AS $$ BEGIN RETURN QUERY - WITH temp_result AS - (SELECT jr.id, jr.job_execution_id, jr.source_id, jr.entity_type, jr.entity_id, jr.entity_hrid, - CASE - WHEN jr.action_type = 'PARSE' THEN 'PARSED' - WHEN jr.error != '' OR jr.action_type IN ('NON_MATCH', 'MATCH') THEN 'DISCARDED' - WHEN jr.action_type = 'CREATE' THEN 'CREATED' - WHEN jr.action_type = 'UPDATE' THEN 'UPDATED' - END AS action_type, - jr.action_status, jr.action_date, jr.source_record_order, jr.error, jr.title, jr.tenant_id, - jr.instance_id, jr.holdings_id, jr.order_id, jr.permanent_location_id - FROM journal_records jr - INNER JOIN ( - SELECT jr_inner.entity_type AS entity_type_max, jr_inner.entity_id AS entity_id_max, - jr_inner.action_status AS action_status_max, MAX(jr_inner.error) AS error_max, - (ARRAY_AGG(jr_inner.id ORDER BY ARRAY_POSITION(ARRAY['CREATE', 'UPDATE', 'NON_MATCH', 'MATCH'], jr_inner.action_type)))[1] AS id_max - FROM journal_records jr_inner - WHERE jr_inner.job_execution_id = jobExecutionId AND jr_inner.source_id = recordId - AND jr_inner.entity_type NOT IN ('EDIFACT', 'INVOICE') - AND (jr_inner.action_type != 'MATCH' OR (jr_inner.action_type = 'MATCH' AND NOT EXISTS ( - SELECT 1 - FROM journal_records jr_sub - WHERE jr_sub.job_execution_id = jobExecutionId AND jr_sub.source_id = recordId - AND jr_sub.action_type NOT IN ('MATCH', 'PARSE') - ) - )) - GROUP BY jr_inner.entity_type, jr_inner.entity_id, jr_inner.action_status - ) AS action_type_by_source ON jr.id = action_type_by_source.id_max - ) - (SELECT - COALESCE(marc.job_execution_id,instances.job_execution_id,holdings.job_execution_id,items.job_execution_id) AS job_execution_id, - COALESCE(marc.source_id, instances.source_id, holdings.source_id, items.source_id, authority.source_id) as incoming_record_id, - marc_entity_id::uuid AS source_id, - COALESCE(marc.source_record_order,instances.source_record_order,holdings.source_record_order,items.source_record_order) AS source_record_order, - COALESCE(marc.title,instances.title,holdings.title,items.title) AS title, - marc.action_type AS source_record_action_status, - marc.error AS source_entity_error, - marc.tenant_id AS source_record_tenant_id, - - instances.action_type AS instance_action_status, - COALESCE(instances.entity_id,holdings.instance_id,items.instance_id) AS instance_entity_id, - instances.entity_hrid AS instance_entity_hrid, - instances.error AS instance_entity_error, - instances.tenant_id AS instance_entity_tenant_id, - - holdings.action_type AS holdings_action_status, - holdings.entity_hrid AS holdings_entity_hrid, - COALESCE(holdings.entity_id,items.holdings_id) AS holdings_entity_id, - holdings.permanent_location_id AS holdings_permanent_location_id, - holdings.error AS holdings_entity_error, - - items.action_type AS item_action_status, - items.entity_hrid AS item_entity_hrid, - items.entity_id AS item_entity_id, - items.error AS item_entity_error, - - authority.action_type AS authority_action_status, - authority.entity_id AS authority_entity_id, - authority.error AS authority_entity_error, - - po_lines.action_type AS po_line_action_status, - po_lines.entity_id AS po_line_entity_id, - po_lines.entity_hrid AS po_line_entity_hrid, - po_lines.error AS po_line_entity_error, - po_lines.order_id AS order_entity_id, - - null AS invoice_action_status, - null AS invoice_entity_id, - null AS invoice_entity_hrid, - null AS invoice_entity_error, - null AS invoice_line_action_status, - null AS invoice_line_entity_id, - null AS invoice_line_entity_hrid, - null AS invoice_line_entity_error - FROM (SELECT temp_result.source_id FROM temp_result WHERE action_type = 'PARSED') as parsed - INNER JOIN - (SELECT temp_result.job_execution_id, entity_id, temp_result.title, temp_result.source_record_order, action_type, error, temp_result.source_id, temp_result.tenant_id - FROM temp_result WHERE entity_type IN ('MARC_BIBLIOGRAPHIC', 'MARC_HOLDINGS', 'MARC_AUTHORITY', 'PO_LINE') AND entity_id IS NOT NULL - UNION ALL - SELECT temp_result.job_execution_id, entity_id, temp_result.title, temp_result.source_record_order, action_type, error, temp_result.source_id, temp_result.tenant_id - FROM temp_result - WHERE entity_type IN ('MARC_BIBLIOGRAPHIC', 'MARC_HOLDINGS', 'MARC_AUTHORITY', 'PO_LINE') AND entity_id IS NULL AND NOT EXISTS - (SELECT 1 - FROM temp_result as tr2 - WHERE tr2.entity_type IN ('MARC_BIBLIOGRAPHIC', 'MARC_HOLDINGS', 'MARC_AUTHORITY', 'PO_LINE') AND tr2.source_id = temp_result.source_id and tr2.entity_id IS NOT NULL)) AS marc - ON marc.source_id = parsed.source_id - LEFT JOIN - (SELECT action_type, entity_id, temp_result.source_id, entity_hrid, error, temp_result.job_execution_id, temp_result.title, temp_result.source_record_order, temp_result.tenant_id - FROM temp_result WHERE entity_type = 'INSTANCE' AND entity_id IS NOT NULL - UNION ALL - SELECT action_type, entity_id, temp_result.source_id, entity_hrid, error, temp_result.job_execution_id, temp_result.title, temp_result.source_record_order, temp_result.tenant_id - FROM temp_result - WHERE entity_type = 'INSTANCE' AND entity_id IS NULL AND NOT EXISTS - (SELECT 1 - FROM temp_result as tr2 - WHERE tr2.entity_type = 'INSTANCE' AND tr2.source_id = temp_result.source_id and tr2.entity_id IS NOT NULL)) AS instances - ON marc.source_id = instances.source_id - LEFT JOIN - (SELECT action_type, entity_id, temp_result.source_id, error, temp_result.job_execution_id, temp_result.title, temp_result.source_record_order - FROM temp_result WHERE entity_type = 'AUTHORITY') AS authority - ON authority.source_id = marc.source_id - LEFT JOIN - (SELECT action_type,entity_id,entity_hrid,temp_result.source_id,error,order_id,temp_result.job_execution_id,temp_result.title,temp_result.source_record_order - FROM temp_result WHERE entity_type = 'PO_LINE') AS po_lines - ON po_lines.source_id = marc.source_id - FULL JOIN - (SELECT tmp.action_type, tmp.entity_type, tmp.entity_id, tmp.entity_hrid, tmp.error, tmp.instance_id, - tmp.permanent_location_id, tmp.job_execution_id, tmp.source_id, tmp.title, tmp.source_record_order - FROM temp_result tmp - INNER JOIN - (SELECT - CASE - WHEN EXISTS (SELECT condition_result.entity_id FROM temp_result condition_result - WHERE (condition_result.action_type='CREATED' AND condition_result.entity_type='HOLDINGS') - OR - (condition_result.action_type='DISCARDED' AND condition_result.error != '' AND condition_result.entity_type='HOLDINGS')) - THEN - (SELECT deep_nested.id - FROM temp_result deep_nested - WHERE - (deep_nested.action_type='CREATED' AND deep_nested.id = nested_result.id) - OR - (deep_nested.action_type='DISCARDED' AND deep_nested.error != '' AND deep_nested.id = nested_result.id)) - ELSE - nested_result.id - END - FROM temp_result nested_result) AS joining_table - ON tmp.id = joining_table.id - WHERE tmp.entity_type='HOLDINGS') - AS holdings - ON instances.entity_id = holdings.instance_id - FULL JOIN - (SELECT tmp.action_type, tmp.entity_id, tmp.holdings_id, tmp.entity_hrid, tmp.error, tmp.instance_id, - tmp.job_execution_id, tmp.source_id, tmp.title, tmp.source_record_order - FROM temp_result tmp - INNER JOIN - (SELECT - CASE - WHEN EXISTS (SELECT condition_result.entity_id FROM temp_result condition_result - WHERE (condition_result.action_type IN ('CREATED','UPDATED') AND condition_result.entity_type='ITEM') - OR - (condition_result.action_type='DISCARDED' AND condition_result.error != '' AND condition_result.entity_type='ITEM')) - THEN - (SELECT deep_nested.id - FROM temp_result deep_nested - WHERE - (deep_nested.action_type IN ('CREATED','UPDATED') AND deep_nested.id = nested_result.id) - OR - (deep_nested.action_type='DISCARDED' AND deep_nested.error != '' AND deep_nested.id = nested_result.id)) - ELSE - nested_result.id - END - FROM temp_result nested_result) AS joining_table - ON tmp.id = joining_table.id - WHERE tmp.entity_type='ITEM') AS items - ON holdings.entity_id = items.holdings_id - LEFT JOIN ( - SELECT entity_id AS marc_entity_id, temp_result.source_id AS marc_source_id - FROM temp_result WHERE entity_type IN ('MARC_BIBLIOGRAPHIC', 'MARC_HOLDINGS', 'MARC_AUTHORITY') AND entity_id IS NOT NULL - ) AS marc_identifiers ON marc.source_id = marc_identifiers.marc_source_id - ORDER BY holdings.entity_hrid) + WITH temp_result AS ( + SELECT jr.id, jr.job_execution_id, jr.source_id, jr.entity_type, jr.entity_id, jr.entity_hrid, + CASE + WHEN jr.action_type = 'PARSE' THEN 'PARSED' + WHEN jr.error != '' OR jr.action_type IN ('NON_MATCH', 'MATCH') THEN 'DISCARDED' + WHEN jr.action_type = 'CREATE' THEN 'CREATED' + WHEN jr.action_type = 'UPDATE' THEN 'UPDATED' + END AS action_type, + jr.action_status, jr.action_date, jr.source_record_order, jr.error, jr.title, jr.tenant_id, + jr.instance_id, jr.holdings_id, jr.order_id, jr.permanent_location_id + FROM journal_records jr + INNER JOIN ( + SELECT jr_inner.entity_type AS entity_type_max, jr_inner.entity_id AS entity_id_max, + jr_inner.action_status AS action_status_max, MAX(jr_inner.error) AS error_max, + (ARRAY_AGG(jr_inner.id ORDER BY ARRAY_POSITION(ARRAY['CREATE', 'UPDATE', 'NON_MATCH', 'MATCH'], jr_inner.action_type)))[1] AS id_max + FROM journal_records jr_inner + WHERE jr_inner.job_execution_id = jobExecutionId AND jr_inner.source_id = recordId + AND jr_inner.entity_type NOT IN ('EDIFACT', 'INVOICE') + AND (jr_inner.action_type != 'MATCH' OR (jr_inner.action_type = 'MATCH' AND NOT EXISTS ( + SELECT 1 + FROM journal_records jr_sub + WHERE jr_sub.job_execution_id = jobExecutionId AND jr_sub.source_id = recordId + AND jr_sub.action_type NOT IN ('MATCH', 'PARSE') + ))) + GROUP BY jr_inner.entity_type, jr_inner.entity_id, jr_inner.action_status + ) AS action_type_by_source ON jr.id = action_type_by_source.id_max + ), + marc_data AS ( + SELECT temp_result.job_execution_id, entity_id, temp_result.title, temp_result.source_record_order, action_type, error, + temp_result.source_id, tenant_id + FROM temp_result + WHERE entity_type IN ('MARC_BIBLIOGRAPHIC', 'MARC_HOLDINGS', 'MARC_AUTHORITY', 'PO_LINE') + ), + instance_data AS ( + SELECT action_type, entity_id, temp_result.source_id, entity_hrid, error, temp_result.job_execution_id, temp_result.title, + temp_result.source_record_order, tenant_id + FROM temp_result + WHERE entity_type = 'INSTANCE' + ), + authority_data AS ( + SELECT action_type, entity_id, temp_result.source_id, error, temp_result.job_execution_id, temp_result.title, + temp_result.source_record_order + FROM temp_result + WHERE entity_type = 'AUTHORITY' + ), + po_line_data AS ( + SELECT action_type, entity_id, entity_hrid, temp_result.source_id, error, order_id, temp_result.job_execution_id, + temp_result.title, temp_result.source_record_order + FROM temp_result + WHERE entity_type = 'PO_LINE' + ) + ( + SELECT + COALESCE(marc.job_execution_id, instances.job_execution_id, holdings.job_execution_id, items.job_execution_id) AS job_execution_id, + COALESCE(marc.source_id, instances.source_id, holdings.source_id, items.source_id, authority.source_id) AS incoming_record_id, + marc.entity_id::uuid AS source_id, + COALESCE(marc.source_record_order, instances.source_record_order, holdings.source_record_order, items.source_record_order) AS source_record_order, + COALESCE(marc.title, instances.title, holdings.title, items.title) AS title, + marc.action_type AS source_record_action_status, + marc.error AS source_entity_error, + marc.tenant_id AS source_record_tenant_id, + instances.action_type AS instance_action_status, + COALESCE(instances.entity_id, holdings.instance_id, items.instance_id) AS instance_entity_id, + instances.entity_hrid AS instance_entity_hrid, + instances.error AS instance_entity_error, + instances.tenant_id AS instance_entity_tenant_id, + holdings.action_type AS holdings_action_status, + holdings.entity_hrid AS holdings_entity_hrid, + COALESCE(holdings.entity_id, items.holdings_id) AS holdings_entity_id, + holdings.permanent_location_id AS holdings_permanent_location_id, + holdings.error AS holdings_entity_error, + items.action_type AS item_action_status, + items.entity_hrid AS item_entity_hrid, + items.entity_id AS item_entity_id, + items.error AS item_entity_error, + authority.action_type AS authority_action_status, + authority.entity_id AS authority_entity_id, + authority.error AS authority_entity_error, + po_lines.action_type AS po_line_action_status, + po_lines.entity_id AS po_line_entity_id, + po_lines.entity_hrid AS po_line_entity_hrid, + po_lines.error AS po_line_entity_error, + po_lines.order_id AS order_entity_id, + NULL AS invoice_action_status, + NULL AS invoice_entity_id, + NULL AS invoice_entity_hrid, + NULL AS invoice_entity_error, + NULL AS invoice_line_action_status, + NULL AS invoice_line_entity_id, + NULL AS invoice_line_entity_hrid, + NULL AS invoice_line_entity_error + FROM marc_data marc + INNER JOIN ( + SELECT temp_result.source_id + FROM temp_result + WHERE action_type = 'PARSED' + ) parsed ON marc.source_id = parsed.source_id + LEFT JOIN instance_data instances ON marc.source_id = instances.source_id + LEFT JOIN authority_data authority ON authority.source_id = marc.source_id + LEFT JOIN po_line_data po_lines ON po_lines.source_id = marc.source_id + FULL JOIN ( + SELECT tmp.action_type, tmp.entity_type, tmp.entity_id, tmp.entity_hrid, tmp.error, tmp.instance_id, tmp.permanent_location_id, tmp.job_execution_id, tmp.source_id, tmp.title, tmp.source_record_order + FROM temp_result tmp + INNER JOIN ( + SELECT + CASE + WHEN EXISTS (SELECT condition_result.entity_id FROM temp_result condition_result + WHERE (condition_result.action_type='CREATED' AND condition_result.entity_type='HOLDINGS') OR + (condition_result.action_type='DISCARDED' AND condition_result.error != '' AND condition_result.entity_type='HOLDINGS')) + THEN ( + SELECT deep_nested.id + FROM temp_result deep_nested + WHERE (deep_nested.action_type='CREATED' AND deep_nested.id = nested_result.id) OR + (deep_nested.action_type='DISCARDED' AND deep_nested.error != '' AND deep_nested.id = nested_result.id)) + ELSE + nested_result.id + END + FROM temp_result nested_result + ) AS joining_table ON tmp.id = joining_table.id + WHERE tmp.entity_type='HOLDINGS' + ) holdings ON instances.entity_id = holdings.instance_id + FULL JOIN ( + SELECT tmp.action_type, tmp.entity_id, tmp.holdings_id, tmp.entity_hrid, tmp.error, tmp.instance_id, + tmp.job_execution_id, tmp.source_id, tmp.title, tmp.source_record_order + FROM temp_result tmp + INNER JOIN ( + SELECT + CASE + WHEN EXISTS (SELECT condition_result.entity_id FROM temp_result condition_result + WHERE (condition_result.action_type IN ('CREATED','UPDATED') AND condition_result.entity_type='ITEM') OR + (condition_result.action_type='DISCARDED' AND condition_result.error != '' AND condition_result.entity_type='ITEM')) + THEN ( + SELECT deep_nested.id + FROM temp_result deep_nested + WHERE (deep_nested.action_type IN ('CREATED','UPDATED') AND deep_nested.id = nested_result.id) OR + (deep_nested.action_type='DISCARDED' AND deep_nested.error != '' AND deep_nested.id = nested_result.id)) + ELSE + nested_result.id + END + FROM temp_result nested_result + ) AS joining_table ON tmp.id = joining_table.id + WHERE tmp.entity_type='ITEM' + ) items ON holdings.entity_id = items.holdings_id + LEFT JOIN ( + SELECT entity_id AS marc_entity_id, temp_result.source_id AS marc_source_id + FROM temp_result + WHERE entity_type IN ('MARC_BIBLIOGRAPHIC', 'MARC_HOLDINGS', 'MARC_AUTHORITY') AND entity_id IS NOT NULL + ) AS marc_identifiers ON marc.source_id = marc_identifiers.marc_source_id + ORDER BY holdings.entity_hrid + ) UNION - SELECT invoice_line_info.job_execution_id, - records_actions.source_id as incoming_record_id, - records_actions.source_id as source_id, - records_actions.source_record_order, - invoice_line_info.title, - CASE WHEN edifact_errors_number != 0 THEN 'DISCARDED' - WHEN edifact_actions[array_length(edifact_actions, 1)] = 'CREATE' THEN 'CREATED' - END AS source_record_action_status, - records_actions.source_record_error[1], - records_actions.source_record_tenant_id, - null AS instance_action_status, - null AS instance_entity_id, - null AS instance_entity_hrid, - null AS instance_entity_error, - null AS instance_entity_tenant_id, - null AS holdings_action_status, - null AS holdings_entity_hrid, - null AS holdings_entity_id, - null AS holdings_permanent_location_id, - null AS holdings_entity_error, - null AS item_action_status, - null AS item_entity_hrid, - null AS item_entity_id, - null AS item_entity_error, - null AS authority_action_status, - null AS authority_entity_id, - null AS authority_entity_error, - null AS po_line_action_status, - null AS po_line_entity_id, - null AS po_line_entity_hrid, - null AS po_line_entity_error, - null AS order_entity_id, + SELECT invoice_line_info.job_execution_id, records_actions.source_id AS incoming_record_id, records_actions.source_id AS source_id, records_actions.source_record_order, invoice_line_info.title, + CASE WHEN edifact_errors_number != 0 THEN 'DISCARDED' WHEN edifact_actions[array_length(edifact_actions, 1)] = 'CREATE' THEN 'CREATED' END AS source_record_action_status, + records_actions.source_record_error[1], records_actions.source_record_tenant_id, NULL AS instance_action_status, NULL AS instance_entity_id, NULL AS instance_entity_hrid, + NULL AS instance_entity_error, NULL AS instance_entity_tenant_id, NULL AS holdings_action_status, NULL AS holdings_entity_hrid, NULL AS holdings_entity_id, + NULL AS holdings_permanent_location_id, NULL AS holdings_entity_error, NULL AS item_action_status, NULL AS item_entity_hrid, NULL AS item_entity_id, + NULL AS item_entity_error, NULL AS authority_action_status, NULL AS authority_entity_id, NULL AS authority_entity_error, NULL AS po_line_action_status, + NULL AS po_line_entity_id, NULL AS po_line_entity_hrid, NULL AS po_line_entity_error, NULL AS order_entity_id, get_entity_status(records_actions.invoice_actions, records_actions.invoice_errors_number) AS invoice_action_status, - records_actions.invoice_entity_id, - records_actions.invoice_entity_hrid, - records_actions.invoice_entity_error[1], - CASE WHEN action_status = 'ERROR' THEN 'DISCARDED' - WHEN inv_line_actions = 'CREATE' THEN 'CREATED' - END AS invoice_line_action_status, - invoice_line_info.invoice_line_entity_id, - invoice_line_info.invoice_line_entity_hrid, - invoice_line_info.invoice_line_entity_error + records_actions.invoice_entity_id, records_actions.invoice_entity_hrid, records_actions.invoice_entity_error[1], + CASE WHEN action_status = 'ERROR' THEN 'DISCARDED' WHEN inv_line_actions = 'CREATE' THEN 'CREATED' END AS invoice_line_action_status, + invoice_line_info.invoice_line_entity_id, invoice_line_info.invoice_line_entity_hrid, invoice_line_info.invoice_line_entity_error FROM ( - SELECT journal_records.source_id, - journal_records.job_execution_id, - journal_records.title, - journal_records.action_type AS inv_line_actions, - action_status, - entity_hrid AS invoice_line_entity_hrid, - entity_id AS invoice_line_entity_id, - error AS invoice_line_entity_error + SELECT journal_records.source_id, journal_records.job_execution_id, journal_records.title, journal_records.action_type AS inv_line_actions, action_status, + entity_hrid AS invoice_line_entity_hrid, entity_id AS invoice_line_entity_id, error AS invoice_line_entity_error FROM journal_records WHERE journal_records.id = recordId AND journal_records.entity_type = 'INVOICE' AND journal_records.title != 'INVOICE' ) AS invoice_line_info LEFT JOIN LATERAL ( - SELECT journal_records.source_id, - journal_records.source_record_order, + SELECT journal_records.source_id, journal_records.source_record_order, array_agg(action_type) FILTER (WHERE entity_type = 'EDIFACT') AS edifact_actions, count(journal_records.source_id) FILTER (WHERE entity_type = 'EDIFACT' AND journal_records.error != '') AS edifact_errors_number, array_agg(error) FILTER (WHERE entity_type = 'EDIFACT') AS source_record_error, journal_records.tenant_id AS source_record_tenant_id, - array_agg(action_type) FILTER (WHERE entity_type = 'INVOICE' AND journal_records.title = 'INVOICE') AS invoice_actions, count(journal_records.source_id) FILTER (WHERE entity_type = 'INVOICE' AND journal_records.title = 'INVOICE' AND journal_records.error != '') AS invoice_errors_number, array_agg(entity_hrid) FILTER (WHERE entity_type = 'INVOICE' AND journal_records.title = 'INVOICE') AS invoice_entity_hrid, @@ -235,7 +186,7 @@ BEGIN array_agg(error) FILTER (WHERE entity_type = 'INVOICE' AND journal_records.title = 'INVOICE') AS invoice_entity_error FROM journal_records WHERE journal_records.source_id = invoice_line_info.source_id AND (entity_type = 'EDIFACT' OR journal_records.title = 'INVOICE') - GROUP BY journal_records.source_id, journal_records.job_execution_id,journal_records.source_record_order, journal_records.tenant_id + GROUP BY journal_records.source_id, journal_records.job_execution_id, journal_records.source_record_order, journal_records.tenant_id ) AS records_actions ON TRUE; END; $$ LANGUAGE plpgsql; From 0a164dcbbcb82d9c8bc0f3cb93d950fd9040e88a Mon Sep 17 00:00:00 2001 From: aliaksandr_fedasiuk Date: Fri, 20 Sep 2024 00:51:39 +0500 Subject: [PATCH 16/25] MODSOURMAN-1204: fix tests --- .../migration_to_partitioning_logs.sql | 243 ++++++++++++++++++ .../MetaDataProviderJobLogEntriesAPITest.java | 1 + 2 files changed, 244 insertions(+) diff --git a/mod-source-record-manager-server/src/main/resources/templates/db_scripts/migration_to_partitioning_logs.sql b/mod-source-record-manager-server/src/main/resources/templates/db_scripts/migration_to_partitioning_logs.sql index 8a4b9ceb4..70638e83e 100644 --- a/mod-source-record-manager-server/src/main/resources/templates/db_scripts/migration_to_partitioning_logs.sql +++ b/mod-source-record-manager-server/src/main/resources/templates/db_scripts/migration_to_partitioning_logs.sql @@ -46,3 +46,246 @@ CREATE INDEX journal_records_et_action_type_idx ON ${myuniversity}_${mymodule}.j ALTER TABLE ${myuniversity}_${mymodule}.journal_records RENAME TO journal_records_backup; ALTER TABLE ${myuniversity}_${mymodule}.journal_records_entity_type RENAME TO journal_records; + +DROP FUNCTION IF EXISTS get_record_processing_log_bak(uuid, uuid); + +CREATE OR REPLACE FUNCTION get_record_processing_log_bak(jobExecutionId uuid, recordId uuid) + RETURNS TABLE(job_execution_id uuid, incoming_record_id uuid, source_id uuid, source_record_order integer, title text, source_record_action_status text, source_entity_error text, source_record_tenant_id text, instance_action_status text, instance_entity_id text, instance_entity_hrid text, instance_entity_error text, instance_entity_tenant_id text, holdings_action_status text, holdings_entity_hrid text, holdings_entity_id text, holdings_permanent_location_id text, holdings_entity_error text, item_action_status text, item_entity_hrid text, item_entity_id text, item_entity_error text, authority_action_status text, authority_entity_id text, authority_entity_error text, po_line_action_status text, po_line_entity_id text, po_line_entity_hrid text, po_line_entity_error text, order_entity_id text, invoice_action_status text, invoice_entity_id text[], invoice_entity_hrid text[], invoice_entity_error text, invoice_line_action_status text, invoice_line_entity_id text, invoice_line_entity_hrid text, invoice_line_entity_error text) +AS $$ +BEGIN + RETURN QUERY + WITH temp_result AS (SELECT id, journal_records_backup.job_execution_id, journal_records_backup.source_id, journal_records_backup.entity_type, journal_records_backup.entity_id, journal_records_backup.entity_hrid, + CASE WHEN action_type = 'PARSE' + THEN 'PARSED' + WHEN error_max != '' OR action_type = 'NON_MATCH' + THEN 'DISCARDED' + WHEN action_type = 'CREATE' + THEN 'CREATED' + WHEN action_type = 'UPDATE' + THEN 'UPDATED' + END AS action_type, journal_records_backup.action_status, journal_records_backup.action_date, journal_records_backup.source_record_order, journal_records_backup.error, journal_records_backup.title, journal_records_backup.tenant_id, journal_records_backup.instance_id, journal_records_backup.holdings_id, journal_records_backup.order_id, journal_records_backup.permanent_location_id + FROM journal_records_backup + INNER JOIN + (SELECT entity_type as entity_type_max, entity_id as entity_id_max,action_status as action_status_max, max(error) AS error_max,(array_agg(id ORDER BY array_position(array['CREATE', 'UPDATE', 'NON_MATCH'], action_type)))[1] AS id_max + FROM journal_records_backup + WHERE journal_records_backup.job_execution_id = jobExecutionId AND journal_records_backup.source_id = recordId AND journal_records_backup.entity_type NOT IN ('EDIFACT', 'INVOICE') AND action_type != 'MATCH' + GROUP BY entity_type,entity_id,action_status) AS action_type_by_source ON journal_records_backup.id = action_type_by_source.id_max + UNION ALL + SELECT id, journal_records_backup.job_execution_id, journal_records_backup.source_id, journal_records_backup.entity_type, journal_records_backup.entity_id, journal_records_backup.entity_hrid, + CASE WHEN error_max != '' OR action_type = 'MATCH' THEN 'DISCARDED' + END AS action_type, journal_records_backup.action_status, journal_records_backup.action_date, journal_records_backup.source_record_order, journal_records_backup.error, journal_records_backup.title, journal_records_backup.tenant_id, journal_records_backup.instance_id, journal_records_backup.holdings_id, journal_records_backup.order_id, journal_records_backup.permanent_location_id + FROM journal_records_backup + INNER JOIN + (SELECT entity_type as entity_type_max, entity_id as entity_id_max,action_status as action_status_max, max(error) AS error_max,(array_agg(id ORDER BY array_position(array['NON_MATCH', 'MATCH'], action_type)))[1] AS id_max + FROM journal_records_backup + WHERE journal_records_backup.job_execution_id = jobExecutionId AND journal_records_backup.source_id = recordId AND journal_records_backup.entity_type NOT IN ('EDIFACT', 'INVOICE') AND action_type = 'MATCH' + AND NOT EXISTS (SELECT 1 FROM journal_records_backup WHERE journal_records_backup.job_execution_id = jobExecutionId AND journal_records_backup.source_id = recordId AND action_type NOT IN ('MATCH', 'PARSE')) + GROUP BY entity_type,entity_id,action_status) AS action_type_by_source ON journal_records_backup.id = action_type_by_source.id_max) + (SELECT + COALESCE(marc.job_execution_id,instances.job_execution_id,holdings.job_execution_id,items.job_execution_id) AS job_execution_id, + COALESCE(marc.source_id, instances.source_id, holdings.source_id, items.source_id, authority.source_id) as incoming_record_id, + marc_entity_id::uuid AS source_id, + COALESCE(marc.source_record_order,instances.source_record_order,holdings.source_record_order,items.source_record_order) AS source_record_order, + COALESCE(marc.title,instances.title,holdings.title,items.title) AS title, + marc.action_type AS source_record_action_status, + marc.error AS source_entity_error, + marc.tenant_id AS source_record_tenant_id, + + instances.action_type AS instance_action_status, + COALESCE(instances.entity_id,holdings.instance_id,items.instance_id) AS instance_entity_id, + instances.entity_hrid AS instance_entity_hrid, + instances.error AS instance_entity_error, + instances.tenant_id AS instance_entity_tenant_id, + + holdings.action_type AS holdings_action_status, + holdings.entity_hrid AS holdings_entity_hrid, + COALESCE(holdings.entity_id,items.holdings_id) AS holdings_entity_id, + holdings.permanent_location_id AS holdings_permanent_location_id, + holdings.error AS holdings_entity_error, + + items.action_type AS item_action_status, + items.entity_hrid AS item_entity_hrid, + items.entity_id AS item_entity_id, + items.error AS item_entity_error, + + authority.action_type AS authority_action_status, + authority.entity_id AS authority_entity_id, + authority.error AS authority_entity_error, + + po_lines.action_type AS po_line_action_status, + po_lines.entity_id AS po_line_entity_id, + po_lines.entity_hrid AS po_line_entity_hrid, + po_lines.error AS po_line_entity_error, + po_lines.order_id AS order_entity_id, + + null AS invoice_action_status, + null AS invoice_entity_id, + null AS invoice_entity_hrid, + null AS invoice_entity_error, + null AS invoice_line_action_status, + null AS invoice_line_entity_id, + null AS invoice_line_entity_hrid, + null AS invoice_line_entity_error + FROM (SELECT temp_result.source_id FROM temp_result WHERE action_type = 'PARSED') as parsed + LEFT JOIN + (SELECT temp_result.job_execution_id, entity_id, temp_result.title, temp_result.source_record_order, action_type, error, temp_result.source_id, temp_result.tenant_id + FROM temp_result WHERE entity_type IN ('MARC_BIBLIOGRAPHIC', 'MARC_HOLDINGS', 'MARC_AUTHORITY', 'PO_LINE') AND entity_id IS NOT NULL + UNION ALL + SELECT temp_result.job_execution_id, entity_id, temp_result.title, temp_result.source_record_order, action_type, error, temp_result.source_id, temp_result.tenant_id + FROM temp_result + WHERE entity_type IN ('MARC_BIBLIOGRAPHIC', 'MARC_HOLDINGS', 'MARC_AUTHORITY', 'PO_LINE') AND entity_id IS NULL AND NOT EXISTS + (SELECT 1 + FROM temp_result as tr2 + WHERE tr2.entity_type IN ('MARC_BIBLIOGRAPHIC', 'MARC_HOLDINGS', 'MARC_AUTHORITY', 'PO_LINE') AND tr2.source_id = temp_result.source_id and tr2.entity_id IS NOT NULL)) AS marc + ON marc.source_id = parsed.source_id + LEFT JOIN + (SELECT action_type, entity_id, temp_result.source_id, entity_hrid, error, temp_result.job_execution_id, temp_result.title, temp_result.source_record_order, temp_result.tenant_id + FROM temp_result WHERE entity_type = 'INSTANCE' AND entity_id IS NOT NULL + UNION ALL + SELECT action_type, entity_id, temp_result.source_id, entity_hrid, error, temp_result.job_execution_id, temp_result.title, temp_result.source_record_order, temp_result.tenant_id + FROM temp_result + WHERE entity_type = 'INSTANCE' AND entity_id IS NULL AND NOT EXISTS + (SELECT 1 + FROM temp_result as tr2 + WHERE tr2.entity_type = 'INSTANCE' AND tr2.source_id = temp_result.source_id and tr2.entity_id IS NOT NULL)) AS instances + ON marc.source_id = instances.source_id + LEFT JOIN + (SELECT action_type, entity_id, temp_result.source_id, error, temp_result.job_execution_id, temp_result.title, temp_result.source_record_order + FROM temp_result WHERE entity_type = 'AUTHORITY') AS authority + ON authority.source_id = marc.source_id + LEFT JOIN + (SELECT action_type,entity_id,entity_hrid,temp_result.source_id,error,order_id,temp_result.job_execution_id,temp_result.title,temp_result.source_record_order + FROM temp_result WHERE entity_type = 'PO_LINE') AS po_lines + ON po_lines.source_id = marc.source_id + FULL JOIN + (SELECT tmp.action_type, tmp.entity_type, tmp.entity_id, tmp.entity_hrid, tmp.error, tmp.instance_id, + tmp.permanent_location_id, tmp.job_execution_id, tmp.source_id, tmp.title, tmp.source_record_order + FROM temp_result tmp + INNER JOIN + (SELECT + CASE + WHEN EXISTS (SELECT condition_result.entity_id FROM temp_result condition_result + WHERE (condition_result.action_type='CREATED' AND condition_result.entity_type='HOLDINGS') + OR + (condition_result.action_type='DISCARDED' AND condition_result.error != '' AND condition_result.entity_type='HOLDINGS')) + THEN + (SELECT deep_nested.id + FROM temp_result deep_nested + WHERE + (deep_nested.action_type='CREATED' AND deep_nested.id = nested_result.id) + OR + (deep_nested.action_type='DISCARDED' AND deep_nested.error != '' AND deep_nested.id = nested_result.id)) + ELSE + nested_result.id + END + FROM temp_result nested_result) AS joining_table + ON tmp.id = joining_table.id + WHERE tmp.entity_type='HOLDINGS') + AS holdings + ON instances.entity_id = holdings.instance_id + FULL JOIN + (SELECT tmp.action_type, tmp.entity_id, tmp.holdings_id, tmp.entity_hrid, tmp.error, tmp.instance_id, + tmp.job_execution_id, tmp.source_id, tmp.title, tmp.source_record_order + FROM temp_result tmp + INNER JOIN + (SELECT + CASE + WHEN EXISTS (SELECT condition_result.entity_id FROM temp_result condition_result + WHERE (condition_result.action_type IN ('CREATED','UPDATED') AND condition_result.entity_type='ITEM') + OR + (condition_result.action_type='DISCARDED' AND condition_result.error != '' AND condition_result.entity_type='ITEM')) + THEN + (SELECT deep_nested.id + FROM temp_result deep_nested + WHERE + (deep_nested.action_type IN ('CREATED','UPDATED') AND deep_nested.id = nested_result.id) + OR + (deep_nested.action_type='DISCARDED' AND deep_nested.error != '' AND deep_nested.id = nested_result.id)) + ELSE + nested_result.id + END + FROM temp_result nested_result) AS joining_table + ON tmp.id = joining_table.id + WHERE tmp.entity_type='ITEM') AS items + ON holdings.entity_id = items.holdings_id + LEFT JOIN ( + SELECT entity_id AS marc_entity_id, temp_result.source_id AS marc_source_id + FROM temp_result WHERE entity_type IN ('MARC_BIBLIOGRAPHIC', 'MARC_HOLDINGS', 'MARC_AUTHORITY') AND entity_id IS NOT NULL + ) AS marc_identifiers ON marc.source_id = marc_identifiers.marc_source_id + ORDER BY holdings.entity_hrid) + UNION + SELECT invoice_line_info.job_execution_id, + records_actions.source_id as incoming_record_id, + records_actions.source_id as source_id, + records_actions.source_record_order, + invoice_line_info.title, + CASE WHEN edifact_errors_number != 0 THEN 'DISCARDED' + WHEN edifact_actions[array_length(edifact_actions, 1)] = 'CREATE' THEN 'CREATED' + END AS source_record_action_status, + records_actions.source_record_error[1], + records_actions.source_record_tenant_id, + null AS instance_action_status, + null AS instance_entity_id, + null AS instance_entity_hrid, + null AS instance_entity_error, + null AS instance_entity_tenant_id, + null AS holdings_action_status, + null AS holdings_entity_hrid, + null AS holdings_entity_id, + null AS holdings_permanent_location_id, + null AS holdings_entity_error, + null AS item_action_status, + null AS item_entity_hrid, + null AS item_entity_id, + null AS item_entity_error, + null AS authority_action_status, + null AS authority_entity_id, + null AS authority_entity_error, + null AS po_line_action_status, + null AS po_line_entity_id, + null AS po_line_entity_hrid, + null AS po_line_entity_error, + null AS order_entity_id, + get_entity_status(records_actions.invoice_actions, records_actions.invoice_errors_number) AS invoice_action_status, + records_actions.invoice_entity_id, + records_actions.invoice_entity_hrid, + records_actions.invoice_entity_error[1], + CASE WHEN action_status = 'ERROR' THEN 'DISCARDED' + WHEN inv_line_actions = 'CREATE' THEN 'CREATED' + END AS invoice_line_action_status, + invoice_line_info.invoice_line_entity_id, + invoice_line_info.invoice_line_entity_hrid, + invoice_line_info.invoice_line_entity_error + FROM ( + SELECT journal_records_backup.source_id, + journal_records_backup.job_execution_id, + journal_records_backup.title, + journal_records_backup.action_type AS inv_line_actions, + action_status, + entity_hrid AS invoice_line_entity_hrid, + entity_id AS invoice_line_entity_id, + error AS invoice_line_entity_error + FROM journal_records_backup + WHERE journal_records_backup.id = recordId AND journal_records_backup.entity_type = 'INVOICE' AND journal_records_backup.title != 'INVOICE' + ) AS invoice_line_info + LEFT JOIN LATERAL ( + SELECT journal_records_backup.source_id, + journal_records_backup.source_record_order, + array_agg(action_type) FILTER (WHERE entity_type = 'EDIFACT') AS edifact_actions, + count(journal_records_backup.source_id) FILTER (WHERE entity_type = 'EDIFACT' AND journal_records_backup.error != '') AS edifact_errors_number, + array_agg(error) FILTER (WHERE entity_type = 'EDIFACT') AS source_record_error, + journal_records_backup.tenant_id AS source_record_tenant_id, + + array_agg(action_type) FILTER (WHERE entity_type = 'INVOICE' AND journal_records_backup.title = 'INVOICE') AS invoice_actions, + count(journal_records_backup.source_id) FILTER (WHERE entity_type = 'INVOICE' AND journal_records_backup.title = 'INVOICE' AND journal_records_backup.error != '') AS invoice_errors_number, + array_agg(entity_hrid) FILTER (WHERE entity_type = 'INVOICE' AND journal_records_backup.title = 'INVOICE') AS invoice_entity_hrid, + array_agg(entity_id) FILTER (WHERE entity_type = 'INVOICE' AND journal_records_backup.title = 'INVOICE') AS invoice_entity_id, + array_agg(error) FILTER (WHERE entity_type = 'INVOICE' AND journal_records_backup.title = 'INVOICE') AS invoice_entity_error + FROM journal_records_backup + WHERE journal_records_backup.source_id = invoice_line_info.source_id AND (entity_type = 'EDIFACT' OR journal_records_backup.title = 'INVOICE') + GROUP BY journal_records_backup.source_id, journal_records_backup.job_execution_id,journal_records_backup.source_record_order, journal_records.tenant_id + ) AS records_actions ON TRUE; +END; +$$ LANGUAGE plpgsql; + + diff --git a/mod-source-record-manager-server/src/test/java/org/folio/rest/impl/metadataProvider/MetaDataProviderJobLogEntriesAPITest.java b/mod-source-record-manager-server/src/test/java/org/folio/rest/impl/metadataProvider/MetaDataProviderJobLogEntriesAPITest.java index 43786c5e8..f971dd370 100644 --- a/mod-source-record-manager-server/src/test/java/org/folio/rest/impl/metadataProvider/MetaDataProviderJobLogEntriesAPITest.java +++ b/mod-source-record-manager-server/src/test/java/org/folio/rest/impl/metadataProvider/MetaDataProviderJobLogEntriesAPITest.java @@ -454,6 +454,7 @@ public void shouldReturnDiscardedForItemsIfNoItemsCreated(TestContext context) { .get(GET_JOB_EXECUTION_JOURNAL_RECORDS_PATH + "/" + createdJobExecution.getId() + "/records/" + sourceRecordId) .then() .statusCode(HttpStatus.SC_OK) + .log().all() .body("jobExecutionId", is(createdJobExecution.getId())) .body("incomingRecordId", is(sourceRecordId)) .body("sourceRecordTitle", is(recordTitle)) From f7cbbc0de445dfd05cb9ab11be9a3989185c615d Mon Sep 17 00:00:00 2001 From: aliaksandr_fedasiuk Date: Mon, 23 Sep 2024 23:44:35 +0500 Subject: [PATCH 17/25] MODSOURMAN-1204: fix tests --- ...ate_get_record_processing_log_function.sql | 3 ++- .../migration_to_partitioning_logs.sql | 2 +- .../MetaDataProviderJobLogEntriesAPITest.java | 23 ++++++++++--------- 3 files changed, 15 insertions(+), 13 deletions(-) diff --git a/mod-source-record-manager-server/src/main/resources/templates/db_scripts/create_get_record_processing_log_function.sql b/mod-source-record-manager-server/src/main/resources/templates/db_scripts/create_get_record_processing_log_function.sql index 462962bc9..1cc9a71b9 100644 --- a/mod-source-record-manager-server/src/main/resources/templates/db_scripts/create_get_record_processing_log_function.sql +++ b/mod-source-record-manager-server/src/main/resources/templates/db_scripts/create_get_record_processing_log_function.sql @@ -187,6 +187,7 @@ BEGIN FROM journal_records WHERE journal_records.source_id = invoice_line_info.source_id AND (entity_type = 'EDIFACT' OR journal_records.title = 'INVOICE') GROUP BY journal_records.source_id, journal_records.job_execution_id, journal_records.source_record_order, journal_records.tenant_id - ) AS records_actions ON TRUE; + ) AS records_actions ON TRUE + ORDER BY holdings_entity_hrid; END; $$ LANGUAGE plpgsql; diff --git a/mod-source-record-manager-server/src/main/resources/templates/db_scripts/migration_to_partitioning_logs.sql b/mod-source-record-manager-server/src/main/resources/templates/db_scripts/migration_to_partitioning_logs.sql index 70638e83e..fabb21da7 100644 --- a/mod-source-record-manager-server/src/main/resources/templates/db_scripts/migration_to_partitioning_logs.sql +++ b/mod-source-record-manager-server/src/main/resources/templates/db_scripts/migration_to_partitioning_logs.sql @@ -283,7 +283,7 @@ BEGIN array_agg(error) FILTER (WHERE entity_type = 'INVOICE' AND journal_records_backup.title = 'INVOICE') AS invoice_entity_error FROM journal_records_backup WHERE journal_records_backup.source_id = invoice_line_info.source_id AND (entity_type = 'EDIFACT' OR journal_records_backup.title = 'INVOICE') - GROUP BY journal_records_backup.source_id, journal_records_backup.job_execution_id,journal_records_backup.source_record_order, journal_records.tenant_id + GROUP BY journal_records_backup.source_id, journal_records_backup.job_execution_id,journal_records_backup.source_record_order, journal_records_backup.tenant_id ) AS records_actions ON TRUE; END; $$ LANGUAGE plpgsql; diff --git a/mod-source-record-manager-server/src/test/java/org/folio/rest/impl/metadataProvider/MetaDataProviderJobLogEntriesAPITest.java b/mod-source-record-manager-server/src/test/java/org/folio/rest/impl/metadataProvider/MetaDataProviderJobLogEntriesAPITest.java index f971dd370..294d12b07 100644 --- a/mod-source-record-manager-server/src/test/java/org/folio/rest/impl/metadataProvider/MetaDataProviderJobLogEntriesAPITest.java +++ b/mod-source-record-manager-server/src/test/java/org/folio/rest/impl/metadataProvider/MetaDataProviderJobLogEntriesAPITest.java @@ -1028,17 +1028,18 @@ public void shouldReturnHoldingsDataIfHoldingsPoweredByMarcHoldingsWasCreated(Te .get(GET_JOB_EXECUTION_JOURNAL_RECORDS_PATH + "/" + createdJobExecution.getId() + "/records/" + incomingRecordId) .then() .statusCode(HttpStatus.SC_OK) -// .body("jobExecutionId", is(createdJobExecution.getId())) -// .body("incomingRecordId", is(incomingRecordId)) -// .body("sourceRecordId", is(marcHoldingsId)) -// .body("sourceRecordOrder", is("0")) -// .body("relatedHoldingsInfo.size()", is(1)) -// .body("relatedHoldingsInfo[0].actionStatus", is(ActionStatus.CREATED.value())) -// .body("relatedHoldingsInfo[0].id", is(holdingId)) -// .body("relatedHoldingsInfo[0].hrid", is(holdingHrid)) -// .body("relatedHoldingsInfo[0].permanentLocationId", is(holdingPermanentLocationId)) -// .body("relatedHoldingsInfo[0].error", emptyOrNullString()) -// .body("error", emptyOrNullString()) + .log().all() + .body("jobExecutionId", is(createdJobExecution.getId())) + .body("incomingRecordId", is(incomingRecordId)) + .body("sourceRecordId", is(marcHoldingsId)) + .body("sourceRecordOrder", is("0")) + .body("relatedHoldingsInfo.size()", is(1)) + .body("relatedHoldingsInfo[0].actionStatus", is(ActionStatus.CREATED.value())) + .body("relatedHoldingsInfo[0].id", is(holdingId)) + .body("relatedHoldingsInfo[0].hrid", is(holdingHrid)) + .body("relatedHoldingsInfo[0].permanentLocationId", is(holdingPermanentLocationId)) + .body("relatedHoldingsInfo[0].error", emptyOrNullString()) + .body("error", emptyOrNullString()) .extract().response().body().as(RecordProcessingLogDto.class); assertThat(recordProcessingLogSummary.getJobExecutionId(), is(createdJobExecution.getId())); From 06d094cc0070b8f674055acb917228221c31b4ba Mon Sep 17 00:00:00 2001 From: aliaksandr_fedasiuk Date: Tue, 24 Sep 2024 00:43:17 +0500 Subject: [PATCH 18/25] MODSOURMAN-1204: fix tests --- .../db_scripts/create_get_record_processing_log_function.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mod-source-record-manager-server/src/main/resources/templates/db_scripts/create_get_record_processing_log_function.sql b/mod-source-record-manager-server/src/main/resources/templates/db_scripts/create_get_record_processing_log_function.sql index 1cc9a71b9..ff612bf69 100644 --- a/mod-source-record-manager-server/src/main/resources/templates/db_scripts/create_get_record_processing_log_function.sql +++ b/mod-source-record-manager-server/src/main/resources/templates/db_scripts/create_get_record_processing_log_function.sql @@ -188,6 +188,6 @@ BEGIN WHERE journal_records.source_id = invoice_line_info.source_id AND (entity_type = 'EDIFACT' OR journal_records.title = 'INVOICE') GROUP BY journal_records.source_id, journal_records.job_execution_id, journal_records.source_record_order, journal_records.tenant_id ) AS records_actions ON TRUE - ORDER BY holdings_entity_hrid; + ORDER BY source_id, holdings_entity_hrid; END; $$ LANGUAGE plpgsql; From dcfe55ba47d4fd131a4cae527b87ef6fcf92202c Mon Sep 17 00:00:00 2001 From: aliaksandr_fedasiuk Date: Tue, 24 Sep 2024 02:42:53 +0500 Subject: [PATCH 19/25] MODSOURMAN-1204: fix tests --- ...ate_get_record_processing_log_function.sql | 379 ++++++++++-------- .../MetaDataProviderJobLogEntriesAPITest.java | 2 + 2 files changed, 216 insertions(+), 165 deletions(-) diff --git a/mod-source-record-manager-server/src/main/resources/templates/db_scripts/create_get_record_processing_log_function.sql b/mod-source-record-manager-server/src/main/resources/templates/db_scripts/create_get_record_processing_log_function.sql index ff612bf69..9ba29c077 100644 --- a/mod-source-record-manager-server/src/main/resources/templates/db_scripts/create_get_record_processing_log_function.sql +++ b/mod-source-record-manager-server/src/main/resources/templates/db_scripts/create_get_record_processing_log_function.sql @@ -5,180 +5,228 @@ CREATE OR REPLACE FUNCTION get_record_processing_log(jobExecutionId uuid, record AS $$ BEGIN RETURN QUERY - WITH temp_result AS ( - SELECT jr.id, jr.job_execution_id, jr.source_id, jr.entity_type, jr.entity_id, jr.entity_hrid, - CASE - WHEN jr.action_type = 'PARSE' THEN 'PARSED' - WHEN jr.error != '' OR jr.action_type IN ('NON_MATCH', 'MATCH') THEN 'DISCARDED' - WHEN jr.action_type = 'CREATE' THEN 'CREATED' - WHEN jr.action_type = 'UPDATE' THEN 'UPDATED' - END AS action_type, - jr.action_status, jr.action_date, jr.source_record_order, jr.error, jr.title, jr.tenant_id, - jr.instance_id, jr.holdings_id, jr.order_id, jr.permanent_location_id - FROM journal_records jr - INNER JOIN ( - SELECT jr_inner.entity_type AS entity_type_max, jr_inner.entity_id AS entity_id_max, - jr_inner.action_status AS action_status_max, MAX(jr_inner.error) AS error_max, - (ARRAY_AGG(jr_inner.id ORDER BY ARRAY_POSITION(ARRAY['CREATE', 'UPDATE', 'NON_MATCH', 'MATCH'], jr_inner.action_type)))[1] AS id_max - FROM journal_records jr_inner - WHERE jr_inner.job_execution_id = jobExecutionId AND jr_inner.source_id = recordId - AND jr_inner.entity_type NOT IN ('EDIFACT', 'INVOICE') - AND (jr_inner.action_type != 'MATCH' OR (jr_inner.action_type = 'MATCH' AND NOT EXISTS ( - SELECT 1 - FROM journal_records jr_sub - WHERE jr_sub.job_execution_id = jobExecutionId AND jr_sub.source_id = recordId - AND jr_sub.action_type NOT IN ('MATCH', 'PARSE') - ))) - GROUP BY jr_inner.entity_type, jr_inner.entity_id, jr_inner.action_status - ) AS action_type_by_source ON jr.id = action_type_by_source.id_max - ), - marc_data AS ( - SELECT temp_result.job_execution_id, entity_id, temp_result.title, temp_result.source_record_order, action_type, error, - temp_result.source_id, tenant_id - FROM temp_result - WHERE entity_type IN ('MARC_BIBLIOGRAPHIC', 'MARC_HOLDINGS', 'MARC_AUTHORITY', 'PO_LINE') - ), - instance_data AS ( - SELECT action_type, entity_id, temp_result.source_id, entity_hrid, error, temp_result.job_execution_id, temp_result.title, - temp_result.source_record_order, tenant_id - FROM temp_result - WHERE entity_type = 'INSTANCE' - ), - authority_data AS ( - SELECT action_type, entity_id, temp_result.source_id, error, temp_result.job_execution_id, temp_result.title, - temp_result.source_record_order - FROM temp_result - WHERE entity_type = 'AUTHORITY' - ), - po_line_data AS ( - SELECT action_type, entity_id, entity_hrid, temp_result.source_id, error, order_id, temp_result.job_execution_id, - temp_result.title, temp_result.source_record_order - FROM temp_result - WHERE entity_type = 'PO_LINE' - ) - ( - SELECT - COALESCE(marc.job_execution_id, instances.job_execution_id, holdings.job_execution_id, items.job_execution_id) AS job_execution_id, - COALESCE(marc.source_id, instances.source_id, holdings.source_id, items.source_id, authority.source_id) AS incoming_record_id, - marc.entity_id::uuid AS source_id, - COALESCE(marc.source_record_order, instances.source_record_order, holdings.source_record_order, items.source_record_order) AS source_record_order, - COALESCE(marc.title, instances.title, holdings.title, items.title) AS title, - marc.action_type AS source_record_action_status, - marc.error AS source_entity_error, - marc.tenant_id AS source_record_tenant_id, - instances.action_type AS instance_action_status, - COALESCE(instances.entity_id, holdings.instance_id, items.instance_id) AS instance_entity_id, - instances.entity_hrid AS instance_entity_hrid, - instances.error AS instance_entity_error, - instances.tenant_id AS instance_entity_tenant_id, - holdings.action_type AS holdings_action_status, - holdings.entity_hrid AS holdings_entity_hrid, - COALESCE(holdings.entity_id, items.holdings_id) AS holdings_entity_id, - holdings.permanent_location_id AS holdings_permanent_location_id, - holdings.error AS holdings_entity_error, - items.action_type AS item_action_status, - items.entity_hrid AS item_entity_hrid, - items.entity_id AS item_entity_id, - items.error AS item_entity_error, - authority.action_type AS authority_action_status, - authority.entity_id AS authority_entity_id, - authority.error AS authority_entity_error, - po_lines.action_type AS po_line_action_status, - po_lines.entity_id AS po_line_entity_id, - po_lines.entity_hrid AS po_line_entity_hrid, - po_lines.error AS po_line_entity_error, - po_lines.order_id AS order_entity_id, - NULL AS invoice_action_status, - NULL AS invoice_entity_id, - NULL AS invoice_entity_hrid, - NULL AS invoice_entity_error, - NULL AS invoice_line_action_status, - NULL AS invoice_line_entity_id, - NULL AS invoice_line_entity_hrid, - NULL AS invoice_line_entity_error - FROM marc_data marc - INNER JOIN ( - SELECT temp_result.source_id - FROM temp_result - WHERE action_type = 'PARSED' - ) parsed ON marc.source_id = parsed.source_id - LEFT JOIN instance_data instances ON marc.source_id = instances.source_id - LEFT JOIN authority_data authority ON authority.source_id = marc.source_id - LEFT JOIN po_line_data po_lines ON po_lines.source_id = marc.source_id - FULL JOIN ( - SELECT tmp.action_type, tmp.entity_type, tmp.entity_id, tmp.entity_hrid, tmp.error, tmp.instance_id, tmp.permanent_location_id, tmp.job_execution_id, tmp.source_id, tmp.title, tmp.source_record_order - FROM temp_result tmp - INNER JOIN ( - SELECT - CASE - WHEN EXISTS (SELECT condition_result.entity_id FROM temp_result condition_result - WHERE (condition_result.action_type='CREATED' AND condition_result.entity_type='HOLDINGS') OR - (condition_result.action_type='DISCARDED' AND condition_result.error != '' AND condition_result.entity_type='HOLDINGS')) - THEN ( - SELECT deep_nested.id - FROM temp_result deep_nested - WHERE (deep_nested.action_type='CREATED' AND deep_nested.id = nested_result.id) OR - (deep_nested.action_type='DISCARDED' AND deep_nested.error != '' AND deep_nested.id = nested_result.id)) - ELSE - nested_result.id - END - FROM temp_result nested_result - ) AS joining_table ON tmp.id = joining_table.id - WHERE tmp.entity_type='HOLDINGS' - ) holdings ON instances.entity_id = holdings.instance_id - FULL JOIN ( - SELECT tmp.action_type, tmp.entity_id, tmp.holdings_id, tmp.entity_hrid, tmp.error, tmp.instance_id, - tmp.job_execution_id, tmp.source_id, tmp.title, tmp.source_record_order - FROM temp_result tmp - INNER JOIN ( - SELECT - CASE - WHEN EXISTS (SELECT condition_result.entity_id FROM temp_result condition_result - WHERE (condition_result.action_type IN ('CREATED','UPDATED') AND condition_result.entity_type='ITEM') OR - (condition_result.action_type='DISCARDED' AND condition_result.error != '' AND condition_result.entity_type='ITEM')) - THEN ( - SELECT deep_nested.id - FROM temp_result deep_nested - WHERE (deep_nested.action_type IN ('CREATED','UPDATED') AND deep_nested.id = nested_result.id) OR - (deep_nested.action_type='DISCARDED' AND deep_nested.error != '' AND deep_nested.id = nested_result.id)) - ELSE - nested_result.id - END - FROM temp_result nested_result - ) AS joining_table ON tmp.id = joining_table.id - WHERE tmp.entity_type='ITEM' - ) items ON holdings.entity_id = items.holdings_id - LEFT JOIN ( - SELECT entity_id AS marc_entity_id, temp_result.source_id AS marc_source_id - FROM temp_result - WHERE entity_type IN ('MARC_BIBLIOGRAPHIC', 'MARC_HOLDINGS', 'MARC_AUTHORITY') AND entity_id IS NOT NULL - ) AS marc_identifiers ON marc.source_id = marc_identifiers.marc_source_id - ORDER BY holdings.entity_hrid - ) + WITH temp_result AS (SELECT id, journal_records.job_execution_id, journal_records.source_id, journal_records.entity_type, journal_records.entity_id, journal_records.entity_hrid, + CASE WHEN action_type = 'PARSE' + THEN 'PARSED' + WHEN error_max != '' OR action_type = 'NON_MATCH' + THEN 'DISCARDED' + WHEN action_type = 'CREATE' + THEN 'CREATED' + WHEN action_type = 'UPDATE' + THEN 'UPDATED' + END AS action_type, journal_records.action_status, journal_records.action_date, journal_records.source_record_order, journal_records.error, journal_records.title, journal_records.tenant_id, journal_records.instance_id, journal_records.holdings_id, journal_records.order_id, journal_records.permanent_location_id + FROM journal_records + INNER JOIN + (SELECT entity_type as entity_type_max, entity_id as entity_id_max,action_status as action_status_max, max(error) AS error_max,(array_agg(id ORDER BY array_position(array['CREATE', 'UPDATE', 'NON_MATCH'], action_type)))[1] AS id_max + FROM journal_records + WHERE journal_records.job_execution_id = jobExecutionId AND journal_records.source_id = recordId AND journal_records.entity_type NOT IN ('EDIFACT', 'INVOICE') AND action_type != 'MATCH' + GROUP BY entity_type,entity_id,action_status) AS action_type_by_source ON journal_records.id = action_type_by_source.id_max + UNION ALL + SELECT id, journal_records.job_execution_id, journal_records.source_id, journal_records.entity_type, journal_records.entity_id, journal_records.entity_hrid, + CASE WHEN error_max != '' OR action_type = 'MATCH' THEN 'DISCARDED' + END AS action_type, journal_records.action_status, journal_records.action_date, journal_records.source_record_order, journal_records.error, journal_records.title, journal_records.tenant_id, journal_records.instance_id, journal_records.holdings_id, journal_records.order_id, journal_records.permanent_location_id + FROM journal_records + INNER JOIN + (SELECT entity_type as entity_type_max, entity_id as entity_id_max,action_status as action_status_max, max(error) AS error_max,(array_agg(id ORDER BY array_position(array['NON_MATCH', 'MATCH'], action_type)))[1] AS id_max + FROM journal_records + WHERE journal_records.job_execution_id = jobExecutionId AND journal_records.source_id = recordId AND journal_records.entity_type NOT IN ('EDIFACT', 'INVOICE') AND action_type = 'MATCH' + AND NOT EXISTS (SELECT 1 FROM journal_records WHERE journal_records.job_execution_id = jobExecutionId AND journal_records.source_id = recordId AND action_type NOT IN ('MATCH', 'PARSE')) + GROUP BY entity_type,entity_id,action_status) AS action_type_by_source ON journal_records.id = action_type_by_source.id_max) + (SELECT + COALESCE(marc.job_execution_id,instances.job_execution_id,holdings.job_execution_id,items.job_execution_id) AS job_execution_id, + COALESCE(marc.source_id, instances.source_id, holdings.source_id, items.source_id, authority.source_id) as incoming_record_id, + marc_entity_id::uuid AS source_id, + COALESCE(marc.source_record_order,instances.source_record_order,holdings.source_record_order,items.source_record_order) AS source_record_order, + COALESCE(marc.title,instances.title,holdings.title,items.title) AS title, + marc.action_type AS source_record_action_status, + marc.error AS source_entity_error, + marc.tenant_id AS source_record_tenant_id, + + instances.action_type AS instance_action_status, + COALESCE(instances.entity_id,holdings.instance_id,items.instance_id) AS instance_entity_id, + instances.entity_hrid AS instance_entity_hrid, + instances.error AS instance_entity_error, + instances.tenant_id AS instance_entity_tenant_id, + + holdings.action_type AS holdings_action_status, + holdings.entity_hrid AS holdings_entity_hrid, + COALESCE(holdings.entity_id,items.holdings_id) AS holdings_entity_id, + holdings.permanent_location_id AS holdings_permanent_location_id, + holdings.error AS holdings_entity_error, + + items.action_type AS item_action_status, + items.entity_hrid AS item_entity_hrid, + items.entity_id AS item_entity_id, + items.error AS item_entity_error, + + authority.action_type AS authority_action_status, + authority.entity_id AS authority_entity_id, + authority.error AS authority_entity_error, + + po_lines.action_type AS po_line_action_status, + po_lines.entity_id AS po_line_entity_id, + po_lines.entity_hrid AS po_line_entity_hrid, + po_lines.error AS po_line_entity_error, + po_lines.order_id AS order_entity_id, + + null AS invoice_action_status, + null AS invoice_entity_id, + null AS invoice_entity_hrid, + null AS invoice_entity_error, + null AS invoice_line_action_status, + null AS invoice_line_entity_id, + null AS invoice_line_entity_hrid, + null AS invoice_line_entity_error + FROM (SELECT temp_result.source_id FROM temp_result WHERE action_type = 'PARSED') as parsed + LEFT JOIN + (SELECT temp_result.job_execution_id, entity_id, temp_result.title, temp_result.source_record_order, action_type, error, temp_result.source_id, temp_result.tenant_id + FROM temp_result WHERE entity_type IN ('MARC_BIBLIOGRAPHIC', 'MARC_HOLDINGS', 'MARC_AUTHORITY', 'PO_LINE') AND entity_id IS NOT NULL + UNION ALL + SELECT temp_result.job_execution_id, entity_id, temp_result.title, temp_result.source_record_order, action_type, error, temp_result.source_id, temp_result.tenant_id + FROM temp_result + WHERE entity_type IN ('MARC_BIBLIOGRAPHIC', 'MARC_HOLDINGS', 'MARC_AUTHORITY', 'PO_LINE') AND entity_id IS NULL AND NOT EXISTS + (SELECT 1 + FROM temp_result as tr2 + WHERE tr2.entity_type IN ('MARC_BIBLIOGRAPHIC', 'MARC_HOLDINGS', 'MARC_AUTHORITY', 'PO_LINE') AND tr2.source_id = temp_result.source_id and tr2.entity_id IS NOT NULL)) AS marc + ON marc.source_id = parsed.source_id + LEFT JOIN + (SELECT action_type, entity_id, temp_result.source_id, entity_hrid, error, temp_result.job_execution_id, temp_result.title, temp_result.source_record_order, temp_result.tenant_id + FROM temp_result WHERE entity_type = 'INSTANCE' AND entity_id IS NOT NULL + UNION ALL + SELECT action_type, entity_id, temp_result.source_id, entity_hrid, error, temp_result.job_execution_id, temp_result.title, temp_result.source_record_order, temp_result.tenant_id + FROM temp_result + WHERE entity_type = 'INSTANCE' AND entity_id IS NULL AND NOT EXISTS + (SELECT 1 + FROM temp_result as tr2 + WHERE tr2.entity_type = 'INSTANCE' AND tr2.source_id = temp_result.source_id and tr2.entity_id IS NOT NULL)) AS instances + ON marc.source_id = instances.source_id + LEFT JOIN + (SELECT action_type, entity_id, temp_result.source_id, error, temp_result.job_execution_id, temp_result.title, temp_result.source_record_order + FROM temp_result WHERE entity_type = 'AUTHORITY') AS authority + ON authority.source_id = marc.source_id + LEFT JOIN + (SELECT action_type,entity_id,entity_hrid,temp_result.source_id,error,order_id,temp_result.job_execution_id,temp_result.title,temp_result.source_record_order + FROM temp_result WHERE entity_type = 'PO_LINE') AS po_lines + ON po_lines.source_id = marc.source_id + FULL JOIN + (SELECT tmp.action_type, tmp.entity_type, tmp.entity_id, tmp.entity_hrid, tmp.error, tmp.instance_id, + tmp.permanent_location_id, tmp.job_execution_id, tmp.source_id, tmp.title, tmp.source_record_order + FROM temp_result tmp + INNER JOIN + (SELECT + CASE + WHEN EXISTS (SELECT condition_result.entity_id FROM temp_result condition_result + WHERE (condition_result.action_type='CREATED' AND condition_result.entity_type='HOLDINGS') + OR + (condition_result.action_type='DISCARDED' AND condition_result.error != '' AND condition_result.entity_type='HOLDINGS')) + THEN + (SELECT deep_nested.id + FROM temp_result deep_nested + WHERE + (deep_nested.action_type='CREATED' AND deep_nested.id = nested_result.id) + OR + (deep_nested.action_type='DISCARDED' AND deep_nested.error != '' AND deep_nested.id = nested_result.id)) + ELSE + nested_result.id + END + FROM temp_result nested_result) AS joining_table + ON tmp.id = joining_table.id + WHERE tmp.entity_type='HOLDINGS') + AS holdings + ON instances.entity_id = holdings.instance_id + FULL JOIN + (SELECT tmp.action_type, tmp.entity_id, tmp.holdings_id, tmp.entity_hrid, tmp.error, tmp.instance_id, + tmp.job_execution_id, tmp.source_id, tmp.title, tmp.source_record_order + FROM temp_result tmp + INNER JOIN + (SELECT + CASE + WHEN EXISTS (SELECT condition_result.entity_id FROM temp_result condition_result + WHERE (condition_result.action_type IN ('CREATED','UPDATED') AND condition_result.entity_type='ITEM') + OR + (condition_result.action_type='DISCARDED' AND condition_result.error != '' AND condition_result.entity_type='ITEM')) + THEN + (SELECT deep_nested.id + FROM temp_result deep_nested + WHERE + (deep_nested.action_type IN ('CREATED','UPDATED') AND deep_nested.id = nested_result.id) + OR + (deep_nested.action_type='DISCARDED' AND deep_nested.error != '' AND deep_nested.id = nested_result.id)) + ELSE + nested_result.id + END + FROM temp_result nested_result) AS joining_table + ON tmp.id = joining_table.id + WHERE tmp.entity_type='ITEM') AS items + ON holdings.entity_id = items.holdings_id + LEFT JOIN ( + SELECT entity_id AS marc_entity_id, temp_result.source_id AS marc_source_id + FROM temp_result WHERE entity_type IN ('MARC_BIBLIOGRAPHIC', 'MARC_HOLDINGS', 'MARC_AUTHORITY') AND entity_id IS NOT NULL + ) AS marc_identifiers ON marc.source_id = marc_identifiers.marc_source_id + ORDER BY holdings.entity_hrid) UNION - SELECT invoice_line_info.job_execution_id, records_actions.source_id AS incoming_record_id, records_actions.source_id AS source_id, records_actions.source_record_order, invoice_line_info.title, - CASE WHEN edifact_errors_number != 0 THEN 'DISCARDED' WHEN edifact_actions[array_length(edifact_actions, 1)] = 'CREATE' THEN 'CREATED' END AS source_record_action_status, - records_actions.source_record_error[1], records_actions.source_record_tenant_id, NULL AS instance_action_status, NULL AS instance_entity_id, NULL AS instance_entity_hrid, - NULL AS instance_entity_error, NULL AS instance_entity_tenant_id, NULL AS holdings_action_status, NULL AS holdings_entity_hrid, NULL AS holdings_entity_id, - NULL AS holdings_permanent_location_id, NULL AS holdings_entity_error, NULL AS item_action_status, NULL AS item_entity_hrid, NULL AS item_entity_id, - NULL AS item_entity_error, NULL AS authority_action_status, NULL AS authority_entity_id, NULL AS authority_entity_error, NULL AS po_line_action_status, - NULL AS po_line_entity_id, NULL AS po_line_entity_hrid, NULL AS po_line_entity_error, NULL AS order_entity_id, + SELECT invoice_line_info.job_execution_id, + records_actions.source_id as incoming_record_id, + records_actions.source_id as source_id, + records_actions.source_record_order, + invoice_line_info.title, + CASE WHEN edifact_errors_number != 0 THEN 'DISCARDED' + WHEN edifact_actions[array_length(edifact_actions, 1)] = 'CREATE' THEN 'CREATED' + END AS source_record_action_status, + records_actions.source_record_error[1], + records_actions.source_record_tenant_id, + null AS instance_action_status, + null AS instance_entity_id, + null AS instance_entity_hrid, + null AS instance_entity_error, + null AS instance_entity_tenant_id, + null AS holdings_action_status, + null AS holdings_entity_hrid, + null AS holdings_entity_id, + null AS holdings_permanent_location_id, + null AS holdings_entity_error, + null AS item_action_status, + null AS item_entity_hrid, + null AS item_entity_id, + null AS item_entity_error, + null AS authority_action_status, + null AS authority_entity_id, + null AS authority_entity_error, + null AS po_line_action_status, + null AS po_line_entity_id, + null AS po_line_entity_hrid, + null AS po_line_entity_error, + null AS order_entity_id, get_entity_status(records_actions.invoice_actions, records_actions.invoice_errors_number) AS invoice_action_status, - records_actions.invoice_entity_id, records_actions.invoice_entity_hrid, records_actions.invoice_entity_error[1], - CASE WHEN action_status = 'ERROR' THEN 'DISCARDED' WHEN inv_line_actions = 'CREATE' THEN 'CREATED' END AS invoice_line_action_status, - invoice_line_info.invoice_line_entity_id, invoice_line_info.invoice_line_entity_hrid, invoice_line_info.invoice_line_entity_error + records_actions.invoice_entity_id, + records_actions.invoice_entity_hrid, + records_actions.invoice_entity_error[1], + CASE WHEN action_status = 'ERROR' THEN 'DISCARDED' + WHEN inv_line_actions = 'CREATE' THEN 'CREATED' + END AS invoice_line_action_status, + invoice_line_info.invoice_line_entity_id, + invoice_line_info.invoice_line_entity_hrid, + invoice_line_info.invoice_line_entity_error FROM ( - SELECT journal_records.source_id, journal_records.job_execution_id, journal_records.title, journal_records.action_type AS inv_line_actions, action_status, - entity_hrid AS invoice_line_entity_hrid, entity_id AS invoice_line_entity_id, error AS invoice_line_entity_error + SELECT journal_records.source_id, + journal_records.job_execution_id, + journal_records.title, + journal_records.action_type AS inv_line_actions, + action_status, + entity_hrid AS invoice_line_entity_hrid, + entity_id AS invoice_line_entity_id, + error AS invoice_line_entity_error FROM journal_records WHERE journal_records.id = recordId AND journal_records.entity_type = 'INVOICE' AND journal_records.title != 'INVOICE' ) AS invoice_line_info LEFT JOIN LATERAL ( - SELECT journal_records.source_id, journal_records.source_record_order, + SELECT journal_records.source_id, + journal_records.source_record_order, array_agg(action_type) FILTER (WHERE entity_type = 'EDIFACT') AS edifact_actions, count(journal_records.source_id) FILTER (WHERE entity_type = 'EDIFACT' AND journal_records.error != '') AS edifact_errors_number, array_agg(error) FILTER (WHERE entity_type = 'EDIFACT') AS source_record_error, journal_records.tenant_id AS source_record_tenant_id, + array_agg(action_type) FILTER (WHERE entity_type = 'INVOICE' AND journal_records.title = 'INVOICE') AS invoice_actions, count(journal_records.source_id) FILTER (WHERE entity_type = 'INVOICE' AND journal_records.title = 'INVOICE' AND journal_records.error != '') AS invoice_errors_number, array_agg(entity_hrid) FILTER (WHERE entity_type = 'INVOICE' AND journal_records.title = 'INVOICE') AS invoice_entity_hrid, @@ -186,8 +234,9 @@ BEGIN array_agg(error) FILTER (WHERE entity_type = 'INVOICE' AND journal_records.title = 'INVOICE') AS invoice_entity_error FROM journal_records WHERE journal_records.source_id = invoice_line_info.source_id AND (entity_type = 'EDIFACT' OR journal_records.title = 'INVOICE') - GROUP BY journal_records.source_id, journal_records.job_execution_id, journal_records.source_record_order, journal_records.tenant_id + GROUP BY journal_records.source_id, journal_records.job_execution_id,journal_records.source_record_order, journal_records.tenant_id ) AS records_actions ON TRUE - ORDER BY source_id, holdings_entity_hrid; + ORDER BY source_id, holdings_entity_hrid, item_entity_hrid; END; $$ LANGUAGE plpgsql; + diff --git a/mod-source-record-manager-server/src/test/java/org/folio/rest/impl/metadataProvider/MetaDataProviderJobLogEntriesAPITest.java b/mod-source-record-manager-server/src/test/java/org/folio/rest/impl/metadataProvider/MetaDataProviderJobLogEntriesAPITest.java index 294d12b07..3e73ca5b1 100644 --- a/mod-source-record-manager-server/src/test/java/org/folio/rest/impl/metadataProvider/MetaDataProviderJobLogEntriesAPITest.java +++ b/mod-source-record-manager-server/src/test/java/org/folio/rest/impl/metadataProvider/MetaDataProviderJobLogEntriesAPITest.java @@ -129,6 +129,7 @@ public void shouldReturnPoLineWithOrderIdWhenMarcCreate(TestContext context) { .when() .get(GET_JOB_EXECUTION_JOURNAL_RECORDS_PATH + "/" + createdJobExecution.getId()+ "/records/" + sourceRecordId) .then().log().all() + .log().all() .statusCode(HttpStatus.SC_OK) .body("jobExecutionId", is(createdJobExecution.getId())) .body("incomingRecordId", is(sourceRecordId)) @@ -1292,6 +1293,7 @@ public void shouldReturnInstanceCreatedIfInstanceNonMatchAndCreated(TestContext .when() .get(GET_JOB_EXECUTION_JOURNAL_RECORDS_PATH + "/" + createdJobExecution.getId() + "/records/" + incomingRecordId) .then() + .log().all() .statusCode(HttpStatus.SC_OK) .body("jobExecutionId", is(createdJobExecution.getId())) .body("incomingRecordId", is(incomingRecordId)) From 64925e764b96c67cf7a930a2be9e978827303710 Mon Sep 17 00:00:00 2001 From: aliaksandr_fedasiuk Date: Tue, 24 Sep 2024 03:11:17 +0500 Subject: [PATCH 20/25] MODSOURMAN-1204: fix tests --- .../db_scripts/create_get_record_processing_log_function.sql | 2 +- .../metadataProvider/MetaDataProviderJobLogEntriesAPITest.java | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/mod-source-record-manager-server/src/main/resources/templates/db_scripts/create_get_record_processing_log_function.sql b/mod-source-record-manager-server/src/main/resources/templates/db_scripts/create_get_record_processing_log_function.sql index 9ba29c077..b4246a63a 100644 --- a/mod-source-record-manager-server/src/main/resources/templates/db_scripts/create_get_record_processing_log_function.sql +++ b/mod-source-record-manager-server/src/main/resources/templates/db_scripts/create_get_record_processing_log_function.sql @@ -236,7 +236,7 @@ BEGIN WHERE journal_records.source_id = invoice_line_info.source_id AND (entity_type = 'EDIFACT' OR journal_records.title = 'INVOICE') GROUP BY journal_records.source_id, journal_records.job_execution_id,journal_records.source_record_order, journal_records.tenant_id ) AS records_actions ON TRUE - ORDER BY source_id, holdings_entity_hrid, item_entity_hrid; + ORDER BY source_id, holdings_entity_hrid, item_entity_hrid, po_line_entity_hrid, invoice_entity_hrid, invoice_line_entity_hrid, title; END; $$ LANGUAGE plpgsql; diff --git a/mod-source-record-manager-server/src/test/java/org/folio/rest/impl/metadataProvider/MetaDataProviderJobLogEntriesAPITest.java b/mod-source-record-manager-server/src/test/java/org/folio/rest/impl/metadataProvider/MetaDataProviderJobLogEntriesAPITest.java index 3e73ca5b1..76f162407 100644 --- a/mod-source-record-manager-server/src/test/java/org/folio/rest/impl/metadataProvider/MetaDataProviderJobLogEntriesAPITest.java +++ b/mod-source-record-manager-server/src/test/java/org/folio/rest/impl/metadataProvider/MetaDataProviderJobLogEntriesAPITest.java @@ -359,6 +359,7 @@ public void shouldReturnMarcBibAndAllEntitiesWithoutErrors(TestContext context) .when() .get(GET_JOB_EXECUTION_JOURNAL_RECORDS_PATH + "/" + createdJobExecution.getId() + "/records/" + sourceRecordId) .then() + .log().all() .statusCode(HttpStatus.SC_OK) .body("jobExecutionId", is(createdJobExecution.getId())) .body("incomingRecordId", is(sourceRecordId)) From f21b19e64395c20ca55038413b4840ec561119f1 Mon Sep 17 00:00:00 2001 From: aliaksandr_fedasiuk Date: Tue, 24 Sep 2024 03:17:27 +0500 Subject: [PATCH 21/25] MODSOURMAN-1204: code review --- .../migration_to_partitioning_logs.sql | 241 ------------------ .../MetaDataProviderJobLogEntriesAPITest.java | 5 +- 2 files changed, 4 insertions(+), 242 deletions(-) diff --git a/mod-source-record-manager-server/src/main/resources/templates/db_scripts/migration_to_partitioning_logs.sql b/mod-source-record-manager-server/src/main/resources/templates/db_scripts/migration_to_partitioning_logs.sql index fabb21da7..0ac60e363 100644 --- a/mod-source-record-manager-server/src/main/resources/templates/db_scripts/migration_to_partitioning_logs.sql +++ b/mod-source-record-manager-server/src/main/resources/templates/db_scripts/migration_to_partitioning_logs.sql @@ -47,245 +47,4 @@ CREATE INDEX journal_records_et_action_type_idx ON ${myuniversity}_${mymodule}.j ALTER TABLE ${myuniversity}_${mymodule}.journal_records RENAME TO journal_records_backup; ALTER TABLE ${myuniversity}_${mymodule}.journal_records_entity_type RENAME TO journal_records; -DROP FUNCTION IF EXISTS get_record_processing_log_bak(uuid, uuid); - -CREATE OR REPLACE FUNCTION get_record_processing_log_bak(jobExecutionId uuid, recordId uuid) - RETURNS TABLE(job_execution_id uuid, incoming_record_id uuid, source_id uuid, source_record_order integer, title text, source_record_action_status text, source_entity_error text, source_record_tenant_id text, instance_action_status text, instance_entity_id text, instance_entity_hrid text, instance_entity_error text, instance_entity_tenant_id text, holdings_action_status text, holdings_entity_hrid text, holdings_entity_id text, holdings_permanent_location_id text, holdings_entity_error text, item_action_status text, item_entity_hrid text, item_entity_id text, item_entity_error text, authority_action_status text, authority_entity_id text, authority_entity_error text, po_line_action_status text, po_line_entity_id text, po_line_entity_hrid text, po_line_entity_error text, order_entity_id text, invoice_action_status text, invoice_entity_id text[], invoice_entity_hrid text[], invoice_entity_error text, invoice_line_action_status text, invoice_line_entity_id text, invoice_line_entity_hrid text, invoice_line_entity_error text) -AS $$ -BEGIN - RETURN QUERY - WITH temp_result AS (SELECT id, journal_records_backup.job_execution_id, journal_records_backup.source_id, journal_records_backup.entity_type, journal_records_backup.entity_id, journal_records_backup.entity_hrid, - CASE WHEN action_type = 'PARSE' - THEN 'PARSED' - WHEN error_max != '' OR action_type = 'NON_MATCH' - THEN 'DISCARDED' - WHEN action_type = 'CREATE' - THEN 'CREATED' - WHEN action_type = 'UPDATE' - THEN 'UPDATED' - END AS action_type, journal_records_backup.action_status, journal_records_backup.action_date, journal_records_backup.source_record_order, journal_records_backup.error, journal_records_backup.title, journal_records_backup.tenant_id, journal_records_backup.instance_id, journal_records_backup.holdings_id, journal_records_backup.order_id, journal_records_backup.permanent_location_id - FROM journal_records_backup - INNER JOIN - (SELECT entity_type as entity_type_max, entity_id as entity_id_max,action_status as action_status_max, max(error) AS error_max,(array_agg(id ORDER BY array_position(array['CREATE', 'UPDATE', 'NON_MATCH'], action_type)))[1] AS id_max - FROM journal_records_backup - WHERE journal_records_backup.job_execution_id = jobExecutionId AND journal_records_backup.source_id = recordId AND journal_records_backup.entity_type NOT IN ('EDIFACT', 'INVOICE') AND action_type != 'MATCH' - GROUP BY entity_type,entity_id,action_status) AS action_type_by_source ON journal_records_backup.id = action_type_by_source.id_max - UNION ALL - SELECT id, journal_records_backup.job_execution_id, journal_records_backup.source_id, journal_records_backup.entity_type, journal_records_backup.entity_id, journal_records_backup.entity_hrid, - CASE WHEN error_max != '' OR action_type = 'MATCH' THEN 'DISCARDED' - END AS action_type, journal_records_backup.action_status, journal_records_backup.action_date, journal_records_backup.source_record_order, journal_records_backup.error, journal_records_backup.title, journal_records_backup.tenant_id, journal_records_backup.instance_id, journal_records_backup.holdings_id, journal_records_backup.order_id, journal_records_backup.permanent_location_id - FROM journal_records_backup - INNER JOIN - (SELECT entity_type as entity_type_max, entity_id as entity_id_max,action_status as action_status_max, max(error) AS error_max,(array_agg(id ORDER BY array_position(array['NON_MATCH', 'MATCH'], action_type)))[1] AS id_max - FROM journal_records_backup - WHERE journal_records_backup.job_execution_id = jobExecutionId AND journal_records_backup.source_id = recordId AND journal_records_backup.entity_type NOT IN ('EDIFACT', 'INVOICE') AND action_type = 'MATCH' - AND NOT EXISTS (SELECT 1 FROM journal_records_backup WHERE journal_records_backup.job_execution_id = jobExecutionId AND journal_records_backup.source_id = recordId AND action_type NOT IN ('MATCH', 'PARSE')) - GROUP BY entity_type,entity_id,action_status) AS action_type_by_source ON journal_records_backup.id = action_type_by_source.id_max) - (SELECT - COALESCE(marc.job_execution_id,instances.job_execution_id,holdings.job_execution_id,items.job_execution_id) AS job_execution_id, - COALESCE(marc.source_id, instances.source_id, holdings.source_id, items.source_id, authority.source_id) as incoming_record_id, - marc_entity_id::uuid AS source_id, - COALESCE(marc.source_record_order,instances.source_record_order,holdings.source_record_order,items.source_record_order) AS source_record_order, - COALESCE(marc.title,instances.title,holdings.title,items.title) AS title, - marc.action_type AS source_record_action_status, - marc.error AS source_entity_error, - marc.tenant_id AS source_record_tenant_id, - - instances.action_type AS instance_action_status, - COALESCE(instances.entity_id,holdings.instance_id,items.instance_id) AS instance_entity_id, - instances.entity_hrid AS instance_entity_hrid, - instances.error AS instance_entity_error, - instances.tenant_id AS instance_entity_tenant_id, - - holdings.action_type AS holdings_action_status, - holdings.entity_hrid AS holdings_entity_hrid, - COALESCE(holdings.entity_id,items.holdings_id) AS holdings_entity_id, - holdings.permanent_location_id AS holdings_permanent_location_id, - holdings.error AS holdings_entity_error, - - items.action_type AS item_action_status, - items.entity_hrid AS item_entity_hrid, - items.entity_id AS item_entity_id, - items.error AS item_entity_error, - - authority.action_type AS authority_action_status, - authority.entity_id AS authority_entity_id, - authority.error AS authority_entity_error, - - po_lines.action_type AS po_line_action_status, - po_lines.entity_id AS po_line_entity_id, - po_lines.entity_hrid AS po_line_entity_hrid, - po_lines.error AS po_line_entity_error, - po_lines.order_id AS order_entity_id, - - null AS invoice_action_status, - null AS invoice_entity_id, - null AS invoice_entity_hrid, - null AS invoice_entity_error, - null AS invoice_line_action_status, - null AS invoice_line_entity_id, - null AS invoice_line_entity_hrid, - null AS invoice_line_entity_error - FROM (SELECT temp_result.source_id FROM temp_result WHERE action_type = 'PARSED') as parsed - LEFT JOIN - (SELECT temp_result.job_execution_id, entity_id, temp_result.title, temp_result.source_record_order, action_type, error, temp_result.source_id, temp_result.tenant_id - FROM temp_result WHERE entity_type IN ('MARC_BIBLIOGRAPHIC', 'MARC_HOLDINGS', 'MARC_AUTHORITY', 'PO_LINE') AND entity_id IS NOT NULL - UNION ALL - SELECT temp_result.job_execution_id, entity_id, temp_result.title, temp_result.source_record_order, action_type, error, temp_result.source_id, temp_result.tenant_id - FROM temp_result - WHERE entity_type IN ('MARC_BIBLIOGRAPHIC', 'MARC_HOLDINGS', 'MARC_AUTHORITY', 'PO_LINE') AND entity_id IS NULL AND NOT EXISTS - (SELECT 1 - FROM temp_result as tr2 - WHERE tr2.entity_type IN ('MARC_BIBLIOGRAPHIC', 'MARC_HOLDINGS', 'MARC_AUTHORITY', 'PO_LINE') AND tr2.source_id = temp_result.source_id and tr2.entity_id IS NOT NULL)) AS marc - ON marc.source_id = parsed.source_id - LEFT JOIN - (SELECT action_type, entity_id, temp_result.source_id, entity_hrid, error, temp_result.job_execution_id, temp_result.title, temp_result.source_record_order, temp_result.tenant_id - FROM temp_result WHERE entity_type = 'INSTANCE' AND entity_id IS NOT NULL - UNION ALL - SELECT action_type, entity_id, temp_result.source_id, entity_hrid, error, temp_result.job_execution_id, temp_result.title, temp_result.source_record_order, temp_result.tenant_id - FROM temp_result - WHERE entity_type = 'INSTANCE' AND entity_id IS NULL AND NOT EXISTS - (SELECT 1 - FROM temp_result as tr2 - WHERE tr2.entity_type = 'INSTANCE' AND tr2.source_id = temp_result.source_id and tr2.entity_id IS NOT NULL)) AS instances - ON marc.source_id = instances.source_id - LEFT JOIN - (SELECT action_type, entity_id, temp_result.source_id, error, temp_result.job_execution_id, temp_result.title, temp_result.source_record_order - FROM temp_result WHERE entity_type = 'AUTHORITY') AS authority - ON authority.source_id = marc.source_id - LEFT JOIN - (SELECT action_type,entity_id,entity_hrid,temp_result.source_id,error,order_id,temp_result.job_execution_id,temp_result.title,temp_result.source_record_order - FROM temp_result WHERE entity_type = 'PO_LINE') AS po_lines - ON po_lines.source_id = marc.source_id - FULL JOIN - (SELECT tmp.action_type, tmp.entity_type, tmp.entity_id, tmp.entity_hrid, tmp.error, tmp.instance_id, - tmp.permanent_location_id, tmp.job_execution_id, tmp.source_id, tmp.title, tmp.source_record_order - FROM temp_result tmp - INNER JOIN - (SELECT - CASE - WHEN EXISTS (SELECT condition_result.entity_id FROM temp_result condition_result - WHERE (condition_result.action_type='CREATED' AND condition_result.entity_type='HOLDINGS') - OR - (condition_result.action_type='DISCARDED' AND condition_result.error != '' AND condition_result.entity_type='HOLDINGS')) - THEN - (SELECT deep_nested.id - FROM temp_result deep_nested - WHERE - (deep_nested.action_type='CREATED' AND deep_nested.id = nested_result.id) - OR - (deep_nested.action_type='DISCARDED' AND deep_nested.error != '' AND deep_nested.id = nested_result.id)) - ELSE - nested_result.id - END - FROM temp_result nested_result) AS joining_table - ON tmp.id = joining_table.id - WHERE tmp.entity_type='HOLDINGS') - AS holdings - ON instances.entity_id = holdings.instance_id - FULL JOIN - (SELECT tmp.action_type, tmp.entity_id, tmp.holdings_id, tmp.entity_hrid, tmp.error, tmp.instance_id, - tmp.job_execution_id, tmp.source_id, tmp.title, tmp.source_record_order - FROM temp_result tmp - INNER JOIN - (SELECT - CASE - WHEN EXISTS (SELECT condition_result.entity_id FROM temp_result condition_result - WHERE (condition_result.action_type IN ('CREATED','UPDATED') AND condition_result.entity_type='ITEM') - OR - (condition_result.action_type='DISCARDED' AND condition_result.error != '' AND condition_result.entity_type='ITEM')) - THEN - (SELECT deep_nested.id - FROM temp_result deep_nested - WHERE - (deep_nested.action_type IN ('CREATED','UPDATED') AND deep_nested.id = nested_result.id) - OR - (deep_nested.action_type='DISCARDED' AND deep_nested.error != '' AND deep_nested.id = nested_result.id)) - ELSE - nested_result.id - END - FROM temp_result nested_result) AS joining_table - ON tmp.id = joining_table.id - WHERE tmp.entity_type='ITEM') AS items - ON holdings.entity_id = items.holdings_id - LEFT JOIN ( - SELECT entity_id AS marc_entity_id, temp_result.source_id AS marc_source_id - FROM temp_result WHERE entity_type IN ('MARC_BIBLIOGRAPHIC', 'MARC_HOLDINGS', 'MARC_AUTHORITY') AND entity_id IS NOT NULL - ) AS marc_identifiers ON marc.source_id = marc_identifiers.marc_source_id - ORDER BY holdings.entity_hrid) - UNION - SELECT invoice_line_info.job_execution_id, - records_actions.source_id as incoming_record_id, - records_actions.source_id as source_id, - records_actions.source_record_order, - invoice_line_info.title, - CASE WHEN edifact_errors_number != 0 THEN 'DISCARDED' - WHEN edifact_actions[array_length(edifact_actions, 1)] = 'CREATE' THEN 'CREATED' - END AS source_record_action_status, - records_actions.source_record_error[1], - records_actions.source_record_tenant_id, - null AS instance_action_status, - null AS instance_entity_id, - null AS instance_entity_hrid, - null AS instance_entity_error, - null AS instance_entity_tenant_id, - null AS holdings_action_status, - null AS holdings_entity_hrid, - null AS holdings_entity_id, - null AS holdings_permanent_location_id, - null AS holdings_entity_error, - null AS item_action_status, - null AS item_entity_hrid, - null AS item_entity_id, - null AS item_entity_error, - null AS authority_action_status, - null AS authority_entity_id, - null AS authority_entity_error, - null AS po_line_action_status, - null AS po_line_entity_id, - null AS po_line_entity_hrid, - null AS po_line_entity_error, - null AS order_entity_id, - get_entity_status(records_actions.invoice_actions, records_actions.invoice_errors_number) AS invoice_action_status, - records_actions.invoice_entity_id, - records_actions.invoice_entity_hrid, - records_actions.invoice_entity_error[1], - CASE WHEN action_status = 'ERROR' THEN 'DISCARDED' - WHEN inv_line_actions = 'CREATE' THEN 'CREATED' - END AS invoice_line_action_status, - invoice_line_info.invoice_line_entity_id, - invoice_line_info.invoice_line_entity_hrid, - invoice_line_info.invoice_line_entity_error - FROM ( - SELECT journal_records_backup.source_id, - journal_records_backup.job_execution_id, - journal_records_backup.title, - journal_records_backup.action_type AS inv_line_actions, - action_status, - entity_hrid AS invoice_line_entity_hrid, - entity_id AS invoice_line_entity_id, - error AS invoice_line_entity_error - FROM journal_records_backup - WHERE journal_records_backup.id = recordId AND journal_records_backup.entity_type = 'INVOICE' AND journal_records_backup.title != 'INVOICE' - ) AS invoice_line_info - LEFT JOIN LATERAL ( - SELECT journal_records_backup.source_id, - journal_records_backup.source_record_order, - array_agg(action_type) FILTER (WHERE entity_type = 'EDIFACT') AS edifact_actions, - count(journal_records_backup.source_id) FILTER (WHERE entity_type = 'EDIFACT' AND journal_records_backup.error != '') AS edifact_errors_number, - array_agg(error) FILTER (WHERE entity_type = 'EDIFACT') AS source_record_error, - journal_records_backup.tenant_id AS source_record_tenant_id, - - array_agg(action_type) FILTER (WHERE entity_type = 'INVOICE' AND journal_records_backup.title = 'INVOICE') AS invoice_actions, - count(journal_records_backup.source_id) FILTER (WHERE entity_type = 'INVOICE' AND journal_records_backup.title = 'INVOICE' AND journal_records_backup.error != '') AS invoice_errors_number, - array_agg(entity_hrid) FILTER (WHERE entity_type = 'INVOICE' AND journal_records_backup.title = 'INVOICE') AS invoice_entity_hrid, - array_agg(entity_id) FILTER (WHERE entity_type = 'INVOICE' AND journal_records_backup.title = 'INVOICE') AS invoice_entity_id, - array_agg(error) FILTER (WHERE entity_type = 'INVOICE' AND journal_records_backup.title = 'INVOICE') AS invoice_entity_error - FROM journal_records_backup - WHERE journal_records_backup.source_id = invoice_line_info.source_id AND (entity_type = 'EDIFACT' OR journal_records_backup.title = 'INVOICE') - GROUP BY journal_records_backup.source_id, journal_records_backup.job_execution_id,journal_records_backup.source_record_order, journal_records_backup.tenant_id - ) AS records_actions ON TRUE; -END; -$$ LANGUAGE plpgsql; - diff --git a/mod-source-record-manager-server/src/test/java/org/folio/rest/impl/metadataProvider/MetaDataProviderJobLogEntriesAPITest.java b/mod-source-record-manager-server/src/test/java/org/folio/rest/impl/metadataProvider/MetaDataProviderJobLogEntriesAPITest.java index 76f162407..aa395f165 100644 --- a/mod-source-record-manager-server/src/test/java/org/folio/rest/impl/metadataProvider/MetaDataProviderJobLogEntriesAPITest.java +++ b/mod-source-record-manager-server/src/test/java/org/folio/rest/impl/metadataProvider/MetaDataProviderJobLogEntriesAPITest.java @@ -34,7 +34,10 @@ import org.folio.dao.JournalRecordDaoImpl; import org.folio.dao.util.PostgresClientFactory; import org.folio.rest.impl.AbstractRestTest; -import org.folio.rest.jaxrs.model.*; +import org.folio.rest.jaxrs.model.ActionStatus; +import org.folio.rest.jaxrs.model.JobExecution; +import org.folio.rest.jaxrs.model.JournalRecord; +import org.folio.rest.jaxrs.model.RecordProcessingLogDto; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; From 226a339da8a5cac869dda50c2a815817ce0a4075 Mon Sep 17 00:00:00 2001 From: Aliaksandr Fedasiuk Date: Mon, 13 Jan 2025 17:30:58 +0500 Subject: [PATCH 22/25] MODSOURMAN-1204: Update version in schema.json --- .../src/main/resources/templates/db_scripts/schema.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mod-source-record-manager-server/src/main/resources/templates/db_scripts/schema.json b/mod-source-record-manager-server/src/main/resources/templates/db_scripts/schema.json index 0d34e3655..e79722c8a 100644 --- a/mod-source-record-manager-server/src/main/resources/templates/db_scripts/schema.json +++ b/mod-source-record-manager-server/src/main/resources/templates/db_scripts/schema.json @@ -315,7 +315,7 @@ { "run": "after", "snippetPath": "migration_to_partitioning_logs.sql", - "fromModuleVersion": "mod-source-record-manager-3.9.0" + "fromModuleVersion": "mod-source-record-manager-3.10.0" } ] } From 9fb02d9fa4f9dd4c99149fe918165ab6d9c24561 Mon Sep 17 00:00:00 2001 From: Aliaksandr Fedasiuk Date: Tue, 14 Jan 2025 17:40:41 +0500 Subject: [PATCH 23/25] Update schema.json --- .../src/main/resources/templates/db_scripts/schema.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mod-source-record-manager-server/src/main/resources/templates/db_scripts/schema.json b/mod-source-record-manager-server/src/main/resources/templates/db_scripts/schema.json index e79722c8a..7d8340b31 100644 --- a/mod-source-record-manager-server/src/main/resources/templates/db_scripts/schema.json +++ b/mod-source-record-manager-server/src/main/resources/templates/db_scripts/schema.json @@ -310,7 +310,7 @@ { "run": "after", "snippetPath": "create_get_record_processing_log_function.sql", - "fromModuleVersion": "mod-source-record-manager-3.9.0" + "fromModuleVersion": "mod-source-record-manager-3.10.0" }, { "run": "after", From cfd9200ee964a006b81f9bf58724c10f9590686d Mon Sep 17 00:00:00 2001 From: Aliaksandr Fedasiuk Date: Tue, 14 Jan 2025 21:16:53 +0500 Subject: [PATCH 24/25] MODSOURMAN-1204: Update migration_to_partitioning_logs.sql --- .../templates/db_scripts/migration_to_partitioning_logs.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/mod-source-record-manager-server/src/main/resources/templates/db_scripts/migration_to_partitioning_logs.sql b/mod-source-record-manager-server/src/main/resources/templates/db_scripts/migration_to_partitioning_logs.sql index 0ac60e363..d2687317d 100644 --- a/mod-source-record-manager-server/src/main/resources/templates/db_scripts/migration_to_partitioning_logs.sql +++ b/mod-source-record-manager-server/src/main/resources/templates/db_scripts/migration_to_partitioning_logs.sql @@ -30,6 +30,7 @@ CREATE TABLE IF NOT EXISTS ${myuniversity}_${mymodule}.journal_records_order PAR CREATE TABLE IF NOT EXISTS ${myuniversity}_${mymodule}.journal_records_invoice PARTITION OF journal_records_entity_type FOR VALUES IN ('INVOICE'); CREATE TABLE IF NOT EXISTS ${myuniversity}_${mymodule}.journal_records_edifact PARTITION OF journal_records_entity_type FOR VALUES IN ('EDIFACT'); CREATE TABLE IF NOT EXISTS ${myuniversity}_${mymodule}.journal_records_empty_entity_type PARTITION OF journal_records_entity_type FOR VALUES IN (''); +CREATE TABLE IF NOT EXISTS ${myuniversity}_${mymodule}.journal_records_null_entity_type PARTITION OF journal_records_entity_type FOR VALUES IN (NULL); INSERT INTO ${myuniversity}_${mymodule}.journal_records_entity_type (id, job_execution_id, source_id, entity_type, entity_id, entity_hrid, action_type, action_status, action_date, source_record_order, error, title, From 12a25119785bebb382d4d4189577c09d86fbd85a Mon Sep 17 00:00:00 2001 From: Aliaksandr Fedasiuk Date: Thu, 16 Jan 2025 15:42:19 +0500 Subject: [PATCH 25/25] MODSOURMAN-1204: add ANALYZE commant into migration_to_partitioning_logs.sql --- .../templates/db_scripts/migration_to_partitioning_logs.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/mod-source-record-manager-server/src/main/resources/templates/db_scripts/migration_to_partitioning_logs.sql b/mod-source-record-manager-server/src/main/resources/templates/db_scripts/migration_to_partitioning_logs.sql index d2687317d..4d238aac3 100644 --- a/mod-source-record-manager-server/src/main/resources/templates/db_scripts/migration_to_partitioning_logs.sql +++ b/mod-source-record-manager-server/src/main/resources/templates/db_scripts/migration_to_partitioning_logs.sql @@ -47,5 +47,6 @@ CREATE INDEX journal_records_et_action_type_idx ON ${myuniversity}_${mymodule}.j ALTER TABLE ${myuniversity}_${mymodule}.journal_records RENAME TO journal_records_backup; ALTER TABLE ${myuniversity}_${mymodule}.journal_records_entity_type RENAME TO journal_records; +--ANALIZE VERBOSE ${myuniversity}_${mymodule}.journal_records;