#1053 new table and function to update

volumes/miovision/sql/function/function-update-open_issues.sql

@@ -1,7 +1,5 @@
---DROP VIEW miovision_api.open_issues;
-CREATE OR REPLACE VIEW miovision_api.open_issues AS
-
---DROP VIEW miovision_api.open_issues;
+--DROP FUNCTION miovision_api.update_open_issues;
+CREATE OR REPLACE FUNCTION miovision_api.update_open_issues() AS
 
 WITH alerts AS (
     SELECT
@@ -13,80 +11,106 @@ WITH alerts AS (
         AND tsrange(alerts.start_time, alerts.end_time)
         && tsrange(ar.range_start, ar.range_end)
     GROUP BY ar.uid
-)
+),
 
-SELECT
-    ar.uid,
-    ar.intersection_uid,
-    i.id AS intersection_id,
-    i.api_name AS intersection_name,
-    ar.classification_uid,
-    CASE
-        WHEN ar.classification_uid = 2 THEN 'Bicycle TMC'
-        WHEN ar.classification_uid = 10 THEN 'Bicycle Approach'
-        WHEN ar.classification_uid IS NULL THEN 'All modes'
-        ELSE c.classification
-    END,
-    ar.leg,
-    ar.range_start::date,
-    (current_timestamp AT TIME ZONE 'EST5EDT')::date - ar.range_start::date AS num_days,
-    ar.notes,
-    SUM(v.volume) AS volume,
-    alerts.alerts
-FROM miovision_api.anomalous_ranges AS ar
---keep rows with null classification_uid
-LEFT JOIN miovision_api.classifications AS c USING (classification_uid)
---omit null intersection_uids. These will go under discontinuities. 
-JOIN miovision_api.intersections AS i USING (intersection_uid)
---find last week volume
-LEFT JOIN miovision_api.volumes AS v
-    ON ar.intersection_uid = v.intersection_uid
-    --volume within the last 7 days and after AR started
-    AND v.datetime_bin >= ar.range_start
-    --prune the partitions
-    AND v.datetime_bin >= current_date - interval '7 days'
-    AND (
-        ar.classification_uid = v.classification_uid
-        OR ar.classification_uid IS NULL
-    )
-    AND (
-        ar.leg = v.leg
-        OR ar.leg IS NULL
-    )
-LEFT JOIN alerts ON alerts.uid = ar.uid
-WHERE
-    ar.problem_level <> 'valid-caveat'
-    --currently active
-    AND (
-        ar.range_end IS NULL
-        OR (
-            ar.notes LIKE '%identified by a daily airflow process%'
-            AND ar.range_end = (current_timestamp AT TIME ZONE 'EST5EDT')::date --today
+open_issues AS (
+    SELECT
+        ar.uid,
+        ar.intersection_uid,
+        i.id AS intersection_id,
+        i.api_name AS intersection_name,
+        ar.classification_uid,
+        CASE
+            WHEN ar.classification_uid = 2 THEN 'Bicycle TMC'
+            WHEN ar.classification_uid = 10 THEN 'Bicycle Approach'
+            WHEN ar.classification_uid IS NULL THEN 'All modes'
+            ELSE c.classification
+        END,
+        ar.leg,
+        ar.range_start::date,
+        (current_timestamp AT TIME ZONE 'EST5EDT')::date - ar.range_start::date AS num_days,
+        ar.notes,
+        SUM(v.volume) AS volume,
+        alerts.alerts
+    FROM miovision_api.anomalous_ranges AS ar
+    --keep rows with null classification_uid
+    LEFT JOIN miovision_api.classifications AS c USING (classification_uid)
+    --omit null intersection_uids. These will go under discontinuities. 
+    JOIN miovision_api.intersections AS i USING (intersection_uid)
+    --find last week volume
+    LEFT JOIN miovision_api.volumes AS v
+        ON ar.intersection_uid = v.intersection_uid
+        --volume within the last 7 days and after AR started
+        AND v.datetime_bin >= ar.range_start
+        --prune the partitions
+        AND v.datetime_bin >= current_date - interval '7 days'
+        AND (
+            ar.classification_uid = v.classification_uid
+            OR ar.classification_uid IS NULL
         )
-    )
-GROUP BY
-    ar.uid,
-    ar.intersection_uid,
-    i.id,
-    i.api_name,
-    ar.classification_uid,
-    c.classification,
-    ar.range_start,
-    ar.notes,
-    alerts.alerts
-ORDER BY
-    ar.intersection_uid,
-    ar.range_start,
-    ar.classification_uid;
+        AND (
+            ar.leg = v.leg
+            OR ar.leg IS NULL
+        )
+    LEFT JOIN alerts ON alerts.uid = ar.uid
+    WHERE
+        ar.problem_level <> 'valid-caveat'
+        --currently active
+        AND (
+            ar.range_end IS NULL
+            OR (
+                ar.notes LIKE '%identified by a daily airflow process%'
+                AND ar.range_end = (current_timestamp AT TIME ZONE 'EST5EDT')::date --today
+            )
+        )
+    GROUP BY
+        ar.uid,
+        ar.intersection_uid,
+        i.id,
+        i.api_name,
+        ar.classification_uid,
+        c.classification,
+        ar.range_start,
+        ar.notes,
+        alerts.alerts
+    ORDER BY
+        ar.intersection_uid,
+        ar.range_start,
+        ar.classification_uid
+),
+
+closed AS (
+    DELETE FROM miovision_api.open_issues_review
+    WHERE uid NOT IN (SELECT uid FROM open_issues)
+)
 
-COMMENT ON VIEW miovision_api.open_issues
-IS '''A view to export open ended anomalous_ranges for communication with Miovision.
-Converts intersection_uid and classification_uid into formats familiar to Miovision
-(intersections.id, classifications.classification). anomalous_ranges.id col can be
-used to link response back to table.''';
+MERGE INTO miovision_api.open_issues_review AS oir
+USING open_issues AS oi 
+ON oir.uid = oi.uid
+WHEN MATCHED THEN
+UPDATE SET
+    intersection_uid = oi.intersection_uid,
+    intersection_id = oi.intersection_id,
+    intersection_name = oi.intersection_name,
+    classification_uid = oi.classification_uid,
+    classification = oi.classification,
+    leg = oi.leg,
+    range_start = oi.range_start,
+    num_days = oi.num_days,
+    notes = oi.notes,
+    volume = oi.volume,
+    alerts = oi.alerts
+WHEN NOT MATCHED THEN
+    INSERT (
+        uid, intersection_uid, intersection_id, intersection_name, classification_uid,
+        classification, leg, range_start, num_days, notes, volume, alerts
+    ) VALUES (
+        oi.uid, oi.intersection_uid, oi.intersection_id, oi.intersection_name,
+        oi.classification_uid, oi.classification, oi.leg, oi.range_start, oi.num_days, oi.notes,
+        oi.volume, oi.alerts
+    );
 
-COMMENT ON COLUMN miovision_api.open_issues.volume
-IS 'Volume recorded within the last 7 days or since anomalous_range began, whichever is later.'
 
-ALTER TABLE miovision_api.open_issues OWNER TO miovision_admins;
-GRANT SELECT ON TABLE miovision_api.open_issues TO bdit_humans;
+COMMENT ON FUNCTION miovision_api.update_open_issues
+IS '''A function to update miovision_api.open_issues.
+Run weekly via miovision_check Airflow DAG.''';

volumes/miovision/sql/table/create-table-open_issues.sql

@@ -0,0 +1,38 @@
+-- Table: miovision_api.open_issues_review
+-- DROP TABLE IF EXISTS miovision_api.open_issues_review;
+
+CREATE TABLE IF NOT EXISTS miovision_api.open_issues_review
+(
+    uid smallint NOT NULL,
+    intersection_uid smallint,
+    intersection_id text COLLATE pg_catalog."default",
+    intersection_name text COLLATE pg_catalog."default",
+    classification_uid smallint,
+    classification text COLLATE pg_catalog."default",
+    leg text COLLATE pg_catalog."default",
+    range_start date,
+    num_days integer,
+    notes text COLLATE pg_catalog."default",
+    volume bigint,
+    alerts text COLLATE pg_catalog."default",
+    logged boolean,
+    CONSTRAINT open_issues_review_pkey PRIMARY KEY (uid)
+)
+
+TABLESPACE pg_default;
+
+ALTER TABLE IF EXISTS miovision_api.open_issues_review
+OWNER to miovision_admins;
+
+REVOKE ALL ON TABLE miovision_api.open_issues_review FROM bdit_humans;
+REVOKE ALL ON TABLE miovision_api.open_issues_review FROM ckousin;
+
+GRANT SELECT ON TABLE miovision_api.open_issues_review TO bdit_humans;
+
+GRANT UPDATE ON TABLE miovision_api.open_issues_review TO ckousin;
+
+GRANT ALL ON TABLE miovision_api.open_issues_review TO miovision_admins;
+
+COMMENT ON TABLE miovision_api.open_issues_review
+IS 'Update this table using `SELECT miovision_api.update_open_issues();`.
+This is performed automatically once a week on Mondays by `miovision_check` Airflow DAG. ';