Merge pull request #300 from yetanalytics/timestamp_updates

Timestamp updates

src/db/postgres/lrsql/postgres/data.clj

@@ -50,3 +50,12 @@
     IPersistentMap
     (set-parameter [^IPersistentMap m ^PreparedStatement stmt ^long i]
       (.setObject stmt i (json->pg-object m)))))
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; Timezone Input
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(def local-tz-input
+  "Returns a properly formatted hug input map to inject a timezone id into a
+  query needing a timezone id"
+  {:tz-id (str "'" u/local-zone-id "'")})

src/db/postgres/lrsql/postgres/record.clj

@@ -59,7 +59,11 @@
       (migrate-xapi-statement-timestamps! tx))
     (when-not (some? (query-xapi-statement-stored-exists tx))
       (alter-xapi-statement-add-stored! tx)
-      (migrate-xapi-statement-stored-times! tx)))
+      (migrate-xapi-statement-stored-times! tx))
+    (when-not (some? (query-state-document-last-modified-is-timestamptz tx))
+      (migrate-state-document-last-modified! tx pd/local-tz-input)
+      (migrate-activity-profile-document-last-modified! tx pd/local-tz-input)
+      (migrate-agent-profile-document-last-modified! tx pd/local-tz-input)))
 
   bp/BackendUtil
   (-txn-retry? [_ ex]

src/db/postgres/lrsql/postgres/sql/ddl.sql

@@ -282,7 +282,7 @@ CREATE TYPE scope_enum AS ENUM (
 ALTER TABLE IF EXISTS credential_to_scope ALTER COLUMN scope TYPE scope_enum USING (scope::scope_enum);
 
 
-/* Migration 2022-05-08-00 - Add timestamp to xapi_statement */
+/* Migration 2023-05-08-00 - Add timestamp to xapi_statement */
 
 -- :name query-xapi-statement-timestamp-exists
 -- :command :query
@@ -300,7 +300,7 @@ ALTER TABLE xapi_statement ADD COLUMN timestamp TIMESTAMPTZ
 -- :doc Backfill `xapi_statement.timestamp` with the values from the payload
 UPDATE xapi_statement SET timestamp = (payload->>'timestamp')::timestamptz WHERE timestamp IS NULL;
 
-/* Migration 2022-05-08-01 - Add stored to xapi_statement */
+/* Migration 2023-05-08-01 - Add stored to xapi_statement */
 
 -- :name query-xapi-statement-stored-exists
 -- :command :query
@@ -317,3 +317,30 @@ ALTER TABLE xapi_statement ADD COLUMN stored TIMESTAMPTZ
 -- :command :execute
 -- :doc Backfill `xapi_statement.stored` with the values from the payload
 UPDATE xapi_statement SET stored = (payload->>'stored')::timestamptz WHERE stored IS NULL;
+
+/* Migration 2023-05-11-00 - Convert timestamps for consistency */
+
+-- :name query-state-document-last-modified-is-timestamptz
+-- :command :query
+-- :result :one
+-- :doc Query to see if `state_document.last_modified` is a timestamp.
+SELECT * FROM information_schema.columns
+WHERE table_name = 'state_document' AND column_name = 'last_modified' AND data_type = 'timestamp with time zone';
+
+-- :name migrate-state-document-last-modified!
+-- :command :execute
+-- :doc Migrate the `state_document.last_modified` to have a timezone, and use the provided timezone to work backwards to Zulu
+ALTER TABLE state_document ALTER COLUMN last_modified TYPE TIMESTAMP WITH TIME ZONE
+USING last_modified AT TIME ZONE :sql:tz-id;
+
+-- :name migrate-activity-profile-document-last-modified!
+-- :command :execute
+-- :doc Migrate the `activity_profile_document.last_modified` to have a timezone, and use the provided timezone to work backwards to Zulu
+ALTER TABLE activity_profile_document ALTER COLUMN last_modified TYPE TIMESTAMP WITH TIME ZONE
+USING last_modified AT TIME ZONE :sql:tz-id;
+
+-- :name migrate-agent-profile-document-last-modified!
+-- :command :execute
+-- :doc Migrate the `agent_profile_document.last_modified` to have a timezone, and use the provided timezone to work backwards to Zulu
+ALTER TABLE agent_profile_document ALTER COLUMN last_modified TYPE TIMESTAMP WITH TIME ZONE
+USING last_modified AT TIME ZONE :sql:tz-id;

src/db/sqlite/lrsql/sqlite/record.clj

@@ -89,6 +89,19 @@
     (when-not (some? (query-xapi-statement-stored-exists tx))
       (alter-xapi-statement-add-stored! tx)
       (migrate-xapi-statement-stored-times! tx))
+    (when-not (some? (query-state-document-last-modified-is-timestamp tx))
+      (migrate-timestamps-state-01! tx)
+      (migrate-timestamps-state-02! tx)
+      (migrate-timestamps-state-03! tx)
+      (migrate-timestamps-state-04! tx)
+      (migrate-timestamps-agent-profile-01! tx)
+      (migrate-timestamps-agent-profile-02! tx)
+      (migrate-timestamps-agent-profile-03! tx)
+      (migrate-timestamps-agent-profile-04! tx)
+      (migrate-timestamps-activity-profile-01! tx)
+      (migrate-timestamps-activity-profile-02! tx)
+      (migrate-timestamps-activity-profile-03! tx)
+      (migrate-timestamps-activity-profile-04! tx))
     (update-schema-simple! tx alter-credential-to-scope-scope-datatype!)
     (log/infof "sqlite schema_version: %d"
                (:schema_version (query-schema-version tx))))

src/db/sqlite/lrsql/sqlite/sql/ddl.sql

@@ -329,7 +329,7 @@ SET sql = 'CREATE TABLE credential_to_scope (
 )'
 WHERE type = 'table' AND name = 'credential_to_scope'
 
-/* Migration 2022-05-08-00 - Add timestamp to xapi_statement */
+/* Migration 2023-05-08-00 - Add timestamp to xapi_statement */
 
 -- :name query-xapi-statement-timestamp-exists
 -- :command :query
@@ -348,7 +348,7 @@ ALTER TABLE xapi_statement ADD COLUMN timestamp TIMESTAMP
 UPDATE xapi_statement SET timestamp = strftime('%Y-%m-%dT%H:%M:%f000000Z', json_extract(payload, '$.timestamp'))
 WHERE timestamp IS NULL;
 
-/* Migration 2022-05-08-01 - Add stored to xapi_statement */
+/* Migration 2023-05-08-01 - Add stored to xapi_statement */
 
 -- :name query-xapi-statement-stored-exists
 -- :command :query
@@ -366,3 +366,63 @@ ALTER TABLE xapi_statement ADD COLUMN stored TIMESTAMP
 -- :doc Backfill `xapi_statement.stored` with the values from the payload
 UPDATE xapi_statement SET stored = strftime('%Y-%m-%dT%H:%M:%f000000Z', json_extract(payload, '$.stored'))
 WHERE stored IS NULL;
+
+
+/* Migration 2023-05-11-00 - Convert timestamps for consistency */
+
+-- :name query-state-document-last-modified-is-timestamp
+-- :command :query
+-- :result :one
+-- :doc Query to see if `state_document.last_modified` is a timestamp.
+SELECT 1 FROM pragma_table_info('state_document') WHERE name = 'last_modified' AND type = 'TIMESTAMP';
+
+-- :name migrate-timestamps-state-01!
+-- :command :execute
+-- :doc Convert `state_document.last_modified` to timestamp - 01
+ALTER TABLE state_document ADD COLUMN last_modified_tmp TIMESTAMP;
+-- :name migrate-timestamps-state-02!
+-- :command :execute
+-- :doc Convert `state_document.last_modified` to timestamp - 02
+UPDATE state_document SET last_modified_tmp = last_modified WHERE last_modified_tmp IS NULL;
+-- :name migrate-timestamps-state-03!
+-- :command :execute
+-- :doc Convert `state_document.last_modified` to timestamp - 03
+ALTER TABLE state_document DROP COLUMN last_modified;
+-- :name migrate-timestamps-state-04!
+-- :command :execute
+-- :doc Convert `state_document.last_modified` to timestamp - 04
+ALTER TABLE state_document RENAME COLUMN last_modified_tmp TO last_modified;
+
+-- :name migrate-timestamps-agent-profile-01!
+-- :command :execute
+-- :doc Convert `agent_profile_document.last_modified` to timestamp - 01
+ALTER TABLE agent_profile_document ADD COLUMN last_modified_tmp TIMESTAMP;
+-- :name migrate-timestamps-agent-profile-02!
+-- :command :execute
+-- :doc Convert `agent_profile_document.last_modified` to timestamp - 02
+UPDATE agent_profile_document SET last_modified_tmp = last_modified WHERE last_modified_tmp IS NULL;
+-- :name migrate-timestamps-agent-profile-03!
+-- :command :execute
+-- :doc Convert `agent_profile_document.last_modified` to timestamp - 03
+ALTER TABLE agent_profile_document DROP COLUMN last_modified;
+-- :name migrate-timestamps-agent-profile-04!
+-- :command :execute
+-- :doc Convert `agent_profile_document.last_modified` to timestamp - 04
+ALTER TABLE agent_profile_document RENAME COLUMN last_modified_tmp TO last_modified;
+
+-- :name migrate-timestamps-activity-profile-01!
+-- :command :execute
+-- :doc Convert `activity_profile_document.last_modified` to timestamp - 01
+ALTER TABLE activity_profile_document ADD COLUMN last_modified_tmp TIMESTAMP;
+-- :name migrate-timestamps-activity-profile-02!
+-- :command :execute
+-- :doc Convert `activity_profile_document.last_modified` to timestamp - 02
+UPDATE activity_profile_document SET last_modified_tmp = last_modified WHERE last_modified_tmp IS NULL;
+-- :name migrate-timestamps-activity-profile-03!
+-- :command :execute
+-- :doc Convert `activity_profile_document.last_modified` to timestamp - 03
+ALTER TABLE activity_profile_document DROP COLUMN last_modified;
+-- :name migrate-timestamps-activity-profile-04!
+-- :command :execute
+-- :doc Convert `activity_profile_document.last_modified` to timestamp - 04
+ALTER TABLE activity_profile_document RENAME COLUMN last_modified_tmp TO last_modified;

src/main/lrsql/util.clj

@@ -11,7 +11,7 @@
             [com.yetanalytics.lrs.xapi.document :refer [json-bytes-gen-fn]]
             [com.yetanalytics.lrs.xapi.statements.timestamp :refer [normalize]]
             [lrsql.spec.common :as cs :refer [instant-spec]])
-  (:import [java.util UUID]
+  (:import [java.util UUID Calendar]
            [java.time Instant]
            [java.io StringReader PushbackReader ByteArrayOutputStream]
            [java.nio.charset Charset]))
@@ -120,6 +120,11 @@
              str-length
              "0000-01-01T00:00:00.000000000Z")))))
 
+(def local-zone-id
+  "Returns the string zoneId for the local calendar's timezone. For use with
+  migrations from zoneless local time."
+  (.getID (.getTimeZone (Calendar/getInstance))))
+
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; UUIDs
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;