Merge pull request #697 from samply/hash

Use Own Class for Resource Hash Instead of ByteString
samply · May 1, 2022 · 21d49df · 21d49df
2 parents d9b3622 + 36e167e
commit 21d49df
Show file tree

Hide file tree

Showing 54 changed files with 580 additions and 366 deletions.
diff --git a/docs/implementation/database.md b/docs/implementation/database.md
@@ -38,16 +38,16 @@ There are two different sets of indices, ones which depend on the database value
 
 ### Indices depending on t
 
-| Name | Key Parts | Value |
-|---|---|---|
+| Name         | Key Parts | Value                         |
+|--------------|-----------|-------------------------------|
 | ResourceAsOf | type id t | content-hash, num-changes, op |
-| TypeAsOf | type t id | content-hash, num-changes, op |
-| SystemAsOf | t type id | content-hash, num-changes, op |
-| TxSuccess | t | instant |
-| TxError | t | anomaly |
-| TByInstant | instant | t |
-| TypeStats | type t | total, num-changes |
-| SystemStats | t | total, num-changes |
+| TypeAsOf     | type t id | content-hash, num-changes, op |
+| SystemAsOf   | t type id | content-hash, num-changes, op |
+| TxSuccess    | t         | instant                       |
+| TxError      | t         | anomaly                       |
+| TByInstant   | instant   | t                             |
+| TypeStats    | type t    | total, num-changes            |
+| SystemStats  | t         | total, num-changes            |
 
 #### ResourceAsOf
 
@@ -115,14 +115,14 @@ The `SystemStats` index keeps track of the total number of resources, and the nu
 
 The indices not depending on `t` directly point to the resource versions by their content hash. 
 
-| Name | Key Parts | Value |
-|---|---|---|
-| SearchParamValueResource | search-param, type, value, id, content-hash | - |
-| ResourceSearchParamValue | type, id, content-hash, search-param, value | - |
-| CompartmentSearchParamValueResource | co-c-hash, co-res-id, sp-c-hash, tid, value, id, hash-prefix | - |
-| CompartmentResource | co-c-hash, co-res-id, tid, id | - |
-| SearchParam | code, tid | id |
-| ActiveSearchParams | id | - |
+| Name                                | Key Parts                                                    | Value |
+|-------------------------------------|--------------------------------------------------------------|-------|
+| SearchParamValueResource            | search-param, type, value, id, content-hash                  | -     |
+| ResourceSearchParamValue            | type, id, content-hash, search-param, value                  | -     |
+| CompartmentSearchParamValueResource | co-c-hash, co-res-id, sp-c-hash, tid, value, id, hash-prefix | -     |
+| CompartmentResource                 | co-c-hash, co-res-id, tid, id                                | -     |
+| SearchParam                         | code, tid                                                    | id    |
+| ActiveSearchParams                  | id                                                           | -     |
 
 #### SearchParamValueResource
 

diff --git a/modules/byte-string/src/blaze/byte_string.clj b/modules/byte-string/src/blaze/byte_string.clj
@@ -46,7 +46,7 @@
   (ByteString/copyFrom ^String s ^Charset charset))
 
 
-(defn from-byte-buffer
+(defn from-byte-buffer!
   "Returns the remaining bytes from `byte-buffer` as byte string."
   {:inline
    (fn

diff --git a/modules/db-resource-store-cassandra/src/blaze/db/resource_store/cassandra.clj b/modules/db-resource-store-cassandra/src/blaze/db/resource_store/cassandra.clj
@@ -3,7 +3,6 @@
     [blaze.anomaly :as ba :refer [when-ok]]
     [blaze.async.comp :as ac :refer [do-sync]]
     [blaze.byte-buffer :as bb]
-    [blaze.byte-string :as bs]
     [blaze.cassandra :as cass]
     [blaze.cassandra.spec]
     [blaze.db.resource-store :as rs]
@@ -47,7 +46,7 @@
 
 (defn- parse-msg [hash cause-msg]
   (format "Error while parsing resource content with hash `%s`: %s"
-          (bs/hex hash) cause-msg))
+          hash cause-msg))
 
 
 (defn- parse-cbor [bytes hash]
@@ -59,8 +58,7 @@
 
 
 (defn- conform-msg [hash]
-  (format "Error while conforming resource content with hash `%s`."
-          (bs/hex hash)))
+  (format "Error while conforming resource content with hash `%s`." hash))
 
 
 (defn- conform-cbor [x hash]
@@ -83,7 +81,7 @@
 
 
 (defn- execute-get* [session statement hash]
-  (-> (execute session "get" (cass/bind statement (bs/hex hash)))
+  (-> (execute session "get" (cass/bind statement (str hash)))
       (ac/then-apply-async #(read-content % hash))
       (ac/exceptionally (partial map-execute-get-error hash))))
 
@@ -100,7 +98,7 @@
 (defn- bind-put [statement hash resource]
   (let [content (bb/wrap (fhir-spec/unform-cbor resource))]
     (prom/observe! resource-bytes (.capacity content))
-    (cass/bind statement (bs/hex hash) content)))
+    (cass/bind statement (str hash) content)))
 
 
 (defn- map-execute-put-error [hash {:fhir/keys [type] :keys [id]} e]

diff --git a/modules/db-resource-store-cassandra/test/blaze/db/resource_store/cassandra_test.clj b/modules/db-resource-store-cassandra/test/blaze/db/resource_store/cassandra_test.clj
@@ -3,7 +3,6 @@
   (:require
     [blaze.async.comp :as ac]
     [blaze.byte-buffer :as bb]
-    [blaze.byte-string :as bs]
     [blaze.cassandra :as cass]
     [blaze.cassandra-spec]
     [blaze.db.resource-store :as rs]
@@ -52,7 +51,7 @@
 
 (defn hash [s]
   (assert (= 1 (count s)))
-  (bs/from-hex (str/repeat s 64)))
+  (hash/from-hex (str/repeat s 64)))
 
 
 (def bound-get-statement (reify BoundStatement))
@@ -115,7 +114,7 @@
             (^PreparedStatement prepare [_ ^SimpleStatement statement]
               (cond
                 (= statement/get-statement statement)
-                (prepared-statement-with [(bs/hex hash)] bound-get-statement)
+                (prepared-statement-with [(str hash)] bound-get-statement)
                 (= (statement/put-statement "TWO") statement)
                 nil
                 :else
@@ -139,7 +138,7 @@
             (^PreparedStatement prepare [_ ^SimpleStatement statement]
               (cond
                 (= statement/get-statement statement)
-                (prepared-statement-with [(bs/hex hash)] bound-get-statement)
+                (prepared-statement-with [(str hash)] bound-get-statement)
                 (= (statement/put-statement "TWO") statement)
                 nil
                 :else
@@ -162,7 +161,7 @@
             (^PreparedStatement prepare [_ ^SimpleStatement statement]
               (cond
                 (= statement/get-statement statement)
-                (prepared-statement-with [(bs/hex hash)] bound-get-statement)
+                (prepared-statement-with [(str hash)] bound-get-statement)
                 (= (statement/put-statement "TWO") statement)
                 nil
                 :else
@@ -183,7 +182,7 @@
             (^PreparedStatement prepare [_ ^SimpleStatement statement]
               (cond
                 (= statement/get-statement statement)
-                (prepared-statement-with [(bs/hex hash)] bound-get-statement)
+                (prepared-statement-with [(str hash)] bound-get-statement)
                 (= (statement/put-statement "TWO") statement)
                 nil
                 :else
@@ -206,7 +205,7 @@
             (^PreparedStatement prepare [_ ^SimpleStatement statement]
               (cond
                 (= statement/get-statement statement)
-                (prepared-statement-with [(bs/hex hash)] bound-get-statement)
+                (prepared-statement-with [(str hash)] bound-get-statement)
                 (= (statement/put-statement "TWO") statement)
                 nil
                 :else
@@ -231,7 +230,7 @@
             (^PreparedStatement prepare [_ ^SimpleStatement statement]
               (cond
                 (= statement/get-statement statement)
-                (prepared-statement-with [(bs/hex hash)] bound-get-statement)
+                (prepared-statement-with [(str hash)] bound-get-statement)
                 (= (statement/put-statement "TWO") statement)
                 nil
                 :else
@@ -255,7 +254,7 @@
             (^PreparedStatement prepare [_ ^SimpleStatement statement]
               (cond
                 (= statement/get-statement statement)
-                (prepared-statement-with [(bs/hex hash)] bound-get-statement)
+                (prepared-statement-with [(str hash)] bound-get-statement)
                 (= (statement/put-statement "TWO") statement)
                 nil
                 :else
@@ -280,7 +279,7 @@
             (^PreparedStatement prepare [_ ^SimpleStatement statement]
               (cond
                 (= statement/get-statement statement)
-                (prepared-statement-with [(bs/hex hash)] bound-get-statement)
+                (prepared-statement-with [(str hash)] bound-get-statement)
                 (= (statement/put-statement "TWO") statement)
                 nil
                 :else
@@ -303,7 +302,7 @@
             (^PreparedStatement prepare [_ ^SimpleStatement statement]
               (cond
                 (= statement/get-statement statement)
-                (prepared-statement-with [(bs/hex hash)] bound-get-statement)
+                (prepared-statement-with [(str hash)] bound-get-statement)
                 (= (statement/put-statement "TWO") statement)
                 nil
                 :else
@@ -343,10 +342,10 @@
             (^PreparedStatement prepare [_ ^SimpleStatement statement]
               (cond
                 (= statement/get-statement statement)
-                (prepared-statement-with [(bs/hex hash)] bound-get-statement)
+                (prepared-statement-with [(str hash)] bound-get-statement)
                 (= (statement/put-statement "TWO") statement)
                 (prepared-statement-with
-                  [(bs/hex hash) encoded-resource]
+                  [(str hash) encoded-resource]
                   bound-put-statement)
                 :else
                 (throw (Error.))))
@@ -368,10 +367,10 @@
             (^PreparedStatement prepare [_ ^SimpleStatement statement]
               (cond
                 (= statement/get-statement statement)
-                (prepared-statement-with [(bs/hex hash)] bound-get-statement)
+                (prepared-statement-with [(str hash)] bound-get-statement)
                 (= (statement/put-statement "TWO") statement)
                 (prepared-statement-with
-                  [(bs/hex hash) encoded-resource]
+                  [(str hash) encoded-resource]
                   bound-put-statement)
                 :else
                 (throw (Error.))))
@@ -397,10 +396,10 @@
             (^PreparedStatement prepare [_ ^SimpleStatement statement]
               (cond
                 (= statement/get-statement statement)
-                (prepared-statement-with [(bs/hex hash)] bound-get-statement)
+                (prepared-statement-with [(str hash)] bound-get-statement)
                 (= (statement/put-statement "TWO") statement)
                 (prepared-statement-with
-                  [(bs/hex hash) encoded-resource]
+                  [(str hash) encoded-resource]
                   bound-put-statement)
                 :else
                 (throw (Error.))))
@@ -426,10 +425,10 @@
             (^PreparedStatement prepare [_ ^SimpleStatement statement]
               (cond
                 (= statement/get-statement statement)
-                (prepared-statement-with [(bs/hex hash)] bound-get-statement)
+                (prepared-statement-with [(str hash)] bound-get-statement)
                 (= (statement/put-statement "TWO") statement)
                 (prepared-statement-with
-                  [(bs/hex hash) encoded-resource]
+                  [(str hash) encoded-resource]
                   bound-put-statement)
                 :else
                 (throw (Error.))))

diff --git a/modules/db-resource-store/src/blaze/db/resource_store/kv.clj b/modules/db-resource-store/src/blaze/db/resource_store/kv.clj
@@ -3,13 +3,14 @@
   (:require
     [blaze.anomaly :as ba :refer [when-ok]]
     [blaze.async.comp :as ac]
-    [blaze.byte-string :as bs]
+    [blaze.byte-buffer :as bb]
     [blaze.coll.core :as coll]
     [blaze.db.kv :as kv]
     [blaze.db.kv.spec]
     [blaze.db.resource-store :as rs]
     [blaze.db.resource-store.kv.spec]
     [blaze.executors :as ex]
+    [blaze.fhir.hash :as hash]
     [blaze.fhir.spec :as fhir-spec]
     [blaze.module :refer [reg-collector]]
     [clojure.spec.alpha :as s]
@@ -34,7 +35,7 @@
 
 (defn- parse-msg [hash cause-msg]
   (format "Error while parsing resource content with hash `%s`: %s"
-          (bs/hex hash) cause-msg))
+          hash cause-msg))
 
 
 (defn- parse-cbor [bytes hash]
@@ -46,8 +47,7 @@
 
 
 (defn- conform-msg [hash]
-  (format "Error while conforming resource content with hash `%s`."
-          (bs/hex hash)))
+  (format "Error while conforming resource content with hash `%s`." hash))
 
 
 (defn- conform-cbor [x hash]
@@ -68,26 +68,26 @@
   (comp
     (map
       (fn [[k v]]
-        (let [hash (bs/from-byte-array k)]
+        (let [hash (hash/from-byte-buffer! (bb/wrap k))]
           (when-ok [resource (parse-and-conform-cbor v hash)]
             [hash resource]))))
     (halt-when ba/anomaly?)))
 
 
 (def ^:private entry-freezer
   (map
-    (fn [[k v]]
-      (let [content (fhir-spec/unform-cbor v)]
+    (fn [[hash resource]]
+      (let [content (fhir-spec/unform-cbor resource)]
         (prom/observe! resource-bytes (alength ^bytes content))
-        [(bs/to-byte-array k) content]))))
+        [(hash/to-byte-array hash) content]))))
 
 
 (defn- get-content [kv-store hash]
-  (kv/get kv-store (bs/to-byte-array hash)))
+  (kv/get kv-store (hash/to-byte-array hash)))
 
 
 (defn- multi-get-content [kv-store hashes]
-  (kv/multi-get kv-store (mapv bs/to-byte-array hashes)))
+  (kv/multi-get kv-store (mapv hash/to-byte-array hashes)))
 
 
 (deftype KvResourceStore [kv-store executor]
@@ -101,6 +101,7 @@
   (-multi-get [_ hashes]
     (log/trace "multi-get" (count hashes) "hash(es)")
     (ac/supply-async
+      ;; TODO - make thawing parallel
       #(transduce entry-thawer conj {} (multi-get-content kv-store hashes))
       executor))
 

diff --git a/modules/db-resource-store/src/blaze/db/resource_store_spec.clj b/modules/db-resource-store/src/blaze/db/resource_store_spec.clj
@@ -9,13 +9,13 @@
 
 
 (s/fdef rs/get
-  :args (s/cat :lookup :blaze.db/resource-store
+  :args (s/cat :store :blaze.db/resource-store
                :hash :blaze.resource/hash)
   :ret ac/completable-future?)
 
 
 (s/fdef rs/multi-get
-  :args (s/cat :lookup :blaze.db/resource-store
+  :args (s/cat :store :blaze.db/resource-store
                :hashes (s/coll-of :blaze.resource/hash))
   :ret ac/completable-future?)