Skip to content

Commit

Permalink
[WIP] Switch to Internal Resource Identifier in the Index
Browse files Browse the repository at this point in the history
  • Loading branch information
alexanderkiel committed Jun 13, 2022
1 parent 8bf8145 commit 4f78f45
Show file tree
Hide file tree
Showing 97 changed files with 1,742 additions and 1,840 deletions.
112 changes: 68 additions & 44 deletions docs/implementation/database.md

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion docs/performance/fhir-search.md
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@ The result is a dataset which consists only of the resource types Patient, Obser

## Controlling and Monitoring the Caches

The size of the resource cache and the resource handle cache can be set by their respective environment variables `DB_RESOURCE_CACHE_SIZE` and `DB_RESOURCE_HANDLE_CACHE_SIZE`. The size denotes the number of resources / resource handles. Because one has to specify a number of resources / resource handles, it's important to know how many bytes a resource / resource handle allocates on the heap. For resource handles, it can be said that they allocate between 272 and 328 bytes depending on the size of the resource id. For resources, the size varies widely. Monitoring of the heap usage is critical.
The size of the resource cache and the resource handle cache can be set by their respective environment variables `DB_RESOURCE_CACHE_SIZE` and `DB_RESOURCE_HANDLE_CACHE_SIZE`. The size denotes the number of resources / resource handles. Because one has to specify a number of resources / resource handles, it's important to know how many bytes a resource / resource handle allocates on the heap. For resource handles, it can be said that they allocate between 152 and 208 bytes depending on the size of the resource id. For resources, the size varies widely. Monitoring of the heap usage is critical.

### Monitoring

Expand Down
29 changes: 11 additions & 18 deletions modules/byte-buffer/src/blaze/byte_buffer.clj
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,12 @@
(.putInt ^ByteBuffer byte-buffer x))


(defn put-5-byte-long!
[byte-buffer ^long x]
(put-byte! byte-buffer (bit-shift-right (unchecked-long x) 32))
(put-int! byte-buffer x))


(defn put-long!
{:inline
(fn [byte-buffer x]
Expand Down Expand Up @@ -220,6 +226,11 @@
(.getInt ^ByteBuffer byte-buffer))


(defn get-5-byte-long! [byte-buffer]
(+ (bit-shift-left (bit-and (get-byte! byte-buffer) 0xFF) 32)
(bit-and (get-int! byte-buffer) 0xFFFFFFFF)))


(defn get-long!
{:inline
(fn [byte-buffer]
Expand All @@ -243,24 +254,6 @@
(.get ^ByteBuffer byte-buffer ^bytes byte-array offset length)))


(defn size-up-to-null [byte-buffer]
(when (pos? (remaining byte-buffer))
(mark! byte-buffer)
(loop [byte (bit-and (long (get-byte! byte-buffer)) 0xFF)
size 0]
(cond
(zero? byte)
(do (reset! byte-buffer)
size)

(pos? (remaining byte-buffer))
(recur (bit-and (long (get-byte! byte-buffer)) 0xFF) (inc size))

:else
(do (reset! byte-buffer)
nil)))))


(defn mismatch
"Finds and returns the relative index of the first mismatch between `a` and
`b`.
Expand Down
50 changes: 14 additions & 36 deletions modules/byte-buffer/test/blaze/byte_buffer_test.clj
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
[blaze.byte-buffer :as bb]
[blaze.test-util :refer [satisfies-prop]]
[clojure.spec.test.alpha :as st]
[clojure.test :as test :refer [deftest is testing]]
[clojure.test :as test :refer [are deftest]]
[clojure.test.check.generators :as gen]
[clojure.test.check.properties :as prop]))

Expand All @@ -26,41 +26,19 @@
(= capacity (bb/limit (bb/allocate capacity))))))


(deftest size-up-to-null-test
(testing "empty buffer"
(let [buf (bb/allocate 0)]
(is (nil? (bb/size-up-to-null buf)))))
(defn- transcode-5-byte-long [x]
(let [buf (bb/allocate 5)]
(bb/put-5-byte-long! buf x)
(bb/flip! buf)
(= x (bb/get-5-byte-long! buf))))

(testing "buffer with only one null byte"
(let [buf (bb/allocate 1)]
(bb/put-byte! buf 0)
(bb/flip! buf)
(is (zero? (bb/size-up-to-null buf)))))

(testing "buffer with only one non-null byte"
(let [buf (bb/allocate 1)]
(bb/put-byte! buf 1)
(bb/flip! buf)
(is (nil? (bb/size-up-to-null buf)))))
(deftest transcode-5-byte-long-test
(are [x] (transcode-5-byte-long x)
0
0xFFFFFFFF
0xFFFFFFFFFF)

(testing "buffer with one non-null and one null byte"
(let [buf (bb/allocate 2)]
(bb/put-byte! buf 1)
(bb/put-byte! buf 0)
(bb/flip! buf)
(is (= 1 (bb/size-up-to-null buf)))))

(testing "buffer with two null bytes"
(let [buf (bb/allocate 2)]
(bb/put-byte! buf 0)
(bb/put-byte! buf 0)
(bb/flip! buf)
(is (zero? (bb/size-up-to-null buf)))))

(testing "buffer with two non-null and one null byte"
(let [buf (bb/allocate 3)]
(bb/put-byte! buf 1)
(bb/put-byte! buf 2)
(bb/put-byte! buf 0)
(bb/flip! buf)
(is (= 2 (bb/size-up-to-null buf))))))
(satisfies-prop 100000
(prop/for-all [x (gen/choose 0 0xFFFFFFFFFF)]
(transcode-5-byte-long x))))
4 changes: 2 additions & 2 deletions modules/db-protocols/src/blaze/db/impl/protocols.clj
Original file line number Diff line number Diff line change
Expand Up @@ -76,8 +76,8 @@
(-compartment-keys [search-param context compartment tid compiled-value])
(-matches? [search-param context resource-handle modifier compiled-values])
(-compartment-ids [_ resolver resource])
(-index-values [_ resolver resource])
(-index-value-compiler [_]))
(-index-values [_ resource-id resolver resource])
(-index-value-compiler [_ resource-id]))


(defprotocol Pull
Expand Down
1 change: 1 addition & 0 deletions modules/db-stub/src/blaze/db/api_stub.clj
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@
:tx-success-index {:reverse-comparator? true}
:tx-error-index nil
:t-by-instant-index {:reverse-comparator? true}
:resource-id-index nil
:resource-as-of-index nil
:type-as-of-index nil
:system-as-of-index nil
Expand Down
3 changes: 2 additions & 1 deletion modules/db-tx-log/deps.edn
Original file line number Diff line number Diff line change
Expand Up @@ -31,4 +31,5 @@
{cloverage/cloverage
{:mvn/version "1.2.4"}}

:main-opts ["-m" "cloverage.coverage" "--codecov" "-p" "src" "-s" "test"]}}}
:main-opts ["-m" "cloverage.coverage" "--codecov" "-p" "src" "-s" "test"
"-e" ".+spec"]}}}
17 changes: 15 additions & 2 deletions modules/db-tx-log/src/blaze/db/tx_log/spec.clj
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,8 @@
[blaze.db.tx-log :as tx-log]
[blaze.fhir.spec]
[blaze.spec]
[clojure.spec.alpha :as s]))
[clojure.spec.alpha :as s]
[clojure.spec.gen.alpha :as gen]))


(s/def :blaze.db/tx-log
Expand All @@ -27,8 +28,20 @@
(s/coll-of :blaze.fhir/local-ref-tuple))


(def ^:private ^:const ^long max-t 0xFFFFFFFFFF)


;; With a 5 byte long `t`, we can create one transaction each millisecond
;; for 34 years. Alone the t-values would need a storage of 5 TB.
(comment
(let [millis-per-year (* 1000 3600 24 365)
five-bytes (long (Math/pow 2 40))]
(double (/ five-bytes millis-per-year))))


;; The point in time `t` of a database value.
(s/def :blaze.db/t
(s/and int? #(<= 0 % 0xFFFFFFFFFFFFFF)))
(s/with-gen (s/and int? #(<= 0 % max-t)) #(gen/choose 0 max-t)))


(s/def :blaze.db.tx-cmd/if-none-exist
Expand Down
2 changes: 1 addition & 1 deletion modules/db-tx-log/test/blaze/db/tx_log/spec_test.clj
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@
(are [x] (s/valid? :blaze.db/t x)
0
1
0xFFFFFFFFFFFFFF))
0xFFFFFFFFFF))


(def patient-hash-0 (hash/generate {:fhir/type :fhir/Patient :id "0"}))
Expand Down
125 changes: 0 additions & 125 deletions modules/db/NOTES.md

This file was deleted.

Loading

0 comments on commit 4f78f45

Please sign in to comment.