datascript.serialize

CHANGELOG.md

@@ -1,3 +1,7 @@
+# WIP
+
+- New namespace: `datascript.serialize`. Prepares database for fast to/from JSON serialization
+
 # 1.1.0
 
 - [ Breaking ] Remove `ISeqable` from `DB`/`FilteredDB` as it was breaking `keys` #393

src/datascript/db.cljc

@@ -150,13 +150,17 @@
 
 (defprotocol IDatom
   (datom-tx [this])
-  (datom-added [this]))
+  (datom-added [this])
+  (datom-get-idx [this])
+  (datom-set-idx [this value]))
 
-(deftype Datom #?(:clj [^int e a v ^int tx ^:unsynchronized-mutable ^int _hash]
-                  :cljs [^number e a v ^number tx ^:mutable ^number _hash])
+(deftype Datom #?(:clj [^int e a v ^int tx ^:unsynchronized-mutable ^int idx ^:unsynchronized-mutable ^int _hash]
+                  :cljs [^number e a v ^number tx ^:mutable ^number idx ^:mutable ^number _hash])
   IDatom
   (datom-tx [d] (if (pos? tx) tx (- tx)))
   (datom-added [d] (pos? tx))
+  (datom-get-idx [_] idx)
+  (datom-set-idx [_ value] (set! idx (int value)))
 
   #?@(:cljs
        [IHash
@@ -224,9 +228,9 @@
 #?(:cljs (goog/exportSymbol "datascript.db.Datom" Datom))
 
 (defn ^Datom datom
-  ([e a v] (Datom. e a v tx0 0))
-  ([e a v tx] (Datom. e a v tx 0))
-  ([e a v tx added] (Datom. e a v (if added tx (- tx)) 0)))
+  ([e a v] (Datom. e a v tx0 0 0))
+  ([e a v tx] (Datom. e a v tx 0 0))
+  ([e a v tx added] (Datom. e a v (if added tx (- tx)) 0 0)))
 
 (defn datom? [x] (instance? Datom x))
 

src/datascript/externs.js

@@ -9,7 +9,7 @@ datascript.db.Datom.prototype.e;
 datascript.db.Datom.prototype.a;
 datascript.db.Datom.prototype.v;
 datascript.db.Datom.prototype.tx;
-
+datascript.db.Datom.prototype.idx;
 
 datascript.impl = {};
 datascript.impl.entity = {};

src/datascript/serialize.cljc

@@ -0,0 +1,211 @@
+(ns datascript.serialize
+  (:refer-clojure :exclude [amap])
+  (:require
+   [clojure.edn :as edn]
+   [clojure.string :as str]
+   [datascript.db :as db #?(:cljs :refer-macros :clj :refer) [raise cond+] #?@(:cljs [:refer [Datom]])]
+   [me.tonsky.persistent-sorted-set :as set]
+   [me.tonsky.persistent-sorted-set.arrays :as arrays])
+  #?(:cljs (:require-macros [datascript.serialize :refer [array dict]]))
+  #?(:clj
+     (:import
+      [datascript.db Datom])))
+
+(def ^:const marker-kw 0)
+(def ^:const marker-other 1)
+
+(defn- if-cljs [env then else]
+  (if (:ns env) then else))
+
+#?(:clj
+   (defmacro array
+     "Platform-native array representation (java.util.List on JVM, Array on JS)"
+     [& args]
+     (if-cljs &env
+       (list* 'js* (str "[" (str/join "," (repeat (count args) "~{}")) "]") args)
+       `(java.util.List/of ~@args))))
+
+#?(:clj
+   (defmacro dict
+     "Platform-native dictionary representation (java.util.Map on JVM, Object on JS)"
+     [& args]
+     (if-cljs &env
+       (list* 'js* (str "{" (str/join "," (repeat (/ (count args) 2) "~{}:~{}")) "}") args)
+       `(array-map ~@args))))
+
+(defn- array-get [d i]
+  #?(:clj  (.get ^java.util.List d (int i))
+     :cljs (arrays/aget d i)))
+
+(defn- dict-get [d k]
+  #?(:clj  (.get ^java.util.Map d k)
+     :cljs (arrays/aget d k)))
+
+(defn- amap [f xs]
+  #?(:clj
+     (let [arr (java.util.ArrayList. (count xs))]
+       (reduce (fn [idx x] (.add arr (f x)) (inc idx)) 0 xs)
+       arr)
+     :cljs
+     (let [arr (js/Array. (count xs))]
+       (reduce (fn [idx x] (arrays/aset arr idx (f x)) (inc idx)) 0 xs)
+       arr)))
+
+(defn- amap-indexed [f xs]
+  #?(:clj
+     (let [arr (java.util.ArrayList. (count xs))]
+       (reduce (fn [idx x] (.add arr (f idx x)) (inc idx)) 0 xs)
+       arr)
+     :cljs
+     (let [arr (js/Array. (count xs))]
+       (reduce (fn [idx x] (arrays/aset arr idx (f idx x)) (inc idx)) 0 xs)
+       arr)))
+
+(defn- attr-comparator
+  "Looks for a datom with attribute exactly bigger than the given one"
+  [^Datom d1 ^Datom d2]
+  (cond 
+    (nil? (.-a d2)) -1
+    (<= (compare (.-a d1) (.-a d2)) 0) -1
+    true 1))
+
+(defn- all-attrs
+  "All attrs in a DB, distinct, sorted"
+  [db]
+  (if (empty? (:aevt db))
+    []
+    (loop [attrs (transient [(:a (first (:aevt db)))])]
+      (let [attr      (nth attrs (dec (count attrs)))
+            left      (db/datom 0 attr nil)
+            right     (db/datom db/emax nil nil)
+            next-attr (:a (first (set/slice (:aevt db) left right attr-comparator)))]
+        (if (some? next-attr)
+          (recur (conj! attrs next-attr))
+          (persistent! attrs))))))
+
+(def ^{:arglists '([kw])} freeze-kw str)
+
+(defn thaw-kw [s]
+  (if (str/starts-with? s ":")
+    (keyword (subs s 1))
+    s))
+
+(defn ^:export serializable
+  "Converts db into a data structure (not string!) that can be fed to JSON
+   serializer of your choice (`js/JSON.stringify` in CLJS, `cheshire.core/generate-string` or
+   `jsonista.core/write-value-as-string` in CLJ).
+
+   Options:
+
+   Non-primitive values will be serialized using optional :freeze-fn (`pr-str` by default).
+
+   Serialized structure breakdown:
+
+   count    :: number    
+   tx0      :: number
+   max-eid  :: number
+   max-tx   :: number
+   schema   :: freezed :schema
+   attrs    :: [keywords ...]
+   keywords :: [keywords ...]
+   eavt     :: [[e a-idx v dtx] ...]
+   a-idx    :: index in attrs
+   v        :: (string | number | boolean | [0 <index in keywords>] | [1 <freezed v>])
+   dtx      :: tx - tx0
+   aevt     :: [<index in eavt> ...]
+   avet     :: [<index in eavt> ...]"
+  ([db]
+   (serializable db {}))
+  ([db {:keys [freeze-fn]
+        :or   {freeze-fn pr-str}}]
+   (let [attrs       (all-attrs db)
+         attrs-map   (into {} (map vector attrs (range)))
+         *kws        (volatile! (transient []))
+         *kw-map     (volatile! (transient {}))
+         write-kw    (fn [kw]
+                       (let [idx (or
+                                   (get @*kw-map kw)
+                                   (let [keywords (vswap! *kws conj! kw)
+                                         idx      (dec (count keywords))]
+                                     (vswap! *kw-map assoc! kw idx)
+                                     idx))]
+                         (array marker-kw idx)))
+         write-other (fn [v] (array marker-other (freeze-fn v)))
+         write-v     (fn [v]
+                       (cond
+                         (string? v)  v
+                         #?@(:clj [(ratio? v) (write-other v)])
+                         (number? v)  v
+                         (boolean? v) v
+                         (keyword? v) (write-kw v)
+                         :else        (write-other v)))
+         eavt        (amap-indexed
+                       (fn [idx ^Datom d]
+                         (db/datom-set-idx d idx)
+                         (let [e  (.-e d)
+                               a  (attrs-map (.-a d))
+                               v  (write-v (.-v d))
+                               tx (- (.-tx d) db/tx0)]
+                           (array e a v tx)))
+                       (:eavt db))
+         aevt        (amap-indexed (fn [_ ^Datom d] (db/datom-get-idx d)) (:aevt db))
+         avet        (amap-indexed (fn [_ ^Datom d] (db/datom-get-idx d)) (:avet db))
+         schema      (freeze-fn (:schema db))
+         attrs       (amap freeze-kw attrs)
+         kws         (amap freeze-kw (persistent! @*kws))]
+       (dict
+         "count"    (count (:eavt db))
+         "tx0"      db/tx0
+         "max-eid"  (:max-eid db)
+         "max-tx"   (:max-tx db)
+         "schema"   schema
+         "attrs"    attrs
+         "keywords" kws
+         "eavt"     eavt
+         "aevt"     aevt
+         "avet"     avet))))
+
+(defn ^:export from-serializable
+  "Creates db from a data structure (not string!) produced by serializable.
+
+   Non-primitive values will be deserialized using optional :thaw-fn
+   (`clojure.edn/read-string` by default).
+
+   :thaw-fn must match :freeze-fn from serializable."
+  ([serializable] 
+   (from-serializable serializable {}))
+  ([serializable {:keys [thaw-fn]
+                  :or   {thaw-fn edn/read-string}}]
+   (let [tx0      (dict-get serializable "tx0")
+         schema   (thaw-fn (dict-get serializable "schema"))
+         _        (#'db/validate-schema schema)
+         attrs    (->> (dict-get serializable "attrs") (mapv thaw-kw))
+         keywords (->> (dict-get serializable "keywords") (mapv thaw-kw))
+         eavt     (->> (dict-get serializable "eavt")
+                    (amap (fn [arr]
+                            (let [e  (array-get arr 0)
+                                  a  (nth attrs (array-get arr 1))
+                                  v  (array-get arr 2)
+                                  v  (cond
+                                       (number? v)  v
+                                       (string? v)  v
+                                       (boolean? v) v
+                                       (arrays/array? v)
+                                       (let [marker (array-get v 0)]
+                                         (case marker
+                                           marker-kw    (array-get keywords (array-get v 1))
+                                           marker-other (thaw-fn (array-get v 1)))))
+                                  tx (+ tx0 (array-get arr 3))]
+                              (db/datom e a v tx))))
+                    #?(:clj .toArray))
+         aevt     (some->> (dict-get serializable "aevt") (amap #(arrays/aget eavt %)) #?(:clj .toArray))
+         avet     (some->> (dict-get serializable "avet") (amap #(arrays/aget eavt %)) #?(:clj .toArray))]
+     (db/map->DB
+       {:schema  schema
+        :rschema (#'db/rschema (merge db/implicit-schema schema))
+        :eavt    (set/from-sorted-array db/cmp-datoms-eavt eavt)
+        :aevt    (set/from-sorted-array db/cmp-datoms-aevt aevt)
+        :avet    (set/from-sorted-array db/cmp-datoms-avet avet)
+        :max-eid (dict-get serializable "max-eid")
+        :max-tx  (dict-get serializable "max-tx")
+        :hash    (atom 0)}))))