From c8524420dfc5e3ac5cbbf806d42f83db4eedb995 Mon Sep 17 00:00:00 2001 From: Gregor Date: Wed, 18 Oct 2023 12:30:11 +0200 Subject: [PATCH 1/8] optionally configure storing mechanism in pickles, v1 --- src/lib/pickles/cache.ml | 107 ++++++++++++++++++++----------- src/lib/pickles/cache.mli | 47 ++++++++++++-- src/lib/pickles/compile.ml | 28 +++++--- src/lib/pickles/compile.mli | 2 +- src/lib/pickles/pickles_intf.mli | 4 +- 5 files changed, 133 insertions(+), 55 deletions(-) diff --git a/src/lib/pickles/cache.ml b/src/lib/pickles/cache.ml index 4c4f5be9324..034b8e33c24 100644 --- a/src/lib/pickles/cache.ml +++ b/src/lib/pickles/cache.ml @@ -27,6 +27,14 @@ module Step = struct [@@warning "-4"] end + type 'header storable = + ('header, Backend.Tick.Keypair.t) Key_cache.Sync.Disk_storable.t + + type 'header vk_storable = + ( 'header + , Kimchi_bindings.Protocol.VerifierIndex.Fp.t ) + Key_cache.Sync.Disk_storable.t + let storable = Key_cache.Sync.Disk_storable.simple Key.Proving.to_string (fun (_, header, _, cs) ~path -> @@ -83,9 +91,8 @@ module Step = struct (Kimchi_bindings.Protocol.VerifierIndex.Fp.write (Some true) x) header path ) ) - let read_or_generate ~prev_challenges cache k_p k_v typ return_typ main = - let s_p = storable in - let s_v = vk_storable in + let read_or_generate ~prev_challenges cache (k_p, s_p) (k_v, s_v) typ + return_typ main = let open Impls.Step in let pk = lazy @@ -155,6 +162,12 @@ module Wrap = struct end end + type 'header storable = + ('header, Backend.Tock.Keypair.t) Key_cache.Sync.Disk_storable.t + + type 'header vk_storable = + ('header, Verification_key.t) Key_cache.Sync.Disk_storable.t + let storable = Key_cache.Sync.Disk_storable.simple Key.Proving.to_string (fun (_, header, cs) ~path -> @@ -182,10 +195,42 @@ module Wrap = struct (Kimchi_bindings.Protocol.Index.Fq.write (Some true) t.index) header path ) ) - let read_or_generate ~prev_challenges cache k_p k_v typ return_typ main = + let vk_storable = + Key_cache.Sync.Disk_storable.simple Key.Verification.to_string + (fun (_, header, _cs) ~path -> + Or_error.try_with_join (fun () -> + let open Or_error.Let_syntax in + let%map header_read, index = + Snark_keys_header.read_with_header + ~read_data:(fun ~offset:_ path -> + Binable.of_string + (module Verification_key.Stable.Latest) + (In_channel.read_all path) ) + path + in + [%test_eq: int] header.header_version header_read.header_version ; + [%test_eq: Snark_keys_header.Kind.t] header.kind header_read.kind ; + [%test_eq: Snark_keys_header.Constraint_constants.t] + header.constraint_constants header_read.constraint_constants ; + [%test_eq: string] header.constraint_system_hash + header_read.constraint_system_hash ; + index ) ) + (fun (_, header, _) t path -> + Or_error.try_with (fun () -> + Snark_keys_header.write_with_header + ~expected_max_size_log2:33 (* 8 GB should be enough *) + ~append_data:(fun path -> + Out_channel.with_file ~append:true path ~f:(fun file -> + Out_channel.output_string file + (Binable.to_string + (module Verification_key.Stable.Latest) + t ) ) ) + header path ) ) + + let read_or_generate ~prev_challenges cache (k_p, s_p) (k_v, s_v) typ + return_typ main = let module Vk = Verification_key in let open Impls.Wrap in - let s_p = storable in let pk = lazy (let k = Lazy.force k_p in @@ -209,40 +254,6 @@ module Wrap = struct let vk = lazy (let k_v = Lazy.force k_v in - let s_v = - Key_cache.Sync.Disk_storable.simple Key.Verification.to_string - (fun (_, header, _cs) ~path -> - Or_error.try_with_join (fun () -> - let open Or_error.Let_syntax in - let%map header_read, index = - Snark_keys_header.read_with_header - ~read_data:(fun ~offset:_ path -> - Binable.of_string - (module Vk.Stable.Latest) - (In_channel.read_all path) ) - path - in - [%test_eq: int] header.header_version - header_read.header_version ; - [%test_eq: Snark_keys_header.Kind.t] header.kind - header_read.kind ; - [%test_eq: Snark_keys_header.Constraint_constants.t] - header.constraint_constants - header_read.constraint_constants ; - [%test_eq: string] header.constraint_system_hash - header_read.constraint_system_hash ; - index ) ) - (fun (_, header, _) t path -> - Or_error.try_with (fun () -> - Snark_keys_header.write_with_header - ~expected_max_size_log2:33 (* 8 GB should be enough *) - ~append_data:(fun path -> - Out_channel.with_file ~append:true path ~f:(fun file -> - Out_channel.output_string file - (Binable.to_string (module Vk.Stable.Latest) t) ) - ) - header path ) ) - in match Key_cache.Sync.read cache s_v k_v with | Ok (vk, d) -> (vk, d) @@ -265,3 +276,21 @@ module Wrap = struct in (pk, vk) end + +module Spec = struct + type ('step_pk_header, 'step_vk_header, 'wrap_pk_header, 'wrap_vk_header) t = + { cache : Key_cache.Spec.t list + ; step_storable : 'step_pk_header Step.storable + ; step_vk_storable : 'step_vk_header Step.vk_storable + ; wrap_storable : 'wrap_pk_header Wrap.storable + ; wrap_vk_storable : 'wrap_vk_header Wrap.vk_storable + } + + let default = + { cache = [] + ; step_storable = Step.storable + ; step_vk_storable = Step.vk_storable + ; wrap_storable = Wrap.storable + ; wrap_vk_storable = Wrap.vk_storable + } +end \ No newline at end of file diff --git a/src/lib/pickles/cache.mli b/src/lib/pickles/cache.mli index c52e7ff2c52..809bcf9c486 100644 --- a/src/lib/pickles/cache.mli +++ b/src/lib/pickles/cache.mli @@ -20,11 +20,23 @@ module Step : sig end end + type 'header storable = + ('header, Backend.Tick.Keypair.t) Key_cache.Sync.Disk_storable.t + + type 'header vk_storable = + ( 'header + , Kimchi_bindings.Protocol.VerifierIndex.Fp.t ) + Key_cache.Sync.Disk_storable.t + + val storable : Key.Proving.t storable + + val vk_storable : Key.Verification.t vk_storable + val read_or_generate : prev_challenges:int -> Key_cache.Spec.t list - -> Key.Proving.t lazy_t - -> Key.Verification.t lazy_t + -> 'pk_header lazy_t * 'pk_header storable + -> 'vk_header lazy_t * 'vk_header vk_storable -> ('a, 'b) Impls.Step.Typ.t -> ('c, 'd) Impls.Step.Typ.t -> ('a -> unit -> 'c) @@ -59,11 +71,21 @@ module Wrap : sig end end + type 'header storable = + ('header, Backend.Tock.Keypair.t) Key_cache.Sync.Disk_storable.t + + type 'header vk_storable = + ('header, Verification_key.t) Key_cache.Sync.Disk_storable.t + + val storable : Key.Proving.t storable + + val vk_storable : Key.Verification.t vk_storable + val read_or_generate : prev_challenges:Core_kernel.Int.t -> Key_cache.Spec.t list - -> Key.Proving.t Core_kernel.Lazy.t - -> Key.Verification.t Core_kernel.Lazy.t + -> 'pk_header lazy_t * 'pk_header storable + -> 'vk_header lazy_t * 'vk_header vk_storable -> ('a, 'b) Impls.Wrap.Typ.t -> ('c, 'd) Impls.Wrap.Typ.t -> ('a -> unit -> 'c) @@ -74,3 +96,20 @@ module Wrap : sig * [> `Cache_hit | `Generated_something | `Locally_generated ] ) lazy_t end + +module Spec : sig + type ('step_pk_header, 'step_vk_header, 'wrap_pk_header, 'wrap_vk_header) t = + { cache : Key_cache.Spec.t list + ; step_storable : 'step_pk_header Step.storable + ; step_vk_storable : 'step_vk_header Step.vk_storable + ; wrap_storable : 'wrap_pk_header Wrap.storable + ; wrap_vk_storable : 'wrap_vk_header Wrap.vk_storable + } + + val default : + ( Step.Key.Proving.t + , Step.Key.Verification.t + , Wrap.Key.Proving.t + , Wrap.Key.Verification.t ) + t +end \ No newline at end of file diff --git a/src/lib/pickles/compile.ml b/src/lib/pickles/compile.ml index f85f4b271a6..a554b036b3b 100644 --- a/src/lib/pickles/compile.ml +++ b/src/lib/pickles/compile.ml @@ -339,7 +339,7 @@ struct let compile : type var value prev_varss prev_valuess widthss heightss max_proofs_verified branches. self:(var, value, max_proofs_verified, branches) Tag.t - -> cache:Key_cache.Spec.t list + -> cache:_ Cache.Spec.t -> proof_cache:Proof_cache.t option -> ?disk_keys: (Cache.Step.Key.Verification.t, branches) Vector.t @@ -378,10 +378,17 @@ struct * _ * _ * _ = - fun ~self ~cache ~proof_cache ?disk_keys - ?(return_early_digest_exception = false) ?override_wrap_domain - ?override_wrap_main ~branches:(module Branches) ~max_proofs_verified - ~name ~constraint_constants ~public_input ~auxiliary_typ ~choices () -> + fun ~self + ~cache: + { cache + ; step_storable + ; step_vk_storable + ; wrap_storable + ; wrap_vk_storable + } ~proof_cache ?disk_keys ?(return_early_digest_exception = false) + ?override_wrap_domain ?override_wrap_main ~branches:(module Branches) + ~max_proofs_verified ~name ~constraint_constants ~public_input + ~auxiliary_typ ~choices () -> let snark_keys_header kind constraint_system_hash = { Snark_keys_header.header_version = Snark_keys_header.header_version ; kind @@ -595,7 +602,7 @@ struct Common.time "step read or generate" (fun () -> Cache.Step.read_or_generate ~prev_challenges:(Nat.to_int (fst b.proofs_verified)) - cache k_p k_v + cache (k_p, step_storable) (k_v, step_vk_storable) (Snarky_backendless.Typ.unit ()) typ main ) in @@ -671,7 +678,10 @@ struct let r = Common.time "wrap read or generate " (fun () -> Cache.Wrap.read_or_generate (* Due to Wrap_hack *) - ~prev_challenges:2 cache disk_key_prover disk_key_verifier typ + ~prev_challenges:2 cache + (disk_key_prover, wrap_storable) + (disk_key_verifier, wrap_vk_storable) + typ (Snarky_backendless.Typ.unit ()) main ) in @@ -937,7 +947,7 @@ end let compile_with_wrap_main_override_promise : type var value a_var a_value ret_var ret_value auxiliary_var auxiliary_value prev_varss prev_valuess widthss heightss max_proofs_verified branches. ?self:(var, value, max_proofs_verified, branches) Tag.t - -> ?cache:Key_cache.Spec.t list + -> ?cache:_ Cache.Spec.t -> ?proof_cache:Proof_cache.t -> ?disk_keys: (Cache.Step.Key.Verification.t, branches) Vector.t @@ -991,7 +1001,7 @@ let compile_with_wrap_main_override_promise : (* This function is an adapter between the user-facing Pickles.compile API and the underlying Make(_).compile function which builds the circuits. *) - fun ?self ?(cache = []) ?proof_cache ?disk_keys + fun ?self ?(cache = Cache.Spec.default) ?proof_cache ?disk_keys ?(return_early_digest_exception = false) ?override_wrap_domain ?override_wrap_main ~public_input ~auxiliary_typ ~branches ~max_proofs_verified ~name ~constraint_constants ~choices () -> diff --git a/src/lib/pickles/compile.mli b/src/lib/pickles/compile.mli index 8adb2de2955..f26f7d4795c 100644 --- a/src/lib/pickles/compile.mli +++ b/src/lib/pickles/compile.mli @@ -269,7 +269,7 @@ type ('max_proofs_verified, 'branches, 'prev_varss) wrap_main_generic = to each inductive rule. *) val compile_with_wrap_main_override_promise : ?self:('var, 'value, 'max_proofs_verified, 'branches) Tag.t - -> ?cache:Key_cache.Spec.t list + -> ?cache:_ Cache.Spec.t -> ?proof_cache:Proof_cache.t -> ?disk_keys: (Cache.Step.Key.Verification.t, 'branches) Vector.t diff --git a/src/lib/pickles/pickles_intf.mli b/src/lib/pickles/pickles_intf.mli index 11898ca4bdc..52b63f342dc 100644 --- a/src/lib/pickles/pickles_intf.mli +++ b/src/lib/pickles/pickles_intf.mli @@ -365,7 +365,7 @@ module type S = sig to each inductive rule. *) val compile_promise : ?self:('var, 'value, 'max_proofs_verified, 'branches) Tag.t - -> ?cache:Key_cache.Spec.t list + -> ?cache:_ Cache.Spec.t -> ?proof_cache:Proof_cache.t -> ?disk_keys: (Cache.Step.Key.Verification.t, 'branches) Vector.t @@ -420,7 +420,7 @@ module type S = sig to each inductive rule. *) val compile : ?self:('var, 'value, 'max_proofs_verified, 'branches) Tag.t - -> ?cache:Key_cache.Spec.t list + -> ?cache:_ Cache.Spec.t -> ?proof_cache:Proof_cache.t -> ?disk_keys: (Cache.Step.Key.Verification.t, 'branches) Vector.t From 2af59c195cba4249183eb64253b8a0bd291fff56 Mon Sep 17 00:00:00 2001 From: Gregor Date: Wed, 18 Oct 2023 12:36:45 +0200 Subject: [PATCH 2/8] remove genericness --- src/lib/pickles/cache.ml | 26 ++++++++--------- src/lib/pickles/cache.mli | 49 ++++++++++++++------------------ src/lib/pickles/compile.ml | 4 +-- src/lib/pickles/compile.mli | 2 +- src/lib/pickles/pickles_intf.mli | 4 +-- 5 files changed, 40 insertions(+), 45 deletions(-) diff --git a/src/lib/pickles/cache.ml b/src/lib/pickles/cache.ml index 034b8e33c24..22203767805 100644 --- a/src/lib/pickles/cache.ml +++ b/src/lib/pickles/cache.ml @@ -27,11 +27,11 @@ module Step = struct [@@warning "-4"] end - type 'header storable = - ('header, Backend.Tick.Keypair.t) Key_cache.Sync.Disk_storable.t + type storable = + (Key.Proving.t, Backend.Tick.Keypair.t) Key_cache.Sync.Disk_storable.t - type 'header vk_storable = - ( 'header + type vk_storable = + ( Key.Verification.t , Kimchi_bindings.Protocol.VerifierIndex.Fp.t ) Key_cache.Sync.Disk_storable.t @@ -162,11 +162,11 @@ module Wrap = struct end end - type 'header storable = - ('header, Backend.Tock.Keypair.t) Key_cache.Sync.Disk_storable.t + type storable = + (Key.Proving.t, Backend.Tock.Keypair.t) Key_cache.Sync.Disk_storable.t - type 'header vk_storable = - ('header, Verification_key.t) Key_cache.Sync.Disk_storable.t + type vk_storable = + (Key.Verification.t, Verification_key.t) Key_cache.Sync.Disk_storable.t let storable = Key_cache.Sync.Disk_storable.simple Key.Proving.to_string @@ -278,12 +278,12 @@ module Wrap = struct end module Spec = struct - type ('step_pk_header, 'step_vk_header, 'wrap_pk_header, 'wrap_vk_header) t = + type t = { cache : Key_cache.Spec.t list - ; step_storable : 'step_pk_header Step.storable - ; step_vk_storable : 'step_vk_header Step.vk_storable - ; wrap_storable : 'wrap_pk_header Wrap.storable - ; wrap_vk_storable : 'wrap_vk_header Wrap.vk_storable + ; step_storable : Step.storable + ; step_vk_storable : Step.vk_storable + ; wrap_storable : Wrap.storable + ; wrap_vk_storable : Wrap.vk_storable } let default = diff --git a/src/lib/pickles/cache.mli b/src/lib/pickles/cache.mli index 809bcf9c486..a3b04eb850c 100644 --- a/src/lib/pickles/cache.mli +++ b/src/lib/pickles/cache.mli @@ -20,23 +20,23 @@ module Step : sig end end - type 'header storable = - ('header, Backend.Tick.Keypair.t) Key_cache.Sync.Disk_storable.t + type storable = + (Key.Proving.t, Backend.Tick.Keypair.t) Key_cache.Sync.Disk_storable.t - type 'header vk_storable = - ( 'header + type vk_storable = + ( Key.Verification.t , Kimchi_bindings.Protocol.VerifierIndex.Fp.t ) Key_cache.Sync.Disk_storable.t - val storable : Key.Proving.t storable + val storable : storable - val vk_storable : Key.Verification.t vk_storable + val vk_storable : vk_storable val read_or_generate : prev_challenges:int -> Key_cache.Spec.t list - -> 'pk_header lazy_t * 'pk_header storable - -> 'vk_header lazy_t * 'vk_header vk_storable + -> Key.Proving.t lazy_t * storable + -> Key.Verification.t lazy_t * vk_storable -> ('a, 'b) Impls.Step.Typ.t -> ('c, 'd) Impls.Step.Typ.t -> ('a -> unit -> 'c) @@ -71,21 +71,21 @@ module Wrap : sig end end - type 'header storable = - ('header, Backend.Tock.Keypair.t) Key_cache.Sync.Disk_storable.t + type storable = + (Key.Proving.t, Backend.Tock.Keypair.t) Key_cache.Sync.Disk_storable.t - type 'header vk_storable = - ('header, Verification_key.t) Key_cache.Sync.Disk_storable.t + type vk_storable = + (Key.Verification.t, Verification_key.t) Key_cache.Sync.Disk_storable.t - val storable : Key.Proving.t storable + val storable : storable - val vk_storable : Key.Verification.t vk_storable + val vk_storable : vk_storable val read_or_generate : prev_challenges:Core_kernel.Int.t -> Key_cache.Spec.t list - -> 'pk_header lazy_t * 'pk_header storable - -> 'vk_header lazy_t * 'vk_header vk_storable + -> Key.Proving.t lazy_t * storable + -> Key.Verification.t lazy_t * vk_storable -> ('a, 'b) Impls.Wrap.Typ.t -> ('c, 'd) Impls.Wrap.Typ.t -> ('a -> unit -> 'c) @@ -98,18 +98,13 @@ module Wrap : sig end module Spec : sig - type ('step_pk_header, 'step_vk_header, 'wrap_pk_header, 'wrap_vk_header) t = + type t = { cache : Key_cache.Spec.t list - ; step_storable : 'step_pk_header Step.storable - ; step_vk_storable : 'step_vk_header Step.vk_storable - ; wrap_storable : 'wrap_pk_header Wrap.storable - ; wrap_vk_storable : 'wrap_vk_header Wrap.vk_storable + ; step_storable : Step.storable + ; step_vk_storable : Step.vk_storable + ; wrap_storable : Wrap.storable + ; wrap_vk_storable : Wrap.vk_storable } - val default : - ( Step.Key.Proving.t - , Step.Key.Verification.t - , Wrap.Key.Proving.t - , Wrap.Key.Verification.t ) - t + val default : t end \ No newline at end of file diff --git a/src/lib/pickles/compile.ml b/src/lib/pickles/compile.ml index a554b036b3b..d8d868618d7 100644 --- a/src/lib/pickles/compile.ml +++ b/src/lib/pickles/compile.ml @@ -339,7 +339,7 @@ struct let compile : type var value prev_varss prev_valuess widthss heightss max_proofs_verified branches. self:(var, value, max_proofs_verified, branches) Tag.t - -> cache:_ Cache.Spec.t + -> cache:Cache.Spec.t -> proof_cache:Proof_cache.t option -> ?disk_keys: (Cache.Step.Key.Verification.t, branches) Vector.t @@ -947,7 +947,7 @@ end let compile_with_wrap_main_override_promise : type var value a_var a_value ret_var ret_value auxiliary_var auxiliary_value prev_varss prev_valuess widthss heightss max_proofs_verified branches. ?self:(var, value, max_proofs_verified, branches) Tag.t - -> ?cache:_ Cache.Spec.t + -> ?cache:Cache.Spec.t -> ?proof_cache:Proof_cache.t -> ?disk_keys: (Cache.Step.Key.Verification.t, branches) Vector.t diff --git a/src/lib/pickles/compile.mli b/src/lib/pickles/compile.mli index f26f7d4795c..c5d14b49302 100644 --- a/src/lib/pickles/compile.mli +++ b/src/lib/pickles/compile.mli @@ -269,7 +269,7 @@ type ('max_proofs_verified, 'branches, 'prev_varss) wrap_main_generic = to each inductive rule. *) val compile_with_wrap_main_override_promise : ?self:('var, 'value, 'max_proofs_verified, 'branches) Tag.t - -> ?cache:_ Cache.Spec.t + -> ?cache:Cache.Spec.t -> ?proof_cache:Proof_cache.t -> ?disk_keys: (Cache.Step.Key.Verification.t, 'branches) Vector.t diff --git a/src/lib/pickles/pickles_intf.mli b/src/lib/pickles/pickles_intf.mli index 52b63f342dc..b0479c4e60c 100644 --- a/src/lib/pickles/pickles_intf.mli +++ b/src/lib/pickles/pickles_intf.mli @@ -365,7 +365,7 @@ module type S = sig to each inductive rule. *) val compile_promise : ?self:('var, 'value, 'max_proofs_verified, 'branches) Tag.t - -> ?cache:_ Cache.Spec.t + -> ?cache:Cache.Spec.t -> ?proof_cache:Proof_cache.t -> ?disk_keys: (Cache.Step.Key.Verification.t, 'branches) Vector.t @@ -420,7 +420,7 @@ module type S = sig to each inductive rule. *) val compile : ?self:('var, 'value, 'max_proofs_verified, 'branches) Tag.t - -> ?cache:_ Cache.Spec.t + -> ?cache:Cache.Spec.t -> ?proof_cache:Proof_cache.t -> ?disk_keys: (Cache.Step.Key.Verification.t, 'branches) Vector.t From 52264d50d976cacf3d32f5e85290d61c2f251f0a Mon Sep 17 00:00:00 2001 From: Gregor Date: Wed, 18 Oct 2023 13:05:57 +0200 Subject: [PATCH 3/8] use additional optional params to stay compatible with existing pickles API --- src/lib/pickles/cache.ml | 16 +++++++--------- src/lib/pickles/cache.mli | 17 ++++++++++------- src/lib/pickles/compile.ml | 32 ++++++++++++++------------------ src/lib/pickles/compile.mli | 3 ++- src/lib/pickles/pickles.ml | 22 +++++++++++----------- src/lib/pickles/pickles_intf.mli | 6 ++++-- 6 files changed, 48 insertions(+), 48 deletions(-) diff --git a/src/lib/pickles/cache.ml b/src/lib/pickles/cache.ml index 22203767805..a473f581762 100644 --- a/src/lib/pickles/cache.ml +++ b/src/lib/pickles/cache.ml @@ -91,8 +91,8 @@ module Step = struct (Kimchi_bindings.Protocol.VerifierIndex.Fp.write (Some true) x) header path ) ) - let read_or_generate ~prev_challenges cache (k_p, s_p) (k_v, s_v) typ - return_typ main = + let read_or_generate ~prev_challenges cache ?(s_p = storable) k_p + ?(s_v = vk_storable) k_v typ return_typ main = let open Impls.Step in let pk = lazy @@ -227,8 +227,8 @@ module Wrap = struct t ) ) ) header path ) ) - let read_or_generate ~prev_challenges cache (k_p, s_p) (k_v, s_v) typ - return_typ main = + let read_or_generate ~prev_challenges cache ?(s_p = storable) k_p + ?(s_v = vk_storable) k_v typ return_typ main = let module Vk = Verification_key in let open Impls.Wrap in let pk = @@ -277,18 +277,16 @@ module Wrap = struct (pk, vk) end -module Spec = struct +module Storables = struct type t = - { cache : Key_cache.Spec.t list - ; step_storable : Step.storable + { step_storable : Step.storable ; step_vk_storable : Step.vk_storable ; wrap_storable : Wrap.storable ; wrap_vk_storable : Wrap.vk_storable } let default = - { cache = [] - ; step_storable = Step.storable + { step_storable = Step.storable ; step_vk_storable = Step.vk_storable ; wrap_storable = Wrap.storable ; wrap_vk_storable = Wrap.vk_storable diff --git a/src/lib/pickles/cache.mli b/src/lib/pickles/cache.mli index a3b04eb850c..270781b477c 100644 --- a/src/lib/pickles/cache.mli +++ b/src/lib/pickles/cache.mli @@ -35,8 +35,10 @@ module Step : sig val read_or_generate : prev_challenges:int -> Key_cache.Spec.t list - -> Key.Proving.t lazy_t * storable - -> Key.Verification.t lazy_t * vk_storable + -> ?s_p:storable + -> Key.Proving.t lazy_t + -> ?s_v:vk_storable + -> Key.Verification.t lazy_t -> ('a, 'b) Impls.Step.Typ.t -> ('c, 'd) Impls.Step.Typ.t -> ('a -> unit -> 'c) @@ -84,8 +86,10 @@ module Wrap : sig val read_or_generate : prev_challenges:Core_kernel.Int.t -> Key_cache.Spec.t list - -> Key.Proving.t lazy_t * storable - -> Key.Verification.t lazy_t * vk_storable + -> ?s_p:storable + -> Key.Proving.t lazy_t + -> ?s_v:vk_storable + -> Key.Verification.t lazy_t -> ('a, 'b) Impls.Wrap.Typ.t -> ('c, 'd) Impls.Wrap.Typ.t -> ('a -> unit -> 'c) @@ -97,10 +101,9 @@ module Wrap : sig lazy_t end -module Spec : sig +module Storables : sig type t = - { cache : Key_cache.Spec.t list - ; step_storable : Step.storable + { step_storable : Step.storable ; step_vk_storable : Step.vk_storable ; wrap_storable : Wrap.storable ; wrap_vk_storable : Wrap.vk_storable diff --git a/src/lib/pickles/compile.ml b/src/lib/pickles/compile.ml index d8d868618d7..6c6b700140b 100644 --- a/src/lib/pickles/compile.ml +++ b/src/lib/pickles/compile.ml @@ -339,7 +339,8 @@ struct let compile : type var value prev_varss prev_valuess widthss heightss max_proofs_verified branches. self:(var, value, max_proofs_verified, branches) Tag.t - -> cache:Cache.Spec.t + -> cache:Key_cache.Spec.t list + -> storables:Cache.Storables.t -> proof_cache:Proof_cache.t option -> ?disk_keys: (Cache.Step.Key.Verification.t, branches) Vector.t @@ -378,14 +379,10 @@ struct * _ * _ * _ = - fun ~self - ~cache: - { cache - ; step_storable - ; step_vk_storable - ; wrap_storable - ; wrap_vk_storable - } ~proof_cache ?disk_keys ?(return_early_digest_exception = false) + fun ~self ~cache + ~storables: + { step_storable; step_vk_storable; wrap_storable; wrap_vk_storable } + ~proof_cache ?disk_keys ?(return_early_digest_exception = false) ?override_wrap_domain ?override_wrap_main ~branches:(module Branches) ~max_proofs_verified ~name ~constraint_constants ~public_input ~auxiliary_typ ~choices () -> @@ -602,7 +599,7 @@ struct Common.time "step read or generate" (fun () -> Cache.Step.read_or_generate ~prev_challenges:(Nat.to_int (fst b.proofs_verified)) - cache (k_p, step_storable) (k_v, step_vk_storable) + cache ~s_p:step_storable k_p ~s_v:step_vk_storable k_v (Snarky_backendless.Typ.unit ()) typ main ) in @@ -678,10 +675,8 @@ struct let r = Common.time "wrap read or generate " (fun () -> Cache.Wrap.read_or_generate (* Due to Wrap_hack *) - ~prev_challenges:2 cache - (disk_key_prover, wrap_storable) - (disk_key_verifier, wrap_vk_storable) - typ + ~prev_challenges:2 cache ~s_p:wrap_storable disk_key_prover + ~s_v:wrap_vk_storable disk_key_verifier typ (Snarky_backendless.Typ.unit ()) main ) in @@ -947,7 +942,8 @@ end let compile_with_wrap_main_override_promise : type var value a_var a_value ret_var ret_value auxiliary_var auxiliary_value prev_varss prev_valuess widthss heightss max_proofs_verified branches. ?self:(var, value, max_proofs_verified, branches) Tag.t - -> ?cache:Cache.Spec.t + -> ?cache:Key_cache.Spec.t list + -> ?storables:Cache.Storables.t -> ?proof_cache:Proof_cache.t -> ?disk_keys: (Cache.Step.Key.Verification.t, branches) Vector.t @@ -1001,8 +997,8 @@ let compile_with_wrap_main_override_promise : (* This function is an adapter between the user-facing Pickles.compile API and the underlying Make(_).compile function which builds the circuits. *) - fun ?self ?(cache = Cache.Spec.default) ?proof_cache ?disk_keys - ?(return_early_digest_exception = false) ?override_wrap_domain + fun ?self ?(cache = []) ?(storables = Cache.Storables.default) ?proof_cache + ?disk_keys ?(return_early_digest_exception = false) ?override_wrap_domain ?override_wrap_main ~public_input ~auxiliary_typ ~branches ~max_proofs_verified ~name ~constraint_constants ~choices () -> let self = @@ -1071,7 +1067,7 @@ let compile_with_wrap_main_override_promise : in let provers, wrap_vk, wrap_disk_key, cache_handle = M.compile ~return_early_digest_exception ~self ~proof_cache ~cache - ?disk_keys ?override_wrap_domain ?override_wrap_main ~branches + ~storables ?disk_keys ?override_wrap_domain ?override_wrap_main ~branches ~max_proofs_verified ~name ~public_input ~auxiliary_typ ~constraint_constants ~choices:(fun ~self -> conv_irs (choices ~self)) diff --git a/src/lib/pickles/compile.mli b/src/lib/pickles/compile.mli index c5d14b49302..a527049d957 100644 --- a/src/lib/pickles/compile.mli +++ b/src/lib/pickles/compile.mli @@ -269,7 +269,8 @@ type ('max_proofs_verified, 'branches, 'prev_varss) wrap_main_generic = to each inductive rule. *) val compile_with_wrap_main_override_promise : ?self:('var, 'value, 'max_proofs_verified, 'branches) Tag.t - -> ?cache:Cache.Spec.t + -> ?cache:Key_cache.Spec.t list + -> ?storables:Cache.Storables.t -> ?proof_cache:Proof_cache.t -> ?disk_keys: (Cache.Step.Key.Verification.t, 'branches) Vector.t diff --git a/src/lib/pickles/pickles.ml b/src/lib/pickles/pickles.ml index 8d5690b17ff..18a29631d4d 100644 --- a/src/lib/pickles/pickles.ml +++ b/src/lib/pickles/pickles.ml @@ -306,22 +306,22 @@ module Make_str (_ : Wire_types.Concrete) = struct let compile_with_wrap_main_override_promise = Compile.compile_with_wrap_main_override_promise - let compile_promise ?self ?cache ?proof_cache ?disk_keys + let compile_promise ?self ?cache ?storables ?proof_cache ?disk_keys ?return_early_digest_exception ?override_wrap_domain ~public_input ~auxiliary_typ ~branches ~max_proofs_verified ~name ~constraint_constants ~choices () = - compile_with_wrap_main_override_promise ?self ?cache ?proof_cache ?disk_keys - ?return_early_digest_exception ?override_wrap_domain ~public_input - ~auxiliary_typ ~branches ~max_proofs_verified ~name ~constraint_constants - ~choices () - - let compile ?self ?cache ?proof_cache ?disk_keys ?override_wrap_domain + compile_with_wrap_main_override_promise ?self ?cache ?storables ?proof_cache + ?disk_keys ?return_early_digest_exception ?override_wrap_domain ~public_input ~auxiliary_typ ~branches ~max_proofs_verified ~name - ~constraint_constants ~choices () = + ~constraint_constants ~choices () + + let compile ?self ?cache ?storables ?proof_cache ?disk_keys + ?override_wrap_domain ~public_input ~auxiliary_typ ~branches + ~max_proofs_verified ~name ~constraint_constants ~choices () = let self, cache_handle, proof_module, provers = - compile_promise ?self ?cache ?proof_cache ?disk_keys ?override_wrap_domain - ~public_input ~auxiliary_typ ~branches ~max_proofs_verified ~name - ~constraint_constants ~choices () + compile_promise ?self ?cache ?storables ?proof_cache ?disk_keys + ?override_wrap_domain ~public_input ~auxiliary_typ ~branches + ~max_proofs_verified ~name ~constraint_constants ~choices () in let rec adjust_provers : type a1 a2 a3 s1 s2_inner. diff --git a/src/lib/pickles/pickles_intf.mli b/src/lib/pickles/pickles_intf.mli index b0479c4e60c..7760372cac5 100644 --- a/src/lib/pickles/pickles_intf.mli +++ b/src/lib/pickles/pickles_intf.mli @@ -365,7 +365,8 @@ module type S = sig to each inductive rule. *) val compile_promise : ?self:('var, 'value, 'max_proofs_verified, 'branches) Tag.t - -> ?cache:Cache.Spec.t + -> ?cache:Key_cache.Spec.t list + -> ?storables:Cache.Storables.t -> ?proof_cache:Proof_cache.t -> ?disk_keys: (Cache.Step.Key.Verification.t, 'branches) Vector.t @@ -420,7 +421,8 @@ module type S = sig to each inductive rule. *) val compile : ?self:('var, 'value, 'max_proofs_verified, 'branches) Tag.t - -> ?cache:Cache.Spec.t + -> ?cache:Key_cache.Spec.t list + -> ?storables:Cache.Storables.t -> ?proof_cache:Proof_cache.t -> ?disk_keys: (Cache.Step.Key.Verification.t, 'branches) Vector.t From 96d8caaae3bdd9c6311b3d3de57f5e5f98eb6325 Mon Sep 17 00:00:00 2001 From: Gregor Date: Wed, 18 Oct 2023 14:27:37 +0200 Subject: [PATCH 4/8] expose pickles cache and to_string for pk/vk headers --- src/lib/pickles/cache.mli | 6 ++++++ src/lib/pickles/pickles.ml | 1 + src/lib/pickles/pickles_intf.mli | 1 + 3 files changed, 8 insertions(+) diff --git a/src/lib/pickles/cache.mli b/src/lib/pickles/cache.mli index 270781b477c..b2b4c670769 100644 --- a/src/lib/pickles/cache.mli +++ b/src/lib/pickles/cache.mli @@ -8,6 +8,8 @@ module Step : sig * Snark_keys_header.t * int * Backend.Tick.R1CS_constraint_system.t + + val to_string : t -> string end module Verification : sig @@ -17,6 +19,8 @@ module Step : sig * Snark_keys_header.t * int * Core_kernel.Md5.t + + val to_string : t -> string end end @@ -57,6 +61,8 @@ module Wrap : sig Core_kernel.Type_equal.Id.Uid.t * Snark_keys_header.t * Backend.Tock.R1CS_constraint_system.t + + val to_string : t -> string end module Verification : sig diff --git a/src/lib/pickles/pickles.ml b/src/lib/pickles/pickles.ml index 18a29631d4d..aa03b65c2fb 100644 --- a/src/lib/pickles/pickles.ml +++ b/src/lib/pickles/pickles.ml @@ -44,6 +44,7 @@ module Make_str (_ : Wire_types.Concrete) = struct module Step_main_inputs = Step_main_inputs module Step_verifier = Step_verifier module Proof_cache = Proof_cache + module Cache = Cache exception Return_digest = Compile.Return_digest diff --git a/src/lib/pickles/pickles_intf.mli b/src/lib/pickles/pickles_intf.mli index 7760372cac5..91b67ea4028 100644 --- a/src/lib/pickles/pickles_intf.mli +++ b/src/lib/pickles/pickles_intf.mli @@ -16,6 +16,7 @@ module type S = sig module Step_verifier = Step_verifier module Common = Common module Proof_cache = Proof_cache + module Cache = Cache exception Return_digest of Md5.t From 2f5070fc0d89cac03e154d85fee89cdaba41e205 Mon Sep 17 00:00:00 2001 From: Gregor Date: Fri, 20 Oct 2023 21:04:05 +0200 Subject: [PATCH 5/8] linebreaks --- src/lib/pickles/cache.ml | 2 +- src/lib/pickles/cache.mli | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib/pickles/cache.ml b/src/lib/pickles/cache.ml index a473f581762..ffae599284e 100644 --- a/src/lib/pickles/cache.ml +++ b/src/lib/pickles/cache.ml @@ -291,4 +291,4 @@ module Storables = struct ; wrap_storable = Wrap.storable ; wrap_vk_storable = Wrap.vk_storable } -end \ No newline at end of file +end diff --git a/src/lib/pickles/cache.mli b/src/lib/pickles/cache.mli index b2b4c670769..3409acc0a4b 100644 --- a/src/lib/pickles/cache.mli +++ b/src/lib/pickles/cache.mli @@ -116,4 +116,4 @@ module Storables : sig } val default : t -end \ No newline at end of file +end From d4e042f7a1eea8c8f3d72c64756d1cef0263383a Mon Sep 17 00:00:00 2001 From: Gregor Date: Fri, 20 Oct 2023 22:15:29 +0200 Subject: [PATCH 6/8] add verification key of yojson --- src/lib/pickles/pickles_intf.mli | 2 +- src/lib/pickles/verification_key.ml | 14 ++++++++++++-- src/lib/pickles/verification_key.mli | 4 ++-- 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/src/lib/pickles/pickles_intf.mli b/src/lib/pickles/pickles_intf.mli index 91b67ea4028..2088d037da6 100644 --- a/src/lib/pickles/pickles_intf.mli +++ b/src/lib/pickles/pickles_intf.mli @@ -38,7 +38,7 @@ module type S = sig [%%versioned: module Stable : sig module V2 : sig - type t [@@deriving to_yojson] + type t [@@deriving to_yojson, of_yojson] end end] diff --git a/src/lib/pickles/verification_key.ml b/src/lib/pickles/verification_key.ml index 79fca91df0f..cee31573c0e 100644 --- a/src/lib/pickles/verification_key.ml +++ b/src/lib/pickles/verification_key.ml @@ -100,6 +100,11 @@ module Verifier_index_json = struct verifier_index_to_yojson fp (fun _ -> `Null) (polycomm_to_yojson (or_infinity_to_yojson fq)) + + let of_yojson fp fq = + verifier_index_of_yojson fp + (fun _ -> Ok (Backend.Tock.Keypair.load_urs ())) + (polycomm_of_yojson (or_infinity_of_yojson fq)) end module Data = struct @@ -139,10 +144,13 @@ module Stable = struct (Impls.Wrap.Verification_key.t [@to_yojson Verifier_index_json.to_yojson Backend.Tock.Field.to_yojson - Backend.Tick.Field.to_yojson] ) + Backend.Tick.Field.to_yojson] + [@of_yojson + Verifier_index_json.of_yojson Backend.Tock.Field.of_yojson + Backend.Tick.Field.of_yojson] ) ; data : Data.t } - [@@deriving fields, to_yojson] + [@@deriving fields, to_yojson, of_yojson] let to_latest = Fn.id @@ -208,6 +216,8 @@ end] let to_yojson = Stable.Latest.to_yojson +let of_yojson = Stable.Latest.of_yojson + let dummy_commitments g = let open Plonk_types in { Plonk_verification_key_evals.sigma_comm = diff --git a/src/lib/pickles/verification_key.mli b/src/lib/pickles/verification_key.mli index b1e289df5ce..a5d8d6844f8 100644 --- a/src/lib/pickles/verification_key.mli +++ b/src/lib/pickles/verification_key.mli @@ -19,7 +19,7 @@ module Stable : sig ; index : Impls.Wrap.Verification_key.t ; data : Data.t } - [@@deriving fields, to_yojson, bin_shape, bin_io] + [@@deriving fields, to_yojson, of_yojson, bin_shape, bin_io] include Pickles_types.Sigs.VERSIONED end @@ -33,7 +33,7 @@ type t = Stable.Latest.t = ; index : Impls.Wrap.Verification_key.t ; data : Data.t } -[@@deriving fields, to_yojson] +[@@deriving fields, to_yojson, of_yojson] val dummy_commitments : 'a -> 'a Pickles_types.Plonk_verification_key_evals.t From 574b275d0f4e2e7a6f05aad2211057022c80db5d Mon Sep 17 00:00:00 2001 From: Gregor Date: Wed, 25 Oct 2023 15:41:30 +0200 Subject: [PATCH 7/8] move storables module into pickles --- src/lib/pickles/cache.ml | 16 ---------------- src/lib/pickles/cache.mli | 11 ----------- src/lib/pickles/compile.ml | 22 +++++++++++++++++++--- src/lib/pickles/compile.mli | 13 ++++++++++++- src/lib/pickles/pickles.ml | 1 + src/lib/pickles/pickles_intf.mli | 8 ++++++-- 6 files changed, 38 insertions(+), 33 deletions(-) diff --git a/src/lib/pickles/cache.ml b/src/lib/pickles/cache.ml index ffae599284e..75e6c55c580 100644 --- a/src/lib/pickles/cache.ml +++ b/src/lib/pickles/cache.ml @@ -276,19 +276,3 @@ module Wrap = struct in (pk, vk) end - -module Storables = struct - type t = - { step_storable : Step.storable - ; step_vk_storable : Step.vk_storable - ; wrap_storable : Wrap.storable - ; wrap_vk_storable : Wrap.vk_storable - } - - let default = - { step_storable = Step.storable - ; step_vk_storable = Step.vk_storable - ; wrap_storable = Wrap.storable - ; wrap_vk_storable = Wrap.vk_storable - } -end diff --git a/src/lib/pickles/cache.mli b/src/lib/pickles/cache.mli index 3409acc0a4b..7603f94be71 100644 --- a/src/lib/pickles/cache.mli +++ b/src/lib/pickles/cache.mli @@ -106,14 +106,3 @@ module Wrap : sig * [> `Cache_hit | `Generated_something | `Locally_generated ] ) lazy_t end - -module Storables : sig - type t = - { step_storable : Step.storable - ; step_vk_storable : Step.vk_storable - ; wrap_storable : Wrap.storable - ; wrap_vk_storable : Wrap.vk_storable - } - - val default : t -end diff --git a/src/lib/pickles/compile.ml b/src/lib/pickles/compile.ml index 6c6b700140b..b840b0c2b46 100644 --- a/src/lib/pickles/compile.ml +++ b/src/lib/pickles/compile.ml @@ -227,6 +227,22 @@ type ('max_proofs_verified, 'branches, 'prev_varss) wrap_main_generic = *) } +module Storables = struct + type t = + { step_storable : Cache.Step.storable + ; step_vk_storable : Cache.Step.vk_storable + ; wrap_storable : Cache.Wrap.storable + ; wrap_vk_storable : Cache.Wrap.vk_storable + } + + let default = + { step_storable = Cache.Step.storable + ; step_vk_storable = Cache.Step.vk_storable + ; wrap_storable = Cache.Wrap.storable + ; wrap_vk_storable = Cache.Wrap.vk_storable + } +end + module Make (Arg_var : Statement_var_intf) (Arg_value : Statement_value_intf) @@ -340,7 +356,7 @@ struct type var value prev_varss prev_valuess widthss heightss max_proofs_verified branches. self:(var, value, max_proofs_verified, branches) Tag.t -> cache:Key_cache.Spec.t list - -> storables:Cache.Storables.t + -> storables:Storables.t -> proof_cache:Proof_cache.t option -> ?disk_keys: (Cache.Step.Key.Verification.t, branches) Vector.t @@ -943,7 +959,7 @@ let compile_with_wrap_main_override_promise : type var value a_var a_value ret_var ret_value auxiliary_var auxiliary_value prev_varss prev_valuess widthss heightss max_proofs_verified branches. ?self:(var, value, max_proofs_verified, branches) Tag.t -> ?cache:Key_cache.Spec.t list - -> ?storables:Cache.Storables.t + -> ?storables:Storables.t -> ?proof_cache:Proof_cache.t -> ?disk_keys: (Cache.Step.Key.Verification.t, branches) Vector.t @@ -997,7 +1013,7 @@ let compile_with_wrap_main_override_promise : (* This function is an adapter between the user-facing Pickles.compile API and the underlying Make(_).compile function which builds the circuits. *) - fun ?self ?(cache = []) ?(storables = Cache.Storables.default) ?proof_cache + fun ?self ?(cache = []) ?(storables = Storables.default) ?proof_cache ?disk_keys ?(return_early_digest_exception = false) ?override_wrap_domain ?override_wrap_main ~public_input ~auxiliary_typ ~branches ~max_proofs_verified ~name ~constraint_constants ~choices () -> diff --git a/src/lib/pickles/compile.mli b/src/lib/pickles/compile.mli index a527049d957..0e6a145f804 100644 --- a/src/lib/pickles/compile.mli +++ b/src/lib/pickles/compile.mli @@ -264,13 +264,24 @@ type ('max_proofs_verified, 'branches, 'prev_varss) wrap_main_generic = *) } +module Storables : sig + type t = + { step_storable : Cache.Step.storable + ; step_vk_storable : Cache.Step.vk_storable + ; wrap_storable : Cache.Wrap.storable + ; wrap_vk_storable : Cache.Wrap.vk_storable + } + + val default : t +end + (** This compiles a series of inductive rules defining a set into a proof system for proving membership in that set, with a prover corresponding to each inductive rule. *) val compile_with_wrap_main_override_promise : ?self:('var, 'value, 'max_proofs_verified, 'branches) Tag.t -> ?cache:Key_cache.Spec.t list - -> ?storables:Cache.Storables.t + -> ?storables:Storables.t -> ?proof_cache:Proof_cache.t -> ?disk_keys: (Cache.Step.Key.Verification.t, 'branches) Vector.t diff --git a/src/lib/pickles/pickles.ml b/src/lib/pickles/pickles.ml index aa03b65c2fb..f9d87c62194 100644 --- a/src/lib/pickles/pickles.ml +++ b/src/lib/pickles/pickles.ml @@ -45,6 +45,7 @@ module Make_str (_ : Wire_types.Concrete) = struct module Step_verifier = Step_verifier module Proof_cache = Proof_cache module Cache = Cache + module Storables = Compile.Storables exception Return_digest = Compile.Return_digest diff --git a/src/lib/pickles/pickles_intf.mli b/src/lib/pickles/pickles_intf.mli index 2088d037da6..83bac67504d 100644 --- a/src/lib/pickles/pickles_intf.mli +++ b/src/lib/pickles/pickles_intf.mli @@ -276,6 +276,10 @@ module type S = sig val generate_or_load : t -> Dirty.t end + module Storables : sig + type t = Compile.Storables.t + end + module Side_loaded : sig module Verification_key : sig [%%versioned: @@ -367,7 +371,7 @@ module type S = sig val compile_promise : ?self:('var, 'value, 'max_proofs_verified, 'branches) Tag.t -> ?cache:Key_cache.Spec.t list - -> ?storables:Cache.Storables.t + -> ?storables:Storables.t -> ?proof_cache:Proof_cache.t -> ?disk_keys: (Cache.Step.Key.Verification.t, 'branches) Vector.t @@ -423,7 +427,7 @@ module type S = sig val compile : ?self:('var, 'value, 'max_proofs_verified, 'branches) Tag.t -> ?cache:Key_cache.Spec.t list - -> ?storables:Cache.Storables.t + -> ?storables:Storables.t -> ?proof_cache:Proof_cache.t -> ?disk_keys: (Cache.Step.Key.Verification.t, 'branches) Vector.t From 5aff374a360e5e33470905563505c46134fca65c Mon Sep 17 00:00:00 2001 From: Gregor Date: Mon, 30 Oct 2023 16:49:27 +0100 Subject: [PATCH 8/8] submodules --- src/lib/crypto/proof-systems | 2 +- src/lib/snarkyjs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib/crypto/proof-systems b/src/lib/crypto/proof-systems index b62865a375e..dab2bde4329 160000 --- a/src/lib/crypto/proof-systems +++ b/src/lib/crypto/proof-systems @@ -1 +1 @@ -Subproject commit b62865a375ef9c962cbbcdcdf6a9e49d82f7fe4e +Subproject commit dab2bde432991496baf14b214b1f982be82201a4 diff --git a/src/lib/snarkyjs b/src/lib/snarkyjs index 2308b743fb8..29144ff3e5c 160000 --- a/src/lib/snarkyjs +++ b/src/lib/snarkyjs @@ -1 +1 @@ -Subproject commit 2308b743fb84b7a5de821b96bfac3d34e1fe736f +Subproject commit 29144ff3e5cd9ac8f5137da7083706ca2d0e209e