From a15520392e04c6acb1e1cce4a176aacdfcf6c752 Mon Sep 17 00:00:00 2001 From: Shuning Date: Fri, 13 Jun 2025 17:27:14 +0800 Subject: [PATCH 01/12] feat: add disco package --- core/Cargo.lock | 444 +++++++++++++++++++++++++++- core/Cargo.toml | 5 +- core/disco/Cargo.toml | 42 +++ core/disco/build.rs | 54 ++++ core/disco/src/client.rs | 67 +++++ core/disco/src/error.rs | 21 ++ core/disco/src/lib.rs | 9 + core/disco/src/protobuf/common.rs | 8 + core/disco/src/protobuf/domainpb.rs | 112 +++++++ core/disco/src/protobuf/mod.rs | 5 + core/disco/src/protobuf/msgpb.rs | 70 +++++ core/disco/src/protobuf/nodepb.rs | 102 +++++++ core/disco/src/protobuf/portalpb.rs | 63 ++++ core/disco/src/utils.rs | 69 +++++ core/disco/src/wasm.rs | 45 +++ core/protobuf/disco/domain.proto | 106 +++++++ core/protobuf/disco/msg.proto | 41 +++ core/protobuf/disco/node.proto | 103 +++++++ core/protobuf/disco/portal.proto | 46 +++ 19 files changed, 1404 insertions(+), 8 deletions(-) create mode 100644 core/disco/Cargo.toml create mode 100644 core/disco/build.rs create mode 100644 core/disco/src/client.rs create mode 100644 core/disco/src/error.rs create mode 100644 core/disco/src/lib.rs create mode 100644 core/disco/src/protobuf/common.rs create mode 100644 core/disco/src/protobuf/domainpb.rs create mode 100644 core/disco/src/protobuf/mod.rs create mode 100644 core/disco/src/protobuf/msgpb.rs create mode 100644 core/disco/src/protobuf/nodepb.rs create mode 100644 core/disco/src/protobuf/portalpb.rs create mode 100644 core/disco/src/utils.rs create mode 100644 core/disco/src/wasm.rs create mode 100644 core/protobuf/disco/domain.proto create mode 100644 core/protobuf/disco/msg.proto create mode 100644 core/protobuf/disco/node.proto create mode 100644 core/protobuf/disco/portal.proto diff --git a/core/Cargo.lock b/core/Cargo.lock index b08e94a9..7928edd5 100644 --- a/core/Cargo.lock +++ b/core/Cargo.lock @@ -67,6 +67,21 @@ version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" +[[package]] +name = "android-tzdata" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + [[package]] name = "ansi_term" version = "0.12.1" @@ -228,6 +243,12 @@ dependencies = [ "syn 2.0.100", ] +[[package]] +name = "asn1_der" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "155a5a185e42c6b77ac7b88a15143d930a9e9727a5b7b77eed417404ab15c247" + [[package]] name = "async-io" version = "2.4.0" @@ -543,6 +564,20 @@ dependencies = [ "zeroize", ] +[[package]] +name = "chrono" +version = "0.4.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c469d952047f47f91b68d1cba3f10d63c11d73e4636f24f08daf0278abf01c4d" +dependencies = [ + "android-tzdata", + "iana-time-zone", + "js-sys", + "num-traits", + "wasm-bindgen", + "windows-link", +] + [[package]] name = "cipher" version = "0.4.4" @@ -1025,6 +1060,12 @@ version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" +[[package]] +name = "fixedbitset" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" + [[package]] name = "fnv" version = "1.0.7" @@ -1037,6 +1078,21 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + [[package]] name = "form_urlencoded" version = "1.2.1" @@ -1533,6 +1589,30 @@ dependencies = [ "tracing", ] +[[package]] +name = "iana-time-zone" +version = "0.1.63" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0c919e5debc312ad217002b8048a17b7d83f80703865bbfcfebb0458b0b27d8" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core 0.61.2", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + [[package]] name = "icu_collections" version = "1.5.0" @@ -1801,6 +1881,15 @@ version = "1.70.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.15" @@ -1832,6 +1921,20 @@ dependencies = [ "simple_asn1", ] +[[package]] +name = "k256" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6e3919bbaa2945715f0bb6d3934a173d1e9a59ac23767fbaaef277265a7411b" +dependencies = [ + "cfg-if", + "ecdsa", + "elliptic-curve", + "once_cell", + "sha2", + "signature", +] + [[package]] name = "lazy_static" version = "1.5.0" @@ -2050,19 +2153,23 @@ dependencies = [ [[package]] name = "libp2p-identity" -version = "0.2.10" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "257b5621d159b32282eac446bed6670c39c7dc68a200a992d8f056afa0066f6d" +checksum = "fbb68ea10844211a59ce46230909fd0ea040e8a192454d4cc2ee0d53e12280eb" dependencies = [ + "asn1_der", "bs58", "ed25519-dalek", "hkdf", + "k256", "multihash", + "p256", "quick-protobuf", "rand 0.8.5", + "sec1", "serde", "sha2", - "thiserror 1.0.69", + "thiserror 2.0.12", "tracing", "zeroize", ] @@ -2660,6 +2767,12 @@ dependencies = [ "unsigned-varint", ] +[[package]] +name = "multimap" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" + [[package]] name = "multistream-select" version = "0.13.0" @@ -2673,6 +2786,23 @@ dependencies = [ "unsigned-varint", ] +[[package]] +name = "native-tls" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87de3442987e9dbec73158d5c715e7ad9072fda936bb03d19d7fa10e00520f0e" +dependencies = [ + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + [[package]] name = "netlink-packet-core" version = "0.7.0" @@ -2859,6 +2989,50 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" +[[package]] +name = "openssl" +version = "0.10.73" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8505734d46c8ab1e19a1dce3aef597ad87dcb4c37e7188231769bd6bd51cebf8" +dependencies = [ + "bitflags 2.9.0", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", +] + +[[package]] +name = "openssl-probe" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" + +[[package]] +name = "openssl-sys" +version = "0.9.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90096e2e47630d78b7d1c20952dc621f957103f8bc2c8359ec81290d75238571" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + [[package]] name = "overload" version = "0.1.1" @@ -2961,6 +3135,16 @@ version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" +[[package]] +name = "petgraph" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772" +dependencies = [ + "fixedbitset", + "indexmap", +] + [[package]] name = "phf" version = "0.11.3" @@ -3021,6 +3205,12 @@ dependencies = [ "spki", ] +[[package]] +name = "pkg-config" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" + [[package]] name = "polling" version = "3.7.4" @@ -3065,6 +3255,33 @@ version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "350e9b48cbc6b0e028b0473b114454c6316e57336ee184ceab6e53f72c178b3e" +[[package]] +name = "posemesh-disco" +version = "0.1.0" +dependencies = [ + "base64 0.22.1", + "chrono", + "console_error_panic_hook", + "futures", + "hex", + "k256", + "libp2p-identity", + "posemesh-utils", + "prost", + "prost-build", + "prost-types", + "thiserror 2.0.12", + "tiny-keccak", + "tokio", + "tokio-tungstenite", + "tracing", + "tracing-wasm", + "uuid", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "posemesh-domain" version = "0.1.0" @@ -3080,6 +3297,7 @@ dependencies = [ "libp2p", "mockall", "pb-rs", + "posemesh-disco", "posemesh-networking", "posemesh-runtime", "posemesh-utils", @@ -3148,7 +3366,9 @@ version = "0.1.0" dependencies = [ "futures", "gloo-timers 0.3.0", + "hex", "js-sys", + "libp2p-identity", "tokio", "tracing", "wasm-bindgen", @@ -3229,6 +3449,16 @@ dependencies = [ "termtree", ] +[[package]] +name = "prettyplease" +version = "0.2.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dee91521343f4c5c6a63edd65e54f31f5c92fe8978c40a4282f8372194c6a7d" +dependencies = [ + "proc-macro2", + "syn 2.0.100", +] + [[package]] name = "primeorder" version = "0.13.6" @@ -3270,6 +3500,58 @@ dependencies = [ "syn 2.0.100", ] +[[package]] +name = "prost" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-build" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" +dependencies = [ + "heck 0.5.0", + "itertools", + "log", + "multimap", + "once_cell", + "petgraph", + "prettyplease", + "prost", + "prost-types", + "regex", + "syn 2.0.100", + "tempfile", +] + +[[package]] +name = "prost-derive" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" +dependencies = [ + "anyhow", + "itertools", + "proc-macro2", + "quote", + "syn 2.0.100", +] + +[[package]] +name = "prost-types" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52c2c1bf36ddb1a1c396b3601a3cec27c2462e45f07c386894ec3ccf5332bd16" +dependencies = [ + "prost", +] + [[package]] name = "quick-error" version = "1.2.3" @@ -3771,6 +4053,15 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "schannel" +version = "0.1.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f29ebaa345f945cec9fbbc532eb307f0fdad8161f281b6369539c8d84876b3d" +dependencies = [ + "windows-sys 0.59.0", +] + [[package]] name = "scopeguard" version = "1.2.0" @@ -3813,6 +4104,29 @@ dependencies = [ "zeroize", ] +[[package]] +name = "security-framework" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" +dependencies = [ + "bitflags 2.9.0", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49db231d56a190491cb4aeda9527f1ad45345af50b0851622a7adb8c03b01c32" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "semver" version = "1.0.26" @@ -4331,6 +4645,15 @@ dependencies = [ "time-core", ] +[[package]] +name = "tiny-keccak" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" +dependencies = [ + "crunchy", +] + [[package]] name = "tinystr" version = "0.7.6" @@ -4395,6 +4718,16 @@ dependencies = [ "syn 2.0.100", ] +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", +] + [[package]] name = "tokio-postgres" version = "0.7.13" @@ -4421,6 +4754,20 @@ dependencies = [ "whoami", ] +[[package]] +name = "tokio-tungstenite" +version = "0.26.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a9daff607c6d2bf6c16fd681ccb7eecc83e4e2cdc1ca067ffaadfca5de7f084" +dependencies = [ + "futures-util", + "log", + "native-tls", + "tokio", + "tokio-native-tls", + "tungstenite", +] + [[package]] name = "tokio-util" version = "0.7.14" @@ -4553,6 +4900,24 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "tungstenite" +version = "0.26.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4793cb5e56680ecbb1d843515b23b6de9a75eb04b66643e256a396d43be33c13" +dependencies = [ + "bytes", + "data-encoding", + "http 1.3.1", + "httparse", + "log", + "native-tls", + "rand 0.9.1", + "sha1", + "thiserror 2.0.12", + "utf-8", +] + [[package]] name = "turn" version = "0.7.1" @@ -4669,6 +5034,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "utf-8" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" + [[package]] name = "utf16_iter" version = "1.0.5" @@ -4689,9 +5060,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.16.0" +version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "458f7a779bf54acc9f347480ac654f68407d3aab21269a6e3c9f922acd9e2da9" +checksum = "3cf4199d1e5d15ddd86a694e4d0dffa9c323ce759fea589f00fef9d81cc1931d" dependencies = [ "getrandom 0.3.2", "js-sys", @@ -4704,6 +5075,12 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + [[package]] name = "vec_map" version = "0.8.2" @@ -5170,7 +5547,7 @@ version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "efc5cf48f83140dcaab716eeaea345f9e93d0018fb81162753a3f76c3397b538" dependencies = [ - "windows-core", + "windows-core 0.53.0", "windows-targets 0.52.6", ] @@ -5180,10 +5557,45 @@ version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9dcc5b895a6377f1ab9fa55acedab1fd5ac0db66ad1e6c7f47e28a22e446a5dd" dependencies = [ - "windows-result", + "windows-result 0.1.2", "windows-targets 0.52.6", ] +[[package]] +name = "windows-core" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0fdd3ddb90610c7638aa2b3a3ab2904fb9e5cdbecc643ddb3647212781c4ae3" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result 0.3.4", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a47fddd13af08290e67f4acabf4b459f647552718f683a7b415d290ac744a836" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", +] + +[[package]] +name = "windows-interface" +version = "0.59.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd9211b69f8dcdfa817bfd14bf1c97c9188afa36f4750130fcdf3f400eca9fa8" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", +] + [[package]] name = "windows-link" version = "0.1.1" @@ -5199,6 +5611,24 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "windows-result" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56f42bd332cc6c8eac5af113fc0c1fd6a8fd2aa08a0119358686e5160d0586c6" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56e6c93f3a0c3b36176cb1327a4958a0353d5d166c2a35cb268ace15e91d3b57" +dependencies = [ + "windows-link", +] + [[package]] name = "windows-sys" version = "0.48.0" diff --git a/core/Cargo.toml b/core/Cargo.toml index b863b123..ca890cc2 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -8,7 +8,7 @@ members = [ "examples/relay", "examples/test-concurrent", "utils" -] +, "disco"] resolver = "2" [workspace.package] @@ -39,10 +39,13 @@ posemesh-networking = { path = "networking" } posemesh-runtime = { path = "runtime" } posemesh-domain = { path = "domain" } posemesh-utils = { path = "utils" } +posemesh-disco = { path = "disco" } async-trait = "0.1.88" thiserror = "2.0.12" mockall = "0.13.1" wasm-bindgen-test = "0.3.50" +libp2p-identity = "0.2.11" +hex = "0.4.3" [profile.release] strip = true diff --git a/core/disco/Cargo.toml b/core/disco/Cargo.toml new file mode 100644 index 00000000..6f6870ae --- /dev/null +++ b/core/disco/Cargo.toml @@ -0,0 +1,42 @@ +[package] +name = "posemesh-disco" +version = "0.1.0" +edition = "2021" +rust-version.workspace = true + +[build-dependencies] +prost-build = "0.13.5" + +[dependencies] +prost = "0.13" +prost-types = "0.13.5" +thiserror = { workspace = true } +tracing = { workspace = true } +futures = { workspace = true } +chrono = "0.4.41" +libp2p-identity = { workspace = true, features = ["ecdsa"] } +hex = "0.4.3" +uuid = { version = "1.17.0", features = ["v4"] } +base64 = "0.22.1" +tiny-keccak = { version = "2.0.2", features = ["keccak"] } +k256 = { version = "0.13.4", features = ["ecdsa"] } + +[target.'cfg(not(target_family="wasm"))'.dependencies] +tokio = { workspace = true, features = ["full"] } +tokio-tungstenite = { version = "*", features = ["native-tls"] } + +[target.'cfg(target_family="wasm")'.dependencies] +wasm-bindgen = { workspace = true } +wasm-bindgen-futures = { workspace = true } +tracing-wasm = { workspace = true } +console_error_panic_hook = { workspace = true } +web-sys = { version = "0.3.77", features = ["ErrorEvent", "WebSocket", "MessageEvent"] } + +[dev-dependencies] +posemesh-utils = { workspace = true } + +[features] +default = [] +c = [] +node = [] +node-c = [] diff --git a/core/disco/build.rs b/core/disco/build.rs new file mode 100644 index 00000000..3da533e4 --- /dev/null +++ b/core/disco/build.rs @@ -0,0 +1,54 @@ +use std::{path::{Path, PathBuf}, fs}; + +fn main() { + let out_dir = Path::new("./src").join("protobuf"); + + let in_dir = PathBuf::from("../protobuf").join("disco"); + // Re-run this build.rs if the protos dir changes (i.e. a new file is added) + println!("cargo:rerun-if-changed={}", in_dir.to_str().unwrap()); + + // Find all *.proto files in the `in_dir` and add them to the list of files + let mut protos = Vec::new(); + let proto_ext = Some(Path::new("proto").as_os_str()); + fs::create_dir_all(&in_dir).expect("cant create input dir"); + let dir = fs::read_dir(in_dir.clone()).unwrap(); + + let mut mod_rs = String::new(); + for entry in dir { + let path = entry.unwrap().path(); + if path.extension() == proto_ext { + // Re-run this build.rs if any of the files in the protos dir change + println!("cargo:rerun-if-changed={}", path.to_str().unwrap()); + protos.push(path.clone()); + mod_rs.push_str(&format!("pub mod {}pb;\n", path.file_stem().unwrap().to_string_lossy().to_string())); + // mod_rs.push_str(&format!(" include!(concat!(\"{}\", \"/{}.rs\"));\n", out_dir.as_os_str().to_string_lossy().to_string(), path.file_stem().unwrap().to_string_lossy().to_string())); + } + } + + // Add common proto files + let common_dir = PathBuf::from("../protobuf").join("common"); + println!("cargo:rerun-if-changed={}", common_dir.to_str().unwrap()); + + let common_dir_entries = fs::read_dir(common_dir.clone()).unwrap(); + for entry in common_dir_entries { + let path = entry.unwrap().path(); + if path.extension() == proto_ext { + println!("cargo:rerun-if-changed={}", path.to_str().unwrap()); + protos.push(path.clone()); + } + } + mod_rs.push_str("pub mod common;\n"); + + // Delete all old generated files before re-generating new ones + if out_dir.exists() { + std::fs::remove_dir_all(&out_dir).unwrap(); + } + std::fs::DirBuilder::new().create(&out_dir).unwrap(); + + // Configure and run prost-build + let mut config = prost_build::Config::new(); + config.out_dir(&out_dir); + config.compile_protos(&protos, &[in_dir, common_dir]).unwrap(); + + fs::write(out_dir.join("mod.rs"), mod_rs).expect("Failed to write mod.rs"); +} diff --git a/core/disco/src/client.rs b/core/disco/src/client.rs new file mode 100644 index 00000000..21059977 --- /dev/null +++ b/core/disco/src/client.rs @@ -0,0 +1,67 @@ +use base64::{alphabet::STANDARD, engine::general_purpose, Engine}; +use libp2p_identity::secp256k1::Keypair; +use prost::Message; +use tokio::{net::TcpStream, spawn}; +use tokio_tungstenite::{MaybeTlsStream, WebSocketStream}; +use crate::{error::DiscoveryError, protobuf::common::Capability, utils::{handle_message, new_register_node_request_v1}}; +use futures::{stream::SplitSink, StreamExt, SinkExt}; + +async fn setup_ws(url: &str, registration_credentials: &str) -> Result>, DiscoveryError> { + use tokio_tungstenite::{connect_async, tungstenite::client::IntoClientRequest}; + + let mut req = url.into_client_request().unwrap(); + req.headers_mut().insert("Authorization", format!("Basic {}", registration_credentials).parse().unwrap()); + + let (socket, _) = connect_async(req).await?; + Ok(socket) +} + +pub struct DiscoClient { + ws: SplitSink>, tokio_tungstenite::tungstenite::Message>, + keypair: Keypair, + centralized_id: String, +} + +impl DiscoClient { + pub async fn new(keypair: &Keypair, disco_url: &str, registration_credentials: &str) -> Result { + let decoded = general_purpose::STANDARD.decode(registration_credentials).map_err(|_| DiscoveryError::InvalidCredentials)?; + let decoded_str = String::from_utf8(decoded).map_err(|_| DiscoveryError::InvalidCredentials)?; + let parts: Vec<&str> = decoded_str.split(':').collect(); + if parts.len() != 2 { + return Err(DiscoveryError::InvalidCredentials); + } + let centralized_id = parts[0].to_string(); + + let ws = setup_ws(disco_url, registration_credentials).await?; + let (writer, mut reader) = ws.split(); + spawn(async move { + while let Some(msg) = reader.next().await { + match msg { + Ok(msg) => { + if msg.is_close() { + tracing::warn!("Disco connection closed: {:?}", msg); + continue; + } + if !msg.is_binary() { + tracing::error!("Message is expected to be in binary"); + continue; + } + if let Err(e) = handle_message(&msg.into_data()) { + tracing::error!("Failed to parse msg: {:?}", e); + } + } + Err(e) => { + tracing::error!("Failed to load message from ws: {:?}", e); + } + } + } + }); + Ok(DiscoClient { ws: writer, keypair: keypair.clone(), centralized_id }) + } + pub async fn register_compatible(&mut self, addrs: Vec, capabilities: &[Capability]) -> Result<(), DiscoveryError> { + let message = new_register_node_request_v1(&self.centralized_id, &self.keypair, addrs); + let message = tokio_tungstenite::tungstenite::Message::binary(message.encode_to_vec()); + self.ws.send(message).await?; + Ok(()) + } +} diff --git a/core/disco/src/error.rs b/core/disco/src/error.rs new file mode 100644 index 00000000..2c43ac84 --- /dev/null +++ b/core/disco/src/error.rs @@ -0,0 +1,21 @@ +use prost::{DecodeError, EncodeError}; + +#[derive(Debug, thiserror::Error)] +pub enum DiscoveryError { + #[cfg(target_family = "wasm")] + #[error("Failed to open socket: {0}")] + OpenSocketError(String), + #[cfg(not(target_family = "wasm"))] + #[error("Failed to open socket: {0}")] + OpenSocketError(#[from] tokio_tungstenite::tungstenite::Error), + #[error("Failed to decode ws meesage: {0}")] + DecodeError(#[from] DecodeError), + #[error("Failed to encode ws message: {0}")] + EncodeError(#[from] EncodeError), + #[error("Invalid ws url: {0}")] + InvalidUrl(String), + #[error("Invalid registration credential")] + InvalidCredentials, + #[error("Failed to sign message: {0}")] + SigningError(#[from] k256::ecdsa::signature::Error), +} diff --git a/core/disco/src/lib.rs b/core/disco/src/lib.rs new file mode 100644 index 00000000..392dd956 --- /dev/null +++ b/core/disco/src/lib.rs @@ -0,0 +1,9 @@ +#[cfg(target_family="wasm")] +mod wasm; + +mod protobuf; +pub mod error; +mod utils; + +#[cfg(not(target_family="wasm"))] +pub mod client; diff --git a/core/disco/src/protobuf/common.rs b/core/disco/src/protobuf/common.rs new file mode 100644 index 00000000..2a4738fb --- /dev/null +++ b/core/disco/src/protobuf/common.rs @@ -0,0 +1,8 @@ +// This file is @generated by prost-build. +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Capability { + #[prost(string, required, tag = "1")] + pub endpoint: ::prost::alloc::string::String, + #[prost(int32, required, tag = "2")] + pub capacity: i32, +} diff --git a/core/disco/src/protobuf/domainpb.rs b/core/disco/src/protobuf/domainpb.rs new file mode 100644 index 00000000..fb1bcba9 --- /dev/null +++ b/core/disco/src/protobuf/domainpb.rs @@ -0,0 +1,112 @@ +// This file is @generated by prost-build. +/// Request sent to notify domain manager that a domain was created on it. +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct CreateDomainRequest { + /// ID of the domain being created. + #[prost(string, tag = "1")] + pub domain_id: ::prost::alloc::string::String, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct CreateDomainResponse { + /// Whether the request was received fine. + #[prost(bool, tag = "1")] + pub ok: bool, + /// Error text. + #[prost(string, tag = "2")] + pub error: ::prost::alloc::string::String, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct DeleteDomainRequest { + /// ID of the domain being deleted. + #[prost(string, tag = "1")] + pub domain_id: ::prost::alloc::string::String, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct DeleteDomainResponse { + /// Whether the request was received fine. + #[prost(bool, tag = "1")] + pub ok: bool, + /// Error text. + #[prost(string, tag = "2")] + pub error: ::prost::alloc::string::String, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ClearDomainPosesRequest { + /// ID of the domain being cleared. + #[prost(string, tag = "1")] + pub domain_id: ::prost::alloc::string::String, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ClearDomainPosesResponse { + /// Whether the request was received fine. + #[prost(bool, tag = "1")] + pub ok: bool, + /// Error text. + #[prost(string, tag = "2")] + pub error: ::prost::alloc::string::String, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct RemovePortalsFromDomainRequest { + /// ID of the domain from which portals are removed. + #[prost(string, tag = "1")] + pub domain_id: ::prost::alloc::string::String, + /// IDs of portals to remove from domain. + #[prost(string, repeated, tag = "2")] + pub portal_ids: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct RemovePortalsFromDomainResponse { + /// Whether the request was received fine. + #[prost(bool, tag = "1")] + pub ok: bool, + /// Error text. + #[prost(string, tag = "2")] + pub error: ::prost::alloc::string::String, +} +/// Request sent to old and new domain managers when manager is updated. +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct UpdateDomainManagerRequest { + /// ID of the domain. + #[prost(string, tag = "1")] + pub domain_id: ::prost::alloc::string::String, + /// ID of the new domain manager node. + #[prost(string, tag = "2")] + pub new_manager_node_id: ::prost::alloc::string::String, + /// ID of the old domain manager node. + #[prost(string, tag = "3")] + pub old_manager_node_id: ::prost::alloc::string::String, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct UpdateDomainManagerResponse { + /// Whether the request was received fine. + #[prost(bool, tag = "1")] + pub ok: bool, + /// Error text. + #[prost(string, tag = "2")] + pub error: ::prost::alloc::string::String, +} +/// Request sent to old and new domain managers when domain manager update is cancelled. +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct CancelUpdateDomainManagerRequest { + /// ID of the domain. + #[prost(string, tag = "1")] + pub domain_id: ::prost::alloc::string::String, + /// ID of the would-be new domain manager node if not cancelled. + #[prost(string, tag = "2")] + pub new_manager_node_id: ::prost::alloc::string::String, + /// ID of the old domain manager node. + #[prost(string, tag = "3")] + pub old_manager_node_id: ::prost::alloc::string::String, + /// Reason of the cancellation. + #[prost(string, tag = "4")] + pub reason: ::prost::alloc::string::String, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct CancelUpdateDomainManagerResponse { + /// Whether the request was received fine. + #[prost(bool, tag = "1")] + pub ok: bool, + /// Error text. + #[prost(string, tag = "2")] + pub error: ::prost::alloc::string::String, +} diff --git a/core/disco/src/protobuf/mod.rs b/core/disco/src/protobuf/mod.rs new file mode 100644 index 00000000..86c12b93 --- /dev/null +++ b/core/disco/src/protobuf/mod.rs @@ -0,0 +1,5 @@ +pub mod portalpb; +pub mod domainpb; +pub mod nodepb; +pub mod msgpb; +pub mod common; diff --git a/core/disco/src/protobuf/msgpb.rs b/core/disco/src/protobuf/msgpb.rs new file mode 100644 index 00000000..8385806c --- /dev/null +++ b/core/disco/src/protobuf/msgpb.rs @@ -0,0 +1,70 @@ +// This file is @generated by prost-build. +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Msg { + /// The request ID (only used for requests and their corresponding responses). + #[prost(string, tag = "1")] + pub req_id: ::prost::alloc::string::String, + #[prost( + oneof = "msg::Payload", + tags = "200, 201, 202, 203, 204, 205, 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, 400, 401, 402, 403" + )] + pub payload: ::core::option::Option, +} +/// Nested message and enum types in `Msg`. +pub mod msg { + #[derive(Clone, PartialEq, ::prost::Oneof)] + pub enum Payload { + #[prost(message, tag = "200")] + RegisterNodeRequest(super::super::nodepb::RegisterNodeRequest), + #[prost(message, tag = "201")] + RegisterNodeResponse(super::super::nodepb::RegisterNodeResponse), + #[prost(message, tag = "202")] + ConnectivityCheckRequest(super::super::nodepb::ConnectivityCheckRequest), + #[prost(message, tag = "203")] + ConnectivityCheckResponse(super::super::nodepb::ConnectivityCheckResponse), + #[prost(message, tag = "204")] + NodeInDomainUpdateRequest(super::super::nodepb::NodeInDomainUpdateRequest), + #[prost(message, tag = "205")] + NodeInDomainUpdateResponse(super::super::nodepb::NodeInDomainUpdateResponse), + #[prost(message, tag = "300")] + CreateDomainRequest(super::super::domainpb::CreateDomainRequest), + #[prost(message, tag = "301")] + CreateDomainResponse(super::super::domainpb::CreateDomainResponse), + #[prost(message, tag = "302")] + DeleteDomainRequest(super::super::domainpb::DeleteDomainRequest), + #[prost(message, tag = "303")] + DeleteDomainResponse(super::super::domainpb::DeleteDomainResponse), + #[prost(message, tag = "304")] + ClearDomainPosesRequest(super::super::domainpb::ClearDomainPosesRequest), + #[prost(message, tag = "305")] + ClearDomainPosesResponse(super::super::domainpb::ClearDomainPosesResponse), + #[prost(message, tag = "306")] + RemovePortalsFromDomainRequest( + super::super::domainpb::RemovePortalsFromDomainRequest, + ), + #[prost(message, tag = "307")] + RemovePortalsFromDomainResponse( + super::super::domainpb::RemovePortalsFromDomainResponse, + ), + #[prost(message, tag = "308")] + UpdateDomainManagerRequest(super::super::domainpb::UpdateDomainManagerRequest), + #[prost(message, tag = "309")] + UpdateDomainManagerResponse(super::super::domainpb::UpdateDomainManagerResponse), + #[prost(message, tag = "310")] + CancelUpdateDomainManagerRequest( + super::super::domainpb::CancelUpdateDomainManagerRequest, + ), + #[prost(message, tag = "311")] + CancelUpdateDomainManagerResponse( + super::super::domainpb::CancelUpdateDomainManagerResponse, + ), + #[prost(message, tag = "400")] + DeletePortalRequest(super::super::portalpb::DeletePortalRequest), + #[prost(message, tag = "401")] + DeletePortalResponse(super::super::portalpb::DeletePortalResponse), + #[prost(message, tag = "402")] + UpdatePortalSizeRequest(super::super::portalpb::UpdatePortalSizeRequest), + #[prost(message, tag = "403")] + UpdatePortalSizeResponse(super::super::portalpb::UpdatePortalSizeResponse), + } +} diff --git a/core/disco/src/protobuf/nodepb.rs b/core/disco/src/protobuf/nodepb.rs new file mode 100644 index 00000000..8f5b4990 --- /dev/null +++ b/core/disco/src/protobuf/nodepb.rs @@ -0,0 +1,102 @@ +// This file is @generated by prost-build. +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct PortalInDomain { + #[prost(string, tag = "1")] + pub portal_id: ::prost::alloc::string::String, + #[prost(string, tag = "2")] + pub short_id: ::prost::alloc::string::String, + #[prost(string, tag = "3")] + pub domain_id: ::prost::alloc::string::String, + #[prost(message, optional, tag = "4")] + pub created_at: ::core::option::Option<::prost_types::Timestamp>, +} +/// Connectivity check result for a single address. +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ConnectivityCheckResult { + /// Node address in p2p multiaddr format. + #[prost(string, tag = "1")] + pub address: ::prost::alloc::string::String, + /// Whether the node is reachable using this address. + #[prost(bool, tag = "2")] + pub reachable: bool, +} +/// Request sent from a node to Discovery Service in order to register. +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct RegisterNodeRequest { + /// The node version. + #[prost(string, tag = "1")] + pub version: ::prost::alloc::string::String, + #[prost(string, tag = "2")] + pub signature: ::prost::alloc::string::String, + #[prost(string, tag = "3")] + pub timestamp: ::prost::alloc::string::String, + #[prost(string, tag = "4")] + pub public_key: ::prost::alloc::string::String, + /// Capabilities in JSON object format. + #[prost(string, tag = "5")] + pub capabilities: ::prost::alloc::string::String, + /// Addresses of the node, in p2p multiaddr format. + #[prost(string, repeated, tag = "6")] + pub addrs: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, +} +/// Response sent from Discovery Service to a node with registration result. +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct RegisterNodeResponse { + /// The node ID. + #[prost(string, tag = "1")] + pub node_id: ::prost::alloc::string::String, + /// Organization ID of the node. + #[prost(string, tag = "2")] + pub organization_id: ::prost::alloc::string::String, + /// IDs of domains which the node should is part of. + #[prost(string, repeated, tag = "3")] + pub domain_ids: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, + /// Portals in domains. + #[prost(message, repeated, tag = "4")] + pub portals_in_domains: ::prost::alloc::vec::Vec, + /// IP under which the node is available. + #[prost(string, tag = "5")] + pub external_ip: ::prost::alloc::string::String, + /// Results of the connectivity checks. + #[prost(message, repeated, tag = "6")] + pub results: ::prost::alloc::vec::Vec, +} +/// Request sent from a node to Discovery Service to request a connectivity check. +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ConnectivityCheckRequest { + /// Addresses of the node, in p2p multiaddr format. + #[prost(string, repeated, tag = "1")] + pub addrs: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, +} +/// Response sent from Discovery Service to a node with connectivity check results. +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ConnectivityCheckResponse { + /// Whether the request was received/processed fine. + #[prost(bool, tag = "1")] + pub ok: bool, + /// Error text. + #[prost(string, tag = "2")] + pub error: ::prost::alloc::string::String, + /// IP under which the node is available. + #[prost(string, tag = "3")] + pub external_ip: ::prost::alloc::string::String, + /// Results of the connectivity checks. + #[prost(message, repeated, tag = "4")] + pub results: ::prost::alloc::vec::Vec, +} +/// Request sent from a node to Discovery Service to add the node to the domain. +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct NodeInDomainUpdateRequest { + #[prost(string, tag = "1")] + pub domain_id: ::prost::alloc::string::String, +} +/// Response sent from Discovery Service to a node with NodeInDomainUpdate result. +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct NodeInDomainUpdateResponse { + /// Whether the request was received/processed fine. + #[prost(bool, tag = "1")] + pub ok: bool, + /// Error text. + #[prost(string, tag = "2")] + pub error: ::prost::alloc::string::String, +} diff --git a/core/disco/src/protobuf/portalpb.rs b/core/disco/src/protobuf/portalpb.rs new file mode 100644 index 00000000..d614dd77 --- /dev/null +++ b/core/disco/src/protobuf/portalpb.rs @@ -0,0 +1,63 @@ +// This file is @generated by prost-build. +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct DeletePortalRequest { + /// ID of the portal being deleted. + #[prost(string, tag = "1")] + pub portal_id: ::prost::alloc::string::String, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct DeletePortalResponse { + /// Whether the request was received fine. + #[prost(bool, tag = "1")] + pub ok: bool, + /// Error text. + #[prost(string, tag = "2")] + pub error: ::prost::alloc::string::String, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct UpdatePortalSizeRequest { + /// ID of the domain being created. + #[prost(string, tag = "1")] + pub portal_id: ::prost::alloc::string::String, + /// New size of portal. + #[prost(double, tag = "2")] + pub size: f64, + /// The action to do regarding the portal. + #[prost(enumeration = "UpdatePortalSizeAction", tag = "3")] + pub action: i32, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct UpdatePortalSizeResponse { + /// Whether the request was received fine. + #[prost(bool, tag = "1")] + pub ok: bool, + /// Error text. + #[prost(string, tag = "2")] + pub error: ::prost::alloc::string::String, +} +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] +#[repr(i32)] +pub enum UpdatePortalSizeAction { + RemovePortalFromDomains = 0, + DeletePoses = 1, +} +impl UpdatePortalSizeAction { + /// String value of the enum field names used in the ProtoBuf definition. + /// + /// The values are not transformed in any way and thus are considered stable + /// (if the ProtoBuf definition does not change) and safe for programmatic use. + pub fn as_str_name(&self) -> &'static str { + match self { + Self::RemovePortalFromDomains => "REMOVE_PORTAL_FROM_DOMAINS", + Self::DeletePoses => "DELETE_POSES", + } + } + /// Creates an enum from field names used in the ProtoBuf definition. + pub fn from_str_name(value: &str) -> ::core::option::Option { + match value { + "REMOVE_PORTAL_FROM_DOMAINS" => Some(Self::RemovePortalFromDomains), + "DELETE_POSES" => Some(Self::DeletePoses), + _ => None, + } + } +} diff --git a/core/disco/src/utils.rs b/core/disco/src/utils.rs new file mode 100644 index 00000000..cd95a895 --- /dev/null +++ b/core/disco/src/utils.rs @@ -0,0 +1,69 @@ +use crate::{error::DiscoveryError, protobuf::{msgpb::{msg::Payload, Msg}, nodepb::RegisterNodeRequest}}; +use k256::ecdsa::{Signature, SigningKey}; +use libp2p_identity::secp256k1::Keypair; +use prost::Message; +use uuid::Uuid; + +pub fn handle_message(bytes: &[u8]) -> Result<(), DiscoveryError> { + let msg = Msg::decode(bytes)?; + match msg.payload { + Some(Payload::RegisterNodeResponse(res)) => { + tracing::info!("received RegisterNodeResponse: {:?}", res); + }, + _ => {} + } + Ok(()) +} + +pub fn new_register_node_request_v1(centralized_id: &str, keypair: &Keypair, addrs: Vec) -> Msg { + let timestamp = chrono::Utc::now().format("%Y-%m-%dT%H:%M:%S.%fZ").to_string(); + let public_key = hex::encode(keypair.public().to_bytes_uncompressed()); + + let message = format!("{}{}", centralized_id, timestamp); + let signature = sign_message(&message, keypair).unwrap(); + + return Msg { + req_id: Uuid::new_v4().to_string(), + payload: Some(Payload::RegisterNodeRequest(RegisterNodeRequest { + version: "v0.0.0".to_string(), + signature, + timestamp, + public_key, + capabilities: "{\"capabilities\": []}".to_string(), + addrs + })) + }; +} + +pub fn sign_message(message: &str, keypair: &Keypair) -> Result { + use tiny_keccak::Hasher; + // Hash the message using Keccak-256 + let mut hasher = tiny_keccak::Keccak::v256(); + hasher.update(message.as_bytes()); + let mut hash = [0u8; 32]; + hasher.finalize(&mut hash); + + // Convert private key bytes to SigningKey + let secret_key = keypair.secret().to_bytes(); + let signing_key = SigningKey::from_slice(&secret_key)?; + + // Sign the hash and return signature bytes + let signature: Signature = signing_key.sign_prehash_recoverable(&hash)?.0; + let signature = signature.to_bytes().to_vec(); + let hex_signature = hex::encode(signature); + Ok(format!("0x{}", hex_signature)) +} + + +#[cfg(test)] +mod tests { + use super::*; + use posemesh_utils::parse_secp256k1_private_key; + + #[test] + fn test_sign_message() { + let keypair = parse_secp256k1_private_key(Some("2719d2c0d35fa8a6dce9622e480764ecc0428dd10c70cc52ec0349351989d27a"), None).unwrap(); + let signature = sign_message("test message", &keypair).unwrap(); + assert_eq!(signature, "0x2dcb35d237f7a1d954aceffbaf7cf6e2d36f947c4a83785117c322b7bab031a8180a24829c9a80fc690de79624088a0fa7f62af48407151818299076ecb08af7"); + } +} diff --git a/core/disco/src/wasm.rs b/core/disco/src/wasm.rs new file mode 100644 index 00000000..cbd78562 --- /dev/null +++ b/core/disco/src/wasm.rs @@ -0,0 +1,45 @@ + +use wasm_bindgen::prelude::*; + +use crate::{error::DiscoveryError}; + +fn setup_ws(url: &str) -> Result { + use wasm_bindgen::{prelude::Closure, JsCast}; + use web_sys::MessageEvent; + let ws = web_sys::WebSocket::new(url) + .map_err(|e| DiscoveryError::OpenSocketError(e.as_string().unwrap()))?; + + let onopen_callback = Closure::::new(move |e: MessageEvent| { + use web_sys::js_sys; + if let Ok(abuf) = e.data().dyn_into::() { + use crate::utils::handle_message; + + tracing::info!("message event, received arraybuffer: {:?}", abuf); + let array = js_sys::Uint8Array::new(&abuf); + let len = array.byte_length() as usize; + tracing::info!("Arraybuffer received {}bytes", len); + if let Err(err) = handle_message(&array.to_vec()) { + tracing::error!("{:?}", err); + } + } + }); + ws.set_onopen(Some(onopen_callback.as_ref().unchecked_ref())); + onopen_callback.forget(); + + let onerror_callback = Closure::::new(move |e: web_sys::ErrorEvent| { + tracing::info!("ws error: {:?}", e); + }); + ws.set_onerror(Some(onerror_callback.as_ref().unchecked_ref())); + onerror_callback.forget(); + + Ok(ws) +} + +#[wasm_bindgen] +pub struct DiscoClient { + ws: Option +} + +#[wasm_bindgen] +impl DiscoClient { +} diff --git a/core/protobuf/disco/domain.proto b/core/protobuf/disco/domain.proto new file mode 100644 index 00000000..5945c1b1 --- /dev/null +++ b/core/protobuf/disco/domain.proto @@ -0,0 +1,106 @@ +syntax = "proto3"; + +package domainpb; + +option go_package = "pkg/msgpb/domainpb"; +option csharp_namespace = "Auki.Posemesh.Domain.Protobuf.Gen"; +option objc_class_prefix = "Domainpb"; + +// Request sent to notify domain manager that a domain was created on it. +message CreateDomainRequest { + // ID of the domain being created. + string domain_id = 1; +} + +message CreateDomainResponse { + // Whether the request was received fine. + bool ok = 1; + + // Error text. + string error = 2; +} + +message DeleteDomainRequest { + // ID of the domain being deleted. + string domain_id = 1; +} + +message DeleteDomainResponse { + // Whether the request was received fine. + bool ok = 1; + + // Error text. + string error = 2; +} + +message ClearDomainPosesRequest { + // ID of the domain being cleared. + string domain_id = 1; +} + +message ClearDomainPosesResponse { + // Whether the request was received fine. + bool ok = 1; + + // Error text. + string error = 2; +} + +message RemovePortalsFromDomainRequest { + // ID of the domain from which portals are removed. + string domain_id = 1; + + // IDs of portals to remove from domain. + repeated string portal_ids = 2; +} + +message RemovePortalsFromDomainResponse { + // Whether the request was received fine. + bool ok = 1; + + // Error text. + string error = 2; +} + +// Request sent to old and new domain managers when manager is updated. +message UpdateDomainManagerRequest { + // ID of the domain. + string domain_id = 1; + + // ID of the new domain manager node. + string new_manager_node_id = 2; + + // ID of the old domain manager node. + string old_manager_node_id = 3; +} + +message UpdateDomainManagerResponse { + // Whether the request was received fine. + bool ok = 1; + + // Error text. + string error = 2; +} + +// Request sent to old and new domain managers when domain manager update is cancelled. +message CancelUpdateDomainManagerRequest { + // ID of the domain. + string domain_id = 1; + + // ID of the would-be new domain manager node if not cancelled. + string new_manager_node_id = 2; + + // ID of the old domain manager node. + string old_manager_node_id = 3; + + // Reason of the cancellation. + string reason = 4; +} + +message CancelUpdateDomainManagerResponse { + // Whether the request was received fine. + bool ok = 1; + + // Error text. + string error = 2; +} diff --git a/core/protobuf/disco/msg.proto b/core/protobuf/disco/msg.proto new file mode 100644 index 00000000..99b49105 --- /dev/null +++ b/core/protobuf/disco/msg.proto @@ -0,0 +1,41 @@ +syntax = "proto3"; + +package msgpb; + +option go_package = "pkg/msgpb"; + +import "domain.proto"; +import "portal.proto"; +import "node.proto"; + +message Msg { + // The request ID (only used for requests and their corresponding responses). + string req_id = 1; + + oneof payload { + nodepb.RegisterNodeRequest register_node_request = 200; + nodepb.RegisterNodeResponse register_node_response = 201; + nodepb.ConnectivityCheckRequest connectivity_check_request = 202; + nodepb.ConnectivityCheckResponse connectivity_check_response = 203; + nodepb.NodeInDomainUpdateRequest node_in_domain_update_request = 204; + nodepb.NodeInDomainUpdateResponse node_in_domain_update_response = 205; + + domainpb.CreateDomainRequest create_domain_request = 300; + domainpb.CreateDomainResponse create_domain_response = 301; + domainpb.DeleteDomainRequest delete_domain_request = 302; + domainpb.DeleteDomainResponse delete_domain_response = 303; + domainpb.ClearDomainPosesRequest clear_domain_poses_request = 304; + domainpb.ClearDomainPosesResponse clear_domain_poses_response = 305; + domainpb.RemovePortalsFromDomainRequest remove_portals_from_domain_request = 306; + domainpb.RemovePortalsFromDomainResponse remove_portals_from_domain_response = 307; + domainpb.UpdateDomainManagerRequest update_domain_manager_request = 308; + domainpb.UpdateDomainManagerResponse update_domain_manager_response = 309; + domainpb.CancelUpdateDomainManagerRequest cancel_update_domain_manager_request = 310; + domainpb.CancelUpdateDomainManagerResponse cancel_update_domain_manager_response = 311; + + portalpb.DeletePortalRequest delete_portal_request = 400; + portalpb.DeletePortalResponse delete_portal_response = 401; + portalpb.UpdatePortalSizeRequest update_portal_size_request = 402; + portalpb.UpdatePortalSizeResponse update_portal_size_response = 403; + } +} diff --git a/core/protobuf/disco/node.proto b/core/protobuf/disco/node.proto new file mode 100644 index 00000000..002d58b4 --- /dev/null +++ b/core/protobuf/disco/node.proto @@ -0,0 +1,103 @@ +syntax = "proto3"; + +package nodepb; + +// https://developers.google.com/protocol-buffers/docs/reference/java/com/google/protobuf/Timestamp +import "google/protobuf/timestamp.proto"; + +option go_package = "pkg/msgpb/nodepb"; +option csharp_namespace = "Auki.Posemesh.Node.Protobuf.Gen"; +option objc_class_prefix = "Nodepb"; + +message PortalInDomain { + string portal_id = 1; + + string short_id = 2; + + string domain_id = 3; + + google.protobuf.Timestamp created_at = 4; +} + +// Connectivity check result for a single address. +message ConnectivityCheckResult { + // Node address in p2p multiaddr format. + string address = 1; + + // Whether the node is reachable using this address. + bool reachable = 2; +} + +// Request sent from a node to Discovery Service in order to register. +message RegisterNodeRequest { + // The node version. + string version = 1; + + string signature = 2; + + string timestamp = 3; + + string public_key = 4; + + // Capabilities in JSON object format. + string capabilities = 5; + + // Addresses of the node, in p2p multiaddr format. + repeated string addrs = 6; +} + +// Response sent from Discovery Service to a node with registration result. +message RegisterNodeResponse { + // The node ID. + string node_id = 1; + + // Organization ID of the node. + string organization_id = 2; + + // IDs of domains which the node should is part of. + repeated string domain_ids = 3; + + // Portals in domains. + repeated PortalInDomain portals_in_domains = 4; + + // IP under which the node is available. + string external_ip = 5; + + // Results of the connectivity checks. + repeated ConnectivityCheckResult results = 6; +} + +// Request sent from a node to Discovery Service to request a connectivity check. +message ConnectivityCheckRequest { + // Addresses of the node, in p2p multiaddr format. + repeated string addrs = 1; +} + +// Response sent from Discovery Service to a node with connectivity check results. +message ConnectivityCheckResponse { + // Whether the request was received/processed fine. + bool ok = 1; + + // Error text. + string error = 2; + + // IP under which the node is available. + string external_ip = 3; + + // Results of the connectivity checks. + repeated ConnectivityCheckResult results = 4; +} + +// Request sent from a node to Discovery Service to add the node to the domain. +message NodeInDomainUpdateRequest { + string domain_id = 1; +} + +// Response sent from Discovery Service to a node with NodeInDomainUpdate result. +message NodeInDomainUpdateResponse { + // Whether the request was received/processed fine. + bool ok = 1; + + // Error text. + string error = 2; +} diff --git a/core/protobuf/disco/portal.proto b/core/protobuf/disco/portal.proto new file mode 100644 index 00000000..387b84ed --- /dev/null +++ b/core/protobuf/disco/portal.proto @@ -0,0 +1,46 @@ +syntax = "proto3"; + +package portalpb; + +option go_package = "pkg/msgpb/portalpb"; +option csharp_namespace = "Auki.Posemesh.Portal.Protobuf.Gen"; +option objc_class_prefix = "Portalpb"; + +message DeletePortalRequest { + // ID of the portal being deleted. + string portal_id = 1; +} + +message DeletePortalResponse { + // Whether the request was received fine. + bool ok = 1; + + // Error text. + string error = 2; +} + +enum UpdatePortalSizeAction { + REMOVE_PORTAL_FROM_DOMAINS = 0; + DELETE_POSES = 1; + + reserved 100 to max; +} + +message UpdatePortalSizeRequest { + // ID of the domain being created. + string portal_id = 1; + + // New size of portal. + double size = 2; + + // The action to do regarding the portal. + UpdatePortalSizeAction action = 3; +} + +message UpdatePortalSizeResponse { + // Whether the request was received fine. + bool ok = 1; + + // Error text. + string error = 2; +} From ec31f4365ad0b58e907043e533d0836a2ecde535 Mon Sep 17 00:00:00 2001 From: Shuning Date: Thu, 19 Jun 2025 09:36:34 +0800 Subject: [PATCH 02/12] register with disco and refactor on domain cluster --- .vscode/settings.json | 3 +- core/.cargo/config.toml | 2 + core/Cargo.lock | 672 ++++++++---------- core/Cargo.toml | 2 +- core/disco/Cargo.toml | 15 +- core/disco/src/client.rs | 30 +- core/disco/src/error.rs | 5 +- core/disco/src/lib.rs | 2 +- core/disco/src/utils.rs | 40 +- core/disco/src/wasm.rs | 11 +- core/domain/Cargo.toml | 13 +- core/domain/build.rs | 15 +- core/domain/examples/browser/.env | 2 +- core/domain/examples/browser/src/index.js | 3 +- core/domain/src/capabilities/domain_data.rs | 2 +- core/domain/src/cluster.rs | 613 ++++++++++------ core/domain/src/datastore/common.rs | 19 +- core/domain/src/datastore/remote.rs | 91 +-- core/domain/src/message.rs | 13 +- core/domain/src/protobuf/common.rs | 51 ++ core/domain/src/protobuf/discovery.rs | 40 +- core/domain/src/protobuf/mod.rs | 1 + core/domain/src/spatial/reconstruction.rs | 7 +- core/domain/src/wasm.rs | 75 +- core/examples/client/src/main.rs | 21 +- core/examples/test-concurrent/src/main.rs | 18 +- core/networking/Cargo.toml | 2 +- core/networking/src/client.rs | 48 +- core/networking/src/event.rs | 2 +- core/networking/src/libp2p.rs | 60 +- core/protobuf/common/common.proto | 8 + .../{ => domain-cluster}/discovery.proto | 7 +- .../{ => domain-cluster}/domain_data.proto | 0 core/protobuf/{ => domain-cluster}/task.proto | 0 core/utils/Cargo.toml | 8 + core/utils/src/crypto.rs | 153 ++++ core/utils/src/lib.rs | 22 +- 37 files changed, 1240 insertions(+), 836 deletions(-) create mode 100644 core/.cargo/config.toml create mode 100644 core/domain/src/protobuf/common.rs create mode 100644 core/protobuf/common/common.proto rename core/protobuf/{ => domain-cluster}/discovery.proto (61%) rename core/protobuf/{ => domain-cluster}/domain_data.proto (100%) rename core/protobuf/{ => domain-cluster}/task.proto (100%) create mode 100644 core/utils/src/crypto.rs diff --git a/.vscode/settings.json b/.vscode/settings.json index 25771eb7..659c1c4b 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -24,5 +24,6 @@ "**/sdk/platform/Web/src/**/*.hpp": "cpp", "**/sdk/platform/Web/src/**/*.c": "c", "**/sdk/platform/Web/src/**/*.cpp": "cpp" - } + }, + "rust-analyzer.cargo.target": "wasm32-unknown-unknown" } diff --git a/core/.cargo/config.toml b/core/.cargo/config.toml new file mode 100644 index 00000000..2e07606d --- /dev/null +++ b/core/.cargo/config.toml @@ -0,0 +1,2 @@ +[target.wasm32-unknown-unknown] +rustflags = ['--cfg', 'getrandom_backend="wasm_js"'] diff --git a/core/Cargo.lock b/core/Cargo.lock index 7928edd5..5b5573ad 100644 --- a/core/Cargo.lock +++ b/core/Cargo.lock @@ -13,9 +13,9 @@ dependencies = [ [[package]] name = "adler2" -version = "2.0.0" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" [[package]] name = "aead" @@ -93,9 +93,9 @@ dependencies = [ [[package]] name = "anstream" -version = "0.6.18" +version = "0.6.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8acc5369981196006228e28809f761875c0327210a891e941f4c683b3a99529b" +checksum = "301af1932e46185686725e0fad2f8f2aa7da69dd70bf6ecc44d6b703844a3933" dependencies = [ "anstyle", "anstyle-parse", @@ -108,44 +108,44 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.10" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9" +checksum = "862ed96ca487e809f1c8e5a8447f6ee2cf102f846893800b20cebdf541fc6bbd" [[package]] name = "anstyle-parse" -version = "0.2.6" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b2d16507662817a6a20a9ea92df6652ee4f94f914589377d69f3b21bc5798a9" +checksum = "4e7644824f0aa2c7b9384579234ef10eb7efb6a0deb83f9630a49594dd9c15c2" dependencies = [ "utf8parse", ] [[package]] name = "anstyle-query" -version = "1.1.2" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79947af37f4177cfead1110013d678905c37501914fba0efea834c3fe9a8d60c" +checksum = "6c8bdeb6047d8983be085bab0ba1472e6dc604e7041dbf6fcd5e71523014fae9" dependencies = [ "windows-sys 0.59.0", ] [[package]] name = "anstyle-wincon" -version = "3.0.7" +version = "3.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca3534e77181a9cc07539ad51f2141fe32f6c3ffd4df76db8ad92346b003ae4e" +checksum = "403f75924867bb1033c59fbf0797484329750cfbe3c4325cd33127941fabc882" dependencies = [ "anstyle", - "once_cell", + "once_cell_polyfill", "windows-sys 0.59.0", ] [[package]] name = "anyhow" -version = "1.0.97" +version = "1.0.98" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcfed56ad506cb2c684a14971b8861fdc3baaaae314b9e5f9bb532cbe3ba7a4f" +checksum = "e16d2d3311acee920a9eb8d33b8cbc1787ce4a264e85f964c2404b969bdcd487" [[package]] name = "arc-swap" @@ -217,8 +217,8 @@ checksum = "965c2d33e53cb6b267e148a4cb0760bc01f4904c1cd4bb4002a085bb016d1490" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", - "synstructure 0.13.1", + "syn 2.0.103", + "synstructure 0.13.2", ] [[package]] @@ -240,20 +240,14 @@ checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.103", ] -[[package]] -name = "asn1_der" -version = "0.7.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "155a5a185e42c6b77ac7b88a15143d930a9e9727a5b7b77eed417404ab15c247" - [[package]] name = "async-io" -version = "2.4.0" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43a2b323ccce0a1d90b449fd71f2a06ca7faa7c54c2751f06c9bd851fc061059" +checksum = "1237c0ae75a0f3765f58910ff9cdd0a12eeb39ab2f4c7de23262f337f0aacbb3" dependencies = [ "async-lock", "cfg-if", @@ -262,7 +256,7 @@ dependencies = [ "futures-lite", "parking", "polling", - "rustix 0.38.44", + "rustix", "slab", "tracing", "windows-sys 0.59.0", @@ -298,7 +292,7 @@ checksum = "e539d3fca749fcee5236ab05e93a52867dd549cc157c8cb7f99595f3cedffdb5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.103", ] [[package]] @@ -350,9 +344,9 @@ checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" [[package]] name = "backtrace" -version = "0.3.74" +version = "0.3.75" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d82cb332cdfaed17ae235a638438ac4d4839913cc2af585c3c6746e8f8bee1a" +checksum = "6806a6321ec58106fea15becdad98371e28d92ccbc7c8f1b3b6dd724fe8f1002" dependencies = [ "addr2line", "cfg-if", @@ -418,9 +412,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.9.0" +version = "2.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c8214115b7bf84099f1309324e63141d4c5d7cc26862f97a0a857dbefe165bd" +checksum = "1b8e56985ec62d17e9c1001dc89c88ecd7dc08e47eba5ec7c29c7b5eeecde967" [[package]] name = "blake2" @@ -460,9 +454,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.17.0" +version = "3.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1628fb46dfa0b37568d12e5edd512553eccf6a22a78e8bde00bb4aed84d5bdbf" +checksum = "793db76d6187cd04dff33004d8e6c9cc4e05cd330500379d2394209271b4aeee" [[package]] name = "byteorder" @@ -494,7 +488,7 @@ version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eadd868a2ce9ca38de7eeafdcec9c7065ef89b42b32f0839278d55f35c54d1ff" dependencies = [ - "clap 4.5.35", + "clap 4.5.40", "heck 0.4.1", "indexmap", "log", @@ -502,16 +496,16 @@ dependencies = [ "quote", "serde", "serde_json", - "syn 2.0.100", + "syn 2.0.103", "tempfile", "toml", ] [[package]] name = "cc" -version = "1.2.18" +version = "1.2.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "525046617d8376e3db1deffb079e91cef90a89fc3ca5c185bbf8c9ecdd15cd5c" +checksum = "d487aa071b5f64da6f19a3e848e3578944b726ee5a4854b82172f02aa876bfdc" dependencies = [ "shlex", ] @@ -530,9 +524,9 @@ dependencies = [ [[package]] name = "cfg-if" -version = "1.0.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +checksum = "9555578bc9e57714c812a1f84e4fc5b4d21fcb063490c624de019f7464c91268" [[package]] name = "cfg_aliases" @@ -606,18 +600,18 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.35" +version = "4.5.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8aa86934b44c19c50f87cc2790e19f54f7a67aedb64101c2e1a2e5ecfb73944" +checksum = "40b6887a1d8685cebccf115538db5c0efe625ccac9696ad45c409d96566e910f" dependencies = [ "clap_builder", ] [[package]] name = "clap_builder" -version = "4.5.35" +version = "4.5.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2414dbb2dd0695280da6ea9261e327479e9d37b0630f6b53ba2a11c60c679fd9" +checksum = "e0c66c08ce9f0c698cbce5c0279d0bb6ac936d8674174fe48f736533b964f59e" dependencies = [ "anstream", "anstyle", @@ -627,9 +621,9 @@ dependencies = [ [[package]] name = "clap_lex" -version = "0.7.4" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f46ad14479a25103f283c0f10005961cf086d8dc42205bb44c46ac563475dca6" +checksum = "b94f61472cee1439c0b966b47e3aca9ae07e45d070759512cd390ea2bebc6675" [[package]] name = "client-example" @@ -646,9 +640,9 @@ dependencies = [ [[package]] name = "colorchoice" -version = "1.0.3" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" +checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" [[package]] name = "concurrent-queue" @@ -711,9 +705,9 @@ dependencies = [ [[package]] name = "crc" -version = "3.2.1" +version = "3.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69e6e4d7b33a94f0991c26729976b10ebde1d34c3ee82408fb536164fa10d636" +checksum = "9710d3b3739c2e349eb44fe848ad0b7c8cb1e42bd87ee49371df2f7acaf3e675" dependencies = [ "crc-catalog", ] @@ -792,20 +786,20 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.103", ] [[package]] name = "data-encoding" -version = "2.8.0" +version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "575f75dfd25738df5b91b8e43e14d44bda14637a58fae779fd2b064f8bf3e010" +checksum = "2a2330da5de22e8a3cb63252ce2abb30116bf5265e89c0e01bc17015ce30a476" [[package]] name = "data-encoding-macro" -version = "0.1.17" +version = "0.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f9724adfcf41f45bf652b3995837669d73c4d49a1b5ac1ff82905ac7d9b5558" +checksum = "47ce6c96ea0102f01122a185683611bd5ac8d99e62bc59dd12e6bda344ee673d" dependencies = [ "data-encoding", "data-encoding-macro-internal", @@ -813,19 +807,19 @@ dependencies = [ [[package]] name = "data-encoding-macro-internal" -version = "0.1.15" +version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18e4fdb82bd54a12e42fb58a800dcae6b9e13982238ce2296dc3570b92148e1f" +checksum = "8d162beedaa69905488a8da94f5ac3edb4dd4788b732fadb7bd120b2625c1976" dependencies = [ "data-encoding", - "syn 2.0.100", + "syn 2.0.103", ] [[package]] name = "der" -version = "0.7.9" +version = "0.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f55bf8e7b65898637379c1b74eb1551107c8294ed26d855ceb9fd1a09cfc9bc0" +checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" dependencies = [ "const-oid", "pem-rfc7468", @@ -889,7 +883,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.103", ] [[package]] @@ -979,7 +973,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.103", ] [[package]] @@ -1003,9 +997,9 @@ checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" [[package]] name = "errno" -version = "0.3.11" +version = "0.3.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "976dd42dc7e85965fe702eb8164f21f450704bdde31faefd6471dba214cb594e" +checksum = "cea14ef9355e3beab063703aa9dab15afd25f0667c341310c1e5274bb1d0da18" dependencies = [ "libc", "windows-sys 0.59.0", @@ -1185,7 +1179,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.103", ] [[package]] @@ -1195,7 +1189,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8f2f12607f92c69b12ed746fabf9ca4f5c482cba46679c1a75b874ed7c26adb" dependencies = [ "futures-io", - "rustls 0.23.26", + "rustls 0.23.27", "rustls-pki-types", ] @@ -1263,22 +1257,22 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.15" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" +checksum = "335ff9f135e4384c8150d6f27c6daed433577f86b4750418338c01a1a2528592" dependencies = [ "cfg-if", "js-sys", "libc", - "wasi 0.11.0+wasi-snapshot-preview1", + "wasi 0.11.1+wasi-snapshot-preview1", "wasm-bindgen", ] [[package]] name = "getrandom" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73fea8450eea4bac3940448fb7ae50d91f034f941199fcd9d909a5a07aa455f0" +checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4" dependencies = [ "cfg-if", "js-sys", @@ -1341,9 +1335,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.8" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5017294ff4bb30944501348f6f8e42e6ad28f42c8bbef7a74029aff064a4e3c2" +checksum = "a9421a676d1b147b16b82c9225157dc629087ef8ec4d5e2960f9437a90dac0a5" dependencies = [ "atomic-waker", "bytes", @@ -1360,9 +1354,9 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.15.2" +version = "0.15.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" +checksum = "5971ac85611da7067dbfcabef3c70ebb5606018acd9e2a3903a0da507521e0d5" dependencies = [ "allocator-api2", "equivalent", @@ -1392,15 +1386,9 @@ dependencies = [ [[package]] name = "hermit-abi" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" - -[[package]] -name = "hermit-abi" -version = "0.4.0" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbf6a919d6cf397374f7dfeeea91d974c7c0a7221d0d0f4f20d859d329e53fcc" +checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" [[package]] name = "hex" @@ -1478,17 +1466,6 @@ dependencies = [ "digest", ] -[[package]] -name = "hostname" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a56f203cd1c76362b69e3863fd987520ac36cf70a8c92627449b2f64a8cf7d65" -dependencies = [ - "cfg-if", - "libc", - "windows-link", -] - [[package]] name = "http" version = "0.2.12" @@ -1571,12 +1548,13 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.11" +version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "497bbc33a26fdd4af9ed9c70d63f61cf56a938375fbb32df34db9b1cd6d643f2" +checksum = "dc2fdfdbff08affe55bb779f33b053aa1fe5dd5b54c257343c17edfa55711bdb" dependencies = [ "bytes", "futures-channel", + "futures-core", "futures-util", "http 1.3.1", "http-body", @@ -1615,21 +1593,22 @@ dependencies = [ [[package]] name = "icu_collections" -version = "1.5.0" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db2fa452206ebee18c4b5c2274dbf1de17008e874b4dc4f0aea9d01ca79e4526" +checksum = "200072f5d0e3614556f94a9930d5dc3e0662a652823904c3a75dc3b0af7fee47" dependencies = [ "displaydoc", + "potential_utf", "yoke", "zerofrom", "zerovec", ] [[package]] -name = "icu_locid" -version = "1.5.0" +name = "icu_locale_core" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13acbb8371917fc971be86fc8057c41a64b521c184808a698c02acc242dbf637" +checksum = "0cde2700ccaed3872079a65fb1a78f6c0a36c91570f28755dda67bc8f7d9f00a" dependencies = [ "displaydoc", "litemap", @@ -1638,31 +1617,11 @@ dependencies = [ "zerovec", ] -[[package]] -name = "icu_locid_transform" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01d11ac35de8e40fdeda00d9e1e9d92525f3f9d887cdd7aa81d727596788b54e" -dependencies = [ - "displaydoc", - "icu_locid", - "icu_locid_transform_data", - "icu_provider", - "tinystr", - "zerovec", -] - -[[package]] -name = "icu_locid_transform_data" -version = "1.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7515e6d781098bf9f7205ab3fc7e9709d34554ae0b21ddbcb5febfa4bc7df11d" - [[package]] name = "icu_normalizer" -version = "1.5.0" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19ce3e0da2ec68599d193c93d088142efd7f9c5d6fc9b803774855747dc6a84f" +checksum = "436880e8e18df4d7bbc06d58432329d6458cc84531f7ac5f024e93deadb37979" dependencies = [ "displaydoc", "icu_collections", @@ -1670,67 +1629,54 @@ dependencies = [ "icu_properties", "icu_provider", "smallvec", - "utf16_iter", - "utf8_iter", - "write16", "zerovec", ] [[package]] name = "icu_normalizer_data" -version = "1.5.1" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5e8338228bdc8ab83303f16b797e177953730f601a96c25d10cb3ab0daa0cb7" +checksum = "00210d6893afc98edb752b664b8890f0ef174c8adbb8d0be9710fa66fbbf72d3" [[package]] name = "icu_properties" -version = "1.5.1" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93d6020766cfc6302c15dbbc9c8778c37e62c14427cb7f6e601d849e092aeef5" +checksum = "016c619c1eeb94efb86809b015c58f479963de65bdb6253345c1a1276f22e32b" dependencies = [ "displaydoc", "icu_collections", - "icu_locid_transform", + "icu_locale_core", "icu_properties_data", "icu_provider", - "tinystr", + "potential_utf", + "zerotrie", "zerovec", ] [[package]] name = "icu_properties_data" -version = "1.5.1" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85fb8799753b75aee8d2a21d7c14d9f38921b54b3dbda10f5a3c7a7b82dba5e2" +checksum = "298459143998310acd25ffe6810ed544932242d3f07083eee1084d83a71bd632" [[package]] name = "icu_provider" -version = "1.5.0" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ed421c8a8ef78d3e2dbc98a973be2f3770cb42b606e3ab18d6237c4dfde68d9" +checksum = "03c80da27b5f4187909049ee2d72f276f0d9f99a42c306bd0131ecfe04d8e5af" dependencies = [ "displaydoc", - "icu_locid", - "icu_provider_macros", + "icu_locale_core", "stable_deref_trait", "tinystr", "writeable", "yoke", "zerofrom", + "zerotrie", "zerovec", ] -[[package]] -name = "icu_provider_macros" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.100", -] - [[package]] name = "idna" version = "1.0.3" @@ -1744,9 +1690,9 @@ dependencies = [ [[package]] name = "idna_adapter" -version = "1.2.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "daca1df1c957320b2cf139ac61e7bd64fed304c5040df000a745aa1de3b4ef71" +checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344" dependencies = [ "icu_normalizer", "icu_properties", @@ -1943,9 +1889,9 @@ checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] name = "libc" -version = "0.2.171" +version = "0.2.173" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c19937216e9d3aa9956d9bb8dfc0b0c8beb6058fc4f7a4dc4d850edf86a237d6" +checksum = "d8cfeafaffdbc32176b64fb251369d52ea9f0a8fbc6f8759edffef7b525d64bb" [[package]] name = "libp2p" @@ -1956,7 +1902,7 @@ dependencies = [ "either", "futures", "futures-timer", - "getrandom 0.2.15", + "getrandom 0.2.16", "libp2p-allow-block-list", "libp2p-autonat", "libp2p-connection-limits", @@ -2110,7 +2056,7 @@ dependencies = [ "fnv", "futures", "futures-ticker", - "getrandom 0.2.15", + "getrandom 0.2.16", "hex_fmt", "instant", "libp2p-core", @@ -2157,16 +2103,12 @@ version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fbb68ea10844211a59ce46230909fd0ea040e8a192454d4cc2ee0d53e12280eb" dependencies = [ - "asn1_der", "bs58", "ed25519-dalek", "hkdf", - "k256", "multihash", - "p256", "quick-protobuf", "rand 0.8.5", - "sec1", "serde", "sha2", "thiserror 2.0.12", @@ -2301,7 +2243,7 @@ dependencies = [ "quinn", "rand 0.8.5", "ring 0.17.14", - "rustls 0.23.26", + "rustls 0.23.27", "socket2", "thiserror 1.0.69", "tokio", @@ -2374,7 +2316,7 @@ dependencies = [ "fnv", "futures", "futures-timer", - "getrandom 0.2.15", + "getrandom 0.2.16", "libp2p-core", "libp2p-identity", "libp2p-swarm-derive", @@ -2398,7 +2340,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.103", ] [[package]] @@ -2428,7 +2370,7 @@ dependencies = [ "libp2p-identity", "rcgen", "ring 0.17.14", - "rustls 0.23.26", + "rustls 0.23.27", "rustls-webpki 0.101.7", "thiserror 1.0.69", "x509-parser 0.16.0", @@ -2507,7 +2449,7 @@ source = "git+https://github.com/aukilabs/rust-libp2p.git?branch=fix/gossipsub-w dependencies = [ "bytes", "futures", - "getrandom 0.2.15", + "getrandom 0.2.16", "hex", "js-sys", "libp2p-core", @@ -2569,7 +2511,7 @@ dependencies = [ "thiserror 1.0.69", "tracing", "yamux 0.12.1", - "yamux 0.13.4", + "yamux 0.13.5", ] [[package]] @@ -2578,12 +2520,6 @@ version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" -[[package]] -name = "linux-raw-sys" -version = "0.4.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" - [[package]] name = "linux-raw-sys" version = "0.9.4" @@ -2592,15 +2528,15 @@ checksum = "cd945864f07fe9f5371a27ad7b52a172b4b499999f1d97574c9fa68373937e12" [[package]] name = "litemap" -version = "0.7.5" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23fb14cb19457329c82206317a5663005a4d404783dc74f4252769b0d5f42856" +checksum = "241eaef5fd12c88705a01fc1066c48c4b36e0dd4377dcdc7ec3942cea7a69956" [[package]] name = "lock_api" -version = "0.4.12" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17" +checksum = "96936507f153605bddfcda068dd804796c84324ed2510809e5b2a624c81da765" dependencies = [ "autocfg", "scopeguard", @@ -2630,6 +2566,12 @@ dependencies = [ "linked-hash-map", ] +[[package]] +name = "lru-slab" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" + [[package]] name = "matchers" version = "0.1.0" @@ -2651,9 +2593,9 @@ dependencies = [ [[package]] name = "memchr" -version = "2.7.4" +version = "2.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" +checksum = "32a282da65faaf38286cf3be983213fcf1d2e2a58700e808f83f4ea9a4804bc0" [[package]] name = "memoffset" @@ -2682,22 +2624,22 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "miniz_oxide" -version = "0.8.8" +version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3be647b768db090acb35d5ec5db2b0e1f1de11133ca123b9eacf5137868f892a" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" dependencies = [ "adler2", ] [[package]] name = "mio" -version = "1.0.3" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2886843bf800fba2e3377cff24abf6379b4c4d5c6681eaf9ea5b0d15090450bd" +checksum = "78bed444cc8a2160f01cbcf811ef18cac863ad68ae8ca62092e8db51d51c761c" dependencies = [ "libc", - "wasi 0.11.0+wasi-snapshot-preview1", - "windows-sys 0.52.0", + "wasi 0.11.1+wasi-snapshot-preview1", + "windows-sys 0.59.0", ] [[package]] @@ -2723,7 +2665,7 @@ dependencies = [ "cfg-if", "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.103", ] [[package]] @@ -2942,11 +2884,11 @@ dependencies = [ [[package]] name = "num_cpus" -version = "1.16.0" +version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" +checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b" dependencies = [ - "hermit-abi 0.3.9", + "hermit-abi 0.5.2", "libc", ] @@ -2983,6 +2925,12 @@ version = "1.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" +[[package]] +name = "once_cell_polyfill" +version = "1.70.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4895175b425cb1f87721b59f0f286c2092bd4af812243672510e1ac53e2e0ad" + [[package]] name = "opaque-debug" version = "0.3.1" @@ -2995,7 +2943,7 @@ version = "0.10.73" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8505734d46c8ab1e19a1dce3aef597ad87dcb4c37e7188231769bd6bd51cebf8" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.9.1", "cfg-if", "foreign-types", "libc", @@ -3012,7 +2960,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.103", ] [[package]] @@ -3071,9 +3019,9 @@ checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" [[package]] name = "parking_lot" -version = "0.12.3" +version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27" +checksum = "70d58bf43669b5795d1576d0641cfb6fbb2057bf629506267a92807158584a13" dependencies = [ "lock_api", "parking_lot_core", @@ -3081,9 +3029,9 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.10" +version = "0.9.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" +checksum = "bc838d2a56b5b1a6c25f55575dfc605fabb63bb2365f6c2353ef9159aa69e4a5" dependencies = [ "cfg-if", "libc", @@ -3180,7 +3128,7 @@ checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.103", ] [[package]] @@ -3213,15 +3161,15 @@ checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" [[package]] name = "polling" -version = "3.7.4" +version = "3.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a604568c3202727d1507653cb121dbd627a58684eb09a820fd746bee38b4442f" +checksum = "b53a684391ad002dd6a596ceb6c74fd004fdce75f4be2e3f615068abbea5fd50" dependencies = [ "cfg-if", "concurrent-queue", - "hermit-abi 0.4.0", + "hermit-abi 0.5.2", "pin-project-lite", - "rustix 0.38.44", + "rustix", "tracing", "windows-sys 0.59.0", ] @@ -3251,9 +3199,9 @@ dependencies = [ [[package]] name = "portable-atomic" -version = "1.11.0" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "350e9b48cbc6b0e028b0473b114454c6316e57336ee184ceab6e53f72c178b3e" +checksum = "f84267b20a16ea918e43c6a88433c2d54fa145c92a811b5b047ccbe153674483" [[package]] name = "posemesh-disco" @@ -3263,15 +3211,13 @@ dependencies = [ "chrono", "console_error_panic_hook", "futures", + "getrandom 0.3.3", "hex", - "k256", - "libp2p-identity", "posemesh-utils", "prost", "prost-build", "prost-types", "thiserror 2.0.12", - "tiny-keccak", "tokio", "tokio-tungstenite", "tracing", @@ -3292,6 +3238,7 @@ dependencies = [ "cbindgen", "console_error_panic_hook", "futures", + "getrandom 0.3.3", "hex", "jsonwebtoken", "libp2p", @@ -3368,7 +3315,9 @@ dependencies = [ "gloo-timers 0.3.0", "hex", "js-sys", - "libp2p-identity", + "k256", + "thiserror 2.0.12", + "tiny-keccak", "tokio", "tracing", "wasm-bindgen", @@ -3408,6 +3357,15 @@ dependencies = [ "uuid", ] +[[package]] +name = "potential_utf" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5a7c30837279ca13e7c867e9e40053bc68740f988cb07f7ca6df43cc734b585" +dependencies = [ + "zerovec", +] + [[package]] name = "powerfmt" version = "0.2.0" @@ -3451,12 +3409,12 @@ dependencies = [ [[package]] name = "prettyplease" -version = "0.2.33" +version = "0.2.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9dee91521343f4c5c6a63edd65e54f31f5c92fe8978c40a4282f8372194c6a7d" +checksum = "6837b9e10d61f45f987d50808f83d1ee3d206c66acf650c3e4ae2e1f6ddedf55" dependencies = [ "proc-macro2", - "syn 2.0.100", + "syn 2.0.103", ] [[package]] @@ -3470,9 +3428,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.94" +version = "1.0.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a31971752e70b8b2686d7e46ec17fb38dad4051d94024c88df49b667caea9c84" +checksum = "02b3e5e68a3a1a02aad3ec490a98007cbc13c37cbe84a3cd7b8e406d76e7f778" dependencies = [ "unicode-ident", ] @@ -3497,7 +3455,7 @@ checksum = "440f724eba9f6996b75d63681b0a92b06947f1457076d503a4d2e2c8f56442b8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.103", ] [[package]] @@ -3526,7 +3484,7 @@ dependencies = [ "prost", "prost-types", "regex", - "syn 2.0.100", + "syn 2.0.103", "tempfile", ] @@ -3540,7 +3498,7 @@ dependencies = [ "itertools", "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.103", ] [[package]] @@ -3594,9 +3552,9 @@ dependencies = [ [[package]] name = "quinn" -version = "0.11.7" +version = "0.11.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3bd15a6f2967aef83887dcb9fec0014580467e33720d073560cf015a5683012" +checksum = "626214629cda6781b6dc1d316ba307189c85ba657213ce642d9c77670f8202c8" dependencies = [ "bytes", "cfg_aliases", @@ -3605,7 +3563,7 @@ dependencies = [ "quinn-proto", "quinn-udp", "rustc-hash", - "rustls 0.23.26", + "rustls 0.23.27", "socket2", "thiserror 2.0.12", "tokio", @@ -3615,16 +3573,17 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.10" +version = "0.11.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b820744eb4dc9b57a3398183639c511b5a26d2ed702cedd3febaa1393caa22cc" +checksum = "49df843a9161c85bb8aae55f101bc0bac8bcafd637a620d9122fd7e0b2f7422e" dependencies = [ "bytes", - "getrandom 0.3.2", + "getrandom 0.3.3", + "lru-slab", "rand 0.9.1", "ring 0.17.14", "rustc-hash", - "rustls 0.23.26", + "rustls 0.23.27", "rustls-pki-types", "slab", "thiserror 2.0.12", @@ -3635,9 +3594,9 @@ dependencies = [ [[package]] name = "quinn-udp" -version = "0.5.11" +version = "0.5.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "541d0f57c6ec747a90738a52741d3221f7960e8ac2f0ff4b1a63680e033b4ab5" +checksum = "ee4e529991f949c5e25755532370b8af5d114acae52326361d68d47af64aa842" dependencies = [ "cfg_aliases", "libc", @@ -3709,7 +3668,7 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ - "getrandom 0.2.15", + "getrandom 0.2.16", ] [[package]] @@ -3718,7 +3677,7 @@ version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38" dependencies = [ - "getrandom 0.3.2", + "getrandom 0.3.3", ] [[package]] @@ -3736,11 +3695,11 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.5.11" +version = "0.5.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2f103c6d277498fbceb16e84d317e2a400f160f46904d5f5410848c829511a3" +checksum = "0d04b7d0ee6b4a0207a0a7adb104d23ecb0b47d6beae7152d0fa34b692b29fd6" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.9.1", ] [[package]] @@ -3800,12 +3759,9 @@ dependencies = [ [[package]] name = "resolv-conf" -version = "0.7.1" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48375394603e3dd4b2d64371f7148fd8c7baa2680e28741f2cb8d23b59e3d4c4" -dependencies = [ - "hostname", -] +checksum = "95325155c684b1c89f7765e30bc1c42e4a6da51ca513615660cb8a62ef9a88e3" [[package]] name = "rfc6979" @@ -3840,7 +3796,7 @@ checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" dependencies = [ "cc", "cfg-if", - "getrandom 0.2.15", + "getrandom 0.2.16", "libc", "untrusted 0.9.0", "windows-sys 0.52.0", @@ -3912,9 +3868,9 @@ dependencies = [ [[package]] name = "rustc-demangle" -version = "0.1.24" +version = "0.1.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" +checksum = "989e6739f80c4ad5b13e0fd7fe89531180375b18520cc8c82080e4dc4035b84f" [[package]] name = "rustc-hash" @@ -3942,27 +3898,14 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.44" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" -dependencies = [ - "bitflags 2.9.0", - "errno", - "libc", - "linux-raw-sys 0.4.15", - "windows-sys 0.59.0", -] - -[[package]] -name = "rustix" -version = "1.0.5" +version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d97817398dd4bb2e6da002002db259209759911da105da92bec29ccb12cf58bf" +checksum = "c71e83d6afe7ff64890ec6b71d6a69bb8a610ab78ce364b3352876bb4c801266" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.9.1", "errno", "libc", - "linux-raw-sys 0.9.4", + "linux-raw-sys", "windows-sys 0.59.0", ] @@ -3980,25 +3923,26 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.26" +version = "0.23.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df51b5869f3a441595eac5e8ff14d486ff285f7b8c0df8770e49c3b56351f0f0" +checksum = "730944ca083c1c233a75c09f199e973ca499344a2b7ba9e755c457e86fb4a321" dependencies = [ "once_cell", "ring 0.17.14", "rustls-pki-types", - "rustls-webpki 0.103.1", + "rustls-webpki 0.103.3", "subtle", "zeroize", ] [[package]] name = "rustls-pki-types" -version = "1.11.0" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "917ce264624a4b4db1c364dcc35bfca9ded014d0a958cd47ad3e960e988ea51c" +checksum = "229a4a4c221013e7e1f1a043678c5cc39fe5171437c88fb47151a21e6f5b5c79" dependencies = [ "web-time", + "zeroize", ] [[package]] @@ -4013,9 +3957,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.103.1" +version = "0.103.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fef8b8769aaccf73098557a87cd1816b4f9c7c16811c9c77142aa695c16f2c03" +checksum = "e4a72fe2bcf7a6ac6fd7d0b9e5cb68aeb7d4c0a0271730218b3e92d43b4eb435" dependencies = [ "ring 0.17.14", "rustls-pki-types", @@ -4024,9 +3968,9 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.20" +version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eded382c5f5f786b989652c49544c4877d9f015cc22e145a5ea8ea66c2921cd2" +checksum = "8a0d197bd2c9dc6e53b84da9556a69ba4cdfab8619eb41a8bd1cc2027a0f6b1d" [[package]] name = "rw-stream-sink" @@ -4110,7 +4054,7 @@ version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.9.1", "core-foundation", "core-foundation-sys", "libc", @@ -4176,7 +4120,7 @@ checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.103", ] [[package]] @@ -4193,9 +4137,9 @@ dependencies = [ [[package]] name = "serde_spanned" -version = "0.6.8" +version = "0.6.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87607cb1398ed59d48732e575a4c28a7a8ebf2454b964fe3f224f2afc07909e1" +checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3" dependencies = [ "serde", ] @@ -4213,9 +4157,9 @@ dependencies = [ [[package]] name = "sha2" -version = "0.10.8" +version = "0.10.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", "cpufeatures", @@ -4239,9 +4183,9 @@ checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" [[package]] name = "signal-hook-registry" -version = "1.4.2" +version = "1.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9e9e0b4211b72e7b8b6e85c807d36c212bdb33ea8587f7569562a84df5465b1" +checksum = "9203b8055f63a2a00e2f593bb0510367fe707d7ff1e5c872de2f537b339e5410" dependencies = [ "libc", ] @@ -4276,18 +4220,15 @@ checksum = "56199f7ddabf13fe5074ce809e7d3f42b42ae711800501b5b16ea82ad029c39d" [[package]] name = "slab" -version = "0.4.9" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" -dependencies = [ - "autocfg", -] +checksum = "04dc19736151f35336d325007ac991178d504a119863a2fcb3758cdb5e52c50d" [[package]] name = "smallvec" -version = "1.15.0" +version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8917285742e9f3e1683f0a9c4e6b57960b7314d0b08d30d1ecd426713ee2eee9" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" [[package]] name = "smol_str" @@ -4317,9 +4258,9 @@ dependencies = [ [[package]] name = "socket2" -version = "0.5.9" +version = "0.5.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f5fd57c80058a56cf5c777ab8a126398ece8e442983605d280a44ce79d0edef" +checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678" dependencies = [ "libc", "windows-sys 0.52.0", @@ -4457,9 +4398,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.100" +version = "2.0.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b09a44accad81e1ba1cd74a32461ba89dee89095ba17b32f5d03683b1b1fc2a0" +checksum = "e4307e30089d6fd6aff212f2da3a1f9e32f3223b1f010fb09b7c95f90f3ca1e8" dependencies = [ "proc-macro2", "quote", @@ -4480,13 +4421,13 @@ dependencies = [ [[package]] name = "synstructure" -version = "0.13.1" +version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.103", ] [[package]] @@ -4495,7 +4436,7 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.9.1", "core-foundation", "system-configuration-sys", ] @@ -4512,14 +4453,14 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.19.1" +version = "3.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7437ac7763b9b123ccf33c338a5cc1bac6f69b45a136c19bdd8a65e3916435bf" +checksum = "e8a64e3985349f2441a1a9ef0b853f869006c3855f2cda6862a94d26ebb9d6a1" dependencies = [ "fastrand", - "getrandom 0.3.2", + "getrandom 0.3.3", "once_cell", - "rustix 1.0.5", + "rustix", "windows-sys 0.59.0", ] @@ -4590,7 +4531,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.103", ] [[package]] @@ -4601,17 +4542,16 @@ checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.103", ] [[package]] name = "thread_local" -version = "1.1.8" +version = "1.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b9ef9bad013ada3808854ceac7b46812a6465ba368859a37e2100283d2d719c" +checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" dependencies = [ "cfg-if", - "once_cell", ] [[package]] @@ -4656,9 +4596,9 @@ dependencies = [ [[package]] name = "tinystr" -version = "0.7.6" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9117f5d4db391c1cf6927e7bea3db74b9a1c1add8f7eda9ffd5364f40f57b82f" +checksum = "5d4f6d1145dcb577acf783d4e601bc1d76a13337bb54e6233add580b07344c8b" dependencies = [ "displaydoc", "zerovec", @@ -4691,9 +4631,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.44.2" +version = "1.45.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6b88822cbe49de4185e3a4cbf8321dd487cf5fe0c5c65695fef6346371e9c48" +checksum = "75ef51a33ef1da925cea3e4eb122833cb377c61439ca401b770f54902b806779" dependencies = [ "backtrace", "bytes", @@ -4715,7 +4655,7 @@ checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.103", ] [[package]] @@ -4756,9 +4696,9 @@ dependencies = [ [[package]] name = "tokio-tungstenite" -version = "0.26.2" +version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a9daff607c6d2bf6c16fd681ccb7eecc83e4e2cdc1ca067ffaadfca5de7f084" +checksum = "489a59b6730eda1b0171fcfda8b121f4bee2b35cba8645ca35c5f7ba3eb736c1" dependencies = [ "futures-util", "log", @@ -4770,9 +4710,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.14" +version = "0.7.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b9590b93e6fcc1739458317cccd391ad3955e2bde8913edf6f95f9e65a8f034" +checksum = "66a539a9ad6d5d281510d5bd368c973d636c02dbf8a67300bfb6b950696ad7df" dependencies = [ "bytes", "futures-core", @@ -4784,9 +4724,9 @@ dependencies = [ [[package]] name = "toml" -version = "0.8.20" +version = "0.8.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd87a5cdd6ffab733b2f74bc4fd7ee5fff6634124999ac278c35fc78c6120148" +checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" dependencies = [ "serde", "serde_spanned", @@ -4796,26 +4736,33 @@ dependencies = [ [[package]] name = "toml_datetime" -version = "0.6.8" +version = "0.6.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41" +checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" dependencies = [ "serde", ] [[package]] name = "toml_edit" -version = "0.22.24" +version = "0.22.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17b4795ff5edd201c7cd6dca065ae59972ce77d1b80fa0a84d94950ece7d1474" +checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" dependencies = [ "indexmap", "serde", "serde_spanned", "toml_datetime", + "toml_write", "winnow", ] +[[package]] +name = "toml_write" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" + [[package]] name = "tower-service" version = "0.3.3" @@ -4835,20 +4782,20 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.28" +version = "0.1.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "395ae124c09f9e6918a2310af6038fba074bcf474ac352496d5910dd59a2226d" +checksum = "1b1ffbcf9c6f6b99d386e7444eb608ba646ae452a36b39737deb9663b610f662" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.103", ] [[package]] name = "tracing-core" -version = "0.1.33" +version = "0.1.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e672c95779cf947c5311f83787af4fa8fffd12fb27e4993211a84bdfd9610f9c" +checksum = "b9d12581f227e93f094d3af2ae690a574abb8a2b9b7a96e7cfe9647b2b617678" dependencies = [ "once_cell", "valuable", @@ -4902,9 +4849,9 @@ checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" [[package]] name = "tungstenite" -version = "0.26.2" +version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4793cb5e56680ecbb1d843515b23b6de9a75eb04b66643e256a396d43be33c13" +checksum = "eadc29d668c91fcc564941132e17b28a7ceb2f3ebf0b9dae3e03fd7a6748eb0d" dependencies = [ "bytes", "data-encoding", @@ -5040,12 +4987,6 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" -[[package]] -name = "utf16_iter" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246" - [[package]] name = "utf8_iter" version = "1.0.4" @@ -5064,7 +5005,7 @@ version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3cf4199d1e5d15ddd86a694e4d0dffa9c323ce759fea589f00fef9d81cc1931d" dependencies = [ - "getrandom 0.3.2", + "getrandom 0.3.3", "js-sys", "wasm-bindgen", ] @@ -5129,9 +5070,9 @@ dependencies = [ [[package]] name = "wasi" -version = "0.11.0+wasi-snapshot-preview1" +version = "0.11.1+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" [[package]] name = "wasi" @@ -5170,7 +5111,7 @@ dependencies = [ "log", "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.103", "wasm-bindgen-shared", ] @@ -5205,7 +5146,7 @@ checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.103", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -5240,7 +5181,7 @@ checksum = "17d5042cc5fa009658f9a7333ef24291b1291a25b6382dd68862a7f3b969f69b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.103", ] [[package]] @@ -5582,7 +5523,7 @@ checksum = "a47fddd13af08290e67f4acabf4b459f647552718f683a7b415d290ac744a836" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.103", ] [[package]] @@ -5593,14 +5534,14 @@ checksum = "bd9211b69f8dcdfa817bfd14bf1c97c9188afa36f4750130fcdf3f400eca9fa8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.103", ] [[package]] name = "windows-link" -version = "0.1.1" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76840935b766e1b0a05c0066835fb9ec80071d4c09a16f6bd5f7e655e3c14c38" +checksum = "5e6ad25900d524eaabdbbb96d20b4311e1e7ae1699af4fb28c17ae66c80d798a" [[package]] name = "windows-result" @@ -5779,9 +5720,9 @@ checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] name = "winnow" -version = "0.7.6" +version = "0.7.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63d3fcd9bba44b03821e7d699eeee959f3126dcc4aa8e4ae18ec617c2a5cea10" +checksum = "74c7b26e3480b707944fc872477815d29a8e429d2f93a1ce000f5fa84a15cbcd" dependencies = [ "memchr", ] @@ -5802,20 +5743,14 @@ version = "0.39.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.9.1", ] -[[package]] -name = "write16" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1890f4022759daae28ed4fe62859b1236caebfc61ede2f63ed4e695f3f6d936" - [[package]] name = "writeable" -version = "0.5.5" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51" +checksum = "ea2f10b9bb0928dfb1b42b65e1f9e36f7f54dbdf08457afefb38afcdec4fa2bb" [[package]] name = "x25519-dalek" @@ -5896,16 +5831,16 @@ dependencies = [ [[package]] name = "yamux" -version = "0.13.4" +version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17610762a1207ee816c6fadc29220904753648aba0a9ed61c7b8336e80a559c4" +checksum = "3da1acad1c2dc53f0dde419115a38bd8221d8c3e47ae9aeceaf453266d29307e" dependencies = [ "futures", "log", "nohash-hasher", "parking_lot", "pin-project", - "rand 0.8.5", + "rand 0.9.1", "static_assertions", "web-time", ] @@ -5921,9 +5856,9 @@ dependencies = [ [[package]] name = "yoke" -version = "0.7.5" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "120e6aef9aa629e3d4f52dc8cc43a015c7724194c97dfaf45180d2daf2b77f40" +checksum = "5f41bb01b8226ef4bfd589436a297c53d118f65921786300e427be8d487695cc" dependencies = [ "serde", "stable_deref_trait", @@ -5933,34 +5868,34 @@ dependencies = [ [[package]] name = "yoke-derive" -version = "0.7.5" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154" +checksum = "38da3c9736e16c5d3c8c597a9aaa5d1fa565d0532ae05e27c24aa62fb32c0ab6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", - "synstructure 0.13.1", + "syn 2.0.103", + "synstructure 0.13.2", ] [[package]] name = "zerocopy" -version = "0.8.24" +version = "0.8.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2586fea28e186957ef732a5f8b3be2da217d65c5969d4b1e17f973ebbe876879" +checksum = "a1702d9583232ddb9174e01bb7c15a2ab8fb1bc6f227aa1233858c351a3ba0cb" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.24" +version = "0.8.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a996a8f63c5c4448cd959ac1bab0aaa3306ccfd060472f85943ee0750f0169be" +checksum = "28a6e20d751156648aa063f3800b706ee209a32c0b4d9f24be3d980b01be55ef" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.103", ] [[package]] @@ -5980,8 +5915,8 @@ checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", - "synstructure 0.13.1", + "syn 2.0.103", + "synstructure 0.13.2", ] [[package]] @@ -6001,14 +5936,25 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.103", +] + +[[package]] +name = "zerotrie" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36f0bbd478583f79edad978b407914f61b2972f5af6fa089686016be8f9af595" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", ] [[package]] name = "zerovec" -version = "0.10.4" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa2b893d79df23bfb12d5461018d408ea19dfafe76c2c7ef6d4eba614f8ff079" +checksum = "4a05eb080e015ba39cc9e23bbe5e7fb04d5fb040350f99f34e338d5fdd294428" dependencies = [ "yoke", "zerofrom", @@ -6017,11 +5963,11 @@ dependencies = [ [[package]] name = "zerovec-derive" -version = "0.10.3" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" +checksum = "5b96237efa0c878c64bd89c436f661be4e46b2f3eff1ebb976f7ef2321d2f58f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.103", ] diff --git a/core/Cargo.toml b/core/Cargo.toml index ca890cc2..f04fe1b1 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -44,8 +44,8 @@ async-trait = "0.1.88" thiserror = "2.0.12" mockall = "0.13.1" wasm-bindgen-test = "0.3.50" -libp2p-identity = "0.2.11" hex = "0.4.3" +uuid = "1.17.0" [profile.release] strip = true diff --git a/core/disco/Cargo.toml b/core/disco/Cargo.toml index 6f6870ae..4a47b4a9 100644 --- a/core/disco/Cargo.toml +++ b/core/disco/Cargo.toml @@ -14,16 +14,14 @@ thiserror = { workspace = true } tracing = { workspace = true } futures = { workspace = true } chrono = "0.4.41" -libp2p-identity = { workspace = true, features = ["ecdsa"] } -hex = "0.4.3" -uuid = { version = "1.17.0", features = ["v4"] } +hex = { workspace=true } +posemesh-utils = { workspace = true, features = ["crypto"] } base64 = "0.22.1" -tiny-keccak = { version = "2.0.2", features = ["keccak"] } -k256 = { version = "0.13.4", features = ["ecdsa"] } [target.'cfg(not(target_family="wasm"))'.dependencies] tokio = { workspace = true, features = ["full"] } tokio-tungstenite = { version = "*", features = ["native-tls"] } +uuid = { workspace = true, features = ["v4"] } [target.'cfg(target_family="wasm")'.dependencies] wasm-bindgen = { workspace = true } @@ -31,12 +29,13 @@ wasm-bindgen-futures = { workspace = true } tracing-wasm = { workspace = true } console_error_panic_hook = { workspace = true } web-sys = { version = "0.3.77", features = ["ErrorEvent", "WebSocket", "MessageEvent"] } +uuid = { workspace = true, features = ["v4", "js"] } -[dev-dependencies] -posemesh-utils = { workspace = true } +[target.'cfg(target_family="wasm")'.dependencies.getrandom] +version = "0.3.3" +features = ["wasm_js"] [features] default = [] c = [] node = [] -node-c = [] diff --git a/core/disco/src/client.rs b/core/disco/src/client.rs index 21059977..15475e63 100644 --- a/core/disco/src/client.rs +++ b/core/disco/src/client.rs @@ -1,11 +1,19 @@ -use base64::{alphabet::STANDARD, engine::general_purpose, Engine}; -use libp2p_identity::secp256k1::Keypair; +use base64::{engine::general_purpose, Engine}; +use posemesh_utils::crypto::{parse_secp256k1_private_key, Secp256k1KeyPair}; use prost::Message; +#[cfg(not(target_family = "wasm"))] use tokio::{net::TcpStream, spawn}; +#[cfg(not(target_family = "wasm"))] use tokio_tungstenite::{MaybeTlsStream, WebSocketStream}; +#[cfg(target_family = "wasm")] +use wasm_bindgen_futures::spawn_local as spawn; +// #[cfg(target_family = "wasm")] +// use crate::wasm::setup_ws; + use crate::{error::DiscoveryError, protobuf::common::Capability, utils::{handle_message, new_register_node_request_v1}}; use futures::{stream::SplitSink, StreamExt, SinkExt}; +#[cfg(not(target_family = "wasm"))] async fn setup_ws(url: &str, registration_credentials: &str) -> Result>, DiscoveryError> { use tokio_tungstenite::{connect_async, tungstenite::client::IntoClientRequest}; @@ -17,13 +25,15 @@ async fn setup_ws(url: &str, registration_credentials: &str) -> Result>, tokio_tungstenite::tungstenite::Message>, - keypair: Keypair, + keypair: Secp256k1KeyPair, centralized_id: String, } impl DiscoClient { - pub async fn new(keypair: &Keypair, disco_url: &str, registration_credentials: &str) -> Result { + pub async fn new(private_key: Option<&str>, private_key_path: Option<&str>, disco_url: &str, registration_credentials: &str) -> Result { + let keypair = parse_secp256k1_private_key(private_key, private_key_path)?; let decoded = general_purpose::STANDARD.decode(registration_credentials).map_err(|_| DiscoveryError::InvalidCredentials)?; let decoded_str = String::from_utf8(decoded).map_err(|_| DiscoveryError::InvalidCredentials)?; let parts: Vec<&str> = decoded_str.split(':').collect(); @@ -32,7 +42,7 @@ impl DiscoClient { } let centralized_id = parts[0].to_string(); - let ws = setup_ws(disco_url, registration_credentials).await?; + let ws = setup_ws(&format!("{}/ws/v1/connect", disco_url), registration_credentials).await?; let (writer, mut reader) = ws.split(); spawn(async move { while let Some(msg) = reader.next().await { @@ -42,6 +52,14 @@ impl DiscoClient { tracing::warn!("Disco connection closed: {:?}", msg); continue; } + if msg.is_ping() { + tracing::info!("Received ping"); + continue; + } + if msg.is_pong() { + tracing::info!("Received pong"); + continue; + } if !msg.is_binary() { tracing::error!("Message is expected to be in binary"); continue; @@ -56,7 +74,7 @@ impl DiscoClient { } } }); - Ok(DiscoClient { ws: writer, keypair: keypair.clone(), centralized_id }) + return Ok(DiscoClient { ws: writer, keypair, centralized_id }); } pub async fn register_compatible(&mut self, addrs: Vec, capabilities: &[Capability]) -> Result<(), DiscoveryError> { let message = new_register_node_request_v1(&self.centralized_id, &self.keypair, addrs); diff --git a/core/disco/src/error.rs b/core/disco/src/error.rs index 2c43ac84..8c13cf72 100644 --- a/core/disco/src/error.rs +++ b/core/disco/src/error.rs @@ -1,3 +1,4 @@ +use posemesh_utils::crypto::CryptoError; use prost::{DecodeError, EncodeError}; #[derive(Debug, thiserror::Error)] @@ -16,6 +17,6 @@ pub enum DiscoveryError { InvalidUrl(String), #[error("Invalid registration credential")] InvalidCredentials, - #[error("Failed to sign message: {0}")] - SigningError(#[from] k256::ecdsa::signature::Error), + #[error("Crypto error: {0}")] + CryptoError(#[from] CryptoError), } diff --git a/core/disco/src/lib.rs b/core/disco/src/lib.rs index 392dd956..bf2654e4 100644 --- a/core/disco/src/lib.rs +++ b/core/disco/src/lib.rs @@ -1,5 +1,5 @@ #[cfg(target_family="wasm")] -mod wasm; +pub mod wasm; mod protobuf; pub mod error; diff --git a/core/disco/src/utils.rs b/core/disco/src/utils.rs index cd95a895..b62dc33f 100644 --- a/core/disco/src/utils.rs +++ b/core/disco/src/utils.rs @@ -1,6 +1,5 @@ use crate::{error::DiscoveryError, protobuf::{msgpb::{msg::Payload, Msg}, nodepb::RegisterNodeRequest}}; -use k256::ecdsa::{Signature, SigningKey}; -use libp2p_identity::secp256k1::Keypair; +use posemesh_utils::crypto::{sign_message, Secp256k1KeyPair}; use prost::Message; use uuid::Uuid; @@ -15,9 +14,9 @@ pub fn handle_message(bytes: &[u8]) -> Result<(), DiscoveryError> { Ok(()) } -pub fn new_register_node_request_v1(centralized_id: &str, keypair: &Keypair, addrs: Vec) -> Msg { +pub fn new_register_node_request_v1(centralized_id: &str, keypair: &Secp256k1KeyPair, addrs: Vec) -> Msg { let timestamp = chrono::Utc::now().format("%Y-%m-%dT%H:%M:%S.%fZ").to_string(); - let public_key = hex::encode(keypair.public().to_bytes_uncompressed()); + let public_key = keypair.public_key_hex(); let message = format!("{}{}", centralized_id, timestamp); let signature = sign_message(&message, keypair).unwrap(); @@ -34,36 +33,3 @@ pub fn new_register_node_request_v1(centralized_id: &str, keypair: &Keypair, add })) }; } - -pub fn sign_message(message: &str, keypair: &Keypair) -> Result { - use tiny_keccak::Hasher; - // Hash the message using Keccak-256 - let mut hasher = tiny_keccak::Keccak::v256(); - hasher.update(message.as_bytes()); - let mut hash = [0u8; 32]; - hasher.finalize(&mut hash); - - // Convert private key bytes to SigningKey - let secret_key = keypair.secret().to_bytes(); - let signing_key = SigningKey::from_slice(&secret_key)?; - - // Sign the hash and return signature bytes - let signature: Signature = signing_key.sign_prehash_recoverable(&hash)?.0; - let signature = signature.to_bytes().to_vec(); - let hex_signature = hex::encode(signature); - Ok(format!("0x{}", hex_signature)) -} - - -#[cfg(test)] -mod tests { - use super::*; - use posemesh_utils::parse_secp256k1_private_key; - - #[test] - fn test_sign_message() { - let keypair = parse_secp256k1_private_key(Some("2719d2c0d35fa8a6dce9622e480764ecc0428dd10c70cc52ec0349351989d27a"), None).unwrap(); - let signature = sign_message("test message", &keypair).unwrap(); - assert_eq!(signature, "0x2dcb35d237f7a1d954aceffbaf7cf6e2d36f947c4a83785117c322b7bab031a8180a24829c9a80fc690de79624088a0fa7f62af48407151818299076ecb08af7"); - } -} diff --git a/core/disco/src/wasm.rs b/core/disco/src/wasm.rs index cbd78562..09135e50 100644 --- a/core/disco/src/wasm.rs +++ b/core/disco/src/wasm.rs @@ -3,7 +3,7 @@ use wasm_bindgen::prelude::*; use crate::{error::DiscoveryError}; -fn setup_ws(url: &str) -> Result { +pub async fn setup_ws(url: &str, credentials: &str) -> Result { use wasm_bindgen::{prelude::Closure, JsCast}; use web_sys::MessageEvent; let ws = web_sys::WebSocket::new(url) @@ -34,12 +34,3 @@ fn setup_ws(url: &str) -> Result { Ok(ws) } - -#[wasm_bindgen] -pub struct DiscoClient { - ws: Option -} - -#[wasm_bindgen] -impl DiscoClient { -} diff --git a/core/domain/Cargo.toml b/core/domain/Cargo.toml index c4a389c6..18b63fdc 100644 --- a/core/domain/Cargo.toml +++ b/core/domain/Cargo.toml @@ -26,16 +26,17 @@ ring = "0.17.14" posemesh-utils = {workspace = true} async-timer = "0.7.4" base64 = "0.22.1" +posemesh-disco = { workspace = true } [dev-dependencies] mockall = {workspace = true} wasm-bindgen-test = {workspace = true} [target.'cfg(not(target_family="wasm"))'.dependencies] -libp2p = { workspace = true, features = [ "tokio", "gossipsub", "mdns", "noise", "macros", "tcp", "yamux", "quic", "serde", "relay", "identify", "kad", "dns", "autonat" ] } +libp2p = { workspace = true, features = [ "tokio", "gossipsub" ] } tokio = { workspace = true, features = ["full"] } posemesh-runtime = { workspace = true } -uuid = { version = "1.13.2", features = ["v4"] } +uuid = { workspace = true, features = ["v4"] } tracing-subscriber = { workspace = true, features = ["env-filter"] } tokio-postgres = { version = "0.7.13", features = ["with-uuid-1", "with-serde_json-1"], optional = true } @@ -44,15 +45,19 @@ wasm-bindgen = { workspace = true } wasm-bindgen-futures = { workspace = true } libp2p = { workspace = true, features = [ "wasm-bindgen", "gossipsub" ] } serde-wasm-bindgen = { workspace = true } -uuid = { version = "1.13.2", features = ["v4", "js"] } +uuid = { workspace = true, features = ["v4", "js"] } tracing-wasm = { workspace = true } console_error_panic_hook = { workspace = true } +[target.'cfg(target_family="wasm")'.dependencies.getrandom] +version = "0.3.3" +features = ["wasm_js"] + [lib] crate-type = ["cdylib", "staticlib", "rlib"] [features] -default = [] +default = ["fs"] c = [] fs = ["dep:tokio-postgres"] diff --git a/core/domain/build.rs b/core/domain/build.rs index 3add8fd0..b75c47f5 100644 --- a/core/domain/build.rs +++ b/core/domain/build.rs @@ -25,7 +25,7 @@ fn main() { let out_dir = Path::new("./src").join("protobuf"); - let in_dir = PathBuf::from("../protobuf"); + let in_dir = PathBuf::from("../protobuf").join("domain-cluster"); // Re-run this build.rs if the protos dir changes (i.e. a new file is added) println!("cargo:rerun-if-changed={}", in_dir.to_str().unwrap()); @@ -44,6 +44,19 @@ fn main() { } } + // Add common proto files + let common_dir = PathBuf::from("../protobuf").join("common"); + println!("cargo:rerun-if-changed={}", common_dir.to_str().unwrap()); + + let common_dir_entries = fs::read_dir(common_dir.clone()).unwrap(); + for entry in common_dir_entries { + let path = entry.unwrap().path(); + if path.extension() == proto_ext { + println!("cargo:rerun-if-changed={}", path.to_str().unwrap()); + protos.push(path); + } + } + // Delete all old generated files before re-generating new ones if out_dir.exists() { std::fs::remove_dir_all(&out_dir).unwrap(); diff --git a/core/domain/examples/browser/.env b/core/domain/examples/browser/.env index 6805dadc..d57aa8c9 100644 --- a/core/domain/examples/browser/.env +++ b/core/domain/examples/browser/.env @@ -1,2 +1,2 @@ -VITE_DOMAIN_MANAGER_ADDRESS=/ip4/127.0.0.1/udp/18001/webrtc-direct/certhash/uEiBTfxTyFjhsuMrFWSEm1z0-hNTMx4nAoQs5kCtKhsU1vw/p2p/12D3KooWLcub9hJZoFfVRMk1q5pKG8xSccS5efTLrVXBZ94Nu8y8 +VITE_DOMAIN_MANAGER_ADDRESS=/ip4/127.0.0.1/udp/18001/webrtc-direct/certhash/uEiAcjja4joJPqJoirThhNcSAz001Sc3kOLQZY-mKzoNP-A/p2p/12D3KooWLcub9hJZoFfVRMk1q5pKG8xSccS5efTLrVXBZ94Nu8y8 VITE_APP_ID=domain-browser-example diff --git a/core/domain/examples/browser/src/index.js b/core/domain/examples/browser/src/index.js index 04509736..73cded3a 100644 --- a/core/domain/examples/browser/src/index.js +++ b/core/domain/examples/browser/src/index.js @@ -330,7 +330,7 @@ async function initializeApp() { } const keepAlive = document.getElementById('keepAlive').checked; - const query = new Query([], [], [], nameRegexp, null, true); + const query = new Query([], [], [], nameRegexp, null, false); downloader = await uploadManager.downloadFiles(query, keepAlive, (file, err) => { if (err) { console.error("Error in downloadFiles", err); @@ -486,7 +486,6 @@ function createTaskTable(tasks) { } if (document.readyState !== 'loading') { - console.log('document is already ready, just execute code here'); initializeApp(); } else { // Initialize the application when the DOM is loaded diff --git a/core/domain/src/capabilities/domain_data.rs b/core/domain/src/capabilities/domain_data.rs index 5d3eb58a..16628718 100644 --- a/core/domain/src/capabilities/domain_data.rs +++ b/core/domain/src/capabilities/domain_data.rs @@ -123,7 +123,7 @@ pub async fn serve_data_v1 Ok(()), Ok(Err(e)) => Err(CapabilityError::ProtobufError(e)), - Err(e) => Err(CapabilityError::DomainError(DomainError::Cancelled("client cancelled".to_string(), e))), + Err(e) => Err(CapabilityError::DomainError(DomainError::Cancelled("libp2p", e))), } } } diff --git a/core/domain/src/cluster.rs b/core/domain/src/cluster.rs index 6987268e..a38d1889 100644 --- a/core/domain/src/cluster.rs +++ b/core/domain/src/cluster.rs @@ -1,9 +1,22 @@ -use libp2p::{gossipsub::TopicHash, PeerId}; -use futures::{channel::{mpsc::{channel, Receiver, SendError, Sender}, oneshot}, AsyncReadExt, SinkExt, StreamExt}; -use posemesh_networking::{client::Client, event, libp2p::{Networking, NetworkingConfig}, AsyncStream}; -use crate::{capabilities, datastore::common::DomainError, message::request_response, protobuf::{discovery::{Capability, JoinClusterRequest, JoinClusterResponse, Node}, task::{self, Job, JobRequest, Status, SubmitJobResponse}}}; -use std::{collections::HashMap, fmt::Error}; -use quick_protobuf::{deserialize_from_slice, serialize_into_vec}; +use async_trait::async_trait; +use libp2p::PeerId; +use futures::{channel::{mpsc::{unbounded, UnboundedReceiver, UnboundedSender}, oneshot}, lock::Mutex, Sink, SinkExt, Stream, StreamExt}; +#[cfg(not(target_family="wasm"))] +use posemesh_networking::AsyncStream; +use posemesh_networking::{client::Client, event, libp2p::{Networking, NetworkingConfig}}; +use posemesh_utils::retry_with_delay; + +#[cfg(not(target_family="wasm"))] +use posemesh_disco::client::DiscoClient; + +#[cfg(not(target_family = "wasm"))] +use tokio::time::sleep; +#[cfg(target_family = "wasm")] +use posemesh_utils::sleep; + +use crate::{datastore::common::DomainError, message::{prefix_size_message, read_prefix_size_message, request_response}, protobuf::{common::Capability, discovery::{JoinClusterRequest, JoinClusterResponse, Node}, task::{self, JobRequest, SubmitJobResponse, Task}}}; +use std::{collections::HashMap, pin::Pin, sync::Arc, task::{Context, Poll}, time::Duration}; +use quick_protobuf::deserialize_from_slice; use posemesh_networking::client::TClient; #[cfg(not(target_family = "wasm"))] @@ -14,45 +27,74 @@ use wasm_bindgen_futures::spawn_local as spawn; pub const UNLIMITED_CAPACITY: i32 = -1; pub const JOIN_CLUSTER_PROTOCOL_V1: &str = "/join/v1"; pub const SUBMIT_JOB_PROTOCOL_V1: &str = "/jobs/v1"; -#[derive(Debug)] -pub enum TaskUpdateResult { - Ok(task::Task), - Err(DomainError), + +// pub trait TaskUpdateHandler: Send { +// fn on_task_update(&self, t: &Task); +// } + +pub struct TaskUpdatesSink { + sender: UnboundedSender, } -#[derive(Debug)] -pub struct TaskUpdateEvent { - pub topic: TopicHash, - pub from: Option, - pub result: TaskUpdateResult, +pub struct TaskUpdatesStream { + receiver: UnboundedReceiver, } -struct InnerDomainCluster { - command_rx: Receiver, - manager: String, - peer: Networking, - jobs: HashMap>, +impl TaskUpdatesStream { + pub fn new(receiver: UnboundedReceiver) -> Self { + TaskUpdatesStream { receiver } + } +} + +pub fn new_task_update_duplex() -> (TaskUpdatesSink, TaskUpdatesStream) { + let (sender, receiver) = unbounded(); + (TaskUpdatesSink { sender }, TaskUpdatesStream { receiver }) } -enum Command { - SubmitJob { - job: JobRequest, - task_updates_channel: Sender, - response: oneshot::Sender, - }, - UpdateTask { - task: task::Task, - }, - MonitorJobs { - response: oneshot::Sender>, +impl Stream for TaskUpdatesStream { + type Item = Task; + + fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + self.receiver.poll_next_unpin(cx) } } +impl Sink for TaskUpdatesSink { + type Error = DomainError; + + fn poll_ready(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + self.sender.poll_ready(cx).map_err(|e| DomainError::SendCommandError(e)) + } + + fn start_send(mut self: Pin<&mut Self>, item: Task) -> Result<(), Self::Error> { + self.sender.start_send(item).map_err(|e| DomainError::SendCommandError(e)) + } + + fn poll_close(self: Pin<&mut Self>, _: &mut Context<'_>) -> Poll> { + self.sender.close_channel(); + Poll::Ready(Ok(())) + } + + fn poll_flush(self: Pin<&mut Self>, _: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } +} +impl TaskUpdatesSink { + fn on_task_update(&self, t: &Task) { + let mut sender = self.sender.clone(); + let task = t.clone(); + spawn(async move { + if let Err(e) = sender.send(task).await { + tracing::error!("Failed to send task update: {:?}", e); + } + }); + } +} async fn join(manager_id: &str, client: Client, id: &str, name: &str, capabilities: &[Capability]) -> Result { request_response::( - client.clone(), + client, manager_id, JOIN_CLUSTER_PROTOCOL_V1, &JoinClusterRequest { @@ -65,149 +107,252 @@ async fn join(manager_id: &str, client: Client, id: &str, name: &str, capabiliti 15000 ).await } +use futures::channel::mpsc::{self, Sender, Receiver}; + +#[derive(Clone)] +pub struct DomainCluster { + command_tx: Sender, + pub manager_id: String, + pub peer: Networking, +} + +enum DomainCommand { + SubmitJob(JobRequest, TaskUpdatesSink, oneshot::Sender>), + MonitorJobs(TaskUpdatesSink, oneshot::Sender>), + OnEvent(event::Event), + OnTaskUpdate(Task), +} + +struct InnerDomainCluster { + manager_id: String, + manager: String, + peer: Networking, + name: String, + command_rx: Receiver, + jobs: Arc>>, + capabilities: Vec, +} + +impl DomainCluster { + pub fn init(manager_id: String, manager: String, peer: Networking, name: String, capabilities: Vec) -> Self { + let (tx, rx) = mpsc::channel(100); + let mut inner = InnerDomainCluster { + manager_id: manager_id.clone(), + manager, + peer: peer.clone(), + name, + command_rx: rx, + jobs: Arc::new(Mutex::new(HashMap::new())), + capabilities, + }; -impl InnerDomainCluster { - fn init(mut self) { - let event_receiver = self.peer.event_receiver.clone(); - #[cfg(not(target_family = "wasm"))] spawn(async move { - loop { - let mut event_receiver = event_receiver.lock().await; - tokio::select! { - Some(command) = self.command_rx.next() => self.handle_command(command).await, - event = event_receiver.next() => self.handle_event(event).await, - else => break, - } - } + inner.run().await; }); - #[cfg(target_family = "wasm")] - spawn(async move { - loop { - let mut event_receiver = event_receiver.lock().await; - futures::select! { - command = self.command_rx.select_next_some() => self.handle_command(command).await, - event = event_receiver.next() => self.handle_event(event).await, - complete => break, - } - } - }) + Self { + command_tx: tx, + manager_id, + peer + } } - async fn handle_command(&mut self, command: Command) { - match command { - Command::SubmitJob { job, task_updates_channel, response } => { - let _ = self.submit_job(&job, task_updates_channel).await; - let _ = response.send(true); - }, - Command::UpdateTask { task } => { - let _ = self.peer.client.publish(task.job_id.clone(), serialize_into_vec(&task).expect("can't serialize task update")).await; - } - Command::MonitorJobs { response } => { - let _ = response.send(self.monitor_jobs().await); - } - } + pub async fn submit_job(&mut self, job: &JobRequest, handler: TaskUpdatesSink) -> Result<(), DomainError> { + let (response_tx, response_rx) = oneshot::channel(); + self.command_tx.send(DomainCommand::SubmitJob(job.clone(), handler, response_tx)).await?; + response_rx.await.map_err(|e| DomainError::Cancelled("submit job", e))? } - async fn handle_event(&mut self, e: Option) { - match e { - Some(event::Event::PubSubMessageReceivedEvent { topic, message, from }) => { - let mut task = deserialize_from_slice::(&message).expect("can't deserialize task"); - if let Some(tx) = self.jobs.get_mut(&topic) { - if let Err(e) = tx.send(TaskUpdateEvent { - topic: topic.clone(), - from: from, - result: TaskUpdateResult::Ok(task.clone()), - }).await { - if SendError::is_disconnected(&e) { - self.jobs.remove(&topic); - return; - } - task.status = Status::FAILED; - if let Err(e) = tx.send(TaskUpdateEvent { - topic: topic.clone(), - from: from, - result: TaskUpdateResult::Ok(task.clone()), - }).await { - tracing::error!("Error sending failed task update: {:?}", e); - if SendError::is_disconnected(&e) { - self.jobs.remove(&topic); - return; + pub async fn monitor_jobs(&mut self, handler: TaskUpdatesSink) -> Result<(), DomainError> { + let (response_tx, response_rx) = oneshot::channel(); + self.command_tx.send(DomainCommand::MonitorJobs(handler, response_tx)).await?; + response_rx.await.map_err(|e| DomainError::Cancelled("monitor jobs", e))? + } +} + +impl InnerDomainCluster { + async fn handle_command(&mut self, cmd: DomainCommand) -> Result<(), DomainError> { + match cmd { + DomainCommand::SubmitJob(job, handler, response) => { + let result = async { + let response = request_response::( + self.peer.client.clone(), + &self.manager_id, + SUBMIT_JOB_PROTOCOL_V1, + &job, + 0 + ).await?; + + let job_id = response.job_id.clone(); + self.peer.client.subscribe(job_id.clone()).await?; + tracing::debug!("Subscribed to job: {:?}", job_id); + let mut jobs = self.jobs.lock().await; + jobs.insert(job_id, handler); + Ok(()) + }.await; + let _ = response.send(result); + Ok(()) + } + + DomainCommand::MonitorJobs(handler, response) => { + let result = async { + let mut stream = self.peer.client.send( + "ack".as_bytes().to_vec(), + &self.manager_id, + "/monitor/v1", + 0 + ).await?; + + spawn(async move { + loop { + match read_prefix_size_message::(&mut stream).await { + Ok(task) => { + handler.on_task_update(&task); + } + Err(e) => { + if let quick_protobuf::Error::Io(e) = e { + if e.kind() == std::io::ErrorKind::UnexpectedEof { + break; + } + tracing::error!("Error loading tasks update from domain manager: {:?}", e); + break; + } + tracing::error!("Error loading tasks update from domain manager: {:?}", e); + break; + } } } - // // TODO: send failed task update with error - // self.peer.publish(topic.to_string().clone(), serialize_into_vec(&task).expect("can't serialize task update")).await.unwrap(); + }); + + Ok(()) + }.await; + let _ = response.send(result); + Ok(()) + } + + DomainCommand::OnEvent(e) => { + match e { + event::Event::NodeUnregistered { node_id } => { + if node_id == self.manager_id { + let name = self.name.clone(); + let jobs = self.jobs.clone(); + let manager_id = self.manager_id.clone(); + let client = self.peer.client.clone(); + let capabilities = self.capabilities.clone(); + let peer_id = self.peer.id.clone(); + + spawn(async move { + use posemesh_utils::INFINITE_RETRIES; + + let mut jobs = jobs.lock().await; + jobs.clear(); + + let _ = posemesh_utils::retry_with_delay(move || { + let manager_id = manager_id.clone(); + let client = client.clone(); + let name = name.clone(); + let capabilities = capabilities.clone(); + let peer_id = peer_id.clone(); + Box::pin(async move { + join(&manager_id, client, &peer_id, &name, &capabilities).await + }) + }, + INFINITE_RETRIES, + Duration::from_secs(60) + ).await; + }); + } } + _ => {} } + Ok(()) + } + + DomainCommand::OnTaskUpdate(t) => { + let mut jobs = self.jobs.lock().await; + if let Some(handler) = jobs.get_mut(&t.job_id) { + handler.on_task_update(&t); + } + Ok(()) } - _ => {} } } - async fn submit_job(&mut self, job: &JobRequest, mut tx: Sender) { - let response = request_response::(self.peer.client.clone(), &self.manager, SUBMIT_JOB_PROTOCOL_V1, job, 0).await; - match response { - Ok(response) => { - self.peer.client.subscribe(response.job_id.clone()).await.expect("can't subscribe to job"); - tracing::debug!("Subscribed to job: {:?}", response.job_id); - self.jobs.insert(TopicHash::from_raw(response.job_id.clone()), tx); - } - Err(e) => { - tracing::error!("Error submitting job: {:?}", e); - tx.close_channel(); + pub async fn run(&mut self) { + while let Some(cmd) = self.command_rx.next().await { + if let Err(e) = self.handle_command(cmd).await { + tracing::error!("Error handling command: {:?}", e); } } } +} - async fn monitor_jobs(&mut self) -> Receiver { - let (mut tx, rx) = channel::(3072); - let mut stream = self.peer.client.send("ack".as_bytes().to_vec(), self.manager.clone(), "/monitor/v1".to_string(), 0).await.expect("monitor jobs"); +#[async_trait] +pub trait EventHandler: Send { + fn handle_event(&mut self, e: event::Event); + fn handle_task_update(&mut self, t: &Task); +} - spawn(async move { - loop { - let mut size_buffer = [0u8; 4]; - if let Err(e) = stream.read_exact(&mut size_buffer).await { - if e.kind() == std::io::ErrorKind::UnexpectedEof { - tx.close_channel(); - break; - } - tracing::error!("Error reading size: {:?}", e); - continue; - } - let size = u32::from_be_bytes(size_buffer); - let mut message_buffer = vec![0u8; size as usize]; - stream.read_exact(&mut message_buffer).await.expect("can't read message"); - let job = deserialize_from_slice::(&message_buffer).expect("can't deserialize job"); - tx.send(job).await.expect("can't send job to monitor"); - } - }); +impl EventHandler for DomainCluster { + fn handle_event(&mut self, e: event::Event) { + let _ = self.command_tx.try_send(DomainCommand::OnEvent(e)); + } - rx + fn handle_task_update(&mut self, t: &Task) { + let _ = self.command_tx.try_send(DomainCommand::OnTaskUpdate(t.clone())); } } +pub fn validate_pubsub_message(topic: &str, message: &[u8], from: Option) -> Result { + let mut task = deserialize_from_slice::(&message)?; + if from.is_none() { + tracing::error!("Received task update from unknown peer: {:?}", task); + return Err(DomainError::InvalidPubsubMessage("task update from unknown peer")); + } + if task.receiver.get_or_insert("".to_string()).to_string() != from.unwrap().to_string() && task.sender != from.unwrap().to_string() { + tracing::error!("Received task update for wrong receiver: {:?}", task); + return Err(DomainError::InvalidPubsubMessage("task update for wrong receiver")); + } + if task.job_id != topic.to_string() { + tracing::error!("Received task update for wrong job: {:?}", task); + return Err(DomainError::InvalidPubsubMessage("task update for wrong job")); + } + Ok(task) +} + #[derive(Clone)] -pub struct DomainCluster { - sender: Sender, +pub struct PosemeshSwarm { pub peer: Networking, - pub manager_id: String, - name: String, + + addresses: Arc>>, + expected_addresses_length: usize, + addresses_ready: Arc>>>>, + #[cfg(not(target_family="wasm"))] + disco_client: Arc>>, + + domains: Arc>>>, + capabilities: Arc>>, } -impl DomainCluster { - pub async fn join( - manager_addr: &str, - node_name: &str, +impl PosemeshSwarm { + #[cfg(not(target_family="wasm"))] + pub async fn as_node(&mut self, disco_url: &str, wallet_private_key: Option<&str>, wallet_private_key_path: Option<&str>, registration_secret: &str, capabilities: &[Capability]) -> Result>, DomainError> { + let disco_client = DiscoClient::new(wallet_private_key, wallet_private_key_path, disco_url, registration_secret).await?; + self.disco_client = Arc::new(Mutex::new(Some(disco_client))); + + self.with_capabilities(capabilities).await + } + pub async fn init( join_as_relay: bool, port: u16, enable_websocket: bool, enable_webrtc: bool, - private_key: Option>, + private_key: Option, private_key_path: Option, - relays: Vec - ) -> Result { + relays: Vec, + ) -> Result { let networking = Networking::new(&NetworkingConfig { - bootstrap_nodes: vec![manager_addr.to_string()], + bootstrap_nodes: vec![], relay_nodes: relays, private_key, private_key_path, @@ -218,76 +363,142 @@ impl DomainCluster { enable_websocket, enable_webrtc, namespace: None, - }).unwrap(); - let domain_manager_id = manager_addr.split("/").last().unwrap().to_string(); - - let (tx, rx) = channel::(3072); - let dc = InnerDomainCluster { - manager: domain_manager_id.clone(), - peer: networking.clone(), - jobs: HashMap::new(), - command_rx: rx, + })?; + #[cfg(not(target_family = "wasm"))] + let mut expected_addresses_len: usize = 2; + #[cfg(target_family = "wasm")] + let mut expected_addresses_len: usize = 0; + if enable_webrtc { + expected_addresses_len+=1; + } + if enable_websocket { + expected_addresses_len+=1; + } + + let dc = PosemeshSwarm { + peer: networking, + #[cfg(not(target_family="wasm"))] + disco_client: Arc::new(Mutex::new(None)), + addresses: Arc::new(Mutex::new(Vec::new())), + expected_addresses_length: expected_addresses_len, + addresses_ready: Arc::new(Mutex::new(None)), + domains: Arc::new(Mutex::new(HashMap::new())), + capabilities: Arc::new(Mutex::new(Vec::new())) }; - dc.init(); - let capabilities = &vec![]; - - let networking_clone = networking.clone(); - let id = networking.id; - tracing::info!("Trying to join cluster {domain_manager_id}"); - join(&domain_manager_id, networking.client, &id, node_name, capabilities).await?; - tracing::info!("Managed to join cluster {domain_manager_id}"); - Ok(DomainCluster { - sender: tx, - peer: networking_clone, - manager_id: domain_manager_id.clone(), - name: node_name.to_string(), - }) + + dc.clone().listen().await; + Ok(dc) } - pub async fn with_capabilities(&mut self, capabilities: &[Capability]) -> Result>, DomainError> { - join(&self.manager_id, self.peer.client.clone(), &self.peer.id, &self.name, capabilities).await?; - let mut streams = Vec::new(); - for capability in capabilities { - let stream = self.peer.client.set_stream_handler(&capability.endpoint).await?; - streams.push(stream); + async fn listen(mut self) -> Vec { + let (tx, rx) = oneshot::channel::>(); + let event_receiver = self.peer.event_receiver.clone(); + if self.expected_addresses_length > 0 { + self.addresses_ready = Arc::new(Mutex::new(Some(tx))); + } else { + let _ = tx.send(vec![]); } - Ok(streams) + #[cfg(not(target_family = "wasm"))] + spawn(async move { + loop { + let mut event_receiver = event_receiver.lock().await; + tokio::select! { + event = event_receiver.next() => self.handle_event(event).await, + else => break, + } + } + }); + + #[cfg(target_family = "wasm")] + spawn(async move { + loop { + let event = { + let mut event_receiver = event_receiver.lock().await; + event_receiver.next().await + }; + self.handle_event(event).await; + } + }); + + rx.await.unwrap() } - pub async fn submit_job(&mut self, job: &JobRequest) -> Receiver { - let (tx, rx) = oneshot::channel::(); - let (updates_tx, updates_rx) = channel::(3072); - let cmd = Command::SubmitJob { - job: job.clone(), - response: tx, - task_updates_channel: updates_tx, - }; - self.sender.send(cmd).await.unwrap_or_else(|_| panic!("can't send command {}", job.name)); - let _ = rx.await.unwrap_or_else(|_| panic!("can't wait for response {}", job.name)); - updates_rx + async fn handle_event(&mut self, e: Option) { + match e { + Some(event::Event::PubSubMessageReceivedEvent { topic, message, from }) => { + let task = validate_pubsub_message(&topic.to_string(), &message, from); + if let Ok(task) = task { + let domains = self.domains.clone(); + spawn(async move { + let mut domains = domains.lock().await; + domains.iter_mut().for_each(|(_, d)| d.handle_task_update(&task)); + }); + } + } + Some(event::Event::NewAddress { address }) => { + let mut addresses = self.addresses.lock().await; + addresses.push(address.to_string()); + if addresses.len() == self.expected_addresses_length { + if let Some(tx) = self.addresses_ready.lock().await.take() { + let _ = tx.send(addresses.clone()); + } + } + } + Some(e) => { + let domains = self.domains.clone(); + spawn(async move { + let mut domains = domains.lock().await; + domains.iter_mut().for_each(|(_, d)| d.handle_event(e.clone())); + }); + } + _ => {} + } } - pub async fn monitor_jobs(&mut self) -> Receiver { - let (tx, rx) = oneshot::channel::>(); - let cmd = Command::MonitorJobs { - response: tx, - }; - self.sender.send(cmd).await.expect("can't send command"); - rx.await.expect("can't wait for response") + #[cfg(not(target_family="wasm"))] + async fn with_capabilities(&mut self, capabilities: &[Capability]) -> Result>, DomainError> { + let mut disco_client_lock = self.disco_client.lock().await; + if let Some(disco_client) = disco_client_lock.as_mut() { + // TODO: can't add capabilities here, because they are using different protobuf builders + disco_client.register_compatible(self.addresses.lock().await.clone(), &vec![]).await?; + drop(disco_client_lock); + let mut streams = Vec::new(); + for capability in capabilities { + let stream = self.peer.client.set_stream_handler(&capability.endpoint).await?; + streams.push(stream); + } + Ok(streams) + } else { + Err(DomainError::RegisterCapabilityError("Disco client not found")) + } } + // TODO: should take domain_id, and disco will find the manager by id + pub async fn join_domain(&mut self, domain_manager_id: &str, dc: impl EventHandler + 'static) -> Result<(), DomainError> { + let mut domains = self.domains.lock().await; + domains.insert(domain_manager_id.to_string(), Box::new(dc)); - pub async fn fail_task(&mut self, task: &task::Task, err: Error) { - let mut t = task.clone(); - t.status = Status::FAILED; - t.output = Some(task::Any { - type_url: "Error".to_string(), - value: serialize_into_vec(&task::Error { - message: format!("{:?}", err), - }).unwrap(), - }); - self.sender.send(Command::UpdateTask { - task: t, - }).await.expect("can't send command"); + Ok(()) } - // pub async fn request_response(&mut self, message: Vec, peer_id: String, protocol: String, timeout: u32) -> Result> +} + +pub async fn join_domain(swarm: &mut PosemeshSwarm, domain_manager: &str, name: &str) -> Result { + let domain_manager_id = if domain_manager.split('/').last().is_none() { + return Err(DomainError::InvalidManagerAddress(domain_manager.to_string())); + } else { + domain_manager.split('/').last().unwrap() + }; + + let peer = swarm.peer.clone(); + + let capabilities = swarm.capabilities.clone(); + let capabilities = capabilities.lock().await; + let dc = DomainCluster::init(domain_manager_id.to_string(), domain_manager.to_string(), peer.clone(), name.to_string(), capabilities.clone()); + let mut parsed_addresses = HashMap::new(); + parsed_addresses.insert(domain_manager_id.to_string(), vec![domain_manager.to_string()]); + swarm.peer.client.bootstrap(parsed_addresses).await?; + sleep(Duration::from_secs(10)).await; + let _ = join(domain_manager_id, peer.client.clone(), &peer.id, name, &capabilities).await?; + drop(capabilities); + swarm.join_domain(domain_manager_id, dc.clone()).await?; + Ok(dc) } diff --git a/core/domain/src/datastore/common.rs b/core/domain/src/datastore/common.rs index 6fb721d2..1f612fc1 100644 --- a/core/domain/src/datastore/common.rs +++ b/core/domain/src/datastore/common.rs @@ -1,12 +1,11 @@ -use std::future::Future; - use crate::{auth::AuthError, protobuf::domain_data::{self, Data}}; use async_trait::async_trait; +use posemesh_disco::error::DiscoveryError; use posemesh_networking::libp2p::NetworkError; use uuid::Uuid; -use futures::{channel::{mpsc::{Receiver, Sender}, oneshot::Canceled}, AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt}; +use futures::{channel::{mpsc::{Receiver, SendError, Sender}, oneshot::Canceled}, AsyncWrite}; use sha2::{Digest, Sha256 as Sha256Hasher}; pub type Reader = Receiver>; @@ -22,8 +21,8 @@ pub const CHUNK_SIZE: usize = 7 * 1024; // receiver over webRTC gets error Custo pub enum DomainError { #[error("Not found")] NotFound(String), - #[error("{0} Cancelled: {1}")] - Cancelled(String, Canceled), + #[error("{0} cancelled: {1}")] + Cancelled(&'static str, Canceled), #[error("IO error: {0}")] Io(#[from]std::io::Error), #[cfg(all(feature="fs", not(target_family="wasm")))] @@ -42,6 +41,16 @@ pub enum DomainError { ProtobufError(#[from] quick_protobuf::Error), #[error("Auth error: {0}")] AuthError(#[from] AuthError), + #[error("Invalid manager address: {0}")] + InvalidManagerAddress(String), + #[error("Disco error: {0}")] + DiscoError(#[from] DiscoveryError), + #[error("Can't register capability: {0}")] + RegisterCapabilityError(&'static str), + #[error("Failed to send command: {0}")] + SendCommandError(#[from] SendError), + #[error("Invalid pubsub message: {0}")] + InvalidPubsubMessage(&'static str), } #[async_trait] diff --git a/core/domain/src/datastore/remote.rs b/core/domain/src/datastore/remote.rs index 7465c18d..24e4f9ef 100644 --- a/core/domain/src/datastore/remote.rs +++ b/core/domain/src/datastore/remote.rs @@ -12,7 +12,7 @@ use std::{collections::HashSet, future::Future, sync::Arc}; use async_trait::async_trait; use posemesh_networking::client::{Client, TClient}; use libp2p::Stream; -use crate::{capabilities::domain_data::{CONSUME_DATA_PROTOCOL_V1, PRODUCE_DATA_PROTOCOL_V1}, cluster::{DomainCluster, TaskUpdateEvent, TaskUpdateResult}, datastore::common::{Datastore, DomainError}, message::{handshake, handshake_then_prefixed_content, prefix_size_message, read_prefix_size_message}, protobuf::{domain_data, task::{self, mod_ResourceRecruitment as ResourceRecruitment, Any, ConsumeDataInputV1, Status, Task}}}; +use crate::{capabilities::domain_data::{CONSUME_DATA_PROTOCOL_V1, PRODUCE_DATA_PROTOCOL_V1}, cluster::{new_task_update_duplex, DomainCluster, TaskUpdatesStream}, datastore::common::{Datastore, DomainError}, message::{handshake, handshake_then_prefixed_content, prefix_size_message, read_prefix_size_message}, protobuf::{domain_data, task::{self, mod_ResourceRecruitment as ResourceRecruitment, Any, ConsumeDataInputV1, Status, Task}}}; use super::common::{hash_chunk, DataConsumer, DomainData, ReliableDataProducer, CHUNK_SIZE}; use futures::{channel::oneshot, lock::Mutex, AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt, StreamExt}; use rs_merkle::{algorithms::Sha256, MerkleTree}; @@ -223,10 +223,10 @@ impl DataConsumer for TaskHandler { if let Some(done_rx) = self.done_rx.take() { match done_rx.await { Ok(res) => res, - Err(e) => Err(DomainError::Cancelled("TaskHandler has been cancelled".to_string(), e)), + Err(e) => Err(DomainError::Cancelled("TaskHandler", e)), } } else { - Err(DomainError::Cancelled("TaskHandler is not initialized".to_string(), Canceled)) + Err(DomainError::Cancelled("TaskHandler", Canceled)) } } } @@ -327,49 +327,37 @@ impl Datastore for RemoteDatastore { nonce: Uuid::new_v4().to_string(), }; - let mut download_task_recv = self.cluster.submit_job(job).await; + let (task_updates_sink, mut task_updates_stream) = new_task_update_duplex(); + self.cluster.submit_job(job, task_updates_sink).await?; let (tx, rx) = oneshot::channel::>(); spawn(async move { - if let Some(update) = download_task_recv.next().await { - match update { - TaskUpdateEvent { - result: TaskUpdateResult::Ok(mut task), - .. - } => match task.status { - Status::PENDING => { - task.status = Status::STARTED; - if let Err(e) = peer.publish(task.job_id.clone(), serialize_into_vec(&task).expect("Failed to serialize message")).await { - tracing::error!("Failed to publish message: {:?}", e); - } - let download_task = download_data(peer.clone(), domain_id.clone(), metadata_only, writer, task, data).await; - tx.send(download_task).expect("Failed to send completion signal"); - return; - }, - Status::FAILED => { - tracing::error!("Failed to download data: {:?}", task); - tx.send(Err(DomainError::Cancelled("Failed to download data".to_string(), Canceled))).expect("Failed to send completion signal"); - return; - }, - _ => { - tracing::debug!("Task status: {:?}", task.status); - tx.send(Err(DomainError::Cancelled("We are not supposed to handle this status".to_string(), Canceled))).expect("Failed to send completion signal"); - return; + if let Some(mut task) = task_updates_stream.next().await { + match task.status { + Status::PENDING => { + task.status = Status::STARTED; + if let Err(e) = peer.publish(task.job_id.clone(), serialize_into_vec(&task).expect("Failed to serialize message")).await { + tracing::error!("Failed to publish message: {:?}", e); } - } - TaskUpdateEvent { - result: TaskUpdateResult::Err(e), - .. - } => { - tracing::error!("Task update failure: {:?}", e); - tx.send(Err(e)).expect("Failed to send completion signal"); + let download_task = download_data(peer.clone(), domain_id.clone(), metadata_only, writer, task, data).await; + tx.send(download_task).expect("Failed to send completion signal"); + return; + }, + Status::FAILED => { + tracing::error!("Failed to download data: {:?}", task); + tx.send(Err(DomainError::Cancelled("Failed to download data", Canceled))).expect("Failed to send completion signal"); + return; + }, + _ => { + tracing::debug!("Task status: {:?}", task.status); + tx.send(Err(DomainError::Cancelled("We are not supposed to handle this status", Canceled))).expect("Failed to send completion signal"); return; } } + } else { + tracing::debug!("task update channel is closed"); + tx.send(Err(DomainError::Cancelled("Task update channel is closed", Canceled))).expect("Failed to send completion signal"); } - - tracing::debug!("task update channel is closed"); - tx.send(Err(DomainError::Cancelled("Task update channel is closed".to_string(), Canceled))).expect("Failed to send completion signal"); }); // TODO: handle more statuses for example domain manager cancel this task by sending a Failed status when it runs out of credits @@ -380,12 +368,15 @@ impl Datastore for RemoteDatastore { match res { Ok(Ok(download_task)) => Ok(Box::new(download_task)), Ok(Err(e)) => Err(e), - Err(e) => Err(DomainError::Cancelled("Failed to download data".to_string(), e)), + Err(e) => Err(DomainError::Cancelled("Failed to download data", e)), } } async fn upsert(&mut self, domain_id: String) -> Result, DomainError>{ - let mut upload_job_recv = self.cluster.submit_job(&task::JobRequest { + let (task_updates_sink, mut task_updates_stream) = new_task_update_duplex(); + + let mut peer = self.cluster.peer.client.clone(); + let job = &task::JobRequest { nonce: Uuid::new_v4().to_string(), domain_id: domain_id.clone(), name: "stream uploading recordings".to_string(), @@ -409,19 +400,16 @@ impl Datastore for RemoteDatastore { receiver: None, } ], - }).await; + }; + self.cluster.submit_job(job, task_updates_sink).await?; - let mut peer = self.cluster.peer.client.clone(); let (tx, rx) = oneshot::channel::>(); let domain_id = domain_id.clone(); spawn(async move{ loop { - let update = upload_job_recv.next().await; + let update = task_updates_stream.next().await; match update { - Some(TaskUpdateEvent { - result: TaskUpdateResult::Ok(mut task), - .. - }) => match task.status { + Some(mut task) => match task.status { Status::PENDING => { task.status = Status::STARTED; @@ -455,13 +443,6 @@ impl Datastore for RemoteDatastore { tracing::debug!("task update channel is closed"); break; } - Some(TaskUpdateEvent { - result: TaskUpdateResult::Err(e), - .. - }) => { - tracing::error!("Task update failure: {:?}", e); - break; - } } } }); @@ -470,7 +451,7 @@ impl Datastore for RemoteDatastore { if let Ok(Some(producer)) = producer { Ok(Box::new(producer)) } else { - Err(DomainError::Cancelled("Failed to upload data".to_string(), Canceled)) + Err(DomainError::Cancelled("Failed to upload data", Canceled)) } } } diff --git a/core/domain/src/message.rs b/core/domain/src/message.rs index 013acba0..b7b01272 100644 --- a/core/domain/src/message.rs +++ b/core/domain/src/message.rs @@ -1,6 +1,6 @@ use std::time::Duration; -use futures::{AsyncRead, AsyncReadExt, AsyncWriteExt}; +use futures::{channel::oneshot::Canceled, AsyncRead, AsyncReadExt, AsyncWriteExt}; use libp2p::Stream; use posemesh_networking::{client::Client, libp2p::NetworkError}; use quick_protobuf::{deserialize_from_slice, serialize_into_vec, MessageRead, MessageWrite}; @@ -60,7 +60,7 @@ pub async fn handshake(mut peer: Client, domain_id: &str, access_token: &str, re let mut upload_stream = peer.send(prefix_size_message(&task::DomainClusterHandshakeRequest{ access_token: access_token.to_string(), domain_id: domain_id.to_string(), - }), receiver.to_string(), endpoint.to_string(), timeout).await?; + }), receiver, endpoint, timeout).await?; let response = read_prefix_size_message::(&mut upload_stream).await?; match response.code { @@ -80,17 +80,16 @@ pub async fn request_response_with_handshake MessageRead<'a> + Send + >(mut peer: Client, receiver: &str, endpoint: &str, request: &Request, timeout_millis: u32) -> Result { - let mut upload_stream = peer.send(prefix_size_message(request), receiver.to_string(), endpoint.to_string(), timeout_millis).await?; - +pub async fn request_response MessageRead<'a> + Send>(mut peer: Client, receiver: &str, endpoint: &str, request: &Request, timeout_millis: u32) -> Result { + let mut upload_stream = peer.send(prefix_size_message(request), receiver, endpoint, timeout_millis).await?; posemesh_utils::timeout(Duration::from_millis(timeout_millis as u64), async move { - let response = read_prefix_size_message::(&mut upload_stream).await.expect("Failed to read response"); + let response = read_prefix_size_message::(&mut upload_stream).await?; Ok(response) }).await? } pub async fn request_response_raw(mut peer:Client, receiver: &str, endpoint: &str, request: &[u8], timeout_millis: u32) -> Result, NetworkError> { - let mut upload_stream = peer.send(request.to_vec(), receiver.to_string(), endpoint.to_string(), timeout_millis).await?; + let mut upload_stream = peer.send(request.to_vec(), receiver, endpoint, timeout_millis).await?; upload_stream.close().await?; let mut response = Vec::new(); upload_stream.read_to_end(&mut response).await?; diff --git a/core/domain/src/protobuf/common.rs b/core/domain/src/protobuf/common.rs new file mode 100644 index 00000000..17fc87d8 --- /dev/null +++ b/core/domain/src/protobuf/common.rs @@ -0,0 +1,51 @@ +// Automatically generated rust module for 'common.proto' file + +#![allow(non_snake_case)] +#![allow(non_upper_case_globals)] +#![allow(non_camel_case_types)] +#![allow(unused_imports)] +#![allow(unknown_lints)] +#![allow(clippy::all)] +#![cfg_attr(rustfmt, rustfmt_skip)] + + +use quick_protobuf::{MessageInfo, MessageRead, MessageWrite, BytesReader, Writer, WriterBackend, Result}; +use quick_protobuf::sizeofs::*; +use super::*; + +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Debug, Default, PartialEq, Clone)] +pub struct Capability { + pub endpoint: String, + pub capacity: i32, +} + +impl<'a> MessageRead<'a> for Capability { + fn from_reader(r: &mut BytesReader, bytes: &'a [u8]) -> Result { + let mut msg = Self::default(); + while !r.is_eof() { + match r.next_tag(bytes) { + Ok(10) => msg.endpoint = r.read_string(bytes)?.to_owned(), + Ok(16) => msg.capacity = r.read_int32(bytes)?, + Ok(t) => { r.read_unknown(bytes, t)?; } + Err(e) => return Err(e), + } + } + Ok(msg) + } +} + +impl MessageWrite for Capability { + fn get_size(&self) -> usize { + 0 + + 1 + sizeof_len((&self.endpoint).len()) + + 1 + sizeof_varint(*(&self.capacity) as u64) + } + + fn write_message(&self, w: &mut Writer) -> Result<()> { + w.write_with_tag(10, |w| w.write_string(&**&self.endpoint))?; + w.write_with_tag(16, |w| w.write_int32(*&self.capacity))?; + Ok(()) + } +} + diff --git a/core/domain/src/protobuf/discovery.rs b/core/domain/src/protobuf/discovery.rs index 7e391ac1..f3103c9d 100644 --- a/core/domain/src/protobuf/discovery.rs +++ b/core/domain/src/protobuf/discovery.rs @@ -13,47 +13,11 @@ use quick_protobuf::{MessageInfo, MessageRead, MessageWrite, BytesReader, Writer use quick_protobuf::sizeofs::*; use super::*; -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Debug, Default, PartialEq, Clone)] -pub struct Capability { - pub endpoint: String, - pub capacity: i32, -} - -impl<'a> MessageRead<'a> for Capability { - fn from_reader(r: &mut BytesReader, bytes: &'a [u8]) -> Result { - let mut msg = Self::default(); - while !r.is_eof() { - match r.next_tag(bytes) { - Ok(10) => msg.endpoint = r.read_string(bytes)?.to_owned(), - Ok(16) => msg.capacity = r.read_int32(bytes)?, - Ok(t) => { r.read_unknown(bytes, t)?; } - Err(e) => return Err(e), - } - } - Ok(msg) - } -} - -impl MessageWrite for Capability { - fn get_size(&self) -> usize { - 0 - + 1 + sizeof_len((&self.endpoint).len()) - + 1 + sizeof_varint(*(&self.capacity) as u64) - } - - fn write_message(&self, w: &mut Writer) -> Result<()> { - w.write_with_tag(10, |w| w.write_string(&**&self.endpoint))?; - w.write_with_tag(16, |w| w.write_int32(*&self.capacity))?; - Ok(()) - } -} - #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Debug, Default, PartialEq, Clone)] pub struct Node { pub id: String, - pub capabilities: Vec, + pub capabilities: Vec, pub name: String, } @@ -63,7 +27,7 @@ impl<'a> MessageRead<'a> for Node { while !r.is_eof() { match r.next_tag(bytes) { Ok(10) => msg.id = r.read_string(bytes)?.to_owned(), - Ok(18) => msg.capabilities.push(r.read_message::(bytes)?), + Ok(18) => msg.capabilities.push(r.read_message::(bytes)?), Ok(26) => msg.name = r.read_string(bytes)?.to_owned(), Ok(t) => { r.read_unknown(bytes, t)?; } Err(e) => return Err(e), diff --git a/core/domain/src/protobuf/mod.rs b/core/domain/src/protobuf/mod.rs index 3be71596..e933900a 100644 --- a/core/domain/src/protobuf/mod.rs +++ b/core/domain/src/protobuf/mod.rs @@ -2,3 +2,4 @@ pub mod discovery; pub mod task; pub mod domain_data; +pub mod common; diff --git a/core/domain/src/spatial/reconstruction.rs b/core/domain/src/spatial/reconstruction.rs index 098dbe57..ba5a0028 100644 --- a/core/domain/src/spatial/reconstruction.rs +++ b/core/domain/src/spatial/reconstruction.rs @@ -1,9 +1,8 @@ -use futures::channel::mpsc::Receiver; use quick_protobuf::serialize_into_vec; -use crate::{cluster::{DomainCluster, TaskUpdateEvent}, protobuf::{domain_data::Query, task}}; +use crate::{cluster::{DomainCluster, TaskUpdatesSink}, datastore::common::DomainError, protobuf::{domain_data::Query, task}}; -pub async fn reconstruction_job(mut domain_cluster: DomainCluster, domain_id: &str, scans: Vec) -> Receiver { +pub async fn reconstruction_job(mut domain_cluster: DomainCluster, domain_id: &str, scans: Vec, handler: TaskUpdatesSink) -> Result<(), DomainError> { let mut uploaded = Vec::::new(); for scan in scans { let input = task::LocalRefinementInputV1 { @@ -72,5 +71,5 @@ pub async fn reconstruction_job(mut domain_cluster: DomainCluster, domain_id: &s tracing::debug!("job has {} tasks", job.tasks.len()); - domain_cluster.submit_job(&job).await + domain_cluster.submit_job(&job, handler).await } diff --git a/core/domain/src/wasm.rs b/core/domain/src/wasm.rs index de6e089c..9631f512 100644 --- a/core/domain/src/wasm.rs +++ b/core/domain/src/wasm.rs @@ -4,7 +4,7 @@ use quick_protobuf::serialize_into_vec; use js_sys::Function; use serde_wasm_bindgen::{from_value, to_value}; use wasm_bindgen::prelude::*; -use crate::{binding_helper::{init_r_remote_storage, initialize_consumer, DataConsumer}, cluster::{DomainCluster as r_DomainCluster, TaskUpdateResult}, datastore::{common::{data_id_generator, Datastore, ReliableDataProducer as r_ReliableDataProducer}, remote::RemoteDatastore as r_RemoteDatastore}, protobuf::domain_data, spatial::reconstruction::reconstruction_job as r_reconstruction_job}; +use crate::{binding_helper::{init_r_remote_storage, initialize_consumer, DataConsumer}, cluster::{join_domain, new_task_update_duplex, DomainCluster as r_DomainCluster, PosemeshSwarm as r_PosemeshSwarm}, datastore::{common::{data_id_generator, Datastore, ReliableDataProducer as r_ReliableDataProducer}, remote::RemoteDatastore as r_RemoteDatastore}, protobuf::domain_data, spatial::reconstruction::reconstruction_job as r_reconstruction_job}; use wasm_bindgen_futures::{future_to_promise, js_sys::{self, Promise, Uint8Array}, spawn_local}; #[derive(Clone)] @@ -109,33 +109,37 @@ pub struct DomainCluster { inner: Arc>, } - #[wasm_bindgen] -pub fn join_cluster(domain_manager_addr: String, name: String, private_key: Option>, private_key_path: Option) -> Promise { +pub fn join_cluster(domain_manager_addr: String, name: String, private_key: Option, private_key_path: Option) -> Promise { let future = async move { - match r_DomainCluster::join(&domain_manager_addr, &name, false, 0, false, false, private_key, private_key_path, vec![]).await { - Ok(cluster) => Ok(JsValue::from(DomainCluster { inner: Arc::new(Mutex::new(cluster)) })), - Err(e) => Err(JsValue::from_str(&format!("failed to join cluster: {}", e))) + match r_PosemeshSwarm::init(false, 0, false, false, private_key, private_key_path, vec![]).await { + Ok(mut swarm) => { + match join_domain(&mut swarm, &domain_manager_addr, &name).await { + Ok(cluster) => Ok(JsValue::from(DomainCluster { inner: Arc::new(Mutex::new(cluster)) })), + Err(e) => Err(JsValue::from_str(&format!("failed to join cluster: {}", e))) + } + } + Err(e) => Err(JsValue::from_str(&format!("failed to init swarm: {}", e))) } }; future_to_promise(future) } -#[wasm_bindgen] -impl DomainCluster { - #[wasm_bindgen] - pub fn monitor(&self, callback: Function) { - let inner = self.inner.clone(); - block_on(async move { - let mut rx = inner.lock().unwrap().monitor_jobs().await; - while let Some(job) = rx.next().await { - let job_bytes = serialize_into_vec(&job).unwrap(); - let js_arr = Uint8Array::from(&job_bytes[..]); - callback.call1(&JsValue::NULL, &js_arr).unwrap(); - } - }); - } -} +// #[wasm_bindgen] +// impl DomainCluster { +// #[wasm_bindgen] +// pub fn monitor(&self, callback: Function) { +// let inner = self.inner.clone(); +// block_on(async move { +// // let mut rx = inner.lock().unwrap().monitor_jobs().await; +// while let Some(job) = rx.next().await { +// let job_bytes = serialize_into_vec(&job).unwrap(); +// let js_arr = Uint8Array::from(&job_bytes[..]); +// callback.call1(&JsValue::NULL, &js_arr).unwrap(); +// } +// }); +// } +// } #[wasm_bindgen] struct ReliableDataProducer { @@ -198,7 +202,7 @@ impl RemoteDatastore { pub fn new(cluster: &DomainCluster) -> Self { let r_domain_cluster = cluster.inner.lock().unwrap(); let cluster = r_domain_cluster.clone(); - Self { inner: init_r_remote_storage(Box::into_raw(Box::new(cluster))) } + Self { inner: r_RemoteDatastore::new(cluster) } } #[wasm_bindgen] @@ -253,23 +257,24 @@ pub fn reconstruction_job(cluster: &DomainCluster, domain_id: String, scans: Vec drop(cluster); future_to_promise(async move { - let mut r = r_reconstruction_job(cluster_clone, &domain_id, scans).await; - spawn_local(async move { - while let Some(task_update) = r.next().await { - match task_update.result { - TaskUpdateResult::Ok(task) => { - tracing::debug!("Task {}-{} update status {:?}", task.job_id, task.name, task.status); - let task_update_bytes = serialize_into_vec(&task).unwrap(); + let (sink, mut stream) = new_task_update_duplex(); + let r = r_reconstruction_job(cluster_clone, &domain_id, scans, sink).await; + match r { + Ok(_) => { + spawn_local(async move { + while let Some(task_update) = stream.next().await { + let task_update_bytes = serialize_into_vec(&task_update).unwrap(); let js_arr = Uint8Array::from(&task_update_bytes[..]); callback.call1(&JsValue::NULL, &js_arr).unwrap(); } - TaskUpdateResult::Err(e) => { - tracing::error!("Error: {}", e); - } - } + }); + + Ok(JsValue::NULL) } - }); - Ok(JsValue::NULL) + Err(e) => { + return Err(JsValue::from_str(&format!("{}", e))); + } + } }) } diff --git a/core/examples/client/src/main.rs b/core/examples/client/src/main.rs index 18f803c3..f4a6e9d9 100644 --- a/core/examples/client/src/main.rs +++ b/core/examples/client/src/main.rs @@ -1,7 +1,7 @@ use futures::{AsyncWrite, StreamExt}; use quick_protobuf::deserialize_from_slice; use std::{collections::HashMap, fs, io::{Error, ErrorKind, Read}, pin::Pin, sync::{Arc, Mutex}, task::{Context, Poll}, vec}; -use posemesh_domain::{cluster::{DomainCluster, TaskUpdateEvent, TaskUpdateResult}, datastore::{common::{data_id_generator, Datastore}, remote::RemoteDatastore}, protobuf::domain_data::{self, Metadata, Query, UpsertMetadata}, spatial::reconstruction::reconstruction_job}; +use posemesh_domain::{cluster::{join_domain, new_task_update_duplex, PosemeshSwarm}, datastore::{common::{data_id_generator, Datastore}, remote::RemoteDatastore}, protobuf::domain_data::{self, Metadata, Query, UpsertMetadata}, spatial::reconstruction::reconstruction_job}; /* * This is a client that wants to do reconstruction in domain cluster @@ -10,7 +10,10 @@ use posemesh_domain::{cluster::{DomainCluster, TaskUpdateEvent, TaskUpdateResult */ #[tokio::main] async fn main() -> Result<(), Box> { - tracing_subscriber::fmt().with_env_filter(tracing_subscriber::EnvFilter::from_default_env()).init(); + tracing_subscriber::fmt() + .with_env_filter(tracing_subscriber::EnvFilter::from_default_env()) + .with_line_number(true) + .init(); let args: Vec = std::env::args().collect(); if args.len() < 5 { println!("Usage: {} ", args[0]); @@ -28,7 +31,11 @@ async fn main() -> Result<(), Box> { let base_path = format!("./volume/{}", name); let private_key_path = format!("{}/pkey", base_path); - let domain_cluster = DomainCluster::join(&domain_manager, &name, false, port, false, false, None, Some(private_key_path), relay).await.expect("failed to join cluster"); + let mut swarm = PosemeshSwarm::init(false, port, false, false, None, Some(private_key_path), relay).await?; + // let wallet_private_key_path = format!("{}/wallet_private_key", base_path); + // swarm.as_node("wss://dds.dev.aukiverse.com", None, Some(&wallet_private_key_path), "MzUwNGI2YzgtODI0ZC00NjZiLThjYzYtMGIyYTYzMWRmNTA3OlpUVmtOVFprTnpjdE56azVZeTAwTkRVMkxXSXdaamd0WmprMU16bGtNMk0wTldZeg==", &[]).await?; + + let domain_cluster = join_domain(&mut swarm, &domain_manager, &name).await?; let mut remote_datastore = RemoteDatastore::new(domain_cluster.clone()); let input_dir = format!("{}/input", base_path); @@ -138,14 +145,12 @@ async fn main() -> Result<(), Box> { println!("producer closed"); - let mut recv = reconstruction_job(domain_cluster, &domain_id, vec![scan]).await; + let (task_update_sink, mut task_update_stream) = new_task_update_duplex(); + reconstruction_job(domain_cluster, &domain_id, vec![scan], task_update_sink).await?; loop { tokio::select! { - Some(TaskUpdateEvent { - result: TaskUpdateResult::Ok(task), - .. - }) = recv.next() => { + Some(task) = task_update_stream.next() => { println!("Received task {} status update: {:?}", task.name, task.status); } else => { diff --git a/core/examples/test-concurrent/src/main.rs b/core/examples/test-concurrent/src/main.rs index e6b1103c..5fff45f0 100644 --- a/core/examples/test-concurrent/src/main.rs +++ b/core/examples/test-concurrent/src/main.rs @@ -25,10 +25,10 @@ async fn main() { let mut bootstrap = Networking::new(&networking).unwrap(); let mut chat_protocol = bootstrap.client.set_stream_handler(protocol).await.unwrap(); - let bootstrap_id = bootstrap.id.clone(); - let _bootstrap_id_clone = bootstrap_id.clone(); - let bootstrap_id_clone_clone = bootstrap_id.clone(); - let bootstrap_id_clone_clone_clone = bootstrap_id.clone(); + let bootstrap_id = bootstrap.id; + let bootstrap_id_clone = bootstrap_id.clone(); + let bootstrap_id_clone_clone = bootstrap_id_clone.clone(); + let bootstrap_id_clone_clone_clone = bootstrap_id_clone_clone.clone(); tokio::spawn(async move { while let Some((peer, mut stream)) = chat_protocol.next().await { let buf = &mut Vec::new(); @@ -62,7 +62,7 @@ async fn main() { let peer_b_cfg = NetworkingConfig { port: 8084, - bootstrap_nodes: vec![format!("/ip4/127.0.0.1/udp/8080/quic-v1/p2p/{}", bootstrap_id_clone_clone.clone())], + bootstrap_nodes: vec![format!("/ip4/127.0.0.1/udp/8080/quic-v1/p2p/{}", bootstrap_id)], enable_relay_server: false, enable_kdht: true, enable_mdns: false, @@ -77,7 +77,7 @@ async fn main() { let peer_c_cfg = NetworkingConfig { port: 8086, - bootstrap_nodes: vec![format!("/ip4/127.0.0.1/udp/8080/quic-v1/p2p/{}", bootstrap_id_clone_clone.clone())], + bootstrap_nodes: vec![format!("/ip4/127.0.0.1/udp/8080/quic-v1/p2p/{}", bootstrap_id)], enable_relay_server: false, enable_kdht: true, enable_mdns: false, @@ -94,7 +94,7 @@ async fn main() { tokio::spawn(async move { // sleep(Duration::from_millis(500)).await; println!("{}: Sending message", peer_b.id); - let mut s = peer_b.client.send(format!("3 - send from {}", peer_b.id).as_bytes().to_vec(), bootstrap_id_clone_clone.clone(), protocol.to_string(), 0).await.unwrap_or_else(|_| panic!("{}: can't send message", peer_b.id)); + let mut s = peer_b.client.send(format!("3 - send from {}", peer_b.id).as_bytes().to_vec(), &bootstrap_id_clone, protocol, 0).await.unwrap_or_else(|_| panic!("{}: can't send message", peer_b.id)); s.flush().await.expect("can't flush stream"); s.close().await.expect("can't close stream"); let buf = &mut Vec::new(); @@ -105,7 +105,7 @@ async fn main() { }); tokio::spawn(async move { println!("{}: Sending message", peer_c.id.clone()); - let mut s = peer_c.client.send(format!("1 - send from {}", peer_c.id).as_bytes().to_vec(), bootstrap_id_clone_clone_clone.clone(), protocol.to_string(), 0).await.unwrap_or_else(|_| panic!("{}: can't send message", peer_c.id)); + let mut s = peer_c.client.send(format!("1 - send from {}", peer_c.id).as_bytes().to_vec(), &bootstrap_id_clone_clone, protocol, 0).await.unwrap_or_else(|_| panic!("{}: can't send message", peer_c.id)); s.flush().await.expect("can't flush stream"); s.close().await.expect("can't close stream"); let buf = &mut Vec::new(); @@ -117,7 +117,7 @@ async fn main() { tokio::spawn(async move { println!("{}: Sending message", peer_a.id); - let mut s = peer_a.client.send(format!("2 - send from {}", peer_a.id).as_bytes().to_vec(), bootstrap_id.clone(), protocol.to_string(), 0).await.unwrap_or_else(|_| panic!("{}: can't send message", peer_a.id)); + let mut s = peer_a.client.send(format!("2 - send from {}", peer_a.id).as_bytes().to_vec(), &bootstrap_id_clone_clone_clone, protocol, 0).await.unwrap_or_else(|_| panic!("{}: can't send message", peer_a.id)); s.flush().await.expect("can't flush stream"); s.close().await.expect("can't close stream"); diff --git a/core/networking/Cargo.toml b/core/networking/Cargo.toml index 98ea6b93..69db1a1a 100644 --- a/core/networking/Cargo.toml +++ b/core/networking/Cargo.toml @@ -24,7 +24,7 @@ libp2p-websocket = { workspace = true } posemesh-runtime = { workspace = true } [target.'cfg(target_family="wasm")'.dependencies] -libp2p = { workspace = true, features = [ "ping", "wasm-bindgen", "macros", "gossipsub", "serde", "identify", "kad", "autonat", "relay", "noise", "yamux", "dcutr" ] } +libp2p = { workspace = true, features = [ "ping", "wasm-bindgen", "macros", "gossipsub", "serde", "identify", "kad", "autonat", "noise", "yamux", "relay", "dcutr" ] } libp2p-webrtc-websys = { workspace = true } libp2p-websocket-websys = { workspace = true } tracing-wasm = { workspace = true } diff --git a/core/networking/src/client.rs b/core/networking/src/client.rs index 952eb76c..d3072daf 100644 --- a/core/networking/src/client.rs +++ b/core/networking/src/client.rs @@ -1,7 +1,7 @@ use async_trait::async_trait; -use libp2p::{PeerId, Stream, StreamProtocol}; +use libp2p::{multiaddr::Protocol, Multiaddr, PeerId, Stream, StreamProtocol}; use libp2p_stream::IncomingStreams; -use std::time::Duration; +use std::{collections::HashMap, time::Duration}; use futures::{channel::{mpsc, oneshot}, SinkExt}; use std::str::FromStr; #[cfg(not(target_family = "wasm"))] @@ -116,11 +116,43 @@ impl Client { receiver.await.map_err(|e| NetworkError::ChannelReceiverError(e)) } + pub async fn bootstrap(&mut self, addresses: HashMap>) -> Result<(), NetworkError> { + let mut parsed_addresses = HashMap::new(); + for (peer_id_str, addrs) in addresses { + let peer_id = PeerId::from_str(&peer_id_str)?; + let multiaddrs = addrs.iter() + .filter(|addr| { + // Filter out circuit addresses + !addr.contains("/p2p-circuit") + }) + .map(|addr| { + let mut maddr = addr.parse::()?; + // Only push peer id if it doesn't end with one + if !matches!(maddr.iter().last(), Some(Protocol::P2p(_))) { + maddr.push(Protocol::P2p(peer_id)); + } + Ok(maddr) + }) + .collect::, NetworkError>>()?; + parsed_addresses.insert(peer_id, multiaddrs); + } + + let (sender, receiver) = oneshot::channel::>(); + self.sender + .send(Command::Bootstrap { addresses: parsed_addresses, sender }) + .await?; + + match receiver.await { + Ok(Ok(())) => Ok(()), + Ok(Err(e)) => Err(e), + Err(e) => Err(NetworkError::ChannelReceiverError(e)), + } + } + // timeout is in milliseconds - pub async fn send(&mut self, message: Vec, peer_id: String, protocol: String, timeout: u32) -> Result { - let peer_id = PeerId::from_str(&peer_id)?; - let pro = StreamProtocol::try_from_owned(protocol)?; - + pub async fn send(&mut self, message: Vec, peer_id: &str, protocol: &str, timeout: u32) -> Result { + let peer_id = PeerId::from_str(peer_id)?; + let pro = StreamProtocol::try_from_owned(protocol.to_string())?; retry_send(self.sender.clone(), message, peer_id, pro, timeout, false).await } } @@ -148,5 +180,9 @@ pub enum Command { }, Cancel { sender: oneshot::Sender<()>, + }, + Bootstrap { + addresses: HashMap>, + sender: oneshot::Sender>, } } diff --git a/core/networking/src/event.rs b/core/networking/src/event.rs index e286adb4..c97cedb5 100644 --- a/core/networking/src/event.rs +++ b/core/networking/src/event.rs @@ -1,7 +1,7 @@ use libp2p::{gossipsub::TopicHash, Multiaddr, PeerId}; use std::error::Error; -#[derive(Debug)] +#[derive(Debug, Clone)] pub enum Event { NodeUnregistered { node_id: String diff --git a/core/networking/src/libp2p.rs b/core/networking/src/libp2p.rs index 0a480226..081b27d9 100644 --- a/core/networking/src/libp2p.rs +++ b/core/networking/src/libp2p.rs @@ -1,8 +1,7 @@ use futures::{channel::{mpsc::{self, channel, Receiver}, oneshot}, executor::block_on, lock::Mutex, AsyncWriteExt, SinkExt, StreamExt, FutureExt}; -use libp2p::{core::muxing::StreamMuxerBox, dcutr, gossipsub::{self, IdentTopic, SubscriptionError}, identity::ParseError, kad::{self, store::MemoryStore, GetClosestPeersOk, ProgressStep, QueryId}, multiaddr::{Multiaddr, Protocol}, noise, ping, swarm::{behaviour::toggle::Toggle, DialError, InvalidProtocol, NetworkBehaviour, SwarmEvent}, yamux, PeerId, Stream, StreamProtocol, Swarm, Transport, TransportError}; +use libp2p::{core::muxing::StreamMuxerBox, dcutr, gossipsub::{self, IdentTopic, SubscriptionError}, identity::{Keypair, ParseError}, kad::{self, store::MemoryStore, GetClosestPeersOk, ProgressStep, QueryId, RoutingUpdate}, multiaddr::{self, Multiaddr, Protocol}, noise, ping, swarm::{behaviour::toggle::Toggle, DialError, InvalidProtocol, NetworkBehaviour, SwarmEvent}, yamux, PeerId, Stream, StreamProtocol, Swarm, Transport, TransportError}; use posemesh_utils::retry_with_delay; use std::{collections::HashMap, error::Error, fmt::{self, Debug, Formatter}, io::{self, Read, Write}, str::FromStr, sync::Arc, time::Duration}; -use rand::{rngs::OsRng, thread_rng}; use libp2p_stream::{self as stream, AlreadyRegistered, IncomingStreams, OpenStreamError}; use crate::{client::{self, Client}, event}; use std::net::{Ipv4Addr, IpAddr}; @@ -57,6 +56,12 @@ pub enum NetworkError { InvalidProtocol(#[from] InvalidProtocol), #[error("Parse error: {0}")] ParseError(#[from] ParseError), + #[error("Multiaddr parse error: {0}")] + MultiaddrParseError(#[from] multiaddr::Error), + #[error("KDHT not enabled")] + KadDHTNotEnabled, + #[error("Failed to bootstrap")] + BootstrapError(&'static str), } fn is_public_ip(ip: IpAddr) -> bool { @@ -122,7 +127,7 @@ pub struct NetworkingConfig { pub bootstrap_nodes: Vec, pub relay_nodes: Vec, pub enable_mdns: bool, - pub private_key: Option>, + pub private_key: Option, pub private_key_path: Option, pub enable_kdht: bool, pub enable_websocket: bool, @@ -167,7 +172,7 @@ struct Libp2p { } #[cfg(not(target_family="wasm"))] -fn keypair_file(private_key_path: &String) -> libp2p::identity::Keypair { +fn keypair_file(private_key_path: &str) -> libp2p::identity::Keypair { let path = Path::new(private_key_path); // Check if the keypair file exists if let Ok(mut file) = fs::File::open(path) { @@ -201,18 +206,18 @@ fn keypair_file(private_key_path: &String) -> libp2p::identity::Keypair { } fn parse_or_create_keypair( - private_key: Option>, - private_key_path: Option, + private_key: Option<&str>, + private_key_path: Option<&str>, ) -> libp2p::identity::Keypair { - let private_key = private_key.unwrap_or_default(); + let private_key = private_key.unwrap_or_default().to_string(); // load private key into keypair - if let Ok(keypair) = libp2p::identity::Keypair::ed25519_from_bytes(private_key) { + if let Ok(keypair) = libp2p::identity::Keypair::ed25519_from_bytes(private_key.as_bytes().to_vec()) { return keypair; } #[cfg(not(target_family="wasm"))] if let Some(key_path) = private_key_path { - return keypair_file(&key_path); + return keypair_file(key_path); } libp2p::identity::Keypair::generate_ed25519() @@ -229,6 +234,8 @@ async fn build_swarm(key: libp2p::identity::Keypair, mut behavior: PosemeshBehav )? .with_quic() .with_other_transport(|id_keys| { + use rand::thread_rng; + Ok(webrtc::tokio::Transport::new( id_keys.clone(), webrtc::tokio::Certificate::generate(&mut thread_rng())?, @@ -315,12 +322,15 @@ fn build_behavior(key: libp2p::identity::Keypair, cfg: &NetworkingConfig) -> Pos #[cfg(not(target_family="wasm"))] if cfg.enable_relay_server { + use rand::rngs::OsRng; let mut relay_config = libp2p::relay::Config::default(); relay_config.max_circuit_bytes = 1024 * 1024 * 1024; // 1GB let relay = libp2p::relay::Behaviour::new(key.public().to_peer_id(), relay_config); behavior.relay = Some(relay).into(); behavior.autonat_server = Some(libp2p::autonat::v2::server::Behaviour::new(OsRng)).into(); } else { + use rand::rngs::OsRng; + behavior.autonat_client = Some(libp2p::autonat::v2::client::Behaviour::new(OsRng,libp2p::autonat::v2::client::Config::default())).into(); behavior.dcutr = Some(libp2p::dcutr::Behaviour::new(key.public().to_peer_id())).into(); } @@ -385,9 +395,7 @@ fn enable_webrtc(port: u16) -> Multiaddr { impl Libp2p { pub async fn new(cfg: &NetworkingConfig, command_receiver: mpsc::Receiver, event_sender: mpsc::Sender) -> Result { - let private_key = cfg.private_key.clone(); - let key = parse_or_create_keypair(private_key, cfg.private_key_path.clone()); - + let key = parse_or_create_keypair(cfg.private_key.as_deref(), cfg.private_key_path.as_deref()); let behaviour = build_behavior(key.clone(), cfg); let mut swarm = build_swarm(key.clone(), behaviour).await.map_err(|e| NetworkError::SwarmInitializationFailed(e))?; @@ -452,7 +460,6 @@ impl Libp2p { } async fn run(mut self) -> Result<(), Box> { - tracing::info!("Starting networking"); #[cfg(not(target_family="wasm"))] loop { @@ -563,7 +570,14 @@ impl Libp2p { SwarmEvent::NewListenAddr { address, .. } => { let local_peer_id = *self.swarm.local_peer_id(); let address = address.clone().with(Protocol::P2p(local_peer_id)); - self.event_sender.send(event::Event::NewAddress { address: address.clone() }).await.expect("failed to send new address"); + + if address.iter().any(|p| match p { + Protocol::Ip4(ip) => ip.is_loopback(), + Protocol::Ip6(ip) => ip.is_loopback(), + _ => false + }) { + self.event_sender.send(event::Event::NewAddress { address: address.clone() }).await.expect("failed to send new address"); + } println!("Local node is listening on {:?}", address); } SwarmEvent::ConnectionEstablished { @@ -750,6 +764,24 @@ impl Libp2p { let _ = cancel_sender.send(()); let _ = sender.send(()); } + client::Command::Bootstrap { addresses, sender } => { + if let Some(kdht) = self.swarm.behaviour_mut().kdht.as_mut() { + for (peer_id, addrs) in addresses { + for addr in addrs { + let res = kdht.add_address(&peer_id, addr); + if RoutingUpdate::Failed == res { + let _ = sender.send(Err(NetworkError::BootstrapError("Failed to bootstrap"))); + return; + } + } + } + + let _ = sender.send(Ok(())); + } else { + tracing::error!("KDHT is not enabled"); + let _ = sender.send(Err(NetworkError::KadDHTNotEnabled)); + } + } } } diff --git a/core/protobuf/common/common.proto b/core/protobuf/common/common.proto new file mode 100644 index 00000000..767fbe7a --- /dev/null +++ b/core/protobuf/common/common.proto @@ -0,0 +1,8 @@ +syntax = "proto2"; + +package common; + +message Capability { + required string endpoint = 1; + required int32 capacity = 2; +} diff --git a/core/protobuf/discovery.proto b/core/protobuf/domain-cluster/discovery.proto similarity index 61% rename from core/protobuf/discovery.proto rename to core/protobuf/domain-cluster/discovery.proto index 254ea434..a1e3f8e4 100644 --- a/core/protobuf/discovery.proto +++ b/core/protobuf/domain-cluster/discovery.proto @@ -2,14 +2,11 @@ syntax = "proto2"; package discovery; -message Capability { - required string endpoint = 1; - required int32 capacity = 2; -} +import "../common/common.proto"; message Node { required string id = 1; - repeated Capability capabilities = 2; + repeated common.Capability capabilities = 2; required string name = 3; } diff --git a/core/protobuf/domain_data.proto b/core/protobuf/domain-cluster/domain_data.proto similarity index 100% rename from core/protobuf/domain_data.proto rename to core/protobuf/domain-cluster/domain_data.proto diff --git a/core/protobuf/task.proto b/core/protobuf/domain-cluster/task.proto similarity index 100% rename from core/protobuf/task.proto rename to core/protobuf/domain-cluster/task.proto diff --git a/core/utils/Cargo.toml b/core/utils/Cargo.toml index 4a41ad4f..369a7c84 100644 --- a/core/utils/Cargo.toml +++ b/core/utils/Cargo.toml @@ -7,6 +7,10 @@ rust-version.workspace = true [dependencies] futures = { workspace = true } tracing = { workspace = true } +hex = { workspace = true, optional = true } +tiny-keccak = { version = "2.0.2", features = ["keccak"], optional = true } +k256 = { version = "0.13.4", features = ["ecdsa"], optional = true } +thiserror = { workspace = true, optional = true } [target.'cfg(not(target_family="wasm"))'.dependencies] tokio = { workspace = true, features = ["full"] } @@ -17,3 +21,7 @@ wasm-bindgen-futures = { workspace = true } web-sys = { version = "0.3", features = ["Window"] } js-sys = "0.3" gloo-timers = { workspace = true, features = ["futures"] } + +[features] +default = ["crypto"] +crypto = ["hex", "tiny-keccak", "k256", "thiserror"] diff --git a/core/utils/src/crypto.rs b/core/utils/src/crypto.rs new file mode 100644 index 00000000..17bdcfd5 --- /dev/null +++ b/core/utils/src/crypto.rs @@ -0,0 +1,153 @@ +use std::{io::Error, io::ErrorKind}; +use k256::ecdsa::{Signature, SigningKey}; + +#[derive(Debug, thiserror::Error)] +pub enum CryptoError { + #[error("IO error: {0}")] + IoError(#[from] std::io::Error), + + #[error("Hex decoding error: {0}")] + HexError(#[from] hex::FromHexError), + + #[error("Signing error: {0}")] + SigningError(#[from] k256::ecdsa::signature::Error), + + #[error("No private key provided")] + NoPrivateKey, +} + + +#[derive(Debug)] +pub struct Secp256k1KeyPair { + signing_key: SigningKey, +} + +impl Secp256k1KeyPair { + pub fn public_key_hex(&self) -> String { + hex::encode(self.signing_key.verifying_key().to_encoded_point(false).as_bytes()) + } +} + +#[cfg(not(target_family="wasm"))] +fn keypair_file(private_key_path: &str) -> Result { + use std::{fs, io::Read, path::Path}; + + let path = Path::new(private_key_path); + // Check if the keypair file exists + let mut file = fs::File::open(path)?; + // Read the keypair from the file + let mut file_private_key = "".to_string(); + file.read_to_string(&mut file_private_key)?; + let private_key = file_private_key.strip_prefix("0x").unwrap_or(&file_private_key).trim(); + let private_key = hex::decode(private_key)?; + let signing_key = SigningKey::from_slice(&private_key)?; + return Ok(Secp256k1KeyPair { signing_key }); +} + +pub fn parse_secp256k1_private_key( + private_key: Option<&str>, + private_key_path: Option<&str>, +) -> Result { + match private_key { + Some(private_key) => { + let removed_0x_prefix = private_key.strip_prefix("0x").unwrap_or(private_key); + let private_key_bytes = hex::decode(removed_0x_prefix)?; + let signing_key = SigningKey::from_slice(&private_key_bytes)?; + + #[cfg(not(target_family="wasm"))] + if let Some(path) = private_key_path { + use std::{fs, path::Path, io::Write}; + + let path = Path::new(path); + if let Some(parent) = path.parent() { + + if let Err(err) = fs::create_dir_all(parent) { + tracing::error!("Failed to create directory: {err}"); + } + + let mut file = fs::File::create(path)?; + file.write_all(private_key.as_bytes())?; + } + } + return Ok(Secp256k1KeyPair { signing_key }); + } + None => { + #[cfg(not(target_family="wasm"))] + if let Some(key_path) = private_key_path { + return keypair_file(key_path); + } + return Err(CryptoError::NoPrivateKey); + } + } +} + + +pub fn sign_message(message: &str, keypair: &Secp256k1KeyPair) -> Result { + use tiny_keccak::Hasher; + // Hash the message using Keccak-256 + let mut hasher = tiny_keccak::Keccak::v256(); + hasher.update(message.as_bytes()); + let mut hash = [0u8; 32]; + hasher.finalize(&mut hash); + + // Sign the hash and return signature bytes + let signature: Signature = keypair.signing_key.sign_prehash_recoverable(&hash).map_err(|e| Error::new(ErrorKind::Other, e))?.0; + let signature = signature.to_bytes().to_vec(); + let hex_signature = hex::encode(signature); + Ok(format!("0x{}", hex_signature)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_parse_or_create_keypair_with_private_key() { + let private_key = "2719d2c0d35fa8a6dce9622e480764ecc0428dd10c70cc52ec0349351989d27a"; + let keypair = parse_secp256k1_private_key(Some(private_key), None).unwrap(); + + // Verify the keypair was created with the correct private key + let expected_bytes = hex::decode(private_key).unwrap(); + assert_eq!(keypair.signing_key.to_bytes().to_vec(), expected_bytes); + } + + #[test] + fn test_parse_or_create_keypair_with_0x_private_key() { + let private_key = "0x2719d2c0d35fa8a6dce9622e480764ecc0428dd10c70cc52ec0349351989d27a"; + let keypair = parse_secp256k1_private_key(Some(private_key), None).unwrap(); + + // Verify the keypair was created with the correct private key + let expected_bytes = hex::decode(private_key.strip_prefix("0x").unwrap()).unwrap(); + assert_eq!(keypair.signing_key.to_bytes().to_vec(), expected_bytes); + } + + #[test] + fn test_parse_ecdsa_private_key_with_no_inputs() { + parse_secp256k1_private_key(None, None).expect_err("it should return error"); + } + + #[test] + fn test_parse_ecdsa_private_key_with_file() { + let temp_dir = std::env::temp_dir(); + let key_path = temp_dir.join("test_key.pkey"); + + // Write private key to file + std::fs::write(&key_path, "2719d2c0d35fa8a6dce9622e480764ecc0428dd10c70cc52ec0349351989d27a").unwrap(); + + let keypair = parse_secp256k1_private_key(None, Some(key_path.to_str().unwrap())).unwrap(); + + // Verify the keypair was created with the correct private key + let expected_bytes = hex::decode("2719d2c0d35fa8a6dce9622e480764ecc0428dd10c70cc52ec0349351989d27a").unwrap(); + assert_eq!(keypair.signing_key.to_bytes().to_vec(), expected_bytes); + + // Cleanup + std::fs::remove_file(key_path).unwrap(); + } + + #[test] + fn test_sign_message() { + let keypair = parse_secp256k1_private_key(Some("2719d2c0d35fa8a6dce9622e480764ecc0428dd10c70cc52ec0349351989d27a"), None).unwrap(); + let signature = sign_message("test message", &keypair).unwrap(); + assert_eq!(signature, "0x2dcb35d237f7a1d954aceffbaf7cf6e2d36f947c4a83785117c322b7bab031a8180a24829c9a80fc690de79624088a0fa7f62af48407151818299076ecb08af7"); + } +} diff --git a/core/utils/src/lib.rs b/core/utils/src/lib.rs index 041f5faa..851442df 100644 --- a/core/utils/src/lib.rs +++ b/core/utils/src/lib.rs @@ -1,5 +1,4 @@ -use std::time::Duration; -use std::{error::Error, io}; +use std::{io::{Error, ErrorKind}, time::Duration}; use futures::{self, Future, FutureExt}; #[cfg(not(target_family = "wasm"))] @@ -14,6 +13,9 @@ use js_sys::Promise; #[cfg(target_family = "wasm")] use wasm_bindgen::JsValue; +#[cfg(feature="crypto")] +pub mod crypto; + #[cfg(target_family = "wasm")] pub async fn sleep(duration: Duration) { let window = web_sys::window().expect("no window"); @@ -40,9 +42,12 @@ pub const INFINITE_RETRIES: u32 = 0; /// # Returns /// * `Ok(T)` - The successful result /// * `Err(E)` - The error from the last attempt if all attempts failed -pub async fn retry_with_delay(mut f: F, max_attempts: u32, delay: Duration) -> Result + + +pub async fn retry_with_delay(mut f: F, max_attempts: u32, delay: Duration) -> Result where - F: FnMut() -> std::pin::Pin> + Send>>, + F: FnMut() -> Fut, + Fut: Future>, E: std::fmt::Debug, { let mut retries = 0; @@ -85,10 +90,9 @@ where } #[cfg(target_family = "wasm")] -pub async fn timeout(duration: Duration, future: F) -> Result +pub async fn timeout(duration: Duration, future: F) -> Result where F: Future + Send, - T: Send, { if duration.is_zero() { return Ok(future.await); @@ -96,12 +100,12 @@ where let timeout_fut = gloo_timers::future::TimeoutFuture::new(duration.as_millis() as u32); futures::select! { result = future.fuse() => Ok(result), - _ = timeout_fut.fuse() => Err(io::Error::new(io::ErrorKind::TimedOut, "Operation timed out")), + _ = timeout_fut.fuse() => Err(Error::new(ErrorKind::TimedOut, "Operation timed out")), } } #[cfg(not(target_family = "wasm"))] -pub async fn timeout(duration: Duration, future: F) -> Result +pub async fn timeout(duration: Duration, future: F) -> Result where F: Future, { @@ -110,5 +114,5 @@ where } tokio::time::timeout(duration, future) .await - .map_err(|_| io::Error::new(io::ErrorKind::TimedOut, "Operation timed out")) + .map_err(|_| Error::new(ErrorKind::TimedOut, "Operation timed out")) } From 021ba616c2d05905f35352d1fd1fe5db0ab027c5 Mon Sep 17 00:00:00 2001 From: Shuning Date: Fri, 20 Jun 2025 10:40:56 +0800 Subject: [PATCH 03/12] only run build.rs when it is not release mode --- .github/workflows/posemesh-sdk.yml | 2 +- .vscode/settings.json | 6 +++++- core/Makefile | 8 ++++---- core/disco/build.rs | 6 +++++- core/domain/build.rs | 5 +++++ 5 files changed, 20 insertions(+), 7 deletions(-) diff --git a/.github/workflows/posemesh-sdk.yml b/.github/workflows/posemesh-sdk.yml index c6f9c40d..f47ec6c3 100644 --- a/.github/workflows/posemesh-sdk.yml +++ b/.github/workflows/posemesh-sdk.yml @@ -195,7 +195,7 @@ jobs: - name: Install Rust run: curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y - + - name: Cache Cargo registry and Git sources (Shared) uses: actions/cache@v4 with: diff --git a/.vscode/settings.json b/.vscode/settings.json index 659c1c4b..ceda74d3 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -25,5 +25,9 @@ "**/sdk/platform/Web/src/**/*.c": "c", "**/sdk/platform/Web/src/**/*.cpp": "cpp" }, - "rust-analyzer.cargo.target": "wasm32-unknown-unknown" + "rust-analyzer.cargo.extraEnv": { + "PROTOC": "/opt/homebrew/bin/protoc", + "PATH": "/opt/homebrew/bin:${env:PATH}" + }, + // "rust-analyzer.cargo.target": "wasm32-unknown-unknown" } diff --git a/core/Makefile b/core/Makefile index cbcc4001..4db6c340 100644 --- a/core/Makefile +++ b/core/Makefile @@ -19,7 +19,7 @@ build-domain-wasm: sed -i '' 's/"name": "posemesh-domain"/"name": "@aukilabs\/posemesh-domain"/' domain/pkg/package.json unit-tests: - wasm-pack test --node domain - wasm-pack test --node networking - wasm-pack test --node utils - cargo test --all-features + wasm-pack test --node --release domain + wasm-pack test --node --release networking + wasm-pack test --node --release utils + cargo test --release --all-features diff --git a/core/disco/build.rs b/core/disco/build.rs index 3da533e4..b6a0b950 100644 --- a/core/disco/build.rs +++ b/core/disco/build.rs @@ -1,6 +1,11 @@ use std::{path::{Path, PathBuf}, fs}; fn main() { + // Only run protobuf generation when not in release mode + if std::env::var("PROFILE").unwrap_or_default() == "release" { + return; + } + let out_dir = Path::new("./src").join("protobuf"); let in_dir = PathBuf::from("../protobuf").join("disco"); @@ -21,7 +26,6 @@ fn main() { println!("cargo:rerun-if-changed={}", path.to_str().unwrap()); protos.push(path.clone()); mod_rs.push_str(&format!("pub mod {}pb;\n", path.file_stem().unwrap().to_string_lossy().to_string())); - // mod_rs.push_str(&format!(" include!(concat!(\"{}\", \"/{}.rs\"));\n", out_dir.as_os_str().to_string_lossy().to_string(), path.file_stem().unwrap().to_string_lossy().to_string())); } } diff --git a/core/domain/build.rs b/core/domain/build.rs index b75c47f5..f2af6851 100644 --- a/core/domain/build.rs +++ b/core/domain/build.rs @@ -5,6 +5,11 @@ use pb_rs::{types::FileDescriptor, ConfigBuilder}; use std::{path::{Path, PathBuf}, fs}; fn main() { + // Only run protobuf generation when not in release mode + if std::env::var("PROFILE").unwrap_or_default() == "release" { + return; + } + let crate_dir = env::var("CARGO_MANIFEST_DIR").unwrap(); cbindgen::Builder::new() From b57eb2d4fdb9282beb7915a452aed6dc16747324 Mon Sep 17 00:00:00 2001 From: Shuning Date: Fri, 20 Jun 2025 10:52:28 +0800 Subject: [PATCH 04/12] disable wasm-opt Caused by: failed to execute : exited with exit status: 1 /Library/Caches/.wasm-pack/wasm-opt-50385c9e73ccee70/bin/wasm-opt domain/pkg/posemesh_domain_bg.wasm -o domain/pkg/posemesh_domain_bg.wasm-opt.wasm -O --- core/domain/Cargo.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/core/domain/Cargo.toml b/core/domain/Cargo.toml index 18b63fdc..849f585b 100644 --- a/core/domain/Cargo.toml +++ b/core/domain/Cargo.toml @@ -63,3 +63,6 @@ fs = ["dep:tokio-postgres"] [profile.release] strip = true + +[package.metadata.wasm-pack.profile.release] +wasm-opt = false From 1b2fa270ce8cabfd1919f01ddbb581097a328beb Mon Sep 17 00:00:00 2001 From: Shuning Date: Fri, 20 Jun 2025 10:57:29 +0800 Subject: [PATCH 05/12] add CARGO_HOME --- .github/workflows/posemesh-sdk.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/posemesh-sdk.yml b/.github/workflows/posemesh-sdk.yml index f47ec6c3..3932029d 100644 --- a/.github/workflows/posemesh-sdk.yml +++ b/.github/workflows/posemesh-sdk.yml @@ -59,6 +59,8 @@ jobs: rust-cache-test- - name: Run unit tests + env: + CARGO_HOME: ~/.cargo run: make unit-tests build_apple_frameworks: From 524391de550fa38cc66b496c353ef4c81ed0f0df Mon Sep 17 00:00:00 2001 From: Shuning Date: Fri, 20 Jun 2025 11:51:27 +0800 Subject: [PATCH 06/12] fix bindings --- .github/workflows/posemesh-sdk.yml | 2 - core/Cargo.lock | 127 +++++++++++++++++++------ core/Makefile | 4 +- core/domain/Cargo.toml | 2 +- core/domain/examples/browser/README.md | 2 +- core/domain/src/binding_helper.rs | 7 +- core/domain/src/c.rs | 4 +- core/domain/src/wasm.rs | 13 +-- core/networking/src/c.rs | 12 +-- core/networking/src/c_compat_wasm.rs | 6 +- 10 files changed, 124 insertions(+), 55 deletions(-) diff --git a/.github/workflows/posemesh-sdk.yml b/.github/workflows/posemesh-sdk.yml index 3932029d..f47ec6c3 100644 --- a/.github/workflows/posemesh-sdk.yml +++ b/.github/workflows/posemesh-sdk.yml @@ -59,8 +59,6 @@ jobs: rust-cache-test- - name: Run unit tests - env: - CARGO_HOME: ~/.cargo run: make unit-tests build_apple_frameworks: diff --git a/core/Cargo.lock b/core/Cargo.lock index 5b5573ad..da86552c 100644 --- a/core/Cargo.lock +++ b/core/Cargo.lock @@ -338,9 +338,9 @@ dependencies = [ [[package]] name = "autocfg" -version = "1.4.0" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" +checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" [[package]] name = "backtrace" @@ -391,9 +391,9 @@ checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" [[package]] name = "base64ct" -version = "1.7.3" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89e25b6adfb930f02d1981565a6e5d9c547ac15a96606256d3b59040e5cd4ca3" +checksum = "55248b47b0caf0546f7988906588779981c43bb1bc9d0c44087278f80cdb44ba" [[package]] name = "bincode" @@ -997,12 +997,12 @@ checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" [[package]] name = "errno" -version = "0.3.12" +version = "0.3.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cea14ef9355e3beab063703aa9dab15afd25f0667c341310c1e5274bb1d0da18" +checksum = "778e2ac28f6c47af28e4907f13ffd1e1ddbd400980a9abd7c8df189bf578a5ad" dependencies = [ "libc", - "windows-sys 0.59.0", + "windows-sys 0.60.2", ] [[package]] @@ -1189,7 +1189,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8f2f12607f92c69b12ed746fabf9ca4f5c482cba46679c1a75b874ed7c26adb" dependencies = [ "futures-io", - "rustls 0.23.27", + "rustls 0.23.28", "rustls-pki-types", ] @@ -1889,9 +1889,9 @@ checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] name = "libc" -version = "0.2.173" +version = "0.2.174" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8cfeafaffdbc32176b64fb251369d52ea9f0a8fbc6f8759edffef7b525d64bb" +checksum = "1171693293099992e19cddea4e8b849964e9846f4acee11b3948bcc337be8776" [[package]] name = "libp2p" @@ -2243,7 +2243,7 @@ dependencies = [ "quinn", "rand 0.8.5", "ring 0.17.14", - "rustls 0.23.27", + "rustls 0.23.28", "socket2", "thiserror 1.0.69", "tokio", @@ -2370,7 +2370,7 @@ dependencies = [ "libp2p-identity", "rcgen", "ring 0.17.14", - "rustls 0.23.27", + "rustls 0.23.28", "rustls-webpki 0.101.7", "thiserror 1.0.69", "x509-parser 0.16.0", @@ -3563,7 +3563,7 @@ dependencies = [ "quinn-proto", "quinn-udp", "rustc-hash", - "rustls 0.23.27", + "rustls 0.23.28", "socket2", "thiserror 2.0.12", "tokio", @@ -3583,7 +3583,7 @@ dependencies = [ "rand 0.9.1", "ring 0.17.14", "rustc-hash", - "rustls 0.23.27", + "rustls 0.23.28", "rustls-pki-types", "slab", "thiserror 2.0.12", @@ -3594,9 +3594,9 @@ dependencies = [ [[package]] name = "quinn-udp" -version = "0.5.12" +version = "0.5.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee4e529991f949c5e25755532370b8af5d114acae52326361d68d47af64aa842" +checksum = "fcebb1209ee276352ef14ff8732e24cc2b02bbac986cd74a4c81bcb2f9881970" dependencies = [ "cfg_aliases", "libc", @@ -3617,9 +3617,9 @@ dependencies = [ [[package]] name = "r-efi" -version = "5.2.0" +version = "5.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74765f6d916ee2faa39bc8e68e4f3ed8949b48cccdac59983d287a7cb71ce9c5" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" [[package]] name = "rand" @@ -3923,9 +3923,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.27" +version = "0.23.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "730944ca083c1c233a75c09f199e973ca499344a2b7ba9e755c457e86fb4a321" +checksum = "7160e3e10bf4535308537f3c4e1641468cd0e485175d6163087c0393c7d46643" dependencies = [ "once_cell", "ring 0.17.14", @@ -4782,9 +4782,9 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.29" +version = "0.1.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b1ffbcf9c6f6b99d386e7444eb608ba646ae452a36b39737deb9663b610f662" +checksum = "81383ab64e72a7a8b8e13130c49e3dab29def6d0c7d76a03087b3cf71c5c6903" dependencies = [ "proc-macro2", "quote", @@ -5597,6 +5597,15 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "windows-sys" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +dependencies = [ + "windows-targets 0.53.2", +] + [[package]] name = "windows-targets" version = "0.48.5" @@ -5621,13 +5630,29 @@ dependencies = [ "windows_aarch64_gnullvm 0.52.6", "windows_aarch64_msvc 0.52.6", "windows_i686_gnu 0.52.6", - "windows_i686_gnullvm", + "windows_i686_gnullvm 0.52.6", "windows_i686_msvc 0.52.6", "windows_x86_64_gnu 0.52.6", "windows_x86_64_gnullvm 0.52.6", "windows_x86_64_msvc 0.52.6", ] +[[package]] +name = "windows-targets" +version = "0.53.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c66f69fcc9ce11da9966ddb31a40968cad001c5bedeb5c2b82ede4253ab48aef" +dependencies = [ + "windows_aarch64_gnullvm 0.53.0", + "windows_aarch64_msvc 0.53.0", + "windows_i686_gnu 0.53.0", + "windows_i686_gnullvm 0.53.0", + "windows_i686_msvc 0.53.0", + "windows_x86_64_gnu 0.53.0", + "windows_x86_64_gnullvm 0.53.0", + "windows_x86_64_msvc 0.53.0", +] + [[package]] name = "windows_aarch64_gnullvm" version = "0.48.5" @@ -5640,6 +5665,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86b8d5f90ddd19cb4a147a5fa63ca848db3df085e25fee3cc10b39b6eebae764" + [[package]] name = "windows_aarch64_msvc" version = "0.48.5" @@ -5652,6 +5683,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7651a1f62a11b8cbd5e0d42526e55f2c99886c77e007179efff86c2b137e66c" + [[package]] name = "windows_i686_gnu" version = "0.48.5" @@ -5664,12 +5701,24 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" +[[package]] +name = "windows_i686_gnu" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1dc67659d35f387f5f6c479dc4e28f1d4bb90ddd1a5d3da2e5d97b42d6272c3" + [[package]] name = "windows_i686_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ce6ccbdedbf6d6354471319e781c0dfef054c81fbc7cf83f338a4296c0cae11" + [[package]] name = "windows_i686_msvc" version = "0.48.5" @@ -5682,6 +5731,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" +[[package]] +name = "windows_i686_msvc" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "581fee95406bb13382d2f65cd4a908ca7b1e4c2f1917f143ba16efe98a589b5d" + [[package]] name = "windows_x86_64_gnu" version = "0.48.5" @@ -5694,6 +5749,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e55b5ac9ea33f2fc1716d1742db15574fd6fc8dadc51caab1c16a3d3b4190ba" + [[package]] name = "windows_x86_64_gnullvm" version = "0.48.5" @@ -5706,6 +5767,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a6e035dd0599267ce1ee132e51c27dd29437f63325753051e71dd9e42406c57" + [[package]] name = "windows_x86_64_msvc" version = "0.48.5" @@ -5718,6 +5785,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486" + [[package]] name = "winnow" version = "0.7.11" @@ -5880,18 +5953,18 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.25" +version = "0.8.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1702d9583232ddb9174e01bb7c15a2ab8fb1bc6f227aa1233858c351a3ba0cb" +checksum = "1039dd0d3c310cf05de012d8a39ff557cb0d23087fd44cad61df08fc31907a2f" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.25" +version = "0.8.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28a6e20d751156648aa063f3800b706ee209a32c0b4d9f24be3d980b01be55ef" +checksum = "9ecf5b4cc5364572d7f4c329661bcc82724222973f2cab6f050a4e5c22f75181" dependencies = [ "proc-macro2", "quote", diff --git a/core/Makefile b/core/Makefile index 4db6c340..da291185 100644 --- a/core/Makefile +++ b/core/Makefile @@ -19,7 +19,5 @@ build-domain-wasm: sed -i '' 's/"name": "posemesh-domain"/"name": "@aukilabs\/posemesh-domain"/' domain/pkg/package.json unit-tests: - wasm-pack test --node --release domain - wasm-pack test --node --release networking - wasm-pack test --node --release utils + wasm-pack test --node --release domain cargo test --release --all-features diff --git a/core/domain/Cargo.toml b/core/domain/Cargo.toml index 849f585b..bc68dbac 100644 --- a/core/domain/Cargo.toml +++ b/core/domain/Cargo.toml @@ -57,7 +57,7 @@ features = ["wasm_js"] crate-type = ["cdylib", "staticlib", "rlib"] [features] -default = ["fs"] +default = [] c = [] fs = ["dep:tokio-postgres"] diff --git a/core/domain/examples/browser/README.md b/core/domain/examples/browser/README.md index a9c5429c..68af36e2 100644 --- a/core/domain/examples/browser/README.md +++ b/core/domain/examples/browser/README.md @@ -20,7 +20,7 @@ cargo install wasm-pack cd core ``` -- (Optional) Installing Protocol Buffers Tools +- Installing Protocol Buffers Tools install - protoc (Protocol Buffers Compiler) diff --git a/core/domain/src/binding_helper.rs b/core/domain/src/binding_helper.rs index 616f49c8..15fe11bd 100644 --- a/core/domain/src/binding_helper.rs +++ b/core/domain/src/binding_helper.rs @@ -13,7 +13,7 @@ use wasm_bindgen_futures::spawn_local as spawn; use wasm_bindgen::prelude::*; -use crate::{cluster::DomainCluster, datastore::{common::{self, DomainError}, remote::RemoteDatastore}, protobuf::domain_data}; +use crate::{cluster::{join_domain, DomainCluster, PosemeshSwarm}, datastore::{common::{self, DomainError}, remote::RemoteDatastore}, protobuf::domain_data}; pub(crate) fn init_r_remote_storage(cluster: *mut DomainCluster) -> RemoteDatastore { unsafe { @@ -26,6 +26,11 @@ pub(crate) fn init_r_remote_storage(cluster: *mut DomainCluster) -> RemoteDatast RemoteDatastore::new(cluster_copy) } } + +pub(crate) async fn init_r_domain_cluster(domain_manager_addr: String, name: String, private_key: Option, private_key_path: Option, relay_nodes: Vec) -> Result { + let mut swarm = PosemeshSwarm::init(false, 0, false, false, private_key, private_key_path, relay_nodes).await?; + join_domain(&mut swarm, &domain_manager_addr, &name).await +} pub struct DomainDataWriter { metadata: Option, content: Option>, diff --git a/core/domain/src/c.rs b/core/domain/src/c.rs index 385a7aa7..fa5ac138 100644 --- a/core/domain/src/c.rs +++ b/core/domain/src/c.rs @@ -7,7 +7,7 @@ use posemesh_runtime::get_runtime; use crate::cluster::DomainCluster; use crate::datastore::common::{self, data_id_generator, Datastore, ReliableDataProducer as r_ReliableDataProducer, DomainError as r_DomainError}; -use crate::binding_helper::{init_r_remote_storage, initialize_consumer, DataConsumer}; +use crate::binding_helper::{init_r_remote_storage, initialize_consumer, init_r_domain_cluster, DataConsumer}; use crate::datastore::remote::RemoteDatastore; use crate::protobuf::domain_data::UpsertMetadata; use crate::protobuf::domain_data::{self, Data}; @@ -224,7 +224,7 @@ pub extern "C" fn init_domain_cluster(domain_manager_addr: *const c_char, name: } }; let cluster = get_runtime().block_on(async move { - return DomainCluster::join(&domain_manager_addr, &name, false, 0, false, false, None, None, nodes).await; + init_r_domain_cluster(domain_manager_addr, name, None, None, nodes).await }).expect("failed to join cluster"); Box::into_raw(Box::new(cluster)) diff --git a/core/domain/src/wasm.rs b/core/domain/src/wasm.rs index 9631f512..4ba30786 100644 --- a/core/domain/src/wasm.rs +++ b/core/domain/src/wasm.rs @@ -4,7 +4,7 @@ use quick_protobuf::serialize_into_vec; use js_sys::Function; use serde_wasm_bindgen::{from_value, to_value}; use wasm_bindgen::prelude::*; -use crate::{binding_helper::{init_r_remote_storage, initialize_consumer, DataConsumer}, cluster::{join_domain, new_task_update_duplex, DomainCluster as r_DomainCluster, PosemeshSwarm as r_PosemeshSwarm}, datastore::{common::{data_id_generator, Datastore, ReliableDataProducer as r_ReliableDataProducer}, remote::RemoteDatastore as r_RemoteDatastore}, protobuf::domain_data, spatial::reconstruction::reconstruction_job as r_reconstruction_job}; +use crate::{binding_helper::{init_r_remote_storage, init_r_domain_cluster, initialize_consumer, DataConsumer}, cluster::{new_task_update_duplex, DomainCluster as r_DomainCluster, PosemeshSwarm as r_PosemeshSwarm}, datastore::{common::{data_id_generator, Datastore, ReliableDataProducer as r_ReliableDataProducer}, remote::RemoteDatastore as r_RemoteDatastore}, protobuf::domain_data, spatial::reconstruction::reconstruction_job as r_reconstruction_job}; use wasm_bindgen_futures::{future_to_promise, js_sys::{self, Promise, Uint8Array}, spawn_local}; #[derive(Clone)] @@ -112,14 +112,9 @@ pub struct DomainCluster { #[wasm_bindgen] pub fn join_cluster(domain_manager_addr: String, name: String, private_key: Option, private_key_path: Option) -> Promise { let future = async move { - match r_PosemeshSwarm::init(false, 0, false, false, private_key, private_key_path, vec![]).await { - Ok(mut swarm) => { - match join_domain(&mut swarm, &domain_manager_addr, &name).await { - Ok(cluster) => Ok(JsValue::from(DomainCluster { inner: Arc::new(Mutex::new(cluster)) })), - Err(e) => Err(JsValue::from_str(&format!("failed to join cluster: {}", e))) - } - } - Err(e) => Err(JsValue::from_str(&format!("failed to init swarm: {}", e))) + match init_r_domain_cluster(domain_manager_addr, name, private_key, private_key_path, vec![]).await { + Ok(cluster) => Ok(JsValue::from(DomainCluster { inner: Arc::new(Mutex::new(cluster)) })), + Err(e) => Err(JsValue::from_str(&format!("failed to join cluster: {}", e))) } }; future_to_promise(future) diff --git a/core/networking/src/c.rs b/core/networking/src/c.rs index d0f873f0..ed0d3d77 100644 --- a/core/networking/src/c.rs +++ b/core/networking/src/c.rs @@ -9,7 +9,7 @@ use posemesh_runtime::get_runtime; pub struct Config { pub bootstraps: *const c_char, // a list of bootstrap nodes separated by comma pub relays: *const c_char, - pub private_key: *const c_uchar, // private key can be null + pub private_key: *const c_char, // private key can be null pub private_key_size: u32, pub private_key_path: *const c_char, // private key path can be null, but if private key is null, private key path must be provided pub enable_mdns: u8, @@ -32,9 +32,9 @@ pub fn to_rust(config: &Config) -> NetworkingConfig { None } else { let private_key = unsafe { - std::slice::from_raw_parts(config.private_key, config.private_key_size as usize) - }; - Some(private_key.to_vec()) + CStr::from_ptr(config.private_key) + }.to_str().expect("Context::new(): config.private_key is not a valid UTF-8 string"); + Some(private_key.to_string()) }; let private_key_path = if config.private_key_path.is_null() { @@ -137,7 +137,7 @@ fn send_message( eprintln!("send_message(): {:?}", error); return 0; } - }.to_string(); + }; let protocol = match unsafe { assert!(!protocol.is_null(), "send_message(): protocol is null"); @@ -148,7 +148,7 @@ fn send_message( eprintln!("send_message(): {:?}", error); return 0; } - }.to_string(); + }; let context = unsafe { assert!(!context.is_null(), "send_message(): context is null"); &mut *context diff --git a/core/networking/src/c_compat_wasm.rs b/core/networking/src/c_compat_wasm.rs index 90da7fcb..ac632203 100644 --- a/core/networking/src/c_compat_wasm.rs +++ b/core/networking/src/c_compat_wasm.rs @@ -7,7 +7,7 @@ use wasm_bindgen_futures::{future_to_promise, js_sys::{Promise, Error}}; pub struct Config { pub bootstraps: Vec, pub relays: Vec, - pub privateKey: Option>, + pub privateKey: Option, } #[wasm_bindgen] @@ -16,7 +16,7 @@ impl Config { pub fn new( bootstraps: Vec, relays: Vec, - private_key: Option>, + private_key: Option, ) -> Self { Self { bootstraps, @@ -67,7 +67,7 @@ pub fn posemeshNetworkingContextSendMessage(context: *mut Networking, message: V }; return future_to_promise(async move { - match networking.client.send(message, peer_id, protocol, timeout).await { + match networking.client.send(message, &peer_id, &protocol, timeout).await { Ok(_) => { Ok(JsValue::from(true)) }, Err(error) => { eprintln!("posemesh_networking_context_send_message(): {:?}", error); From 2bc8097aba6a05860af6cb885ce70fb5bdcbff6d Mon Sep 17 00:00:00 2001 From: Shuning Date: Fri, 20 Jun 2025 12:03:23 +0800 Subject: [PATCH 07/12] fix vscode settings.json for rust-analyzer --- .vscode/settings.json | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index ceda74d3..da0f4ba5 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -27,7 +27,9 @@ }, "rust-analyzer.cargo.extraEnv": { "PROTOC": "/opt/homebrew/bin/protoc", - "PATH": "/opt/homebrew/bin:${env:PATH}" + "PATH": "/opt/homebrew/bin:${env:PATH}:/opt/homebrew/opt/llvm/bin", + "CC": "/opt/homebrew/opt/llvm/bin/clang", + "CXX": "/opt/homebrew/opt/llvm/bin/clang++" }, // "rust-analyzer.cargo.target": "wasm32-unknown-unknown" } From 7bcb22a438cc756aa0bf6b305b16e09ed49edc56 Mon Sep 17 00:00:00 2001 From: Shuning Date: Fri, 20 Jun 2025 12:53:28 +0800 Subject: [PATCH 08/12] fix base64ct version --- core/Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/Cargo.lock b/core/Cargo.lock index da86552c..6836816b 100644 --- a/core/Cargo.lock +++ b/core/Cargo.lock @@ -391,9 +391,9 @@ checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" [[package]] name = "base64ct" -version = "1.8.0" +version = "1.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55248b47b0caf0546f7988906588779981c43bb1bc9d0c44087278f80cdb44ba" +checksum = "89e25b6adfb930f02d1981565a6e5d9c547ac15a96606256d3b59040e5cd4ca3" [[package]] name = "bincode" From d56e1bdccfeafd354aa8edce5233ab656f466aa2 Mon Sep 17 00:00:00 2001 From: Shuning Date: Fri, 20 Jun 2025 13:21:03 +0800 Subject: [PATCH 09/12] upgrade rustup version --- core/scripts/Build-Library.ps1 | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/core/scripts/Build-Library.ps1 b/core/scripts/Build-Library.ps1 index 894841ee..c178ef42 100755 --- a/core/scripts/Build-Library.ps1 +++ b/core/scripts/Build-Library.ps1 @@ -55,7 +55,7 @@ Switch($Platform) { Write-Error -Message "Parameter '-Architecture' is not specified for 'macOS' platform." Exit 1 } - $RustToolchain = '1.81.0' + $RustToolchain = '1.87.0' Switch($Architecture) { 'AMD64' { $RustTarget = 'x86_64-apple-darwin' } 'ARM64' { $RustTarget = 'aarch64-apple-darwin' } @@ -96,7 +96,7 @@ Switch($Platform) { Write-Error -Message "Invalid or unsupported '$Architecture' architecture for 'iOS' platform." Exit 1 } - $RustToolchain = '1.81.0' + $RustToolchain = '1.87.0' $RustTarget = 'aarch64-apple-ios' } 'iOS-Simulator' { @@ -108,7 +108,7 @@ Switch($Platform) { Write-Error -Message "Parameter '-Architecture' is not specified for 'iOS-Simulator' platform." Exit 1 } - $RustToolchain = '1.81.0' + $RustToolchain = '1.87.0' Switch($Architecture) { 'AMD64' { $RustTarget = 'x86_64-apple-ios' } 'ARM64' { $RustTarget = 'aarch64-apple-ios-sim' } @@ -126,7 +126,7 @@ Switch($Platform) { Write-Error -Message "Invalid or unsupported '$Architecture' architecture for 'Web' platform." Exit 1 } - $RustToolchain = '1.81.0' + $RustToolchain = '1.87.0' $RustTarget = 'wasm32-unknown-unknown' $WASMTarget = 'bundler' } @@ -153,7 +153,7 @@ Switch($Platform) { Write-Error -Message "Parameter '-Architecture' is not specified for 'Linux' platform." Exit 1 } - $RustToolchain = '1.81.0' + $RustToolchain = '1.87.0' Switch($Architecture) { 'AMD64' { $RustTarget = 'x86_64-unknown-linux-gnu' } 'ARM64' { From d5388fcdfdfdacb934d84fd639e8bb2c908831be Mon Sep 17 00:00:00 2001 From: Shuning Date: Fri, 20 Jun 2025 13:32:45 +0800 Subject: [PATCH 10/12] diable wasm-opt for base --- core/base/Cargo.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/core/base/Cargo.toml b/core/base/Cargo.toml index f8b87133..f56a8206 100644 --- a/core/base/Cargo.toml +++ b/core/base/Cargo.toml @@ -14,3 +14,6 @@ crate-type = ["cdylib", "staticlib", "rlib"] [features] default = ["posemesh-domain/default", "posemesh-networking/default"] c = ["posemesh-domain/c", "posemesh-networking/c"] + +[package.metadata.wasm-pack.profile.release] +wasm-opt = false \ No newline at end of file From 4114b80ab04b8a30a7366650756d3d216c87e5d7 Mon Sep 17 00:00:00 2001 From: Shuning Date: Fri, 20 Jun 2025 14:05:38 +0800 Subject: [PATCH 11/12] update tokio tungstenite's feature for cross compilation --- core/Cargo.lock | 169 +++++++++--------------------------------- core/disco/Cargo.toml | 2 +- 2 files changed, 36 insertions(+), 135 deletions(-) diff --git a/core/Cargo.lock b/core/Cargo.lock index 6836816b..12a4047f 100644 --- a/core/Cargo.lock +++ b/core/Cargo.lock @@ -1072,21 +1072,6 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" -[[package]] -name = "foreign-types" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" -dependencies = [ - "foreign-types-shared", -] - -[[package]] -name = "foreign-types-shared" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" - [[package]] name = "form_urlencoded" version = "1.2.1" @@ -2480,7 +2465,7 @@ dependencies = [ "thiserror 1.0.69", "tracing", "url", - "webpki-roots", + "webpki-roots 0.25.4", ] [[package]] @@ -2728,23 +2713,6 @@ dependencies = [ "unsigned-varint", ] -[[package]] -name = "native-tls" -version = "0.2.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87de3442987e9dbec73158d5c715e7ad9072fda936bb03d19d7fa10e00520f0e" -dependencies = [ - "libc", - "log", - "openssl", - "openssl-probe", - "openssl-sys", - "schannel", - "security-framework", - "security-framework-sys", - "tempfile", -] - [[package]] name = "netlink-packet-core" version = "0.7.0" @@ -2937,50 +2905,6 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" -[[package]] -name = "openssl" -version = "0.10.73" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8505734d46c8ab1e19a1dce3aef597ad87dcb4c37e7188231769bd6bd51cebf8" -dependencies = [ - "bitflags 2.9.1", - "cfg-if", - "foreign-types", - "libc", - "once_cell", - "openssl-macros", - "openssl-sys", -] - -[[package]] -name = "openssl-macros" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.103", -] - -[[package]] -name = "openssl-probe" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" - -[[package]] -name = "openssl-sys" -version = "0.9.109" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90096e2e47630d78b7d1c20952dc621f957103f8bc2c8359ec81290d75238571" -dependencies = [ - "cc", - "libc", - "pkg-config", - "vcpkg", -] - [[package]] name = "overload" version = "0.1.1" @@ -3153,12 +3077,6 @@ dependencies = [ "spki", ] -[[package]] -name = "pkg-config" -version = "0.3.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" - [[package]] name = "polling" version = "3.8.0" @@ -3997,15 +3915,6 @@ dependencies = [ "winapi-util", ] -[[package]] -name = "schannel" -version = "0.1.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f29ebaa345f945cec9fbbc532eb307f0fdad8161f281b6369539c8d84876b3d" -dependencies = [ - "windows-sys 0.59.0", -] - [[package]] name = "scopeguard" version = "1.2.0" @@ -4048,29 +3957,6 @@ dependencies = [ "zeroize", ] -[[package]] -name = "security-framework" -version = "2.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" -dependencies = [ - "bitflags 2.9.1", - "core-foundation", - "core-foundation-sys", - "libc", - "security-framework-sys", -] - -[[package]] -name = "security-framework-sys" -version = "2.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49db231d56a190491cb4aeda9527f1ad45345af50b0851622a7adb8c03b01c32" -dependencies = [ - "core-foundation-sys", - "libc", -] - [[package]] name = "semver" version = "1.0.26" @@ -4658,16 +4544,6 @@ dependencies = [ "syn 2.0.103", ] -[[package]] -name = "tokio-native-tls" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" -dependencies = [ - "native-tls", - "tokio", -] - [[package]] name = "tokio-postgres" version = "0.7.13" @@ -4694,6 +4570,16 @@ dependencies = [ "whoami", ] +[[package]] +name = "tokio-rustls" +version = "0.26.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e727b36a1a0e8b74c376ac2211e40c2c8af09fb4013c60d910495810f008e9b" +dependencies = [ + "rustls 0.23.28", + "tokio", +] + [[package]] name = "tokio-tungstenite" version = "0.27.0" @@ -4702,10 +4588,12 @@ checksum = "489a59b6730eda1b0171fcfda8b121f4bee2b35cba8645ca35c5f7ba3eb736c1" dependencies = [ "futures-util", "log", - "native-tls", + "rustls 0.23.28", + "rustls-pki-types", "tokio", - "tokio-native-tls", + "tokio-rustls", "tungstenite", + "webpki-roots 0.26.11", ] [[package]] @@ -4858,8 +4746,9 @@ dependencies = [ "http 1.3.1", "httparse", "log", - "native-tls", "rand 0.9.1", + "rustls 0.23.28", + "rustls-pki-types", "sha1", "thiserror 2.0.12", "utf-8", @@ -5016,12 +4905,6 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" -[[package]] -name = "vcpkg" -version = "0.2.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" - [[package]] name = "vec_map" version = "0.8.2" @@ -5210,6 +5093,24 @@ version = "0.25.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f20c57d8d7db6d3b86154206ae5d8fba62dd39573114de97c2cb0578251f8e1" +[[package]] +name = "webpki-roots" +version = "0.26.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" +dependencies = [ + "webpki-roots 1.0.1", +] + +[[package]] +name = "webpki-roots" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8782dd5a41a24eed3a4f40b606249b3e236ca61adf1f25ea4d45c73de122b502" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "webrtc" version = "0.9.0" diff --git a/core/disco/Cargo.toml b/core/disco/Cargo.toml index 4a47b4a9..a625e67c 100644 --- a/core/disco/Cargo.toml +++ b/core/disco/Cargo.toml @@ -20,7 +20,7 @@ base64 = "0.22.1" [target.'cfg(not(target_family="wasm"))'.dependencies] tokio = { workspace = true, features = ["full"] } -tokio-tungstenite = { version = "*", features = ["native-tls"] } +tokio-tungstenite = { version = "*", features = ["rustls-tls-webpki-roots"] } uuid = { workspace = true, features = ["v4"] } [target.'cfg(target_family="wasm")'.dependencies] From e75ecab27a82629979ff4856aa01d8191a2beadd Mon Sep 17 00:00:00 2001 From: Shuning Date: Mon, 30 Jun 2025 15:28:53 +0800 Subject: [PATCH 12/12] create reconstruction job request --- core/domain/src/spatial/reconstruction.rs | 34 +++++++++++++++-------- 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/core/domain/src/spatial/reconstruction.rs b/core/domain/src/spatial/reconstruction.rs index ba5a0028..845dfb38 100644 --- a/core/domain/src/spatial/reconstruction.rs +++ b/core/domain/src/spatial/reconstruction.rs @@ -2,8 +2,13 @@ use quick_protobuf::serialize_into_vec; use crate::{cluster::{DomainCluster, TaskUpdatesSink}, datastore::common::DomainError, protobuf::{domain_data::Query, task}}; -pub async fn reconstruction_job(mut domain_cluster: DomainCluster, domain_id: &str, scans: Vec, handler: TaskUpdatesSink) -> Result<(), DomainError> { +pub fn new_reconstruction_job_request( + domain_id: &str, + scans: Vec, + manager_id: &str, +) -> task::JobRequest { let mut uploaded = Vec::::new(); + for scan in scans { let input = task::LocalRefinementInputV1 { query: Query { @@ -15,6 +20,7 @@ pub async fn reconstruction_job(mut domain_cluster: DomainCluster, domain_id: &s metadata_only: false, }, }; + let task = task::TaskRequest { needs: vec![], resource_recruitment: task::ResourceRecruitment { @@ -30,10 +36,10 @@ pub async fn reconstruction_job(mut domain_cluster: DomainCluster, domain_id: &s min_cpu: Some(0), }, data: Some(task::Any { - type_url: "LocalRefinementInputV1".to_string(), // TODO: use actual type url + type_url: "LocalRefinementInputV1".to_string(), value: serialize_into_vec(&input).expect("cant serialize input"), }), - sender: domain_cluster.manager_id.clone(), + sender: manager_id.to_string(), receiver: None, }; uploaded.push(task); @@ -52,22 +58,26 @@ pub async fn reconstruction_job(mut domain_cluster: DomainCluster, domain_id: &s capability_filters: task::CapabilityFilters { endpoint: "/global-refinement/v1".to_string(), min_gpu: Some(1), - min_cpu: Some(1), + min_cpu: None, }, data: Some(task::Any { - type_url: "GlobalRefinementInputV1".to_string(), // TODO: use actual type url - value: vec![], + type_url: "GlobalRefinementInputV1".to_string(), + value: vec![], // TODO: serialize actual input }), - sender: domain_cluster.manager_id.clone(), + sender: manager_id.to_string(), receiver: None, }); - let job = task::JobRequest { - domain_id: domain_id.to_string(), - name: "refinement job".to_string(), + task::JobRequest { + name: "reconstruction".to_string(), tasks: uploaded, - nonce: "".to_string(), - }; + nonce: "".to_string(), // TODO: generate proper nonce + domain_id: domain_id.to_string(), + } +} + +pub async fn reconstruction_job(mut domain_cluster: DomainCluster, domain_id: &str, scans: Vec, handler: TaskUpdatesSink) -> Result<(), DomainError> { + let job = new_reconstruction_job_request(domain_id, scans, &domain_cluster.manager_id); tracing::debug!("job has {} tasks", job.tasks.len());