diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a6471b48..e545af82 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,22 +9,20 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: actions-rs/toolchain@v1 + - uses: dtolnay/rust-toolchain@master with: - toolchain: 1.81 + toolchain: nightly-2025-02-25 components: rustfmt - override: true - - run: cargo fmt --all --check + - run: cargo +nightly-2025-02-25 fmt --all --check clippy: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: actions-rs/toolchain@v1 + - uses: dtolnay/rust-toolchain@master with: - toolchain: 1.81 + toolchain: 1.85 components: clippy - override: true - uses: Swatinem/rust-cache@v2 - run: cargo clippy --workspace --all-targets --all-features env: @@ -34,9 +32,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: actions-rs/toolchain@v1 + - uses: dtolnay/rust-toolchain@master with: - toolchain: 1.81 - override: true + toolchain: 1.85 - uses: Swatinem/rust-cache@v2 - run: cargo test --workspace --all-features --no-fail-fast diff --git a/Cargo.lock b/Cargo.lock index a47d0e00..862ac0d9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -373,17 +373,19 @@ dependencies = [ [[package]] name = "bothan-api" -version = "0.0.1-beta.1" +version = "0.0.1-beta.2" dependencies = [ + "async-trait", "bothan-binance", + "bothan-bitfinex", "bothan-bybit", "bothan-coinbase", "bothan-coingecko", "bothan-coinmarketcap", "bothan-core", - "bothan-cryptocompare", "bothan-htx", "bothan-kraken", + "bothan-lib", "bothan-okx", "config", "dirs", @@ -397,21 +399,13 @@ dependencies = [ [[package]] name = "bothan-api-cli" -version = "0.0.1-beta.1" +version = "0.0.1-beta.2" dependencies = [ "anyhow", "bothan-api", - "bothan-binance", - "bothan-bybit", "bothan-client", - "bothan-coinbase", - "bothan-coingecko", - "bothan-coinmarketcap", "bothan-core", - "bothan-cryptocompare", - "bothan-htx", - "bothan-kraken", - "bothan-okx", + "bothan-lib", "clap", "dirs", "inquire", @@ -427,10 +421,10 @@ dependencies = [ [[package]] name = "bothan-binance" -version = "0.0.1-beta.1" +version = "0.0.1-beta.2" dependencies = [ "async-trait", - "bothan-core", + "bothan-lib", "futures-util", "humantime-serde", "opentelemetry", @@ -438,7 +432,7 @@ dependencies = [ "rust_decimal", "serde", "serde_json", - "thiserror 2.0.9", + "thiserror 2.0.11", "tokio", "tokio-tungstenite", "tracing", @@ -446,28 +440,44 @@ dependencies = [ "ws-mock", ] +[[package]] +name = "bothan-bitfinex" +version = "0.0.1-beta.2" +dependencies = [ + "async-trait", + "bothan-lib", + "chrono", + "humantime-serde", + "reqwest", + "rust_decimal", + "serde", + "serde_json", + "thiserror 2.0.11", + "tokio", + "url", +] + [[package]] name = "bothan-bybit" -version = "0.0.1-beta.1" +version = "0.0.1-beta.2" dependencies = [ "async-trait", - "bothan-core", + "bothan-lib", "chrono", "futures-util", "rust_decimal", "serde", "serde_json", - "thiserror 2.0.9", + "thiserror 2.0.11", "tokio", "tokio-tungstenite", "tracing", - "tracing-subscriber", "ws-mock", ] [[package]] name = "bothan-client" -version = "0.0.1-beta.1" +version = "0.0.1-beta.2" dependencies = [ "pbjson", "prost", @@ -481,70 +491,74 @@ dependencies = [ [[package]] name = "bothan-coinbase" -version = "0.0.1-beta.1" +version = "0.0.1-beta.2" dependencies = [ "async-trait", - "bothan-core", + "bothan-lib", "chrono", "futures-util", "rust_decimal", "serde", "serde_json", - "thiserror 2.0.9", + "thiserror 2.0.11", "tokio", "tokio-tungstenite", "tracing", - "tracing-subscriber", "ws-mock", ] [[package]] name = "bothan-coingecko" -version = "0.0.1-beta.1" +version = "0.0.1-beta.2" dependencies = [ "async-trait", - "bothan-core", - "chrono", + "bothan-lib", "humantime-serde", "mockito", "reqwest", "rust_decimal", "serde", "serde_json", - "thiserror 2.0.9", + "thiserror 2.0.11", "tokio", - "tracing", - "tracing-subscriber", "url", ] [[package]] name = "bothan-coinmarketcap" -version = "0.0.1-beta.1" +version = "0.0.1-beta.2" dependencies = [ "async-trait", - "bothan-core", + "bothan-lib", "chrono", "humantime-serde", - "itertools 0.13.0", + "itertools 0.14.0", "mockito", "reqwest", "rust_decimal", "serde", "serde_json", - "thiserror 2.0.9", + "thiserror 2.0.11", "tokio", - "tracing", - "tracing-subscriber", "url", ] [[package]] name = "bothan-core" -version = "0.0.1-beta.1" +version = "0.0.1-beta.2" dependencies = [ "async-trait", "bincode", + "bothan-binance", + "bothan-bitfinex", + "bothan-bybit", + "bothan-coinbase", + "bothan-coingecko", + "bothan-coinmarketcap", + "bothan-htx", + "bothan-kraken", + "bothan-lib", + "bothan-okx", "chrono", "derive_more", "ed25519", @@ -559,84 +573,78 @@ dependencies = [ "semver", "serde", "serde_json", - "thiserror 2.0.9", + "thiserror 2.0.11", "tokio", "tracing", ] [[package]] -name = "bothan-cryptocompare" -version = "0.0.1-beta.1" +name = "bothan-htx" +version = "0.0.1-beta.2" dependencies = [ "async-trait", - "bothan-core", + "bothan-lib", + "flate2", "futures-util", "rust_decimal", "serde", "serde_json", - "thiserror 2.0.9", + "thiserror 2.0.11", "tokio", "tokio-tungstenite", "tracing", - "tracing-subscriber", - "url", + "ws-mock", ] [[package]] -name = "bothan-htx" -version = "0.0.1-beta.1" +name = "bothan-kraken" +version = "0.0.1-beta.2" dependencies = [ "async-trait", - "bothan-core", - "flate2", + "bothan-lib", + "chrono", "futures-util", "rust_decimal", "serde", "serde_json", - "thiserror 2.0.9", + "thiserror 2.0.11", "tokio", "tokio-tungstenite", "tracing", - "tracing-subscriber", "ws-mock", ] [[package]] -name = "bothan-kraken" -version = "0.0.1-beta.1" +name = "bothan-lib" +version = "0.0.1-alpha.4" dependencies = [ "async-trait", - "bothan-core", - "chrono", - "futures-util", - "humantime-serde", + "bincode", + "derive_more", + "num-traits", "rust_decimal", "serde", "serde_json", - "thiserror 2.0.9", + "thiserror 2.0.11", "tokio", - "tokio-tungstenite", "tracing", - "tracing-subscriber", - "ws-mock", ] [[package]] name = "bothan-okx" -version = "0.0.1-beta.1" +version = "0.0.1-beta.2" dependencies = [ "async-trait", - "bothan-core", + "bothan-lib", "chrono", "futures-util", "rust_decimal", "serde", "serde_json", - "thiserror 2.0.9", + "thiserror 2.0.11", "tokio", "tokio-tungstenite", "tracing", - "tracing-subscriber", "ws-mock", ] @@ -788,9 +796,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.23" +version = "4.5.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3135e7ec2ef7b10c6ed8950f0f792ed96ee093fa088608f1c76e569722700c84" +checksum = "8acebd8ad879283633b343856142139f2da2317c96b05b4dd6181c61e2480184" dependencies = [ "clap_builder", "clap_derive", @@ -798,9 +806,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.23" +version = "4.5.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30582fc632330df2bd26877bde0c1f4470d57c582bbc070376afcd04d8cb4838" +checksum = "f6ba32cbda51c7e1dfd49acc1457ba1a7dec5b64fe360e828acb13ca8dc9c2f9" dependencies = [ "anstream", "anstyle", @@ -810,9 +818,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.18" +version = "4.5.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ac6a0c7b1a9e9a5186361f67dfa1b88213572f427fb9ab038efb2bd8c582dab" +checksum = "bf4ced95c6f4a675af3da73304b9ac4ed991640c36374e4b46795c49e17cf1ed" dependencies = [ "heck", "proc-macro2", @@ -844,20 +852,20 @@ dependencies = [ [[package]] name = "config" -version = "0.14.1" +version = "0.15.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68578f196d2a33ff61b27fae256c3164f65e36382648e30666dde05b8cc9dfdf" +checksum = "8cf9dc8d4ef88e27a8cb23e85cb116403dedd57f7971964dc4b18ccead548901" dependencies = [ "async-trait", - "convert_case", + "convert_case 0.6.0", "json5", - "nom", "pathdiff", "ron", "rust-ini", "serde", "serde_json", "toml", + "winnow", "yaml-rust2", ] @@ -896,6 +904,15 @@ dependencies = [ "unicode-segmentation", ] +[[package]] +name = "convert_case" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb402b8d4c85569410425650ce3eddc7d698ed96d39a73f941b08fb63082f1e7" +dependencies = [ + "unicode-segmentation", +] + [[package]] name = "core-foundation" version = "0.9.4" @@ -1089,20 +1106,20 @@ dependencies = [ [[package]] name = "derive_more" -version = "1.0.0" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a9b99b9cbbe49445b21764dc0625032a89b145a2642e67603e1c936f5458d05" +checksum = "093242cf7570c207c83073cf82f79706fe7b8317e98620a47d5be7c3d8497678" dependencies = [ "derive_more-impl", ] [[package]] name = "derive_more-impl" -version = "1.0.0" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb7330aeadfbe296029522e6c40f315320aba36fc43a5b3632f3795348f3bd22" +checksum = "bda628edc44c4bb645fbe0f758797143e4e07926f7ebf4e9bdfbd3d2ce621df3" dependencies = [ - "convert_case", + "convert_case 0.7.1", "proc-macro2", "quote", "syn 2.0.94", @@ -1121,23 +1138,23 @@ dependencies = [ [[package]] name = "dirs" -version = "5.0.1" +version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44c45a9d03d6676652bcb5e724c7e988de1acad23a711b5217ab9cbecbec2225" +checksum = "c3e8aa94d75141228480295a7d0e7feb620b1a5ad9f12bc40be62411e38cce4e" dependencies = [ "dirs-sys", ] [[package]] name = "dirs-sys" -version = "0.4.1" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "520f05a5cbd335fae5a99ff7a6ab8627577660ee5cfd6a94a6a929b52ff0321c" +checksum = "e01a3366d27ee9890022452ee61b2b63a67e6f13f58900b651ff5665f0bb1fab" dependencies = [ "libc", "option-ext", "redox_users", - "windows-sys 0.48.0", + "windows-sys 0.59.0", ] [[package]] @@ -1151,9 +1168,9 @@ dependencies = [ [[package]] name = "dyn-clone" -version = "1.0.17" +version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d6ef0072f8a535281e4876be788938b528e9a1d43900b82c2569af7da799125" +checksum = "feeef44e73baff3a26d371801df019877a9866a8c493d315ab00177843314f35" [[package]] name = "ed25519" @@ -1709,6 +1726,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.11" @@ -2052,16 +2078,16 @@ dependencies = [ [[package]] name = "opentelemetry" -version = "0.26.0" +version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "570074cc999d1a58184080966e5bd3bf3a9a4af650c3b05047c2621e7405cd17" +checksum = "236e667b670a5cdf90c258f5a55794ec5ac5027e960c224bff8367a59e1e6426" dependencies = [ "futures-core", "futures-sink", "js-sys", - "once_cell", "pin-project-lite", - "thiserror 1.0.65", + "thiserror 2.0.11", + "tracing", ] [[package]] @@ -2390,13 +2416,13 @@ dependencies = [ [[package]] name = "redox_users" -version = "0.4.6" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba009ff324d1fc1b900bd1fdb31564febe58a8ccc8a6fdbb93b543d33b13ca43" +checksum = "dd6f9d3d47bdd2ad6945c5015a226ec6155d0bcdfd8f7cd29f86b71f8de99d2b" dependencies = [ "getrandom", "libredox", - "thiserror 1.0.65", + "thiserror 2.0.11", ] [[package]] @@ -2553,19 +2579,20 @@ dependencies = [ [[package]] name = "rust-ini" -version = "0.20.0" +version = "0.21.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e0698206bcb8882bf2a9ecb4c1e7785db57ff052297085a6efd4fe42302068a" +checksum = "4e310ef0e1b6eeb79169a1171daf9abcb87a2e17c03bee2c4bb100b55c75409f" dependencies = [ "cfg-if", "ordered-multimap", + "trim-in-place", ] [[package]] name = "rust-librocksdb-sys" -version = "0.30.0+9.8.4" +version = "0.32.0+9.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a81bce962dd4113eae8a848ecb3598f25122cc6be675f815b71a4a2d0a523b85" +checksum = "50146b7fadd68926e9dcb902bf0515783aaaf5f73af26949f188aead5ede8cd0" dependencies = [ "bindgen", "bzip2-sys", @@ -2579,9 +2606,9 @@ dependencies = [ [[package]] name = "rust-rocksdb" -version = "0.34.0" +version = "0.36.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16af4eac13d9b0025c84b4912cda7b5fa59f16c513a1b6ba65ae16cfb9ab10e3" +checksum = "3bf088a714aa3fad699f7dbe06047ca732c09629a2f9b28aa16ca6d3897a4c2f" dependencies = [ "libc", "rust-librocksdb-sys", @@ -2772,9 +2799,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.134" +version = "1.0.138" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d00f4175c42ee48b15416f6193a959ba3a0d67fc699a0db9ad12df9f83991c7d" +checksum = "d434192e7da787e94a6ea7e9670b26a036d0ca41e0b7efb2676dd32bae872949" dependencies = [ "itoa", "memchr", @@ -3095,11 +3122,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.9" +version = "2.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f072643fd0190df67a8bab670c20ef5d8737177d6ac6b2e9a236cb096206b2cc" +checksum = "d452f284b73e6d76dd36758a0c8684b1d5be31f92b89d07fd5822175732206fc" dependencies = [ - "thiserror-impl 2.0.9", + "thiserror-impl 2.0.11", ] [[package]] @@ -3115,9 +3142,9 @@ dependencies = [ [[package]] name = "thiserror-impl" -version = "2.0.9" +version = "2.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b50fa271071aae2e6ee85f842e2e28ba8cd2c5fb67f11fcb1fd70b276f9e7d4" +checksum = "26afc1baea8a989337eeb52b6e72a039780ce45c3edfcc9c5b9d112feeb173c2" dependencies = [ "proc-macro2", "quote", @@ -3279,9 +3306,9 @@ dependencies = [ [[package]] name = "toml" -version = "0.8.19" +version = "0.8.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1ed1f98e3fdc28d6d910e6737ae6ab1a93bf1985935a1193e68f93eeb68d24e" +checksum = "cd87a5cdd6ffab733b2f74bc4fd7ee5fff6634124999ac278c35fc78c6120148" dependencies = [ "serde", "serde_spanned", @@ -3300,9 +3327,9 @@ dependencies = [ [[package]] name = "toml_edit" -version = "0.22.22" +version = "0.22.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ae48d6208a266e853d946088ed816055e556cc6028c5e8e2b84d9fa5dd7c7f5" +checksum = "17b4795ff5edd201c7cd6dca065ae59972ce77d1b80fa0a84d94950ece7d1474" dependencies = [ "indexmap 2.6.0", "serde", @@ -3448,6 +3475,12 @@ dependencies = [ "tracing-log", ] +[[package]] +name = "trim-in-place" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "343e926fc669bc8cde4fa3129ab681c63671bae288b1f1081ceee6d9d37904fc" + [[package]] name = "triomphe" version = "0.1.14" @@ -3917,9 +3950,9 @@ checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] name = "winnow" -version = "0.6.20" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36c1fec1a2bb5866f07c25f68c26e565c4c200aebb96d7e55710c19d3e8ac49b" +checksum = "59690dea168f2198d1a3b0cac23b8063efcd11012f10ae4698f284808c8ef603" dependencies = [ "memchr", ] @@ -3948,9 +3981,9 @@ dependencies = [ [[package]] name = "yaml-rust2" -version = "0.8.1" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8902160c4e6f2fb145dbe9d6760a75e3c9522d8bf796ed7047c85919ac7115f8" +checksum = "2a1a1c0bc9823338a3bdf8c61f994f23ac004c6fa32c08cd152984499b445e8d" dependencies = [ "arraydeque", "encoding_rs", diff --git a/Cargo.toml b/Cargo.toml index b2c4ab1d..822b39ac 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [workspace.package] authors = ["Band Protocol "] -edition = "2021" +edition = "2024" license = "MIT OR Apache-2.0" repository = "https://github.com/bandprotocol/bothan" exclude = [".github/"] @@ -10,31 +10,40 @@ members = [ "bothan-api/client/rust-client", "bothan-api/server", "bothan-api/server-cli", - "bothan-binance", - "bothan-bybit", - "bothan-coinbase", - "bothan-coingecko", - "bothan-coinmarketcap", - "bothan-core", - "bothan-cryptocompare", - "bothan-htx", - "bothan-kraken", - "bothan-okx", + "bothan-*", ] exclude = ["bothan-api", "bothan-api-proxy"] resolver = "2" [workspace.dependencies] +bothan-api = { path = "bothan-api/server" } +bothan-core = { path = "bothan-core" } +bothan-client = { path = "bothan-api/client/rust-client" } +bothan-lib = { path = "bothan-lib" } + +bothan-binance = { path = "bothan-binance" } +bothan-bitfinex = { path = "bothan-bitfinex" } +bothan-bybit = { path = "bothan-bybit" } +bothan-coinbase = { path = "bothan-coinbase" } +bothan-coingecko = { path = "bothan-coingecko" } +bothan-coinmarketcap = { path = "bothan-coinmarketcap" } +bothan-htx = { path = "bothan-htx" } +bothan-kraken = { path = "bothan-kraken" } +bothan-okx = { path = "bothan-okx" } + anyhow = "1.0.86" async-trait = "0.1.77" -chrono = "0.4.38" -derive_more = { version = "1.0.0-beta.6", features = ["full"] } +bincode = "2.0.0-rc.3" +chrono = "0.4.39" +derive_more = { version = "2.0.1", features = ["full"] } +dirs = "6.0.0" futures = "0.3.30" futures-util = "0.3.31" humantime-serde = "1.1.1" -itertools = "0.13.0" +itertools = "0.14.0" mockito = "1.4.0" -opentelemetry = { version = "0.26.0", features = ["metrics"] } +num-traits = "0.2.19" +opentelemetry = { version = "0.28.0", features = ["metrics"] } prost = "0.13.1" protoc-gen-prost = "0.4.0" protoc-gen-tonic = "0.4.1" @@ -43,26 +52,11 @@ reqwest = { version = "0.12.3", features = ["json"] } rust_decimal = "1.10.2" semver = "1.0.23" serde = { version = "1.0.197", features = ["std", "derive", "alloc"] } -serde_json = "1.0.116" -thiserror = "2.0.6" +serde_json = "1.0.138" +thiserror = "2.0.11" tokio = { version = "1.36.0", features = ["full"] } tokio-tungstenite = { version = "0.24.0", features = ["native-tls"] } tonic = "0.12.1" tracing = "0.1.40" tracing-subscriber = { version = "0.3.18", features = ["env-filter"] } url = "2.5.0" - -bothan-core = { path = "bothan-core" } - -bothan-client = { path = "bothan-api/client/rust-client" } -bothan-api = { path = "bothan-api/server" } - -bothan-binance = { path = "bothan-binance" } -bothan-bybit = { path = "bothan-bybit" } -bothan-coinbase = { path = "bothan-coinbase" } -bothan-coingecko = { path = "bothan-coingecko" } -bothan-coinmarketcap = { path = "bothan-coinmarketcap" } -bothan-cryptocompare = { path = "bothan-cryptocompare" } -bothan-htx = { path = "bothan-htx" } -bothan-kraken = { path = "bothan-kraken" } -bothan-okx = { path = "bothan-okx" } diff --git a/bothan-api/client/rust-client/Cargo.toml b/bothan-api/client/rust-client/Cargo.toml index 30c3cd45..c86496c5 100644 --- a/bothan-api/client/rust-client/Cargo.toml +++ b/bothan-api/client/rust-client/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "bothan-client" -version = "0.0.1-beta.1" +version = "0.0.1-beta.2" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/bothan-api/client/rust-client/src/proto.rs b/bothan-api/client/rust-client/src/proto.rs index 229c70be..17af9672 100644 --- a/bothan-api/client/rust-client/src/proto.rs +++ b/bothan-api/client/rust-client/src/proto.rs @@ -1,3 +1,4 @@ +#![allow(clippy::all)] pub mod bothan { pub mod v1 { pub use bothan_service_client::BothanServiceClient; diff --git a/bothan-api/server-cli/Cargo.toml b/bothan-api/server-cli/Cargo.toml index c07478de..e2275729 100644 --- a/bothan-api/server-cli/Cargo.toml +++ b/bothan-api/server-cli/Cargo.toml @@ -1,7 +1,6 @@ [package] name = "bothan-api-cli" -version = "0.0.1-beta.1" -rust-version = "1.81" +version = "0.0.1-beta.2" edition.workspace = true license.workspace = true repository.workspace = true @@ -11,20 +10,12 @@ name = "bothan" path = "src/main.rs" [dependencies] -bothan-client = { workspace = true } -bothan-core = { workspace = true } - bothan-api = { workspace = true } -bothan-binance = { workspace = true } -bothan-bybit = { workspace = true } -bothan-coinbase = { workspace = true } -bothan-coingecko = { workspace = true } -bothan-coinmarketcap = { workspace = true } -bothan-cryptocompare = { workspace = true } -bothan-htx = { workspace = true } -bothan-kraken = { workspace = true } -bothan-okx = { workspace = true } +bothan-core = { workspace = true, features = ["rocksdb"] } +bothan-client = { workspace = true } +bothan-lib = { workspace = true } +dirs = { workspace = true } reqwest = { workspace = true } semver = { workspace = true } serde_json = { workspace = true } @@ -33,8 +24,7 @@ tonic = { workspace = true } tracing = { workspace = true } tracing-subscriber = { workspace = true } -anyhow = "1.0.94" -clap = { version = "4.5.23", features = ["derive"] } +anyhow = "1.0.95" +clap = { version = "4.5.29", features = ["derive"] } inquire = "0.7.5" -dirs = "5.0.1" -toml = "0.8.19" +toml = "0.8.20" diff --git a/bothan-api/server-cli/Dockerfile b/bothan-api/server-cli/Dockerfile index ac6dedfb..6de3209a 100644 --- a/bothan-api/server-cli/Dockerfile +++ b/bothan-api/server-cli/Dockerfile @@ -1,5 +1,5 @@ # Start from the latest Rust image -FROM rust:1.81-bookworm AS builder +FROM rust:1.85-bookworm AS builder WORKDIR /builder # Copy entire source code diff --git a/bothan-api/server-cli/config.example.toml b/bothan-api/server-cli/config.example.toml index 65240b46..6a1040d4 100644 --- a/bothan-api/server-cli/config.example.toml +++ b/bothan-api/server-cli/config.example.toml @@ -52,6 +52,13 @@ url = "wss://stream.binance.com:9443/stream" # Internal channel size for Binance data streams. internal_ch_size = 1000 +# Configuration for Bitfinex. +[manager.crypto.source.bitfinex] +# REST API URL for querying data from Bitfinex. +url = "https://api-pub.bitfinex.com/v2/" +# Update interval for polling data from Bitfinex. +update_interval = "1m" + # Configuration for Bybit. [manager.crypto.source.bybit] # WebSocket URL for streaming data from Bybit. @@ -68,13 +75,13 @@ internal_ch_size = 1000 # Configuration for Coingecko. [manager.crypto.source.coingecko] -# REST API URL for querying data from Coingecko. +# REST API URL for querying data from CoinGecko. url = "https://api.coingecko.com/api/v3/" # API key for authentication (if required). api_key = "" -# User agent string for HTTP requests to Coingecko. +# User agent string for HTTP requests to CoinGecko. user_agent = "Bothan" -# Update interval for pulling data from Coingecko. +# Update interval for polling data from CoinGecko. update_interval = "30s" # Configuration for CoinMarketCap. @@ -86,15 +93,6 @@ api_key = "" # Update interval for pulling data from CoinMarketCap. update_interval = "1m" -# Configuration for CryptoCompare. -[manager.crypto.source.cryptocompare] -# REST API URL for querying data from CryptoCompare. -url = "wss://data-streamer.cryptocompare.com" -# API key for authentication (required for access). -api_key = "" -# Internal channel size for cryptocompare data streams. -internal_ch_size = 1000 - # Configuration for HTX (Huobi). [manager.crypto.source.htx] # WebSocket URL for streaming data from HTX. diff --git a/bothan-api/server-cli/src/commands/config.rs b/bothan-api/server-cli/src/commands/config.rs index 15014627..a3650f39 100644 --- a/bothan-api/server-cli/src/commands/config.rs +++ b/bothan-api/server-cli/src/commands/config.rs @@ -1,10 +1,10 @@ -use anyhow::{anyhow, Context}; -use clap::{Parser, Subcommand}; use std::fs::{create_dir_all, write}; use std::path::PathBuf; -use bothan_api::config::manager::crypto_info::sources::CryptoSourceConfigs; +use anyhow::{Context, anyhow}; use bothan_api::config::AppConfig; +use bothan_api::config::manager::crypto_info::sources::CryptoSourceConfigs; +use clap::{Parser, Subcommand}; use crate::bothan_home_dir; diff --git a/bothan-api/server-cli/src/commands/key.rs b/bothan-api/server-cli/src/commands/key.rs index 309ced23..8f6d0c65 100644 --- a/bothan-api/server-cli/src/commands/key.rs +++ b/bothan-api/server-cli/src/commands/key.rs @@ -1,10 +1,10 @@ -use anyhow::{anyhow, Context}; -use clap::{Parser, Subcommand}; -use inquire::{Password, PasswordDisplayMode}; use std::fs::{create_dir_all, read, read_to_string, write}; +use anyhow::{Context, anyhow}; use bothan_api::config::AppConfig; use bothan_core::monitoring::Signer; +use clap::{Parser, Subcommand}; +use inquire::{Password, PasswordDisplayMode}; #[derive(Parser)] pub struct KeyCli { diff --git a/bothan-api/server-cli/src/commands/request.rs b/bothan-api/server-cli/src/commands/request.rs index 76e00d5a..f3e0ab24 100644 --- a/bothan-api/server-cli/src/commands/request.rs +++ b/bothan-api/server-cli/src/commands/request.rs @@ -1,8 +1,7 @@ use anyhow::Context; -use clap::{Parser, Subcommand}; - use bothan_api::config::AppConfig; use bothan_client::client::GrpcClient; +use clap::{Parser, Subcommand}; #[derive(Parser)] pub struct RequestCli { diff --git a/bothan-api/server-cli/src/commands/start.rs b/bothan-api/server-cli/src/commands/start.rs index 22f441b4..32f6058f 100644 --- a/bothan-api/server-cli/src/commands/start.rs +++ b/bothan-api/server-cli/src/commands/start.rs @@ -1,39 +1,33 @@ use std::collections::HashMap; -use std::fs::{create_dir_all, read_to_string, remove_dir_all, write, File}; +use std::fs::{File, create_dir_all, read_to_string, write}; use std::io::BufReader; use std::path::PathBuf; use std::str::FromStr; use std::sync::Arc; use std::time::Duration; -use anyhow::Context; -use clap::Parser; -use reqwest::header::{HeaderName, HeaderValue}; -use semver::{Version, VersionReq}; -use tonic::transport::Server; -use tracing::{debug, error, info}; - +use anyhow::{Context, bail}; use bothan_api::api::BothanServer; +use bothan_api::config::AppConfig; use bothan_api::config::ipfs::IpfsAuthentication; use bothan_api::config::manager::crypto_info::sources::CryptoSourceConfigs; -use bothan_api::config::AppConfig; use bothan_api::proto::bothan::v1::BothanServiceServer; use bothan_api::{REGISTRY_REQUIREMENT, VERSION}; -use bothan_binance::BinanceWorkerBuilder; -use bothan_bybit::BybitWorkerBuilder; -use bothan_coinbase::CoinbaseWorkerBuilder; -use bothan_coingecko::CoinGeckoWorkerBuilder; -use bothan_coinmarketcap::CoinMarketCapWorkerBuilder; use bothan_core::ipfs::{IpfsClient, IpfsClientBuilder}; use bothan_core::manager::CryptoAssetInfoManager; +use bothan_core::manager::crypto_asset_info::worker::CryptoAssetWorker; +use bothan_core::manager::crypto_asset_info::worker::opts::CryptoAssetWorkerOpts; use bothan_core::monitoring::{Client as MonitoringClient, Signer}; -use bothan_core::registry::{Registry, Valid}; -use bothan_core::store::SharedStore; -use bothan_core::worker::{AssetWorker, AssetWorkerBuilder}; -use bothan_cryptocompare::CryptoCompareWorkerBuilder; -use bothan_htx::HtxWorkerBuilder; -use bothan_kraken::KrakenWorkerBuilder; -use bothan_okx::OkxWorkerBuilder; +use bothan_core::store::rocksdb::RocksDbStore; +use bothan_lib::registry::{Registry, Valid}; +use bothan_lib::store::{RegistryStore, Store}; +use bothan_lib::worker::AssetWorker; +use bothan_lib::worker::error::AssetWorkerError; +use clap::Parser; +use reqwest::header::{HeaderName, HeaderValue}; +use semver::{Version, VersionReq}; +use tonic::transport::Server; +use tracing::{debug, error, info}; #[derive(Parser)] pub struct StartCli { @@ -60,16 +54,14 @@ impl StartCli { Some(p) => { let file = File::open(p).with_context(|| "Failed to open registry file")?; let reader = BufReader::new(file); - serde_json::from_reader(reader).with_context(|| "Failed to parse registry")? + let registry = + serde_json::from_reader(reader).with_context(|| "Failed to parse registry")?; + Some(registry) } - None => Registry::default(), + None => None, }; - let valid_registry = registry - .validate() - .with_context(|| "Failed to validate registry")?; - - let store = init_store(&app_config, valid_registry, self.unsafe_reset).await?; + let store = init_rocks_db_store(&app_config, registry, self.unsafe_reset).await?; let ipfs_client = init_ipfs_client(&app_config).await?; let monitoring_client = init_monitoring_client(&app_config).await?; @@ -86,30 +78,38 @@ impl StartCli { } } -async fn init_store( +async fn init_rocks_db_store( config: &AppConfig, - registry: Registry, + registry: Option>, reset: bool, -) -> anyhow::Result { - if reset { - remove_dir_all(&config.store.path).with_context(|| "Failed to remove store directory")?; - } - - if !config.store.path.is_dir() { - create_dir_all(&config.store.path).with_context(|| "Failed to create home directory")?; - } - - let mut store = SharedStore::new(registry, &config.store.path) - .await - .with_context(|| "Failed to create store")?; - - debug!("store created successfully at {:?}", &config.store.path); +) -> anyhow::Result { + let flush_path = &config.store.path; + + let store = match (reset, flush_path.is_dir()) { + // If reset is true and the path is a directory, remove the directory and create a new store + (true, true) => { + let db = RocksDbStore::new(flush_path)?; + debug!("store reset successfully at {:?}", &flush_path); + db + } + // If no reset, load the store + (false, true) => { + let db = RocksDbStore::load(flush_path)?; + debug!("store loaded successfully at {:?}", &flush_path); + db + } + // If the path does not exist, create the directory and create a new store + (_, false) => { + create_dir_all(flush_path).with_context(|| "Failed to create home directory")?; + let db = RocksDbStore::new(flush_path)?; + debug!("store created successfully at {:?}", &flush_path); + db + } + }; - if !reset { - store - .restore() - .await - .with_context(|| "Failed to restore store state")?; + // If a registry is provided, overwrite the registry + if let Some(registry) = registry { + store.set_registry(registry, "".to_string()).await?; } Ok(store) @@ -165,13 +165,13 @@ async fn init_ipfs_client(config: &AppConfig) -> anyhow::Result { Ok(ipfs_client) } -async fn init_bothan_server( +async fn init_bothan_server( config: &AppConfig, - store: SharedStore, + store: S, ipfs_client: IpfsClient, monitoring_client: Option>, -) -> anyhow::Result> { - let manager_store = SharedStore::create_manager_store(&store); +) -> anyhow::Result>> { + let manager_store = RegistryStore::new(store.clone()); let stale_threshold = config.manager.crypto.stale_threshold; let bothan_version = @@ -179,7 +179,12 @@ async fn init_bothan_server( let registry_version_requirement = VersionReq::from_str(REGISTRY_REQUIREMENT) .with_context(|| "Failed to parse registry version requirement")?; - let workers = init_crypto_workers(&store, &config.manager.crypto.source).await?; + let workers = match init_crypto_workers(&store, &config.manager.crypto.source).await { + Ok(workers) => workers, + Err(e) => { + bail!("failed to initialize workers: {:?}", e); + } + }; let manager = CryptoAssetInfoManager::new( workers, manager_store, @@ -210,79 +215,40 @@ async fn init_bothan_server( Ok(Arc::new(BothanServer::new(manager))) } -async fn init_crypto_workers( - store: &SharedStore, +async fn init_crypto_workers( + store: &S, source: &CryptoSourceConfigs, -) -> anyhow::Result>> { - type Binance = BinanceWorkerBuilder; - type Bybit = BybitWorkerBuilder; - type Coinbase = CoinbaseWorkerBuilder; - type CoinGecko = CoinGeckoWorkerBuilder; - type CoinMarketCap = CoinMarketCapWorkerBuilder; - type CryptoCompare = CryptoCompareWorkerBuilder; - type Htx = HtxWorkerBuilder; - type Kraken = KrakenWorkerBuilder; - type Okx = OkxWorkerBuilder; - +) -> Result>, AssetWorkerError> { let mut workers = HashMap::new(); - if let Some(opts) = &source.binance { - add_worker::(&mut workers, store, opts).await?; - } - - if let Some(opts) = &source.bybit { - add_worker::(&mut workers, store, opts).await?; - } - - if let Some(opts) = &source.coinbase { - add_worker::(&mut workers, store, opts).await?; - } - - if let Some(opts) = &source.coingecko { - add_worker::(&mut workers, store, opts).await?; - } - - if let Some(opts) = &source.coinmarketcap { - add_worker::(&mut workers, store, opts).await?; - } - - if let Some(opts) = &source.cryptocompare { - add_worker::(&mut workers, store, opts).await?; - } - - if let Some(opts) = &source.htx { - add_worker::(&mut workers, store, opts).await?; - } - - if let Some(opts) = &source.kraken { - add_worker::(&mut workers, store, opts).await?; - } - - if let Some(opts) = &source.okx { - add_worker::(&mut workers, store, opts).await?; - } + add_worker(&mut workers, store, &source.binance).await?; + add_worker(&mut workers, store, &source.bitfinex).await?; + add_worker(&mut workers, store, &source.bybit).await?; + add_worker(&mut workers, store, &source.coinbase).await?; + add_worker(&mut workers, store, &source.coingecko).await?; + add_worker(&mut workers, store, &source.coinmarketcap).await?; + add_worker(&mut workers, store, &source.htx).await?; + add_worker(&mut workers, store, &source.kraken).await?; + add_worker(&mut workers, store, &source.okx).await?; Ok(workers) } -async fn add_worker( - workers: &mut HashMap>, - store: &SharedStore, - opts: &B::Opts, -) -> anyhow::Result<()> +async fn add_worker( + workers: &mut HashMap>, + store: &S, + opts: &Option, +) -> Result<(), AssetWorkerError> where - B: AssetWorkerBuilder<'static>, - B::Error: Send + Sync + 'static, - B::Opts: Clone, + S: Store + 'static, + O: Clone + Into, { - let worker_name = B::worker_name(); - let worker_store = SharedStore::create_worker_store(store, worker_name); - let worker = B::new(worker_store, opts.clone()) - .build() - .await - .with_context(|| format!("Failed to build worker {worker_name}"))?; + if let Some(opts) = opts { + let worker = CryptoAssetWorker::build(opts.clone().into(), store).await?; + let worker_name = worker.name(); + workers.insert(worker_name.to_string(), worker); + info!("loaded {} worker", worker_name); + } - workers.insert(worker_name.to_string(), worker); - info!("loaded {} worker", worker_name); Ok(()) } diff --git a/bothan-api/server-cli/src/main.rs b/bothan-api/server-cli/src/main.rs index 475ca4ca..09f3e657 100644 --- a/bothan-api/server-cli/src/main.rs +++ b/bothan-api/server-cli/src/main.rs @@ -1,12 +1,11 @@ use std::path::PathBuf; use std::str::FromStr; +use bothan_api::config::AppConfig; +use bothan_api::config::log::LogLevel; use clap::{Parser, Subcommand}; -use tracing_subscriber::filter::Directive; use tracing_subscriber::EnvFilter; - -use bothan_api::config::log::LogLevel; -use bothan_api::config::AppConfig; +use tracing_subscriber::filter::Directive; use crate::commands::config::ConfigCli; use crate::commands::key::KeyCli; @@ -26,6 +25,9 @@ struct Cli { // global args #[arg(long, global = true)] config: Option, + + #[arg(long, short)] + version: bool, } #[derive(Subcommand)] @@ -50,6 +52,11 @@ async fn main() { } }; + if cli.version { + println!("bothan-api {}", env!("CARGO_PKG_VERSION")); + std::process::exit(0); + } + let config_path = &cli.config.unwrap_or(bothan_home_dir().join("config.toml")); let app_config = if config_path.is_file() { diff --git a/bothan-api/server/Cargo.toml b/bothan-api/server/Cargo.toml index c34a74c6..f216299a 100644 --- a/bothan-api/server/Cargo.toml +++ b/bothan-api/server/Cargo.toml @@ -1,23 +1,26 @@ [package] name = "bothan-api" -version = "0.0.1-beta.1" +version = "0.0.1-beta.2" edition.workspace = true license.workspace = true repository.workspace = true [dependencies] +bothan-lib = { workspace = true } bothan-core = { workspace = true } bothan-binance = { workspace = true } +bothan-bitfinex = { workspace = true } bothan-bybit = { workspace = true } bothan-coinbase = { workspace = true } bothan-coingecko = { workspace = true } bothan-coinmarketcap = { workspace = true } -bothan-cryptocompare = { workspace = true } bothan-htx = { workspace = true } bothan-kraken = { workspace = true } bothan-okx = { workspace = true } +async-trait = { workspace = true } +dirs = { workspace = true } prost = { workspace = true } rust_decimal = { workspace = true } semver = { workspace = true } @@ -25,5 +28,4 @@ serde = { workspace = true } tonic = { workspace = true } tracing = { workspace = true } -config = "0.14.1" -dirs = "5.0.1" +config = "0.15.8" diff --git a/bothan-api/server/src/api/server.rs b/bothan-api/server/src/api/server.rs index 3c1f0463..a5f68cfb 100644 --- a/bothan-api/server/src/api/server.rs +++ b/bothan-api/server/src/api/server.rs @@ -1,12 +1,12 @@ use std::sync::Arc; +use bothan_core::manager::CryptoAssetInfoManager; +use bothan_core::manager::crypto_asset_info::error::{PushMonitoringRecordError, SetRegistryError}; +use bothan_lib::store::Store; use semver::Version; use tonic::{Request, Response, Status}; use tracing::{debug, error, info}; -use bothan_core::manager::crypto_asset_info::error::{PushMonitoringRecordError, SetRegistryError}; -use bothan_core::manager::CryptoAssetInfoManager; - use crate::api::utils::parse_price_state; use crate::proto::bothan::v1::{ BothanService, GetInfoRequest, GetInfoResponse, GetPricesRequest, GetPricesResponse, Price, @@ -17,20 +17,20 @@ use crate::proto::bothan::v1::{ pub const PRECISION: u32 = 9; /// The `BothanServer` struct represents a server that implements the `BothanService` trait. -pub struct BothanServer { - manager: Arc>, +pub struct BothanServer { + manager: Arc>, } -impl BothanServer { +impl BothanServer { /// Creates a new `CryptoQueryServer` instance. - pub fn new(manager: Arc>) -> Self { + pub fn new(manager: Arc>) -> Self { BothanServer { manager } } } // TODO: cleanup logging with span #[tonic::async_trait] -impl BothanService for BothanServer { +impl BothanService for BothanServer { async fn get_info( &self, _: Request, @@ -83,19 +83,19 @@ impl BothanService for BothanServer { Err(Status::invalid_argument("Registry is invalid")) } Err(SetRegistryError::UnsupportedVersion) => { - error!("invalid registry"); - Err(Status::invalid_argument("Registry is invalid")) + error!("unsupported registry version"); + Err(Status::invalid_argument("Registry version is unsupported")) } Err(SetRegistryError::FailedToParse) => { error!("failed to parse registry"); - Err(Status::invalid_argument("Registry is invalid")) + Err(Status::invalid_argument("Unable to parse registry version")) } Err(SetRegistryError::InvalidHash) => { error!("invalid IPFS hash"); Err(Status::invalid_argument("Invalid IPFS hash")) } - Err(SetRegistryError::FailedToSetRegistry(e)) => { - error!("failed to set registry: {e}"); + Err(SetRegistryError::FailedToSetRegistry) => { + error!("failed to set registry"); Err(Status::internal("Failed to set registry")) } } diff --git a/bothan-api/server/src/api/utils.rs b/bothan-api/server/src/api/utils.rs index a98efc83..10cc3c16 100644 --- a/bothan-api/server/src/api/utils.rs +++ b/bothan-api/server/src/api/utils.rs @@ -1,8 +1,7 @@ +use bothan_core::manager::crypto_asset_info::types::PriceState; use rust_decimal::prelude::Zero; use tracing::warn; -use bothan_core::manager::crypto_asset_info::types::PriceState; - use crate::api::server::PRECISION; use crate::proto::bothan::v1::{Price, Status}; diff --git a/bothan-api/server/src/config/manager.rs b/bothan-api/server/src/config/manager.rs index 319de607..93b3f496 100644 --- a/bothan-api/server/src/config/manager.rs +++ b/bothan-api/server/src/config/manager.rs @@ -1,6 +1,5 @@ -use serde::{Deserialize, Serialize}; - use crypto_info::CryptoInfoManagerConfig; +use serde::{Deserialize, Serialize}; pub mod crypto_info; diff --git a/bothan-api/server/src/config/manager/crypto_info/sources.rs b/bothan-api/server/src/config/manager/crypto_info/sources.rs index 3194a86e..eb617dc1 100644 --- a/bothan-api/server/src/config/manager/crypto_info/sources.rs +++ b/bothan-api/server/src/config/manager/crypto_info/sources.rs @@ -1,41 +1,31 @@ use serde::{Deserialize, Serialize}; -use bothan_binance::BinanceWorkerBuilderOpts; -use bothan_bybit::BybitWorkerBuilderOpts; -use bothan_coinbase::CoinbaseWorkerBuilderOpts; -use bothan_coingecko::CoinGeckoWorkerBuilderOpts; -use bothan_coinmarketcap::CoinMarketCapWorkerBuilderOpts; -use bothan_cryptocompare::CryptoCompareWorkerBuilderOpts; -use bothan_htx::HtxWorkerBuilderOpts; -use bothan_kraken::KrakenWorkerBuilderOpts; -use bothan_okx::OkxWorkerBuilderOpts; - /// The configuration for the worker sources. #[derive(Clone, Debug, Default, Serialize, Deserialize)] pub struct CryptoSourceConfigs { - pub binance: Option, - pub bybit: Option, - pub coinbase: Option, - pub coingecko: Option, - pub coinmarketcap: Option, - pub cryptocompare: Option, - pub htx: Option, - pub kraken: Option, - pub okx: Option, + pub binance: Option, + pub bitfinex: Option, + pub bybit: Option, + pub coinbase: Option, + pub coingecko: Option, + pub coinmarketcap: Option, + pub htx: Option, + pub kraken: Option, + pub okx: Option, } impl CryptoSourceConfigs { pub fn with_default_sources() -> Self { CryptoSourceConfigs { - binance: Some(BinanceWorkerBuilderOpts::default()), - bybit: Some(BybitWorkerBuilderOpts::default()), - coinbase: Some(CoinbaseWorkerBuilderOpts::default()), - coingecko: Some(CoinGeckoWorkerBuilderOpts::default()), - coinmarketcap: Some(CoinMarketCapWorkerBuilderOpts::default()), - cryptocompare: Some(CryptoCompareWorkerBuilderOpts::default()), - htx: Some(HtxWorkerBuilderOpts::default()), - kraken: Some(KrakenWorkerBuilderOpts::default()), - okx: Some(OkxWorkerBuilderOpts::default()), + binance: Some(bothan_binance::WorkerOpts::default()), + bitfinex: Some(bothan_bitfinex::WorkerOpts::default()), + bybit: Some(bothan_bybit::WorkerOpts::default()), + coinbase: Some(bothan_coinbase::WorkerOpts::default()), + coingecko: Some(bothan_coingecko::WorkerOpts::default()), + coinmarketcap: Some(bothan_coinmarketcap::WorkerOpts::default()), + htx: Some(bothan_htx::WorkerOpts::default()), + kraken: Some(bothan_kraken::WorkerOpts::default()), + okx: Some(bothan_okx::WorkerOpts::default()), } } } diff --git a/bothan-api/server/src/proto.rs b/bothan-api/server/src/proto.rs index 38b59d76..f12e671f 100644 --- a/bothan-api/server/src/proto.rs +++ b/bothan-api/server/src/proto.rs @@ -1,9 +1,9 @@ +#![allow(clippy::all)] pub mod bothan { pub mod v1 { - use serde::ser::SerializeStruct; - use serde::Serialize; - pub use bothan_service_server::{BothanService, BothanServiceServer}; + use serde::Serialize; + use serde::ser::SerializeStruct; include!("proto/bothan.v1.rs"); impl Price { diff --git a/bothan-binance/Cargo.toml b/bothan-binance/Cargo.toml index e979533c..b8f25ac3 100644 --- a/bothan-binance/Cargo.toml +++ b/bothan-binance/Cargo.toml @@ -1,13 +1,13 @@ [package] name = "bothan-binance" -version = "0.0.1-beta.1" +version = "0.0.1-beta.2" edition.workspace = true license.workspace = true repository.workspace = true [dependencies] async-trait = { workspace = true } -bothan-core = { workspace = true } +bothan-lib = { workspace = true } futures-util = { workspace = true, features = ["sink", "std"] } humantime-serde = { workspace = true } opentelemetry = { workspace = true } diff --git a/bothan-binance/examples/binance_basic.rs b/bothan-binance/examples/binance_basic.rs deleted file mode 100644 index 6f72d0f7..00000000 --- a/bothan-binance/examples/binance_basic.rs +++ /dev/null @@ -1,38 +0,0 @@ -use std::time::Duration; - -use tokio::time::sleep; -use tracing_subscriber::fmt::init; - -use bothan_binance::{BinanceWorkerBuilder, BinanceWorkerBuilderOpts}; -use bothan_core::registry::Registry; -use bothan_core::store::SharedStore; -use bothan_core::worker::{AssetWorker, AssetWorkerBuilder}; - -#[tokio::main] -async fn main() { - init(); - let path = std::env::current_dir().unwrap(); - let registry = Registry::default().validate().unwrap(); - let store = SharedStore::new(registry, path.as_path()).await.unwrap(); - - let worker_store = store.create_worker_store(BinanceWorkerBuilder::worker_name()); - let opts = BinanceWorkerBuilderOpts::default(); - - let worker = BinanceWorkerBuilder::new(worker_store, opts) - .build() - .await - .unwrap(); - - worker - .set_query_ids(vec!["btcusdt".to_string(), "ethusdt".to_string()]) - .await - .unwrap(); - - sleep(Duration::from_secs(2)).await; - - loop { - let data = worker.get_asset("btcusdt").await; - println!("{:?}", data); - sleep(Duration::from_secs(5)).await; - } -} diff --git a/bothan-binance/src/api.rs b/bothan-binance/src/api.rs index 37be4d5b..55979338 100644 --- a/bothan-binance/src/api.rs +++ b/bothan-binance/src/api.rs @@ -1,5 +1,5 @@ pub use error::{ConnectionError, MessageError, SendError}; -pub use websocket::{BinanceWebSocketConnection, BinanceWebSocketConnector}; +pub use websocket::{WebSocketConnection, WebSocketConnector}; pub mod error; pub mod msgs; diff --git a/bothan-binance/src/api/websocket.rs b/bothan-binance/src/api/websocket.rs index 97cdd389..ec148688 100644 --- a/bothan-binance/src/api/websocket.rs +++ b/bothan-binance/src/api/websocket.rs @@ -2,7 +2,7 @@ use futures_util::{SinkExt, StreamExt}; use serde_json::json; use tokio::net::TcpStream; use tokio_tungstenite::tungstenite::Message; -use tokio_tungstenite::{connect_async, MaybeTlsStream, WebSocketStream}; +use tokio_tungstenite::{MaybeTlsStream, WebSocketStream, connect_async}; use crate::api::error::{ConnectionError, MessageError, SendError}; use crate::api::msgs::BinanceResponse; @@ -10,11 +10,11 @@ use crate::api::msgs::BinanceResponse; pub const DEFAULT_URL: &str = "wss://stream.binance.com:9443/stream"; /// A connector for establishing WebSocket connections to Binance. -pub struct BinanceWebSocketConnector { +pub struct WebSocketConnector { url: String, } -impl BinanceWebSocketConnector { +impl WebSocketConnector { /// Creates a new `BinanceWebSocketConnector` with the given URL. /// /// # Examples @@ -34,7 +34,7 @@ impl BinanceWebSocketConnector { /// let connector = BinanceWebSocketConnector::new("wss://example.com/socket"); /// let connection = connector.connect().await?; /// ``` - pub async fn connect(&self) -> Result { + pub async fn connect(&self) -> Result { // Attempt to establish a WebSocket connection. let (wss, resp) = connect_async(self.url.clone()).await?; @@ -45,16 +45,16 @@ impl BinanceWebSocketConnector { } // Return the WebSocket connection. - Ok(BinanceWebSocketConnection::new(wss)) + Ok(WebSocketConnection::new(wss)) } } /// Represents an active WebSocket connection to Binance. -pub struct BinanceWebSocketConnection { +pub struct WebSocketConnection { ws_stream: WebSocketStream>, } -impl BinanceWebSocketConnection { +impl WebSocketConnection { /// Creates a new `BinanceWebSocketConnection` pub fn new(ws_stream: WebSocketStream>) -> Self { Self { ws_stream } @@ -160,9 +160,8 @@ pub(crate) mod test { use tokio::sync::mpsc; use ws_mock::ws_mock_server::{WsMock, WsMockServer}; - use crate::api::msgs::{Data, MiniTickerInfo, StreamResponse}; - use super::*; + use crate::api::msgs::{Data, MiniTickerInfo, StreamResponse}; pub(crate) async fn setup_mock_server() -> WsMockServer { WsMockServer::start().await @@ -172,7 +171,7 @@ pub(crate) mod test { async fn test_recv_ticker() { // Set up the mock server and the WebSocket connector. let server = setup_mock_server().await; - let connector = BinanceWebSocketConnector::new(server.uri().await); + let connector = WebSocketConnector::new(server.uri().await); let (mpsc_send, mpsc_recv) = mpsc::channel::(32); // Create a mock mini ticker response. @@ -211,7 +210,7 @@ pub(crate) mod test { async fn test_recv_ping() { // Set up the mock server and the WebSocket connector. let server = setup_mock_server().await; - let connector = BinanceWebSocketConnector::new(server.uri().await); + let connector = WebSocketConnector::new(server.uri().await); let (mpsc_send, mpsc_recv) = mpsc::channel::(32); // Mount the mock WebSocket server and send a ping message. @@ -231,7 +230,7 @@ pub(crate) mod test { async fn test_recv_close() { // Set up the mock server and the WebSocket connector. let server = setup_mock_server().await; - let connector = BinanceWebSocketConnector::new(server.uri().await); + let connector = WebSocketConnector::new(server.uri().await); let (mpsc_send, mpsc_recv) = mpsc::channel::(32); // Mount the mock WebSocket server and send a close message. diff --git a/bothan-binance/src/lib.rs b/bothan-binance/src/lib.rs index 5f3d4a02..ac1901ea 100644 --- a/bothan-binance/src/lib.rs +++ b/bothan-binance/src/lib.rs @@ -1,8 +1,6 @@ -pub use api::websocket::{BinanceWebSocketConnection, BinanceWebSocketConnector}; -pub use worker::builder::BinanceWorkerBuilder; -pub use worker::error::BuildError; -pub use worker::opts::BinanceWorkerBuilderOpts; -pub use worker::BinanceWorker; +pub use api::websocket::{WebSocketConnection, WebSocketConnector}; +pub use worker::Worker; +pub use worker::opts::WorkerOpts; pub mod api; pub mod worker; diff --git a/bothan-binance/src/worker.rs b/bothan-binance/src/worker.rs index 7dd53635..e4064e1f 100644 --- a/bothan-binance/src/worker.rs +++ b/bothan-binance/src/worker.rs @@ -1,67 +1,87 @@ -use tokio::sync::mpsc::Sender; +use std::collections::HashSet; +use std::sync::Arc; -use bothan_core::store::error::Error as StoreError; -use bothan_core::store::WorkerStore; -use bothan_core::worker::{AssetState, AssetWorker, SetQueryIDError}; +use bothan_lib::store::{Store, WorkerStore}; +use bothan_lib::types::AssetState; +use bothan_lib::worker::AssetWorker; +use bothan_lib::worker::error::AssetWorkerError; +use tokio::sync::mpsc::{Sender, channel}; -use crate::api::websocket::BinanceWebSocketConnector; +use crate::WorkerOpts; +use crate::api::websocket::WebSocketConnector; +use crate::worker::asset_worker::start_asset_worker; mod asset_worker; -pub mod builder; -pub(crate) mod error; pub mod opts; mod types; -/// A worker that fetches and stores the asset information from Binance's API. -pub struct BinanceWorker { - connector: BinanceWebSocketConnector, - store: WorkerStore, +const WORKER_NAME: &str = "binance"; + +pub struct Worker { + inner: Arc>, +} + +struct InnerWorker { + connector: WebSocketConnector, + store: WorkerStore, subscribe_tx: Sender>, unsubscribe_tx: Sender>, } -impl BinanceWorker { - /// Create a new worker with the specified connector, store and channels. - pub fn new( - connector: BinanceWebSocketConnector, - store: WorkerStore, - subscribe_tx: Sender>, - unsubscribe_tx: Sender>, - ) -> Self { - Self { - connector, - store, - subscribe_tx, - unsubscribe_tx, +#[async_trait::async_trait] +impl AssetWorker for Worker { + type Opts = WorkerOpts; + + fn name(&self) -> &'static str { + WORKER_NAME + } + + async fn build(opts: Self::Opts, store: &S) -> Result, AssetWorkerError> { + let url = opts.url; + let ch_size = opts.internal_ch_size; + + let connector = WebSocketConnector::new(url); + let connection = connector.connect().await?; + + let (sub_tx, sub_rx) = channel(ch_size); + let (unsub_tx, unsub_rx) = channel(ch_size); + + let worker_store = WorkerStore::new(store, WORKER_NAME); + let to_sub = worker_store + .get_query_ids() + .await? + .into_iter() + .collect::>(); + + if !to_sub.is_empty() { + sub_tx.send(to_sub).await?; } + + let inner = Arc::new(InnerWorker { + connector, + store: worker_store, + subscribe_tx: sub_tx, + unsubscribe_tx: unsub_tx, + }); + tokio::spawn(start_asset_worker( + Arc::downgrade(&inner), + connection, + sub_rx, + unsub_rx, + )); + + Ok(Worker { inner }) } -} -#[async_trait::async_trait] -impl AssetWorker for BinanceWorker { - /// Fetches the AssetStatus for the given cryptocurrency id. - async fn get_asset(&self, id: &str) -> Result { - self.store.get_asset(&id).await + async fn get_asset(&self, id: &str) -> Result { + Ok(self.inner.store.get_asset(id).await?) } - /// Sets the specified cryptocurrency IDs to the query. If the ids are already in the query set, - /// it will not be resubscribed. - async fn set_query_ids(&self, ids: Vec) -> Result<(), SetQueryIDError> { - let (to_sub, to_unsub) = self - .store - .compute_query_id_differences(ids) - .await - .map_err(|e| SetQueryIDError::new(e.to_string()))?; - - self.subscribe_tx - .send(to_sub) - .await - .map_err(|e| SetQueryIDError::new(e.to_string()))?; - - self.unsubscribe_tx - .send(to_unsub) - .await - .map_err(|e| SetQueryIDError::new(e.to_string()))?; + async fn set_query_ids(&self, ids: HashSet) -> Result<(), AssetWorkerError> { + let diff = self.inner.store.compute_query_id_difference(ids).await?; + + self.inner.subscribe_tx.send(diff.added).await?; + self.inner.unsubscribe_tx.send(diff.removed).await?; Ok(()) } diff --git a/bothan-binance/src/worker/asset_worker.rs b/bothan-binance/src/worker/asset_worker.rs index 1bc21f8e..fa4b14de 100644 --- a/bothan-binance/src/worker/asset_worker.rs +++ b/bothan-binance/src/worker/asset_worker.rs @@ -1,37 +1,36 @@ use std::collections::HashMap; use std::sync::Weak; -use opentelemetry::{global, KeyValue}; +use bothan_lib::store::{Store, WorkerStore}; +use bothan_lib::types::AssetInfo; +use opentelemetry::{KeyValue, global}; use rand::random; -use rust_decimal::prelude::ToPrimitive; use rust_decimal::Decimal; +use rust_decimal::prelude::ToPrimitive; use tokio::select; use tokio::sync::mpsc::Receiver; use tokio::time::{sleep, timeout}; use tracing::{debug, error, info, warn}; -use bothan_core::store::WorkerStore; -use bothan_core::types::AssetInfo; - use crate::api::error::MessageError; use crate::api::msgs::{BinanceResponse, Data, ErrorResponse, SuccessResponse}; -use crate::api::{BinanceWebSocketConnection, BinanceWebSocketConnector}; +use crate::api::{WebSocketConnection, WebSocketConnector}; +use crate::worker::InnerWorker; use crate::worker::types::{DEFAULT_TIMEOUT, METER_NAME, RECONNECT_BUFFER}; -use crate::worker::BinanceWorker; enum Event { Subscribe(Vec), Unsubscribe(Vec), } -pub(crate) async fn start_asset_worker( - worker: Weak, - mut connection: BinanceWebSocketConnection, +pub(crate) async fn start_asset_worker( + inner_worker: Weak>, + mut connection: WebSocketConnection, mut subscribe_rx: Receiver>, mut unsubscribe_rx: Receiver>, ) { let mut subscription_map = HashMap::new(); - while let Some(worker) = worker.upgrade() { + while let Some(worker) = inner_worker.upgrade() { select! { Some(ids) = subscribe_rx.recv() => handle_subscribe_recv(ids, &mut connection, &mut subscription_map).await, Some(ids) = unsubscribe_rx.recv() => handle_unsubscribe_recv(ids, &mut connection, &mut subscription_map).await, @@ -57,7 +56,7 @@ pub(crate) async fn start_asset_worker( async fn handle_subscribe_recv( ids: Vec, - connection: &mut BinanceWebSocketConnection, + connection: &mut WebSocketConnection, subscription_map: &mut HashMap, ) { if ids.is_empty() { @@ -68,7 +67,7 @@ async fn handle_subscribe_recv( let tickers = ids.iter().map(|s| s.as_ref()).collect::>(); let meter = global::meter(METER_NAME); - meter.u64_counter("subscribe_attempt").init().add( + meter.u64_counter("subscribe_attempt").build().add( 1, &[ KeyValue::new("subscription.id", packet_id), @@ -88,7 +87,7 @@ async fn handle_subscribe_recv( error!("failed attempt to subscribe to ids {:?}: {}", ids, e); meter .u64_counter("failed_subscribe_attempt") - .init() + .build() .add(1, &[KeyValue::new("subscription.id", packet_id)]); } } @@ -96,7 +95,7 @@ async fn handle_subscribe_recv( async fn handle_unsubscribe_recv( ids: Vec, - connection: &mut BinanceWebSocketConnection, + connection: &mut WebSocketConnection, subscription_map: &mut HashMap, ) { if ids.is_empty() { @@ -107,7 +106,7 @@ async fn handle_unsubscribe_recv( let tickers = ids.iter().map(|s| s.as_ref()).collect::>(); let meter = global::meter(METER_NAME); - meter.u64_counter("unsubscribe_attempt").init().add( + meter.u64_counter("unsubscribe_attempt").build().add( 1, &[ KeyValue::new("subscription.id", packet_id), @@ -127,17 +126,17 @@ async fn handle_unsubscribe_recv( error!("failed attempt to unsubscribe to ids {:?}: {}", ids, e); meter .u64_counter("failed_unsubscribe_attempt") - .init() + .build() .add(1, &[KeyValue::new("subscription.id", packet_id)]); } } } -async fn handle_connection_recv( +async fn handle_connection_recv( recv_result: Result, - connector: &BinanceWebSocketConnector, - connection: &mut BinanceWebSocketConnection, - store: &WorkerStore, + connector: &WebSocketConnector, + connection: &mut WebSocketConnection, + store: &WorkerStore, subscription_map: &mut HashMap, ) { match recv_result { @@ -156,15 +155,15 @@ async fn handle_connection_recv( } } -async fn handle_reconnect( - connector: &BinanceWebSocketConnector, - connection: &mut BinanceWebSocketConnection, - query_ids: &WorkerStore, +async fn handle_reconnect( + connector: &WebSocketConnector, + connection: &mut WebSocketConnection, + query_ids: &WorkerStore, ) { let mut retry_count: usize = 1; loop { let meter = global::meter(METER_NAME); - meter.u64_counter("reconnect-attempts").init().add(1, &[]); + meter.u64_counter("reconnect-attempts").build().add(1, &[]); warn!("reconnecting: attempt {}", retry_count); @@ -205,8 +204,8 @@ async fn handle_reconnect( } } -async fn process_response( - store: &WorkerStore, +async fn process_response( + store: &WorkerStore, resp: BinanceResponse, subscription_map: &mut HashMap, ) { @@ -218,7 +217,7 @@ async fn process_response( } } -async fn store_data(store: &WorkerStore, data: Data) { +async fn store_data(store: &WorkerStore, data: Data) { match data { Data::MiniTicker(ticker) => { let id = ticker.symbol.to_lowercase(); @@ -230,12 +229,12 @@ async fn store_data(store: &WorkerStore, data: Data) { let timestamp = ticker.event_time / 1000; let asset_info = AssetInfo::new(id.clone(), price, timestamp); - match store.set_asset(&id, asset_info).await { + match store.set_asset_info(asset_info).await { Ok(_) => { info!("stored data for id {}", id); global::meter(METER_NAME) .f64_gauge("asset-prices") - .init() + .build() .record( price.to_f64().unwrap(), // Prices should never be NaN so unwrap here &[KeyValue::new("asset.symbol", id)], @@ -247,8 +246,8 @@ async fn store_data(store: &WorkerStore, data: Data) { } } -async fn process_success( - store: &WorkerStore, +async fn process_success( + store: &WorkerStore, success_response: SuccessResponse, subscription_map: &mut HashMap, ) { @@ -259,7 +258,7 @@ async fn process_success( info!("subscribed to ids {:?}", ids); meter .u64_counter("subscribe_success") - .init() + .build() .add(1, &[KeyValue::new("id", success_response.id)]); if store.add_query_ids(ids).await.is_err() { error!("failed to add query ids to store"); @@ -269,9 +268,9 @@ async fn process_success( info!("unsubscribed to ids {:?}", ids); meter .u64_counter("unsubscribe_success") - .init() + .build() .add(1, &[KeyValue::new("subscription.id", success_response.id)]); - if store.remove_query_ids(ids).await.is_err() { + if store.remove_query_ids(&ids).await.is_err() { error!("failed to remove query ids from store"); }; } @@ -283,7 +282,7 @@ fn process_ping() { debug!("received ping from binance"); global::meter(METER_NAME) .u64_counter("pings") - .init() + .build() .add(1, &[]); } @@ -292,7 +291,7 @@ fn process_error(error: ErrorResponse) { "error code {} received from binance: {}", error.code, error.msg ); - global::meter(METER_NAME).u64_counter("errors").init().add( + global::meter(METER_NAME).u64_counter("errors").build().add( 1, &[ KeyValue::new("msg.code", error.code as i64), diff --git a/bothan-binance/src/worker/builder.rs b/bothan-binance/src/worker/builder.rs deleted file mode 100644 index 32333a7f..00000000 --- a/bothan-binance/src/worker/builder.rs +++ /dev/null @@ -1,94 +0,0 @@ -use std::sync::Arc; - -use tokio::sync::mpsc::channel; - -use bothan_core::store::WorkerStore; -use bothan_core::worker::AssetWorkerBuilder; - -use crate::api::websocket::BinanceWebSocketConnector; -use crate::worker::asset_worker::start_asset_worker; -use crate::worker::error::BuildError; -use crate::worker::opts::BinanceWorkerBuilderOpts; -use crate::worker::BinanceWorker; - -/// Builds a `BinanceWorker` with custom options. -/// Methods can be chained to set the configuration values, and the -/// service is constructed by calling the [`build`](BinanceWorkerBuilder::build) method. -pub struct BinanceWorkerBuilder { - store: WorkerStore, - opts: BinanceWorkerBuilderOpts, -} - -impl BinanceWorkerBuilder { - /// Set the URL for the `BinanceWorker`. - /// The default URL is `DEFAULT_URL`. - pub fn with_url>(mut self, url: T) -> Self { - self.opts.url = url.into(); - self - } - - /// Set the internal channel size for the `BinanceWorker`. - /// The default size is `DEFAULT_CHANNEL_SIZE`. - pub fn with_internal_ch_size(mut self, size: usize) -> Self { - self.opts.internal_ch_size = size; - self - } - - /// Sets the store for the `BinanceWorker`. - /// If not set, the store is created and owned by the worker. - pub fn with_store(mut self, store: WorkerStore) -> Self { - self.store = store; - self - } -} - -#[async_trait::async_trait] -impl<'a> AssetWorkerBuilder<'a> for BinanceWorkerBuilder { - type Opts = BinanceWorkerBuilderOpts; - type Worker = BinanceWorker; - type Error = BuildError; - - /// Returns a new `BinanceWorkerBuilder` with the given options. - fn new(store: WorkerStore, opts: Self::Opts) -> Self { - Self { store, opts } - } - - /// Returns the name of the worker. - fn worker_name() -> &'static str { - "binance" - } - - /// Creates the configured `BinanceWorker`. - async fn build(self) -> Result, Self::Error> { - let url = self.opts.url; - let ch_size = self.opts.internal_ch_size; - - let connector = BinanceWebSocketConnector::new(url); - let connection = connector.connect().await?; - - let (sub_tx, sub_rx) = channel(ch_size); - let (unsub_tx, unsub_rx) = channel(ch_size); - let to_sub = self - .store - .get_query_ids() - .await? - .into_iter() - .collect::>(); - - if !to_sub.is_empty() { - // Unwrap here as the channel is guaranteed to be open - sub_tx.send(to_sub).await.unwrap(); - } - - let worker = Arc::new(BinanceWorker::new(connector, self.store, sub_tx, unsub_tx)); - - tokio::spawn(start_asset_worker( - Arc::downgrade(&worker), - connection, - sub_rx, - unsub_rx, - )); - - Ok(worker) - } -} diff --git a/bothan-binance/src/worker/error.rs b/bothan-binance/src/worker/error.rs deleted file mode 100644 index 8b75135c..00000000 --- a/bothan-binance/src/worker/error.rs +++ /dev/null @@ -1,12 +0,0 @@ -use crate::api; -use bothan_core::store; -use thiserror::Error; - -#[derive(Debug, Error)] -pub enum BuildError { - #[error("failed to connect: {0}")] - FailedToConnect(#[from] api::ConnectionError), - - #[error("store error: {0}")] - StoreError(#[from] store::error::Error), -} diff --git a/bothan-binance/src/worker/opts.rs b/bothan-binance/src/worker/opts.rs index 55dc0934..94bf7e34 100644 --- a/bothan-binance/src/worker/opts.rs +++ b/bothan-binance/src/worker/opts.rs @@ -5,11 +5,12 @@ use crate::worker::types::DEFAULT_CHANNEL_SIZE; /// Options for configuring the `BinanceWorkerBuilder`. /// -/// `BinanceWorkerBuilderOpts` provides a way to specify custom settings for creating a `BinanceWorker`. -/// This struct allows users to set optional parameters such as the WebSocket URL and the internal channel size, -/// which will be used during the construction of the `BinanceWorker`. +/// `BinanceWorkerBuilderOpts` provides a way to specify custom settings for creating a +/// `BinanceWorker`. This struct allows users to set optional parameters such as the WebSocket URL +/// and the internal channel size, which will be used during the construction of the +/// `BinanceWorker`. #[derive(Clone, Debug, Serialize, Deserialize)] -pub struct BinanceWorkerBuilderOpts { +pub struct WorkerOpts { #[serde(default = "default_url")] pub url: String, #[serde(default = "default_internal_ch_size")] @@ -24,7 +25,7 @@ fn default_internal_ch_size() -> usize { DEFAULT_CHANNEL_SIZE } -impl Default for BinanceWorkerBuilderOpts { +impl Default for WorkerOpts { fn default() -> Self { Self { url: default_url(), diff --git a/bothan-bitfinex/Cargo.toml b/bothan-bitfinex/Cargo.toml new file mode 100644 index 00000000..d6c4dd5c --- /dev/null +++ b/bothan-bitfinex/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "bothan-bitfinex" +version = "0.0.1-beta.2" +edition.workspace = true +license.workspace = true +repository.workspace = true + +[dependencies] +async-trait = { workspace = true } +bothan-lib = { workspace = true } +chrono = { workspace = true } +humantime-serde = { workspace = true } +reqwest = { workspace = true } +rust_decimal = { workspace = true } +serde = { workspace = true, features = ["derive"] } +serde_json = { workspace = true } +thiserror = { workspace = true } +tokio = { workspace = true } +url = { workspace = true } + diff --git a/bothan-bitfinex/src/api.rs b/bothan-bitfinex/src/api.rs new file mode 100644 index 00000000..08f8ab96 --- /dev/null +++ b/bothan-bitfinex/src/api.rs @@ -0,0 +1,4 @@ +pub mod builder; +pub mod error; +pub mod msg; +pub mod rest; diff --git a/bothan-bitfinex/src/api/builder.rs b/bothan-bitfinex/src/api/builder.rs new file mode 100644 index 00000000..5833f144 --- /dev/null +++ b/bothan-bitfinex/src/api/builder.rs @@ -0,0 +1,36 @@ +use reqwest::ClientBuilder; +use url::Url; + +use crate::api::error::BuildError; +use crate::api::rest::{DEFAULT_URL, RestApi}; + +pub struct RestApiBuilder { + url: String, +} + +impl RestApiBuilder { + pub fn new>(url: T) -> Self { + RestApiBuilder { url: url.into() } + } + + pub fn with_url>(mut self, url: T) -> Self { + self.url = url.into(); + self + } + + pub fn build(self) -> Result { + let parsed_url = Url::parse(&self.url)?; + + let client = ClientBuilder::new().build()?; + + Ok(RestApi::new(parsed_url, client)) + } +} + +impl Default for RestApiBuilder { + fn default() -> Self { + RestApiBuilder { + url: DEFAULT_URL.into(), + } + } +} diff --git a/bothan-bitfinex/src/api/error.rs b/bothan-bitfinex/src/api/error.rs new file mode 100644 index 00000000..8235fa59 --- /dev/null +++ b/bothan-bitfinex/src/api/error.rs @@ -0,0 +1,19 @@ +use thiserror::Error; + +#[derive(Debug, Error)] +pub enum BuildError { + #[error("invalid url")] + InvalidURL(#[from] url::ParseError), + + #[error("failed to build with error: {0}")] + FailedToBuild(#[from] reqwest::Error), +} + +#[derive(Debug, Error)] +pub enum ProviderError { + #[error("failed to fetch tickers: {0}")] + RequestError(#[from] reqwest::Error), + + #[error("value contains nan")] + InvalidValue, +} diff --git a/bothan-bitfinex/src/api/msg.rs b/bothan-bitfinex/src/api/msg.rs new file mode 100644 index 00000000..2e6961ef --- /dev/null +++ b/bothan-bitfinex/src/api/msg.rs @@ -0,0 +1 @@ +pub mod ticker; diff --git a/bothan-bitfinex/src/api/msg/ticker.rs b/bothan-bitfinex/src/api/msg/ticker.rs new file mode 100644 index 00000000..3419a3f0 --- /dev/null +++ b/bothan-bitfinex/src/api/msg/ticker.rs @@ -0,0 +1,72 @@ +mod funding; +mod spot; + +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(untagged)] +pub enum Ticker { + Funding(funding::Ticker), + Spot(spot::Ticker), +} + +impl Ticker { + pub fn symbol(&self) -> &str { + match self { + Ticker::Funding(t) => &t.symbol, + Ticker::Spot(t) => &t.symbol, + } + } + + pub fn price(&self) -> f64 { + match self { + Ticker::Funding(t) => t.last_price, + Ticker::Spot(t) => t.last_price, + } + } +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn test_parse_tickers_from_array() { + let json = r#"[["tBTCUSD",101530,39.76548266,101540,32.24226311,2680,0.0271063,101550,661.88869229,102760,98740],["fUSD",0.000180427397260274,0.0002,120,35441993.51575242,0.00008219,2,39208.22419296,-0.00005519,-0.5017,0.00005481,406448929.8255126,0.000137,0.000024,null,null,5863426.35928275]]"#; + let ticker: Vec = serde_json::from_str(json).unwrap(); + + let expected = vec![ + Ticker::Spot(spot::Ticker { + symbol: "tBTCUSD".to_string(), + bid: 101530.0, + bid_size: 39.76548266, + ask: 101540.0, + ask_size: 32.24226311, + daily_change: 2680.0, + daily_change_relative: 0.0271063, + last_price: 101550.0, + volume: 661.88869229, + high: 102760.0, + low: 98740.0, + }), + Ticker::Funding(funding::Ticker { + symbol: "fUSD".to_string(), + frr: 0.000180427397260274, + bid: 0.0002, + bid_period: 120, + bid_size: 35441993.51575242, + ask: 0.00008219, + ask_period: 2, + ask_size: 39208.22419296, + daily_change: -0.00005519, + daily_change_relative: -0.5017, + last_price: 0.00005481, + volume: 406448929.8255126, + high: 0.000137, + low: 0.000024, + frr_amount_available: 5863426.35928275, + }), + ]; + assert_eq!(ticker, expected); + } +} diff --git a/bothan-bitfinex/src/api/msg/ticker/funding.rs b/bothan-bitfinex/src/api/msg/ticker/funding.rs new file mode 100644 index 00000000..22677352 --- /dev/null +++ b/bothan-bitfinex/src/api/msg/ticker/funding.rs @@ -0,0 +1,440 @@ +use std::fmt; + +use serde::de::{MapAccess, SeqAccess, Visitor}; +use serde::{Deserialize, Deserializer, Serialize, de}; + +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct Ticker { + // The symbol of the requested ticker data + pub symbol: String, + // Flash Return Rate - average of all fixed rate funding over the last hour + pub frr: f64, + // Price of last highest bid + pub bid: f64, + // Bid period covered (in days) + pub bid_period: i64, + // Sum of the 25 highest bid sizes + pub bid_size: f64, + // Price of last lowest ask + pub ask: f64, + // Ask period covered (in days) + pub ask_period: i64, + // Sum of the 25 lowest ask sizes + pub ask_size: f64, + // The amount that the last price has changed since yesterday + pub daily_change: f64, + // Relative price change since yesterday (*100 for percentage change) + pub daily_change_relative: f64, + // Price of the last trade + pub last_price: f64, + // Daily volume + pub volume: f64, + // Daily high + pub high: f64, + // Daily low + pub low: f64, + // The amount of funding that is available at the Flash Return Rate + pub frr_amount_available: f64, +} + +impl<'de> Deserialize<'de> for Ticker { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + #[derive(Deserialize)] + #[serde(field_identifier, rename_all = "snake_case")] + enum Field { + Symbol, + Frr, + Bid, + BidPeriod, + BidSize, + Ask, + AskPeriod, + AskSize, + DailyChange, + DailyChangeRelative, + LastPrice, + Volume, + High, + Low, + FrrAmountAvailable, + } + + struct FundingTickerVisitor {} + impl<'de> Visitor<'de> for FundingTickerVisitor { + type Value = Ticker; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("tuple with length 15") + } + + fn visit_seq(self, mut seq: V) -> Result + where + V: SeqAccess<'de>, + { + let symbol = seq + .next_element()? + .ok_or_else(|| de::Error::invalid_length(0, &self))?; + let frr = seq + .next_element()? + .ok_or_else(|| de::Error::invalid_length(1, &self))?; + let bid = seq + .next_element()? + .ok_or_else(|| de::Error::invalid_length(2, &self))?; + let bid_period = seq + .next_element()? + .ok_or_else(|| de::Error::invalid_length(3, &self))?; + let bid_size = seq + .next_element()? + .ok_or_else(|| de::Error::invalid_length(4, &self))?; + let ask = seq + .next_element()? + .ok_or_else(|| de::Error::invalid_length(5, &self))?; + let ask_period = seq + .next_element()? + .ok_or_else(|| de::Error::invalid_length(6, &self))?; + let ask_size = seq + .next_element()? + .ok_or_else(|| de::Error::invalid_length(7, &self))?; + let daily_change = seq + .next_element()? + .ok_or_else(|| de::Error::invalid_length(8, &self))?; + let daily_change_relative = seq + .next_element()? + .ok_or_else(|| de::Error::invalid_length(9, &self))?; + let last_price = seq + .next_element()? + .ok_or_else(|| de::Error::invalid_length(10, &self))?; + let volume = seq + .next_element()? + .ok_or_else(|| de::Error::invalid_length(11, &self))?; + let high = seq + .next_element()? + .ok_or_else(|| de::Error::invalid_length(12, &self))?; + let low = seq + .next_element()? + .ok_or_else(|| de::Error::invalid_length(13, &self))?; + // Skip the next two elements as they're reserved and currently only contain nil + let _ = seq + .next_element::>()? + .ok_or_else(|| de::Error::invalid_length(14, &self))?; + let _ = seq + .next_element::>()? + .ok_or_else(|| de::Error::invalid_length(15, &self))?; + let frr_amount_available = seq + .next_element()? + .ok_or_else(|| de::Error::invalid_length(16, &self))?; + + let funding_ticker = Ticker { + symbol, + frr, + bid, + bid_period, + bid_size, + ask, + ask_period, + ask_size, + daily_change, + daily_change_relative, + last_price, + volume, + high, + low, + frr_amount_available, + }; + Ok(funding_ticker) + } + + fn visit_map(self, mut map: A) -> Result + where + A: MapAccess<'de>, + { + let mut symbol = None; + let mut frr = None; + let mut bid = None; + let mut bid_period = None; + let mut bid_size = None; + let mut ask = None; + let mut ask_period = None; + let mut ask_size = None; + let mut daily_change = None; + let mut daily_change_relative = None; + let mut last_price = None; + let mut volume = None; + let mut high = None; + let mut low = None; + let mut frr_amount_available = None; + + while let Some(key) = map.next_key()? { + match key { + Field::Symbol => { + if symbol.is_some() { + return Err(de::Error::duplicate_field("symbol")); + } + symbol = Some(map.next_value()?); + } + Field::Frr => { + if frr.is_some() { + return Err(de::Error::duplicate_field("frr")); + } + frr = Some(map.next_value()?); + } + Field::Bid => { + if bid.is_some() { + return Err(de::Error::duplicate_field("bid")); + } + bid = Some(map.next_value()?); + } + Field::BidPeriod => { + if bid_period.is_some() { + return Err(de::Error::duplicate_field("bid_period")); + } + bid_period = Some(map.next_value()?); + } + Field::BidSize => { + if bid_size.is_some() { + return Err(de::Error::duplicate_field("bid_size")); + } + bid_size = Some(map.next_value()?); + } + Field::Ask => { + if ask.is_some() { + return Err(de::Error::duplicate_field("ask")); + } + ask = Some(map.next_value()?); + } + Field::AskPeriod => { + if ask_period.is_some() { + return Err(de::Error::duplicate_field("ask_period")); + } + ask_period = Some(map.next_value()?); + } + Field::AskSize => { + if ask_size.is_some() { + return Err(de::Error::duplicate_field("ask_size")); + } + ask_size = Some(map.next_value()?); + } + Field::DailyChange => { + if daily_change.is_some() { + return Err(de::Error::duplicate_field("daily_change")); + } + daily_change = Some(map.next_value()?); + } + Field::DailyChangeRelative => { + if daily_change_relative.is_some() { + return Err(de::Error::duplicate_field("daily_change_relative")); + } + daily_change_relative = Some(map.next_value()?); + } + Field::LastPrice => { + if last_price.is_some() { + return Err(de::Error::duplicate_field("last_price")); + } + last_price = Some(map.next_value()?); + } + Field::Volume => { + if volume.is_some() { + return Err(de::Error::duplicate_field("volume")); + } + volume = Some(map.next_value()?); + } + Field::High => { + if high.is_some() { + return Err(de::Error::duplicate_field("high")); + } + high = Some(map.next_value()?); + } + Field::Low => { + if low.is_some() { + return Err(de::Error::duplicate_field("low")); + } + low = Some(map.next_value()?); + } + Field::FrrAmountAvailable => { + if frr_amount_available.is_some() { + return Err(de::Error::duplicate_field("frr_amount_available")); + } + frr_amount_available = Some(map.next_value()?); + } + } + } + + let symbol = symbol.ok_or_else(|| de::Error::missing_field("symbol"))?; + let frr = frr.ok_or_else(|| de::Error::missing_field("frr"))?; + let bid = bid.ok_or_else(|| de::Error::missing_field("bid"))?; + let bid_period = + bid_period.ok_or_else(|| de::Error::missing_field("bid_period"))?; + let bid_size = bid_size.ok_or_else(|| de::Error::missing_field("bid_size"))?; + let ask = ask.ok_or_else(|| de::Error::missing_field("ask"))?; + let ask_period = + ask_period.ok_or_else(|| de::Error::missing_field("ask_period"))?; + let ask_size = ask_size.ok_or_else(|| de::Error::missing_field("ask_size"))?; + let daily_change = + daily_change.ok_or_else(|| de::Error::missing_field("daily_change"))?; + let daily_change_relative = daily_change_relative + .ok_or_else(|| de::Error::missing_field("daily_change_relative"))?; + let last_price = + last_price.ok_or_else(|| de::Error::missing_field("last_price"))?; + let volume = volume.ok_or_else(|| de::Error::missing_field("volume"))?; + let high = high.ok_or_else(|| de::Error::missing_field("high"))?; + let low = low.ok_or_else(|| de::Error::missing_field("low"))?; + let frr_amount_available = frr_amount_available + .ok_or_else(|| de::Error::missing_field("frr_amount_available"))?; + + let ticker = Ticker { + symbol, + frr, + bid, + bid_period, + bid_size, + ask, + ask_period, + ask_size, + daily_change, + daily_change_relative, + last_price, + volume, + high, + low, + frr_amount_available, + }; + + Ok(ticker) + } + } + + const FIELDS: &[&str] = &[ + "symbol", + "frr", + "bid", + "bid_period", + "bid_size", + "ask", + "ask_period", + "ask_size", + "daily_change", + "daily_change_relative", + "last_price", + "volume", + "high", + "low", + "frr_amount_available", + ]; + deserializer.deserialize_struct("FundingTicker", FIELDS, FundingTickerVisitor {}) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_deserialize_funding_ticker_from_array() { + let json = r#"["fUSD",0.00018055342465753425,0.0002,120,35545399.51575242,0.00008219178082191781,2,28117235.06098758,-0.0000278,-0.2528,0.00008219,413386933.358769,0.000137,0.000025,null,null,5817583.43063814]"#; + let funding_ticker: Ticker = serde_json::from_str(json).unwrap(); + + let expected = Ticker { + symbol: "fUSD".to_string(), + frr: 0.00018055342465753425, + bid: 0.0002, + bid_period: 120, + bid_size: 35545399.51575242, + ask: 0.00008219178082191781, + ask_period: 2, + ask_size: 28117235.06098758, + daily_change: -0.0000278, + daily_change_relative: -0.2528, + last_price: 0.00008219, + volume: 413386933.358769, + high: 0.000137, + low: 0.000025, + frr_amount_available: 5817583.43063814, + }; + + assert_eq!(funding_ticker, expected); + } + + #[test] + fn test_deserialize_funding_ticker_from_array_with_invalid_length() { + let json = r#"["fUSD",0.00018055342465753425,0.0002,120,35545399.51575242,0.00008219178082191781,2,28117235.06098758,-0.0000278,-0.2528,0.00008219,413386933.358769,0.000137,0.000025,null,null]"#; + let funding_ticker: Result = serde_json::from_str(json); + + assert_eq!( + funding_ticker.err().unwrap().to_string(), + "invalid length 16, expected tuple with length 15 at line 1 column 178" + ); + } + + #[test] + fn test_deserialize_funding_ticker_from_array_with_invalid_value() { + let json = r#"[1000000,0.00018055342465753425,0.0002,120,35545399.51575242,0.00008219178082191781,2,28117235.06098758,-0.0000278,-0.2528,0.00008219,413386933.358769,0.000137,0.000025,null,null,5817583.43063814]"#; + let funding_ticker: Result = serde_json::from_str(json); + + assert_eq!( + funding_ticker.err().unwrap().to_string(), + "invalid type: integer `1000000`, expected a string at line 1 column 8" + ); + } + + #[test] + fn test_deserialize_funding_ticker_from_map() { + let json = r#"{"symbol":"fUSD","frr":0.00018055342465753425,"bid":0.0002,"bid_period":120,"bid_size":35545399.51575242,"ask":0.00008219178082191781,"ask_period":2,"ask_size":28117235.06098758,"daily_change":-0.0000278,"daily_change_relative":-0.2528,"last_price":0.00008219,"volume":413386933.358769,"high":0.000137,"low":0.000025,"frr_amount_available":5817583.43063814}"#; + let funding_ticker: Ticker = serde_json::from_str(json).unwrap(); + + let expected = Ticker { + symbol: "fUSD".to_string(), + frr: 0.00018055342465753425, + bid: 0.0002, + bid_period: 120, + bid_size: 35545399.51575242, + ask: 0.00008219178082191781, + ask_period: 2, + ask_size: 28117235.06098758, + daily_change: -0.0000278, + daily_change_relative: -0.2528, + last_price: 0.00008219, + volume: 413386933.358769, + high: 0.000137, + low: 0.000025, + frr_amount_available: 5817583.43063814, + }; + + assert_eq!(funding_ticker, expected); + } + + #[test] + fn test_deserialize_funding_ticker_from_map_with_missing_field() { + let json = r#"{"symbol":"fUSD","frr":0.00018055342465753425,"bid":0.0002,"bid_period":120,"bid_size":35545399.51575242,"ask":0.00008219178082191781,"ask_period":2,"ask_size":28117235.06098758,"daily_change":-0.0000278,"daily_change_relative":-0.2528,"last_price":0.00008219,"volume":413386933.358769,"high":0.000137,"low":0.000025}"#; + let funding_ticker: Result = serde_json::from_str(json); + + assert_eq!( + funding_ticker.err().unwrap().to_string(), + "missing field `frr_amount_available` at line 1 column 317" + ); + } + + #[test] + fn test_deserialize_funding_ticker_from_map_with_invalid_key() { + let json = r#"{"abc":"fUSD","frr":0.00018055342465753425,"bid":0.0002,"bid_period":120,"bid_size":35545399.51575242,"ask":0.00008219178082191781,"ask_period":2,"ask_size":28117235.06098758,"daily_change":-0.0000278,"daily_change_relative":-0.2528,"last_price":0.00008219,"volume":413386933.358769,"high":0.000137,"low":0.000025,"frr_amount_available":5817583.43063814}"#; + let funding_ticker: Result = serde_json::from_str(json); + + assert_eq!( + funding_ticker.err().unwrap().to_string(), + "unknown field `abc`, expected one of `symbol`, `frr`, `bid`, `bid_period`, `bid_size`, `ask`, `ask_period`, `ask_size`, `daily_change`, `daily_change_relative`, `last_price`, `volume`, `high`, `low`, `frr_amount_available` at line 1 column 6" + ); + } + + #[test] + fn test_deserialize_funding_ticker_from_map_with_invalid_value() { + let json = r#"{"symbol": 0,"frr":0.00018055342465753425,"bid":0.0002,"bid_period":120,"bid_size":35545399.51575242,"ask":0.00008219178082191781,"ask_period":2,"ask_size":28117235.06098758,"daily_change":-0.0000278,"daily_change_relative":-0.2528,"last_price":0.00008219,"volume":413386933.358769,"high":0.000137,"low":0.000025,"frr_amount_available":5817583.43063814}"#; + let funding_ticker: Result = serde_json::from_str(json); + + assert_eq!( + funding_ticker.err().unwrap().to_string(), + "invalid type: integer `0`, expected a string at line 1 column 12" + ); + } +} diff --git a/bothan-bitfinex/src/api/msg/ticker/spot.rs b/bothan-bitfinex/src/api/msg/ticker/spot.rs new file mode 100644 index 00000000..868a1950 --- /dev/null +++ b/bothan-bitfinex/src/api/msg/ticker/spot.rs @@ -0,0 +1,343 @@ +use std::fmt; + +use serde::de::{MapAccess, SeqAccess, Visitor}; +use serde::{Deserialize, Deserializer, Serialize, de}; + +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct Ticker { + pub symbol: String, + pub bid: f64, + pub bid_size: f64, + pub ask: f64, + pub ask_size: f64, + pub daily_change: f64, + pub daily_change_relative: f64, + pub last_price: f64, + pub volume: f64, + pub high: f64, + pub low: f64, +} + +impl<'de> Deserialize<'de> for Ticker { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + #[derive(Deserialize)] + #[serde(field_identifier, rename_all = "snake_case")] + enum Field { + Symbol, + Bid, + BidSize, + Ask, + AskSize, + DailyChange, + DailyChangeRelative, + LastPrice, + Volume, + High, + Low, + } + + struct SpotTickerVisitor {} + impl<'de> Visitor<'de> for SpotTickerVisitor { + type Value = Ticker; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("tuple with length 11") + } + + fn visit_seq(self, mut seq: V) -> Result + where + V: SeqAccess<'de>, + { + let symbol = seq + .next_element()? + .ok_or_else(|| de::Error::invalid_length(0, &self))?; + let bid = seq + .next_element()? + .ok_or_else(|| de::Error::invalid_length(1, &self))?; + let bid_size = seq + .next_element()? + .ok_or_else(|| de::Error::invalid_length(2, &self))?; + let ask = seq + .next_element()? + .ok_or_else(|| de::Error::invalid_length(3, &self))?; + let ask_size = seq + .next_element()? + .ok_or_else(|| de::Error::invalid_length(4, &self))?; + let daily_change = seq + .next_element()? + .ok_or_else(|| de::Error::invalid_length(5, &self))?; + let daily_change_relative = seq + .next_element()? + .ok_or_else(|| de::Error::invalid_length(6, &self))?; + let last_price = seq + .next_element()? + .ok_or_else(|| de::Error::invalid_length(7, &self))?; + let volume = seq + .next_element()? + .ok_or_else(|| de::Error::invalid_length(8, &self))?; + let high = seq + .next_element()? + .ok_or_else(|| de::Error::invalid_length(9, &self))?; + let low = seq + .next_element()? + .ok_or_else(|| de::Error::invalid_length(10, &self))?; + + let spot_ticker = Ticker { + symbol, + bid, + bid_size, + ask, + ask_size, + daily_change, + daily_change_relative, + last_price, + volume, + high, + low, + }; + Ok(spot_ticker) + } + + fn visit_map(self, mut map: A) -> Result + where + A: MapAccess<'de>, + { + let mut symbol = None; + let mut bid = None; + let mut bid_size = None; + let mut ask = None; + let mut ask_size = None; + let mut daily_change = None; + let mut daily_change_relative = None; + let mut last_price = None; + let mut volume = None; + let mut high = None; + let mut low = None; + + while let Some(key) = map.next_key()? { + match key { + Field::Symbol => { + if symbol.is_some() { + return Err(de::Error::duplicate_field("symbol")); + } + symbol = Some(map.next_value()?); + } + Field::Bid => { + if bid.is_some() { + return Err(de::Error::duplicate_field("bid")); + } + bid = Some(map.next_value()?); + } + Field::BidSize => { + if bid_size.is_some() { + return Err(de::Error::duplicate_field("bid_size")); + } + bid_size = Some(map.next_value()?); + } + Field::Ask => { + if ask.is_some() { + return Err(de::Error::duplicate_field("ask")); + } + ask = Some(map.next_value()?); + } + Field::AskSize => { + if ask_size.is_some() { + return Err(de::Error::duplicate_field("ask_size")); + } + ask_size = Some(map.next_value()?); + } + Field::DailyChange => { + if daily_change.is_some() { + return Err(de::Error::duplicate_field("daily_change")); + } + daily_change = Some(map.next_value()?); + } + Field::DailyChangeRelative => { + if daily_change_relative.is_some() { + return Err(de::Error::duplicate_field("daily_change_relative")); + } + daily_change_relative = Some(map.next_value()?); + } + Field::LastPrice => { + if last_price.is_some() { + return Err(de::Error::duplicate_field("last_price")); + } + last_price = Some(map.next_value()?); + } + Field::Volume => { + if volume.is_some() { + return Err(de::Error::duplicate_field("volume")); + } + volume = Some(map.next_value()?); + } + Field::High => { + if high.is_some() { + return Err(de::Error::duplicate_field("high")); + } + high = Some(map.next_value()?); + } + Field::Low => { + if low.is_some() { + return Err(de::Error::duplicate_field("low")); + } + low = Some(map.next_value()?); + } + } + } + + let symbol = symbol.ok_or_else(|| de::Error::missing_field("symbol"))?; + let bid = bid.ok_or_else(|| de::Error::missing_field("bid"))?; + let bid_size = bid_size.ok_or_else(|| de::Error::missing_field("bid_size"))?; + let ask = ask.ok_or_else(|| de::Error::missing_field("ask"))?; + let ask_size = ask_size.ok_or_else(|| de::Error::missing_field("ask_size"))?; + let daily_change = + daily_change.ok_or_else(|| de::Error::missing_field("daily_change"))?; + let daily_change_relative = daily_change_relative + .ok_or_else(|| de::Error::missing_field("daily_change_relative"))?; + let last_price = + last_price.ok_or_else(|| de::Error::missing_field("last_price"))?; + let volume = volume.ok_or_else(|| de::Error::missing_field("volume"))?; + let high = high.ok_or_else(|| de::Error::missing_field("high"))?; + let low = low.ok_or_else(|| de::Error::missing_field("low"))?; + + let ticker = Ticker { + symbol, + bid, + bid_size, + ask, + ask_size, + daily_change, + daily_change_relative, + last_price, + volume, + high, + low, + }; + Ok(ticker) + } + } + + const FIELDS: &[&str] = &[ + "symbol", + "bid", + "bid_size", + "ask", + "ask_size", + "daily_change", + "daily_change_relative", + "last_price", + "volume", + "high", + "low", + ]; + deserializer.deserialize_struct("SpotTicker", FIELDS, SpotTickerVisitor {}) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_deserialize_spot_ticker_from_array() { + let json = r#"["tBTCUSD",101740,93.86022424,101750,38.06413103,2132,0.02140175,101750,663.27534767,102760,98740]"#; + let spot_ticker: Ticker = serde_json::from_str(json).unwrap(); + + let expected = Ticker { + symbol: "tBTCUSD".to_string(), + bid: 101740.0, + bid_size: 93.86022424, + ask: 101750.0, + ask_size: 38.06413103, + daily_change: 2132.0, + daily_change_relative: 0.02140175, + last_price: 101750.0, + volume: 663.27534767, + high: 102760.0, + low: 98740.0, + }; + + assert_eq!(spot_ticker, expected); + } + + #[test] + fn test_deserialize_spot_ticker_from_array_with_invalid_length() { + let json = r#"["tBTCUSD",101740,93.86022424,101750,38.06413103,2132,0.02140175,101750,663.27534767]"#; + let spot_ticker: Result = serde_json::from_str(json); + + assert_eq!( + spot_ticker.err().unwrap().to_string(), + "invalid length 9, expected tuple with length 11 at line 1 column 85" + ); + } + + #[test] + fn test_deserialize_spot_ticker_from_array_with_invalid_value() { + let json = + r#"[100000,101740,93.86022424,101750,38.06413103,2132,0.02140175,101750,663.27534767]"#; + let spot_ticker: Result = serde_json::from_str(json); + + assert_eq!( + spot_ticker.err().unwrap().to_string(), + "invalid type: integer `100000`, expected a string at line 1 column 7" + ); + } + + #[test] + fn test_deserialize_spot_ticker_from_map() { + let json = r#"{"symbol":"tBTCUSD","bid":101740,"bid_size":93.86022424,"ask":101750,"ask_size":38.06413103,"daily_change":2132,"daily_change_relative":0.02140175,"last_price":101750,"volume":663.27534767,"high":102760,"low":98740.0}"#; + let spot_ticker: Ticker = serde_json::from_str(json).unwrap(); + + let expected = Ticker { + symbol: "tBTCUSD".to_string(), + bid: 101740.0, + bid_size: 93.86022424, + ask: 101750.0, + ask_size: 38.06413103, + daily_change: 2132.0, + daily_change_relative: 0.02140175, + last_price: 101750.0, + volume: 663.27534767, + high: 102760.0, + low: 98740.0, + }; + + assert_eq!(spot_ticker, expected); + } + + #[test] + fn test_deserialize_spot_ticker_from_map_with_missing_field() { + let json = r#"{"bid":101740,"bid_size":93.86022424,"ask":101750,"ask_size":38.06413103,"daily_change":2132,"daily_change_relative":0.02140175,"last_price":101750,"volume":663.27534767,"high":102760,"low":98740.0}"#; + let spot_ticker: Result = serde_json::from_str(json); + + assert_eq!( + spot_ticker.err().unwrap().to_string(), + "missing field `symbol` at line 1 column 198" + ); + } + + #[test] + fn test_deserialize_spot_ticker_from_map_with_invalid_key() { + let json = r#"{"abc":"tBTCUSD","bid":101740,"bid_size":93.86022424,"ask":101750,"ask_size":38.06413103,"daily_change":2132,"daily_change_relative":0.02140175,"last_price":101750,"volume":663.27534767,"high":102760,"low":98740.0}"#; + let spot_ticker: Result = serde_json::from_str(json); + + assert_eq!( + spot_ticker.err().unwrap().to_string(), + "unknown field `abc`, expected one of `symbol`, `bid`, `bid_size`, `ask`, `ask_size`, `daily_change`, `daily_change_relative`, `last_price`, `volume`, `high`, `low` at line 1 column 6" + ); + } + + #[test] + fn test_deserialize_spot_ticker_from_map_with_invalid_value() { + let json = r#"{"symbol": -1000,"bid":101740,"bid_size":93.86022424,"ask":101750,"ask_size":38.06413103,"daily_change":2132,"daily_change_relative":0.02140175,"last_price":101750,"volume":663.27534767,"high":102760,"low":98740.0}"#; + let spot_ticker: Result = serde_json::from_str(json); + + assert_eq!( + spot_ticker.err().unwrap().to_string(), + "invalid type: integer `-1000`, expected a string at line 1 column 16" + ); + } +} diff --git a/bothan-bitfinex/src/api/rest.rs b/bothan-bitfinex/src/api/rest.rs new file mode 100644 index 00000000..8ec4ae32 --- /dev/null +++ b/bothan-bitfinex/src/api/rest.rs @@ -0,0 +1,55 @@ +use bothan_lib::types::AssetInfo; +use bothan_lib::worker::rest::AssetInfoProvider; +use reqwest::{Client, Url}; +use rust_decimal::Decimal; + +use crate::api::error::ProviderError; +use crate::api::msg::ticker::Ticker; + +pub const DEFAULT_URL: &str = "https://api-pub.bitfinex.com/v2/"; + +pub struct RestApi { + url: Url, + client: Client, +} + +impl RestApi { + pub fn new(url: Url, client: Client) -> Self { + Self { url, client } + } + + pub async fn get_tickers>( + &self, + tickers: &[T], + ) -> Result, reqwest::Error> { + let url = format!("{}/tickers", self.url); + let symbols = tickers + .iter() + .map(|t| t.as_ref()) + .collect::>() + .join(","); + let params = vec![("symbols", symbols)]; + + let resp = self.client.get(&url).query(¶ms).send().await?; + resp.error_for_status_ref()?; + resp.json().await + } +} + +#[async_trait::async_trait] +impl AssetInfoProvider for RestApi { + type Error = ProviderError; + + async fn get_asset_info(&self, ids: &[String]) -> Result, Self::Error> { + let timestamp = chrono::Utc::now().timestamp(); + self.get_tickers(ids) + .await? + .into_iter() + .map(|t| { + let price = + Decimal::from_f64_retain(t.price()).ok_or(ProviderError::InvalidValue)?; + Ok(AssetInfo::new(t.symbol().to_string(), price, timestamp)) + }) + .collect() + } +} diff --git a/bothan-bitfinex/src/lib.rs b/bothan-bitfinex/src/lib.rs new file mode 100644 index 00000000..e31a5354 --- /dev/null +++ b/bothan-bitfinex/src/lib.rs @@ -0,0 +1,5 @@ +pub use worker::Worker; +pub use worker::opts::WorkerOpts; + +pub mod api; +pub mod worker; diff --git a/bothan-bitfinex/src/worker.rs b/bothan-bitfinex/src/worker.rs new file mode 100644 index 00000000..44095071 --- /dev/null +++ b/bothan-bitfinex/src/worker.rs @@ -0,0 +1,61 @@ +use std::collections::HashSet; +use std::sync::{Arc, Weak}; + +use bothan_lib::store::{Store, WorkerStore}; +use bothan_lib::types::AssetState; +use bothan_lib::worker::AssetWorker; +use bothan_lib::worker::error::AssetWorkerError; +use bothan_lib::worker::rest::{AssetInfoProvider, start_polling}; + +use crate::api::builder::RestApiBuilder; +use crate::api::error::ProviderError; +use crate::api::rest::RestApi; +use crate::worker::opts::WorkerOpts; + +const WORKER_NAME: &str = "bitfinex"; + +pub mod opts; + +pub struct Worker { + // The `api` is owned by this struct to ensure that any weak references + // are properly cleaned up when the worker is dropped. + #[allow(dead_code)] + api: Arc, + store: WorkerStore, +} + +#[async_trait::async_trait] +impl AssetWorker for Worker { + type Opts = WorkerOpts; + + fn name(&self) -> &'static str { + WORKER_NAME + } + + async fn build(opts: Self::Opts, store: &S) -> Result { + let api = Arc::new(RestApiBuilder::new(&opts.url).build()?); + + let worker_store = WorkerStore::new(store, WORKER_NAME); + + tokio::spawn(start_polling( + opts.update_interval, + Arc::downgrade(&api) as Weak>, + worker_store.clone(), + )); + + let worker = Worker { + api, + store: worker_store, + }; + Ok(worker) + } + + async fn get_asset(&self, id: &str) -> Result { + Ok(self.store.get_asset(id).await?) + } + + async fn set_query_ids(&self, ids: HashSet) -> Result<(), AssetWorkerError> { + self.store.set_query_ids(ids).await?; + Ok(()) + } +} diff --git a/bothan-bitfinex/src/worker/opts.rs b/bothan-bitfinex/src/worker/opts.rs new file mode 100644 index 00000000..a8f37bbd --- /dev/null +++ b/bothan-bitfinex/src/worker/opts.rs @@ -0,0 +1,34 @@ +use std::time::Duration; + +use serde::{Deserialize, Serialize}; + +use crate::api::rest::DEFAULT_URL; + +const DEFAULT_UPDATE_INTERVAL: Duration = Duration::from_secs(60); + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct WorkerOpts { + #[serde(default = "default_url")] + pub url: String, + + #[serde(default = "default_update_interval")] + #[serde(with = "humantime_serde")] + pub update_interval: Duration, +} + +fn default_url() -> String { + DEFAULT_URL.to_string() +} + +fn default_update_interval() -> Duration { + DEFAULT_UPDATE_INTERVAL +} + +impl Default for WorkerOpts { + fn default() -> Self { + Self { + url: default_url(), + update_interval: default_update_interval(), + } + } +} diff --git a/bothan-bybit/Cargo.toml b/bothan-bybit/Cargo.toml index b4b1c007..94db3cbf 100644 --- a/bothan-bybit/Cargo.toml +++ b/bothan-bybit/Cargo.toml @@ -1,13 +1,14 @@ [package] name = "bothan-bybit" -version = "0.0.1-beta.1" +version = "0.0.1-beta.2" edition.workspace = true license.workspace = true repository.workspace = true [dependencies] +bothan-lib = { workspace = true } + async-trait = { workspace = true } -bothan-core = { workspace = true } chrono = { workspace = true } futures-util = { workspace = true } rust_decimal = { workspace = true } @@ -17,7 +18,6 @@ thiserror = { workspace = true } tokio = { workspace = true } tokio-tungstenite = { workspace = true } tracing = { workspace = true } -tracing-subscriber = { workspace = true } [dev-dependencies] ws-mock = { git = "https://github.com/bandprotocol/ws-mock.git", branch = "master" } diff --git a/bothan-bybit/examples/bybit_basic.rs b/bothan-bybit/examples/bybit_basic.rs deleted file mode 100644 index 250fcff0..00000000 --- a/bothan-bybit/examples/bybit_basic.rs +++ /dev/null @@ -1,39 +0,0 @@ -use std::time::Duration; - -use tokio::time::sleep; -use tracing_subscriber::fmt::init; - -use bothan_bybit::{BybitWorkerBuilder, BybitWorkerBuilderOpts}; -use bothan_core::registry::Registry; -use bothan_core::store::SharedStore; -use bothan_core::worker::{AssetWorker, AssetWorkerBuilder}; - -#[tokio::main] -async fn main() { - init(); - let path = std::env::current_dir().unwrap(); - let registry = Registry::default().validate().unwrap(); - let store = SharedStore::new(registry, path.as_path()).await.unwrap(); - - let worker_store = store.create_worker_store(BybitWorkerBuilder::worker_name()); - let opts = BybitWorkerBuilderOpts::default(); - - let worker = BybitWorkerBuilder::new(worker_store, opts) - .build() - .await - .unwrap(); - - worker - .set_query_ids(vec!["BTCUSDT".to_string(), "ETHUSDT".to_string()]) - .await - .unwrap(); - - sleep(Duration::from_secs(2)).await; - - loop { - let btc_data = worker.get_asset("BTCUSDT").await; - let eth_data = worker.get_asset("ETHUSDT").await; - println!("{:?} {:?}", btc_data, eth_data); - sleep(Duration::from_secs(5)).await; - } -} diff --git a/bothan-bybit/src/api.rs b/bothan-bybit/src/api.rs index b8a6a1c5..d054fad1 100644 --- a/bothan-bybit/src/api.rs +++ b/bothan-bybit/src/api.rs @@ -1,5 +1,5 @@ pub use error::{ConnectionError, MessageError, SendError}; -pub use websocket::{BybitWebSocketConnection, BybitWebSocketConnector}; +pub use websocket::{WebSocketConnection, WebSocketConnector}; pub mod error; pub mod types; diff --git a/bothan-bybit/src/api/websocket.rs b/bothan-bybit/src/api/websocket.rs index f01340ea..6a47b351 100644 --- a/bothan-bybit/src/api/websocket.rs +++ b/bothan-bybit/src/api/websocket.rs @@ -2,28 +2,28 @@ use futures_util::stream::{SplitSink, SplitStream}; use futures_util::{SinkExt, StreamExt}; use serde_json::json; use tokio::net::TcpStream; +use tokio_tungstenite::tungstenite::Message; use tokio_tungstenite::tungstenite::error::Error as TungsteniteError; use tokio_tungstenite::tungstenite::http::StatusCode; -use tokio_tungstenite::tungstenite::Message; -use tokio_tungstenite::{connect_async, MaybeTlsStream, WebSocketStream}; +use tokio_tungstenite::{MaybeTlsStream, WebSocketStream, connect_async}; use tracing::warn; use crate::api::error::{ConnectionError, MessageError, SendError}; use crate::api::types::BybitResponse; /// A connector for establishing a WebSocket connection to the Bybit API. -pub struct BybitWebSocketConnector { +pub struct WebSocketConnector { url: String, } -impl BybitWebSocketConnector { +impl WebSocketConnector { /// Creates a new instance of `BybitWebSocketConnector`. pub fn new(url: impl Into) -> Self { Self { url: url.into() } } /// Connects to the Bybit WebSocket API. - pub async fn connect(&self) -> Result { + pub async fn connect(&self) -> Result { let (wss, resp) = connect_async(self.url.clone()).await?; let status = resp.status(); @@ -34,17 +34,17 @@ impl BybitWebSocketConnector { )); } - Ok(BybitWebSocketConnection::new(wss)) + Ok(WebSocketConnection::new(wss)) } } /// Represents an active WebSocket connection to the Bybit API. -pub struct BybitWebSocketConnection { +pub struct WebSocketConnection { sender: SplitSink>, Message>, receiver: SplitStream>>, } -impl BybitWebSocketConnection { +impl WebSocketConnection { /// Creates a new `BybitWebSocketConnection` instance. pub fn new(web_socket_stream: WebSocketStream>) -> Self { let (sender, receiver) = web_socket_stream.split(); @@ -104,10 +104,8 @@ pub(crate) mod test { use tokio::sync::mpsc; use ws_mock::ws_mock_server::{WsMock, WsMockServer}; - // use crate::api::msgs::{Data, MiniTickerInfo, StreamResponse}; - use crate::api::types::{BybitResponse, PublicMessageResponse, PublicTickerResponse, Ticker}; - use super::*; + use crate::api::types::{BybitResponse, PublicMessageResponse, PublicTickerResponse, Ticker}; pub(crate) async fn setup_mock_server() -> WsMockServer { WsMockServer::start().await @@ -117,7 +115,7 @@ pub(crate) mod test { async fn test_recv_public_ticker() { // Set up the mock server and the WebSocket connector. let server = setup_mock_server().await; - let connector = BybitWebSocketConnector::new(server.uri().await); + let connector = WebSocketConnector::new(server.uri().await); let (mpsc_send, mpsc_recv) = mpsc::channel::(32); // Create a mock ticker response. @@ -165,7 +163,7 @@ pub(crate) mod test { async fn test_recv_public_message() { // Set up the mock server and the WebSocket connector. let server = setup_mock_server().await; - let connector = BybitWebSocketConnector::new(server.uri().await); + let connector = WebSocketConnector::new(server.uri().await); let (mpsc_send, mpsc_recv) = mpsc::channel::(32); // Create a mock public message response. @@ -202,7 +200,7 @@ pub(crate) mod test { async fn test_recv_close() { // Set up the mock server and the WebSocket connector. let server = setup_mock_server().await; - let connector = BybitWebSocketConnector::new(server.uri().await); + let connector = WebSocketConnector::new(server.uri().await); let (mpsc_send, mpsc_recv) = mpsc::channel::(32); // Mount the mock WebSocket server and send a close message. diff --git a/bothan-bybit/src/lib.rs b/bothan-bybit/src/lib.rs index 902ccf5a..ac1901ea 100644 --- a/bothan-bybit/src/lib.rs +++ b/bothan-bybit/src/lib.rs @@ -1,8 +1,6 @@ -pub use api::websocket::{BybitWebSocketConnection, BybitWebSocketConnector}; -pub use worker::builder::BybitWorkerBuilder; -pub use worker::error::BuildError; -pub use worker::opts::BybitWorkerBuilderOpts; -pub use worker::BybitWorker; +pub use api::websocket::{WebSocketConnection, WebSocketConnector}; +pub use worker::Worker; +pub use worker::opts::WorkerOpts; pub mod api; pub mod worker; diff --git a/bothan-bybit/src/worker.rs b/bothan-bybit/src/worker.rs index 00e06f02..7727c810 100644 --- a/bothan-bybit/src/worker.rs +++ b/bothan-bybit/src/worker.rs @@ -1,68 +1,89 @@ -use tokio::sync::mpsc::Sender; +use std::collections::HashSet; +use std::sync::Arc; -use bothan_core::store::error::Error as StoreError; -use bothan_core::store::WorkerStore; -use bothan_core::worker::{AssetState, AssetWorker, SetQueryIDError}; +use async_trait::async_trait; +use bothan_lib::store::{Store, WorkerStore}; +use bothan_lib::types::AssetState; +use bothan_lib::worker::AssetWorker; +use bothan_lib::worker::error::AssetWorkerError; +use tokio::sync::mpsc::{Sender, channel}; -use crate::api::websocket::BybitWebSocketConnector; +use crate::WorkerOpts; +use crate::api::websocket::WebSocketConnector; +use crate::worker::asset_worker::start_asset_worker; mod asset_worker; -pub mod builder; -pub(crate) mod error; pub mod opts; mod types; -/// A worker that fetches and stores the asset information from Bybit's API. -pub struct BybitWorker { - connector: BybitWebSocketConnector, - store: WorkerStore, - subscribe_tx: Sender>, - unsubscribe_tx: Sender>, +const WORKER_NAME: &str = "bybit"; + +pub struct Worker { + inner: Arc>, } -impl BybitWorker { - /// Create a new worker with the specified connector, store and channels. - pub fn new( - connector: BybitWebSocketConnector, - store: WorkerStore, - subscribe_tx: Sender>, - unsubscribe_tx: Sender>, - ) -> Self { - Self { - connector, - store, - subscribe_tx, - unsubscribe_tx, +#[async_trait] +impl AssetWorker for Worker { + type Opts = WorkerOpts; + + fn name(&self) -> &'static str { + WORKER_NAME + } + + async fn build(opts: Self::Opts, store: &S) -> Result { + let url = opts.url; + let ch_size = opts.internal_ch_size; + + let connector = WebSocketConnector::new(url); + let connection = connector.connect().await?; + + let (sub_tx, sub_rx) = channel(ch_size); + let (unsub_tx, unsub_rx) = channel(ch_size); + + let worker_store = WorkerStore::new(store, WORKER_NAME); + let to_sub = worker_store + .get_query_ids() + .await? + .into_iter() + .collect::>(); + + if !to_sub.is_empty() { + sub_tx.send(to_sub).await?; } + + let inner = Arc::new(InnerWorker { + connector, + store: worker_store.clone(), + subscribe_tx: sub_tx, + unsubscribe_tx: unsub_tx, + }); + tokio::spawn(start_asset_worker( + Arc::downgrade(&inner), + connection, + sub_rx, + unsub_rx, + )); + + Ok(Worker { inner }) } -} -#[async_trait::async_trait] -impl AssetWorker for BybitWorker { - /// Fetches the AssetStatus for the given cryptocurrency id. - async fn get_asset(&self, id: &str) -> Result { - self.store.get_asset(&id).await + async fn get_asset(&self, id: &str) -> Result { + Ok(self.inner.store.get_asset(id).await?) } - /// Sets the specified cryptocurrency IDs to the query. If the ids are already in the query set, - /// it will not be resubscribed. - async fn set_query_ids(&self, ids: Vec) -> Result<(), SetQueryIDError> { - let (to_sub, to_unsub) = self - .store - .set_query_ids(ids) - .await - .map_err(|e| SetQueryIDError::new(e.to_string()))?; - - self.subscribe_tx - .send(to_sub) - .await - .map_err(|e| SetQueryIDError::new(e.to_string()))?; - - self.unsubscribe_tx - .send(to_unsub) - .await - .map_err(|e| SetQueryIDError::new(e.to_string()))?; + async fn set_query_ids(&self, ids: HashSet) -> Result<(), AssetWorkerError> { + let diff = self.inner.store.compute_query_id_difference(ids).await?; + + self.inner.subscribe_tx.send(diff.added).await?; + self.inner.unsubscribe_tx.send(diff.removed).await?; Ok(()) } } + +struct InnerWorker { + connector: WebSocketConnector, + store: WorkerStore, + subscribe_tx: Sender>, + unsubscribe_tx: Sender>, +} diff --git a/bothan-bybit/src/worker/asset_worker.rs b/bothan-bybit/src/worker/asset_worker.rs index e9bcb6b1..01577367 100644 --- a/bothan-bybit/src/worker/asset_worker.rs +++ b/bothan-bybit/src/worker/asset_worker.rs @@ -1,24 +1,22 @@ use std::sync::Weak; +use bothan_lib::store::{Store, WorkerStore}; +use bothan_lib::types::AssetInfo; use rust_decimal::Decimal; use tokio::select; use tokio::sync::mpsc::Receiver; use tokio::time::{sleep, timeout}; use tracing::{debug, error, info, warn}; -use bothan_core::store::WorkerStore; -use bothan_core::types::AssetInfo; - use crate::api::error::{MessageError, SendError}; -use crate::api::types::{BybitResponse, Ticker, MAX_ARGS}; -use crate::api::{BybitWebSocketConnection, BybitWebSocketConnector}; -use crate::worker::error::WorkerError; +use crate::api::types::{BybitResponse, MAX_ARGS, Ticker}; +use crate::api::{WebSocketConnection, WebSocketConnector}; +use crate::worker::InnerWorker; use crate::worker::types::{DEFAULT_TIMEOUT, RECONNECT_BUFFER}; -use crate::worker::BybitWorker; -pub(crate) async fn start_asset_worker( - worker: Weak, - mut connection: BybitWebSocketConnection, +pub(crate) async fn start_asset_worker( + inner_worker: Weak>, + mut connection: WebSocketConnection, mut subscribe_rx: Receiver>, mut unsubscribe_rx: Receiver>, ) { @@ -27,7 +25,7 @@ pub(crate) async fn start_asset_worker( Some(ids) = subscribe_rx.recv() => handle_subscribe_recv(ids, &mut connection).await, Some(ids) = unsubscribe_rx.recv() => handle_unsubscribe_recv(ids, &mut connection).await, result = timeout(DEFAULT_TIMEOUT, connection.next()) => { - if let Some(worker) = worker.upgrade() { + if let Some(worker) = inner_worker.upgrade() { match result { Err(_) => handle_reconnect(&worker.connector, &mut connection, &worker.store).await, Ok(bybit_result) => handle_connection_recv(bybit_result, &worker.connector, &mut connection, &worker.store).await, @@ -49,10 +47,7 @@ pub(crate) async fn start_asset_worker( debug!("asset worker has been dropped, stopping asset worker"); } -async fn subscribe( - ids: &[String], - connection: &mut BybitWebSocketConnection, -) -> Result<(), SendError> { +async fn subscribe(ids: &[String], connection: &mut WebSocketConnection) -> Result<(), SendError> { if !ids.is_empty() { for batched_ids in ids.chunks(MAX_ARGS as usize) { let symbols = batched_ids @@ -66,7 +61,7 @@ async fn subscribe( Ok(()) } -async fn handle_subscribe_recv(ids: Vec, connection: &mut BybitWebSocketConnection) { +async fn handle_subscribe_recv(ids: Vec, connection: &mut WebSocketConnection) { if let Err(e) = subscribe(&ids, connection).await { error!("failed to subscribe to ids {:?}: {}", ids, e); } else { @@ -76,7 +71,7 @@ async fn handle_subscribe_recv(ids: Vec, connection: &mut BybitWebSocket async fn unsubscribe( ids: &[String], - connection: &mut BybitWebSocketConnection, + connection: &mut WebSocketConnection, ) -> Result<(), SendError> { if !ids.is_empty() { for batched_ids in ids.chunks(MAX_ARGS as usize) { @@ -91,7 +86,7 @@ async fn unsubscribe( Ok(()) } -async fn handle_unsubscribe_recv(ids: Vec, connection: &mut BybitWebSocketConnection) { +async fn handle_unsubscribe_recv(ids: Vec, connection: &mut WebSocketConnection) { if let Err(e) = unsubscribe(&ids, connection).await { error!("failed to unsubscribe to ids {:?}: {}", ids, e); } else { @@ -99,10 +94,10 @@ async fn handle_unsubscribe_recv(ids: Vec, connection: &mut BybitWebSock } } -async fn handle_reconnect( - connector: &BybitWebSocketConnector, - connection: &mut BybitWebSocketConnection, - query_ids: &WorkerStore, +async fn handle_reconnect( + connector: &WebSocketConnector, + connection: &mut WebSocketConnection, + query_ids: &WorkerStore, ) { let mut retry_count: usize = 1; loop { @@ -137,41 +132,45 @@ async fn handle_reconnect( } } -fn parse_ticker(ticker: Ticker) -> Result { +fn parse_ticker(ticker: Ticker) -> Result { let id = ticker.symbol.clone(); let price_value = Decimal::from_str_exact(&ticker.last_price)?; let timestamp = chrono::Utc::now().timestamp(); Ok(AssetInfo::new(id, price_value, timestamp)) } -async fn store_ticker(store: &WorkerStore, ticker: Ticker) -> Result<(), WorkerError> { +async fn store_ticker(store: &WorkerStore, ticker: Ticker) { let id = ticker.symbol.clone(); - store.set_asset(id.clone(), parse_ticker(ticker)?).await?; - debug!("stored data for id {}", id); - Ok(()) + let asset_info = match parse_ticker(ticker) { + Ok(asset_info) => asset_info, + Err(e) => { + error!("failed to parse ticker data for id {}: {}", id, e); + return; + } + }; + + if let Err(e) = store.set_asset_info(asset_info).await { + error!("failed to store ticker data for id {}: {}", id, e); + } else { + debug!("stored data for id {}", id); + } } /// Processes the response from the Bybit API. -async fn process_response(resp: BybitResponse, store: &WorkerStore) { +async fn process_response(resp: BybitResponse, store: &WorkerStore) { match resp { - BybitResponse::PublicTicker(resp) => { - // Assuming MarketData is used as a vector of tickers, update if it's not the case - match store_ticker(store, resp.data).await { - Ok(_) => debug!("saved ticker data"), - Err(e) => error!("failed to save ticker data: {}", e), - } - } + BybitResponse::PublicTicker(resp) => store_ticker(store, resp.data).await, BybitResponse::PublicMessage(resp) => { - debug!("received public message from Bybit: {:?}", resp); + debug!("received public message from Bybit: {:?}", resp) } } } -async fn handle_connection_recv( +async fn handle_connection_recv( recv_result: Result, - connector: &BybitWebSocketConnector, - connection: &mut BybitWebSocketConnection, - store: &WorkerStore, + connector: &WebSocketConnector, + connection: &mut WebSocketConnection, + store: &WorkerStore, ) { match recv_result { Ok(resp) => { diff --git a/bothan-bybit/src/worker/builder.rs b/bothan-bybit/src/worker/builder.rs deleted file mode 100644 index c0d31d18..00000000 --- a/bothan-bybit/src/worker/builder.rs +++ /dev/null @@ -1,98 +0,0 @@ -use std::sync::Arc; - -use tokio::sync::mpsc::channel; - -use crate::api::BybitWebSocketConnector; -use crate::worker::asset_worker::start_asset_worker; -use crate::worker::error::BuildError; -use crate::worker::opts::BybitWorkerBuilderOpts; -use crate::worker::BybitWorker; -use bothan_core::store::WorkerStore; -use bothan_core::worker::AssetWorkerBuilder; - -/// Builds a `BybitWorker` with custom options. -/// Methods can be chained to set the configuration values and the -/// service is constructed by calling the [`build`](BybitWorkerBuilder::build) method. -pub struct BybitWorkerBuilder { - store: WorkerStore, - opts: BybitWorkerBuilderOpts, -} - -impl BybitWorkerBuilder { - /// Returns a new `BybitWorkerBuilder` with the given options. - pub fn new(store: WorkerStore, opts: BybitWorkerBuilderOpts) -> Self { - Self { store, opts } - } - - /// Set the URL for the `BybitWorker`. - /// The default URL is `DEFAULT_URL`. - pub fn with_url>(mut self, url: T) -> Self { - self.opts.url = url.into(); - self - } - - /// Set the internal channel size for the `BybitWorker`. - /// The default size is `DEFAULT_CHANNEL_SIZE`. - pub fn with_internal_ch_size(mut self, size: usize) -> Self { - self.opts.internal_ch_size = size; - self - } - - /// Sets the store for the `BybitWorker`. - /// If not set, the store is created and owned by the worker. - pub fn with_store(mut self, store: WorkerStore) -> Self { - self.store = store; - self - } -} - -#[async_trait::async_trait] -impl<'a> AssetWorkerBuilder<'a> for BybitWorkerBuilder { - type Opts = BybitWorkerBuilderOpts; - type Worker = BybitWorker; - type Error = BuildError; - - /// Returns a new `BybitWorkerBuilder` with the given options. - fn new(store: WorkerStore, opts: Self::Opts) -> Self { - Self { store, opts } - } - - /// Returns the name of the worker. - fn worker_name() -> &'static str { - "bybit" - } - - /// Creates the configured `BybitWorker`. - async fn build(self) -> Result, BuildError> { - let url = self.opts.url; - let ch_size = self.opts.internal_ch_size; - - let connector = BybitWebSocketConnector::new(url); - let connection = connector.connect().await?; - - let (sub_tx, sub_rx) = channel(ch_size); - let (unsub_tx, unsub_rx) = channel(ch_size); - let to_sub = self - .store - .get_query_ids() - .await? - .into_iter() - .collect::>(); - - if !to_sub.is_empty() { - // Unwrap here as the channel is guaranteed to be open - sub_tx.send(to_sub).await.unwrap(); - } - - let worker = Arc::new(BybitWorker::new(connector, self.store, sub_tx, unsub_tx)); - - tokio::spawn(start_asset_worker( - Arc::downgrade(&worker), - connection, - sub_rx, - unsub_rx, - )); - - Ok(worker) - } -} diff --git a/bothan-bybit/src/worker/error.rs b/bothan-bybit/src/worker/error.rs deleted file mode 100644 index 25b5a5a1..00000000 --- a/bothan-bybit/src/worker/error.rs +++ /dev/null @@ -1,24 +0,0 @@ -use crate::api; -use bothan_core::store; -use thiserror::Error; - -#[derive(Error, Debug)] -pub(crate) enum WorkerError { - #[error("failed to parse message")] - Parse(#[from] serde_json::Error), - - #[error("value is not a valid decimal: {0}")] - InvalidDecimal(#[from] rust_decimal::Error), - - #[error("failed to set data to the store: {0}")] - SetFailed(#[from] store::error::Error), -} - -#[derive(Debug, Error)] -pub enum BuildError { - #[error("failed to connect: {0}")] - FailedToConnect(#[from] api::ConnectionError), - - #[error("store error: {0}")] - StoreError(#[from] store::error::Error), -} diff --git a/bothan-bybit/src/worker/opts.rs b/bothan-bybit/src/worker/opts.rs index 0314343d..d18884ca 100644 --- a/bothan-bybit/src/worker/opts.rs +++ b/bothan-bybit/src/worker/opts.rs @@ -6,10 +6,10 @@ use crate::worker::types::DEFAULT_CHANNEL_SIZE; /// Options for configuring the `BybitWorkerBuilder`. /// /// `BybitWorkerBuilderOpts` provides a way to specify custom settings for creating a `BybitWorker`. -/// This struct allows users to set optional parameters such as the WebSocket URL and the internal channel size, -/// which will be used during the construction of the `BybitWorker`. +/// This struct allows users to set optional parameters such as the WebSocket URL and the internal +/// channel size, which will be used during the construction of the `BybitWorker`. #[derive(Clone, Debug, Serialize, Deserialize)] -pub struct BybitWorkerBuilderOpts { +pub struct WorkerOpts { #[serde(default = "default_url")] pub url: String, #[serde(default = "default_internal_ch_size")] @@ -24,7 +24,7 @@ fn default_internal_ch_size() -> usize { DEFAULT_CHANNEL_SIZE } -impl Default for BybitWorkerBuilderOpts { +impl Default for WorkerOpts { fn default() -> Self { Self { url: default_url(), diff --git a/bothan-coinbase/Cargo.toml b/bothan-coinbase/Cargo.toml index 4b33601b..1621b52e 100644 --- a/bothan-coinbase/Cargo.toml +++ b/bothan-coinbase/Cargo.toml @@ -1,13 +1,14 @@ [package] name = "bothan-coinbase" -version = "0.0.1-beta.1" +version = "0.0.1-beta.2" edition.workspace = true license.workspace = true repository.workspace = true [dependencies] +bothan-lib = { workspace = true } + async-trait = { workspace = true } -bothan-core = { workspace = true } chrono = { workspace = true } futures-util = { workspace = true } rust_decimal = { workspace = true } @@ -17,7 +18,6 @@ thiserror = { workspace = true } tokio = { workspace = true } tokio-tungstenite = { workspace = true } tracing = { workspace = true } -tracing-subscriber = { workspace = true } [dev-dependencies] ws-mock = { git = "https://github.com/bandprotocol/ws-mock.git", branch = "master" } diff --git a/bothan-coinbase/examples/coinbase_basic.rs b/bothan-coinbase/examples/coinbase_basic.rs deleted file mode 100644 index 96d8ea90..00000000 --- a/bothan-coinbase/examples/coinbase_basic.rs +++ /dev/null @@ -1,39 +0,0 @@ -use std::time::Duration; - -use tokio::time::sleep; -use tracing_subscriber::fmt::init; - -use bothan_coinbase::{CoinbaseWorkerBuilder, CoinbaseWorkerBuilderOpts}; -use bothan_core::registry::Registry; -use bothan_core::store::SharedStore; -use bothan_core::worker::{AssetWorker, AssetWorkerBuilder}; - -#[tokio::main] -async fn main() { - init(); - let path = std::env::current_dir().unwrap(); - let registry = Registry::default().validate().unwrap(); - let store = SharedStore::new(registry, path.as_path()).await.unwrap(); - - let worker_store = store.create_worker_store(CoinbaseWorkerBuilder::worker_name()); - let opts = CoinbaseWorkerBuilderOpts::default(); - - let worker = CoinbaseWorkerBuilder::new(worker_store, opts) - .build() - .await - .unwrap(); - - worker - .set_query_ids(vec!["BTC-USD".to_string(), "ETH-USD".to_string()]) - .await - .unwrap(); - - sleep(Duration::from_secs(2)).await; - - loop { - let btc_data = worker.get_asset("BTC-USD").await; - let eth_data = worker.get_asset("ETH-USD").await; - println!("{:?} {:?}", btc_data, eth_data); - sleep(Duration::from_secs(5)).await; - } -} diff --git a/bothan-coinbase/src/api.rs b/bothan-coinbase/src/api.rs index 3d074c8d..366a9493 100644 --- a/bothan-coinbase/src/api.rs +++ b/bothan-coinbase/src/api.rs @@ -1,6 +1,6 @@ pub use error::{ConnectionError, MessageError, SendError}; pub use types::channels::ticker::Ticker; -pub use websocket::{CoinbaseWebSocketConnection, CoinbaseWebSocketConnector}; +pub use websocket::{WebSocketConnection, WebSocketConnector}; pub mod error; pub mod types; diff --git a/bothan-coinbase/src/api/websocket.rs b/bothan-coinbase/src/api/websocket.rs index 7f99b638..8495e6b4 100644 --- a/bothan-coinbase/src/api/websocket.rs +++ b/bothan-coinbase/src/api/websocket.rs @@ -1,10 +1,10 @@ use futures_util::stream::{SplitSink, SplitStream}; use futures_util::{SinkExt, StreamExt}; use tokio::net::TcpStream; +use tokio_tungstenite::tungstenite::Message; use tokio_tungstenite::tungstenite::error::Error as TungsteniteError; use tokio_tungstenite::tungstenite::http::StatusCode; -use tokio_tungstenite::tungstenite::Message; -use tokio_tungstenite::{connect_async, MaybeTlsStream, WebSocketStream}; +use tokio_tungstenite::{MaybeTlsStream, WebSocketStream, connect_async}; use tracing::warn; use crate::api::error::{ConnectionError, MessageError, SendError}; @@ -13,18 +13,18 @@ use crate::api::types::request::{Request, RequestType}; use crate::api::types::{CoinbaseResponse, DEFAULT_URL}; /// A connector for establishing a WebSocket connection to the Coinbase API. -pub struct CoinbaseWebSocketConnector { +pub struct WebSocketConnector { url: String, } -impl CoinbaseWebSocketConnector { +impl WebSocketConnector { /// Creates a new `CoinbaseWebSocketConnector`. pub fn new(url: impl Into) -> Self { Self { url: url.into() } } /// Connects to the WebSocket and returns a `CoinbaseWebSocketConnection`. - pub async fn connect(&self) -> Result { + pub async fn connect(&self) -> Result { let (wss, resp) = connect_async(self.url.clone()).await?; let status = resp.status(); @@ -35,11 +35,11 @@ impl CoinbaseWebSocketConnector { )); } - Ok(CoinbaseWebSocketConnection::new(wss)) + Ok(WebSocketConnection::new(wss)) } } -impl Default for CoinbaseWebSocketConnector { +impl Default for WebSocketConnector { /// Creates a default `CoinbaseWebSocketConnector` with the default URL. fn default() -> Self { Self { @@ -49,12 +49,12 @@ impl Default for CoinbaseWebSocketConnector { } /// A connection to the Coinbase WebSocket API. -pub struct CoinbaseWebSocketConnection { +pub struct WebSocketConnection { sender: SplitSink>, Message>, receiver: SplitStream>>, } -impl CoinbaseWebSocketConnection { +impl WebSocketConnection { /// Creates a new `CoinbaseWebSocketConnection`. pub fn new(web_socket_stream: WebSocketStream>) -> Self { let (sender, receiver) = web_socket_stream.split(); @@ -126,10 +126,9 @@ pub(crate) mod test { use tokio::sync::mpsc; use ws_mock::ws_mock_server::{WsMock, WsMockServer}; - use crate::api::types::CoinbaseResponse; - use crate::api::Ticker; - use super::*; + use crate::api::Ticker; + use crate::api::types::CoinbaseResponse; pub(crate) async fn setup_mock_server() -> WsMockServer { WsMockServer::start().await @@ -139,7 +138,7 @@ pub(crate) mod test { async fn test_recv_ticker() { // Set up the mock server and the WebSocket connector. let server = setup_mock_server().await; - let connector = CoinbaseWebSocketConnector::new(server.uri().await); + let connector = WebSocketConnector::new(server.uri().await); let (mpsc_send, mpsc_recv) = mpsc::channel::(32); // Create a mock ticker response. @@ -183,7 +182,7 @@ pub(crate) mod test { async fn test_recv_close() { // Set up the mock server and the WebSocket connector. let server = setup_mock_server().await; - let connector = CoinbaseWebSocketConnector::new(server.uri().await); + let connector = WebSocketConnector::new(server.uri().await); let (mpsc_send, mpsc_recv) = mpsc::channel::(32); // Mount the mock WebSocket server and send a close message. diff --git a/bothan-coinbase/src/lib.rs b/bothan-coinbase/src/lib.rs index b60477b3..ac1901ea 100644 --- a/bothan-coinbase/src/lib.rs +++ b/bothan-coinbase/src/lib.rs @@ -1,8 +1,6 @@ -pub use api::websocket::{CoinbaseWebSocketConnection, CoinbaseWebSocketConnector}; -pub use worker::builder::CoinbaseWorkerBuilder; -pub use worker::error::BuildError; -pub use worker::opts::CoinbaseWorkerBuilderOpts; -pub use worker::CoinbaseWorker; +pub use api::websocket::{WebSocketConnection, WebSocketConnector}; +pub use worker::Worker; +pub use worker::opts::WorkerOpts; pub mod api; pub mod worker; diff --git a/bothan-coinbase/src/worker.rs b/bothan-coinbase/src/worker.rs index 61ae3b3f..e56b1b1b 100644 --- a/bothan-coinbase/src/worker.rs +++ b/bothan-coinbase/src/worker.rs @@ -1,68 +1,90 @@ -use tokio::sync::mpsc::Sender; +use std::collections::HashSet; +use std::sync::Arc; -use bothan_core::store::error::Error as StoreError; -use bothan_core::store::WorkerStore; -use bothan_core::worker::{AssetState, AssetWorker, SetQueryIDError}; +use async_trait::async_trait; +use bothan_lib::store::{Store, WorkerStore}; +use bothan_lib::types::AssetState; +use bothan_lib::worker::AssetWorker; +use bothan_lib::worker::error::AssetWorkerError; +use tokio::sync::mpsc::{Sender, channel}; -use crate::api::websocket::CoinbaseWebSocketConnector; +use crate::WorkerOpts; +use crate::api::websocket::WebSocketConnector; +use crate::worker::asset_worker::start_asset_worker; mod asset_worker; -pub mod builder; -pub(crate) mod error; pub mod opts; mod types; -/// A worker that fetches and stores the asset information from Coinbase's API. -pub struct CoinbaseWorker { - connector: CoinbaseWebSocketConnector, - store: WorkerStore, - subscribe_tx: Sender>, - unsubscribe_tx: Sender>, +const WORKER_NAME: &str = "coinbase"; + +pub struct Worker { + inner: Arc>, } -impl CoinbaseWorker { - /// Create a new worker with the specified connector, store and channels. - pub fn new( - connector: CoinbaseWebSocketConnector, - store: WorkerStore, - subscribe_tx: Sender>, - unsubscribe_tx: Sender>, - ) -> Self { - Self { - connector, - store, - subscribe_tx, - unsubscribe_tx, +#[async_trait] +impl AssetWorker for Worker { + type Opts = WorkerOpts; + + fn name(&self) -> &'static str { + WORKER_NAME + } + + async fn build(opts: Self::Opts, store: &S) -> Result { + let url = opts.url; + let ch_size = opts.internal_ch_size; + + let connector = WebSocketConnector::new(url); + let connection = connector.connect().await?; + + let (sub_tx, sub_rx) = channel(ch_size); + let (unsub_tx, unsub_rx) = channel(ch_size); + + let worker_store = WorkerStore::new(store, WORKER_NAME); + let to_sub = worker_store + .get_query_ids() + .await? + .into_iter() + .collect::>(); + + if !to_sub.is_empty() { + sub_tx.send(to_sub).await?; } + + let inner = Arc::new(InnerWorker { + connector, + store: worker_store.clone(), + subscribe_tx: sub_tx, + unsubscribe_tx: unsub_tx, + }); + + tokio::spawn(start_asset_worker( + Arc::downgrade(&inner), + connection, + sub_rx, + unsub_rx, + )); + + Ok(Worker { inner }) } -} -#[async_trait::async_trait] -impl AssetWorker for CoinbaseWorker { - /// Fetches the AssetStatus for the given cryptocurrency id. - async fn get_asset(&self, id: &str) -> Result { - self.store.get_asset(&id).await + async fn get_asset(&self, id: &str) -> Result { + Ok(self.inner.store.get_asset(id).await?) } - /// Sets the specified cryptocurrency IDs to the query. If the ids are already in the query set, - /// it will not be resubscribed. - async fn set_query_ids(&self, ids: Vec) -> Result<(), SetQueryIDError> { - let (to_sub, to_unsub) = self - .store - .set_query_ids(ids) - .await - .map_err(|e| SetQueryIDError::new(e.to_string()))?; - - self.subscribe_tx - .send(to_sub) - .await - .map_err(|e| SetQueryIDError::new(e.to_string()))?; - - self.unsubscribe_tx - .send(to_unsub) - .await - .map_err(|e| SetQueryIDError::new(e.to_string()))?; + async fn set_query_ids(&self, ids: HashSet) -> Result<(), AssetWorkerError> { + let diff = self.inner.store.compute_query_id_difference(ids).await?; + + self.inner.subscribe_tx.send(diff.added).await?; + self.inner.unsubscribe_tx.send(diff.removed).await?; Ok(()) } } + +struct InnerWorker { + connector: WebSocketConnector, + store: WorkerStore, + subscribe_tx: Sender>, + unsubscribe_tx: Sender>, +} diff --git a/bothan-coinbase/src/worker/asset_worker.rs b/bothan-coinbase/src/worker/asset_worker.rs index 866eeb73..039d81b9 100644 --- a/bothan-coinbase/src/worker/asset_worker.rs +++ b/bothan-coinbase/src/worker/asset_worker.rs @@ -1,25 +1,23 @@ use std::sync::Weak; +use bothan_lib::store::{Store, WorkerStore}; +use bothan_lib::types::AssetInfo; use rust_decimal::Decimal; use tokio::select; use tokio::sync::mpsc::Receiver; use tokio::time::{sleep, timeout}; use tracing::{debug, error, info, warn}; -use bothan_core::store::WorkerStore; -use bothan_core::types::AssetInfo; - use crate::api::error::{MessageError, SendError}; -use crate::api::types::channels::Channel; use crate::api::types::CoinbaseResponse; -use crate::api::{CoinbaseWebSocketConnection, CoinbaseWebSocketConnector, Ticker}; -use crate::worker::error::WorkerError; +use crate::api::types::channels::Channel; +use crate::api::{Ticker, WebSocketConnection, WebSocketConnector}; +use crate::worker::InnerWorker; use crate::worker::types::{DEFAULT_TIMEOUT, RECONNECT_BUFFER}; -use crate::worker::CoinbaseWorker; -pub(crate) async fn start_asset_worker( - worker: Weak, - mut connection: CoinbaseWebSocketConnection, +pub(crate) async fn start_asset_worker( + inner_worker: Weak>, + mut connection: WebSocketConnection, mut subscribe_rx: Receiver>, mut unsubscribe_rx: Receiver>, ) { @@ -28,7 +26,7 @@ pub(crate) async fn start_asset_worker( Some(ids) = subscribe_rx.recv() => handle_subscribe_recv(ids, &mut connection).await, Some(ids) = unsubscribe_rx.recv() => handle_unsubscribe_recv(ids, &mut connection).await, result = timeout(DEFAULT_TIMEOUT, connection.next()) => { - if let Some(worker) = worker.upgrade() { + if let Some(worker) = inner_worker.upgrade() { match result { Err(_) => handle_reconnect(&worker.connector, &mut connection, &worker.store).await, Ok(coinbase_result) => handle_connection_recv(coinbase_result, &worker.connector, &mut connection, &worker.store).await, @@ -50,10 +48,7 @@ pub(crate) async fn start_asset_worker( debug!("asset worker has been dropped, stopping asset worker"); } -async fn subscribe( - ids: &[String], - connection: &mut CoinbaseWebSocketConnection, -) -> Result<(), SendError> { +async fn subscribe(ids: &[String], connection: &mut WebSocketConnection) -> Result<(), SendError> { if !ids.is_empty() { let ids_vec = ids.iter().map(|s| s.as_str()).collect::>(); connection @@ -64,7 +59,7 @@ async fn subscribe( Ok(()) } -async fn handle_subscribe_recv(ids: Vec, connection: &mut CoinbaseWebSocketConnection) { +async fn handle_subscribe_recv(ids: Vec, connection: &mut WebSocketConnection) { if let Err(e) = subscribe(&ids, connection).await { error!("failed to subscribe to ids {:?}: {}", ids, e); } else { @@ -74,7 +69,7 @@ async fn handle_subscribe_recv(ids: Vec, connection: &mut CoinbaseWebSoc async fn unsubscribe( ids: &[String], - connection: &mut CoinbaseWebSocketConnection, + connection: &mut WebSocketConnection, ) -> Result<(), SendError> { if !ids.is_empty() { connection @@ -88,7 +83,7 @@ async fn unsubscribe( Ok(()) } -async fn handle_unsubscribe_recv(ids: Vec, connection: &mut CoinbaseWebSocketConnection) { +async fn handle_unsubscribe_recv(ids: Vec, connection: &mut WebSocketConnection) { if let Err(e) = unsubscribe(&ids, connection).await { error!("failed to unsubscribe to ids {:?}: {}", ids, e); } else { @@ -96,10 +91,10 @@ async fn handle_unsubscribe_recv(ids: Vec, connection: &mut CoinbaseWebS } } -async fn handle_reconnect( - connector: &CoinbaseWebSocketConnector, - connection: &mut CoinbaseWebSocketConnection, - query_ids: &WorkerStore, +async fn handle_reconnect( + connector: &WebSocketConnector, + connection: &mut WebSocketConnection, + query_ids: &WorkerStore, ) { let mut retry_count: usize = 1; loop { @@ -134,27 +129,32 @@ async fn handle_reconnect( } } -fn parse_ticker(ticker: &Ticker) -> Result { +fn parse_ticker(ticker: &Ticker) -> Result { let id = ticker.product_id.clone(); let price_value = Decimal::from_str_exact(&ticker.price)?; let timestamp = chrono::Utc::now().timestamp(); Ok(AssetInfo::new(id, price_value, timestamp)) } -async fn store_ticker(store: &WorkerStore, ticker: &Ticker) -> Result<(), WorkerError> { +async fn store_ticker(store: &WorkerStore, ticker: &Ticker) { let id = ticker.product_id.clone(); - store.set_asset(id.clone(), parse_ticker(ticker)?).await?; - debug!("stored data for id {}", id); - Ok(()) + let asset_info = match parse_ticker(ticker) { + Ok(ticker) => ticker, + Err(e) => { + error!("failed to parse ticker: {}", e); + return; + } + }; + match store.set_asset_info(asset_info).await { + Ok(_) => debug!("stored data for id {}", id), + Err(e) => error!("failed to save data: {}", e), + } } /// Processes the response from the Coinbase API. -async fn process_response(resp: CoinbaseResponse, store: &WorkerStore) { +async fn process_response(resp: CoinbaseResponse, store: &WorkerStore) { match resp { - CoinbaseResponse::Ticker(ticker) => match store_ticker(store, &ticker).await { - Ok(_) => debug!("saved data"), - Err(e) => error!("failed to save data: {}", e), - }, + CoinbaseResponse::Ticker(ticker) => store_ticker(store, &ticker).await, CoinbaseResponse::Ping => debug!("received ping"), CoinbaseResponse::Subscriptions(_) => { info!("received request response"); @@ -162,11 +162,11 @@ async fn process_response(resp: CoinbaseResponse, store: &WorkerStore) { } } -async fn handle_connection_recv( +async fn handle_connection_recv( recv_result: Result, - connector: &CoinbaseWebSocketConnector, - connection: &mut CoinbaseWebSocketConnection, - store: &WorkerStore, + connector: &WebSocketConnector, + connection: &mut WebSocketConnection, + store: &WorkerStore, ) { match recv_result { Ok(resp) => { diff --git a/bothan-coinbase/src/worker/builder.rs b/bothan-coinbase/src/worker/builder.rs deleted file mode 100644 index 88343fca..00000000 --- a/bothan-coinbase/src/worker/builder.rs +++ /dev/null @@ -1,99 +0,0 @@ -use std::sync::Arc; - -use tokio::sync::mpsc::channel; - -use crate::api::CoinbaseWebSocketConnector; -use crate::worker::asset_worker::start_asset_worker; -use crate::worker::error::BuildError; -use crate::worker::opts::CoinbaseWorkerBuilderOpts; -use crate::worker::CoinbaseWorker; -use bothan_core::store::WorkerStore; -use bothan_core::worker::AssetWorkerBuilder; - -/// Builds a `CoinbaseWorker` with custom options. -/// Methods can be chained to set the configuration values and the -/// service is constructed by calling the [`build`](CoinbaseWorkerBuilder::build) method. -pub struct CoinbaseWorkerBuilder { - store: WorkerStore, - opts: CoinbaseWorkerBuilderOpts, -} - -impl CoinbaseWorkerBuilder { - /// Returns a new `CoinbaseWorkerBuilder` with the given options. - pub fn new(store: WorkerStore, opts: CoinbaseWorkerBuilderOpts) -> Self { - Self { store, opts } - } - - /// Set the URL for the `CoinbaseWorker`. - /// The default URL is `DEFAULT_URL`. - pub fn with_url>(mut self, url: T) -> Self { - self.opts.url = url.into(); - self - } - - /// Set the internal channel size for the `CoinbaseWorker`. - /// The default size is `DEFAULT_CHANNEL_SIZE`. - pub fn with_internal_ch_size(mut self, size: usize) -> Self { - self.opts.internal_ch_size = size; - self - } - - /// Sets the store for the `CoinbaseWorker`. - /// If not set, the store is created and owned by the worker. - pub fn with_store(mut self, store: WorkerStore) -> Self { - self.store = store; - self - } -} - -#[async_trait::async_trait] -impl<'a> AssetWorkerBuilder<'a> for CoinbaseWorkerBuilder { - type Opts = CoinbaseWorkerBuilderOpts; - type Worker = CoinbaseWorker; - type Error = BuildError; - - /// Returns a new `CoinbaseWorkerBuilder` with the given options. - fn new(store: WorkerStore, opts: Self::Opts) -> Self { - Self { store, opts } - } - - /// Returns the name of the worker. - fn worker_name() -> &'static str { - "coinbase" - } - - /// Creates the configured `CoinbaseWorker`. - async fn build(self) -> Result, BuildError> { - let url = self.opts.url; - let ch_size = self.opts.internal_ch_size; - - let connector = CoinbaseWebSocketConnector::new(url); - let connection = connector.connect().await?; - - let (sub_tx, sub_rx) = channel(ch_size); - let (unsub_tx, unsub_rx) = channel(ch_size); - - let to_sub = self - .store - .get_query_ids() - .await? - .into_iter() - .collect::>(); - - if !to_sub.is_empty() { - // Unwrap here as the channel is guaranteed to be open - sub_tx.send(to_sub).await.unwrap(); - } - - let worker = Arc::new(CoinbaseWorker::new(connector, self.store, sub_tx, unsub_tx)); - - tokio::spawn(start_asset_worker( - Arc::downgrade(&worker), - connection, - sub_rx, - unsub_rx, - )); - - Ok(worker) - } -} diff --git a/bothan-coinbase/src/worker/error.rs b/bothan-coinbase/src/worker/error.rs deleted file mode 100644 index ff6cdc33..00000000 --- a/bothan-coinbase/src/worker/error.rs +++ /dev/null @@ -1,21 +0,0 @@ -use crate::api; -use bothan_core::store; -use thiserror::Error; - -#[derive(Error, Debug)] -pub(crate) enum WorkerError { - #[error("value is not a valid decimal: {0}")] - InvalidDecimal(#[from] rust_decimal::Error), - - #[error("failed to set data to the store: {0}")] - SetFailed(#[from] store::error::Error), -} - -#[derive(Debug, Error)] -pub enum BuildError { - #[error("failed to connect: {0}")] - FailedToConnect(#[from] api::ConnectionError), - - #[error("store error: {0}")] - StoreError(#[from] store::error::Error), -} diff --git a/bothan-coinbase/src/worker/opts.rs b/bothan-coinbase/src/worker/opts.rs index 09202aae..5e873bbe 100644 --- a/bothan-coinbase/src/worker/opts.rs +++ b/bothan-coinbase/src/worker/opts.rs @@ -5,11 +5,12 @@ use crate::worker::types::DEFAULT_CHANNEL_SIZE; /// Options for configuring the `CoinbaseWorkerBuilder`. /// -/// `CoinbaseWorkerBuilderOpts` provides a way to specify custom settings for creating a `CoinbaseWorker`. -/// This struct allows users to set optional parameters such as the WebSocket URL and the internal channel size, -/// which will be used during the construction of the `CoinbaseWorker`. +/// `CoinbaseWorkerBuilderOpts` provides a way to specify custom settings for creating a +/// `CoinbaseWorker`. This struct allows users to set optional parameters such as the WebSocket URL +/// and the internal channel size, which will be used during the construction of the +/// `CoinbaseWorker`. #[derive(Clone, Debug, Serialize, Deserialize)] -pub struct CoinbaseWorkerBuilderOpts { +pub struct WorkerOpts { #[serde(default = "default_url")] pub url: String, #[serde(default = "default_internal_ch_size")] @@ -24,7 +25,7 @@ fn default_internal_ch_size() -> usize { DEFAULT_CHANNEL_SIZE } -impl Default for CoinbaseWorkerBuilderOpts { +impl Default for WorkerOpts { fn default() -> Self { Self { url: default_url(), diff --git a/bothan-coingecko/Cargo.toml b/bothan-coingecko/Cargo.toml index 3a274dc5..dcf11025 100644 --- a/bothan-coingecko/Cargo.toml +++ b/bothan-coingecko/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "bothan-coingecko" -version = "0.0.1-beta.1" +version = "0.0.1-beta.2" edition.workspace = true license.workspace = true repository.workspace = true @@ -8,17 +8,15 @@ repository.workspace = true # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +bothan-lib = { workspace = true } + async-trait = { workspace = true } -bothan-core = { workspace = true } -chrono = { workspace = true } humantime-serde = { workspace = true } reqwest = { workspace = true } rust_decimal = { workspace = true } serde = { workspace = true } thiserror = { workspace = true } -tokio = { workspace = true } -tracing = { workspace = true } -tracing-subscriber = { workspace = true } +tokio = { workspace = true, features = ["rt", "rt-multi-thread", "macros"] } url = { workspace = true } [dev-dependencies] diff --git a/bothan-coingecko/examples/coingecko_basic.rs b/bothan-coingecko/examples/coingecko_basic.rs deleted file mode 100644 index c6d8f972..00000000 --- a/bothan-coingecko/examples/coingecko_basic.rs +++ /dev/null @@ -1,38 +0,0 @@ -use std::time::Duration; - -use tokio::time::sleep; -use tracing_subscriber::fmt::init; - -use bothan_coingecko::{CoinGeckoWorkerBuilder, CoinGeckoWorkerBuilderOpts}; -use bothan_core::registry::Registry; -use bothan_core::store::SharedStore; -use bothan_core::worker::{AssetWorker, AssetWorkerBuilder}; - -#[tokio::main] -async fn main() { - init(); - let path = std::env::current_dir().unwrap(); - let store = SharedStore::new(Registry::default().validate().unwrap(), path.as_path()) - .await - .unwrap(); - let worker_store = store.create_worker_store(CoinGeckoWorkerBuilder::worker_name()); - let opts = CoinGeckoWorkerBuilderOpts::default(); - - let worker = CoinGeckoWorkerBuilder::new(worker_store, opts) - .build() - .await - .unwrap(); - - worker - .set_query_ids(vec!["bitcoin".to_string()]) - .await - .unwrap(); - - sleep(Duration::from_secs(2)).await; - - loop { - let data = worker.get_asset("bitcoin").await; - println!("{:?}", data); - tokio::time::sleep(Duration::from_secs(5)).await; - } -} diff --git a/bothan-coingecko/src/api.rs b/bothan-coingecko/src/api.rs index e52e69c5..6771bcac 100644 --- a/bothan-coingecko/src/api.rs +++ b/bothan-coingecko/src/api.rs @@ -1,5 +1,5 @@ -pub use builder::CoinGeckoRestAPIBuilder; -pub use rest::CoinGeckoRestAPI; +pub use builder::RestApiBuilder; +pub use rest::RestApi; pub mod builder; pub mod error; diff --git a/bothan-coingecko/src/api/builder.rs b/bothan-coingecko/src/api/builder.rs index 84bd36cf..2119c9a5 100644 --- a/bothan-coingecko/src/api/builder.rs +++ b/bothan-coingecko/src/api/builder.rs @@ -1,23 +1,22 @@ -use reqwest::header::{HeaderMap, HeaderValue}; use reqwest::ClientBuilder; +use reqwest::header::{HeaderMap, HeaderValue}; use url::Url; +use crate::api::RestApi; use crate::api::error::BuildError; use crate::api::types::{API_KEY_HEADER, DEFAULT_PRO_URL, DEFAULT_URL, DEFAULT_USER_AGENT}; -use crate::api::CoinGeckoRestAPI; /// Builder for creating instances of `CoinGeckoRestAPI`. /// Methods can be chained to set the configuration values and the -/// service is constructed by calling the [`build`](CoinGeckoRestAPIBuilder::build) method. +/// service is constructed by calling the [`build`](RestApiBuilder::build) method. /// /// # Example /// ``` -/// use bothan_coingecko::api::CoinGeckoRestAPIBuilder; -/// +/// use bothan_coingecko::api::RestApiBuilder; /// /// #[tokio::main] /// async fn main() { -/// let mut api = CoinGeckoRestAPIBuilder::default() +/// let mut api = RestApiBuilder::default() /// .with_url("https://api.coingecko.com/api/v3") /// .with_api_key("your_api_key") /// .with_user_agent("your_user_agent") @@ -27,26 +26,23 @@ use crate::api::CoinGeckoRestAPI; /// // use api ... /// } /// ``` -pub struct CoinGeckoRestAPIBuilder { +pub struct RestApiBuilder { + url: Option, user_agent: String, - url: String, api_key: Option, - // Flag to determine if the URL was modified by the user. - mod_url: bool, } -impl CoinGeckoRestAPIBuilder { - pub fn new(user_agent: T, url: U, api_key: Option) -> Self +impl RestApiBuilder { + pub fn new(url: Option, user_agent: U, api_key: Option) -> Self where T: Into, U: Into, V: Into, { - CoinGeckoRestAPIBuilder { + RestApiBuilder { + url: url.map(|v| v.into()), user_agent: user_agent.into(), - url: url.into(), api_key: api_key.map(Into::into), - mod_url: true, } } @@ -54,8 +50,7 @@ impl CoinGeckoRestAPIBuilder { /// If not specified, the default URL is `DEFAULT_URL` when no API key is provided, /// and `DEFAULT_PRO_URL` when an API key is provided. pub fn with_url>(mut self, url: T) -> Self { - self.url = url.into(); - self.mod_url = true; + self.url = Some(url.into()); self } @@ -74,48 +69,37 @@ impl CoinGeckoRestAPIBuilder { } /// Builds the `CoinGeckoRestAPI` instance. - pub fn build(self) -> Result { + pub fn build(self) -> Result { let mut headers = HeaderMap::new(); - let agent = match HeaderValue::from_str(&self.user_agent) { - Ok(agent) => agent, - Err(_) => return Err(BuildError::InvalidHeaderValue(self.user_agent)), - }; + let agent = HeaderValue::from_str(&self.user_agent)?; headers.insert("User-Agent", agent); - let url = match (&self.mod_url, &self.api_key) { - (true, _) => &self.url, - (false, Some(_)) => DEFAULT_PRO_URL, - (false, None) => DEFAULT_URL, + let url = match (&self.url, &self.api_key) { + (Some(url), _) => url, + (None, Some(_)) => DEFAULT_PRO_URL, + (None, None) => DEFAULT_URL, }; let parsed_url = Url::parse(url)?; if let Some(key) = &self.api_key { - let mut api_key = match HeaderValue::from_str(key) { - Ok(key) => key, - Err(_) => return Err(BuildError::InvalidHeaderValue(key.clone())), - }; + let mut api_key = HeaderValue::from_str(key)?; api_key.set_sensitive(true); headers.insert(API_KEY_HEADER, api_key); } - let client = ClientBuilder::new() - .default_headers(headers) - .build() - .map_err(|e| BuildError::BuildFailed(e.to_string()))?; - - Ok(CoinGeckoRestAPI::new(parsed_url, client)) + let client = ClientBuilder::new().default_headers(headers).build()?; + Ok(RestApi::new(parsed_url, client)) } } -impl Default for CoinGeckoRestAPIBuilder { +impl Default for RestApiBuilder { /// Creates a default `CoinGeckoRestAPIBuilder` instance with default values. fn default() -> Self { - CoinGeckoRestAPIBuilder { - url: DEFAULT_URL.into(), + RestApiBuilder { + url: None, api_key: None, user_agent: DEFAULT_USER_AGENT.into(), - mod_url: false, } } } diff --git a/bothan-coingecko/src/api/error.rs b/bothan-coingecko/src/api/error.rs index 9bec7a99..f6f438fa 100644 --- a/bothan-coingecko/src/api/error.rs +++ b/bothan-coingecko/src/api/error.rs @@ -1,23 +1,22 @@ -#[derive(Clone, Debug, PartialEq, thiserror::Error)] -pub enum BuildError { - #[error("invalid header value: {0}")] - InvalidHeaderValue(String), +use thiserror::Error; +#[derive(Debug, Error)] +pub enum BuildError { #[error("invalid url")] InvalidURL(#[from] url::ParseError), - #[error("build failed with error: {0}")] - BuildFailed(String), -} + #[error("invalid header value")] + InvalidHeaderValue(#[from] reqwest::header::InvalidHeaderValue), -#[derive(Clone, Debug, PartialEq, thiserror::Error)] -pub enum SendError { - #[error("failed to send request with error: {0}")] - FailedRequest(String), + #[error("failed to build with error: {0}")] + FailedToBuild(#[from] reqwest::Error), +} - #[error("received non-2xx http status: {0}")] - UnsuccessfulResponse(reqwest::StatusCode), +#[derive(Debug, Error)] +pub enum ProviderError { + #[error("failed to fetch tickers: {0}")] + RequestError(#[from] reqwest::Error), - #[error("failed to parse response with error: {0}")] - ParseResponseFailed(String), + #[error("value contains nan")] + InvalidValue, } diff --git a/bothan-coingecko/src/api/rest.rs b/bothan-coingecko/src/api/rest.rs index 89614f3b..ee0eefae 100644 --- a/bothan-coingecko/src/api/rest.rs +++ b/bothan-coingecko/src/api/rest.rs @@ -1,25 +1,28 @@ use std::collections::HashMap; +use bothan_lib::types::AssetInfo; +use bothan_lib::worker::rest::AssetInfoProvider; use reqwest::{Client, RequestBuilder, Url}; +use rust_decimal::Decimal; use serde::de::DeserializeOwned; -use crate::api::error::SendError; +use crate::api::error::ProviderError; use crate::api::types::{Coin, Price}; /// A client for interacting with the CoinGecko REST API. -pub struct CoinGeckoRestAPI { +pub struct RestApi { url: Url, client: Client, } -impl CoinGeckoRestAPI { +impl RestApi { /// Creates a new instance of `CoinGeckoRestAPI`. pub fn new(url: Url, client: Client) -> Self { Self { url, client } } /// Retrieves a list of coins from the CoinGecko API. - pub async fn get_coins_list(&self) -> Result, SendError> { + pub async fn get_coins_list(&self) -> Result, reqwest::Error> { let url = format!("{}coins/list", self.url); let builder = self.client.get(url); @@ -30,7 +33,7 @@ impl CoinGeckoRestAPI { pub async fn get_simple_price_usd>( &self, ids: &[T], - ) -> Result, SendError> { + ) -> Result, reqwest::Error> { let url = format!("{}simple/price", self.url); let joined_ids = ids .iter() @@ -51,35 +54,41 @@ impl CoinGeckoRestAPI { } } -async fn request(request_builder: RequestBuilder) -> Result { - let response = request_builder - .send() - .await - .map_err(|e| SendError::FailedRequest(e.to_string()))?; +async fn request( + request_builder: RequestBuilder, +) -> Result { + let response = request_builder.send().await?.error_for_status()?; - let status = response.status(); - if !status.is_success() { - return Err(SendError::UnsuccessfulResponse(status)); - } + response.json::().await +} - response - .json::() - .await - .map_err(|e| SendError::ParseResponseFailed(e.to_string())) +#[async_trait::async_trait] +impl AssetInfoProvider for RestApi { + type Error = ProviderError; + + async fn get_asset_info(&self, ids: &[String]) -> Result, Self::Error> { + self.get_simple_price_usd(ids) + .await? + .into_iter() + .map(|(id, p)| { + let price = Decimal::from_f64_retain(p.usd).ok_or(ProviderError::InvalidValue)?; + Ok(AssetInfo::new(id, price, p.last_updated_at)) + }) + .collect() + } } #[cfg(test)] -pub(crate) mod test { +mod test { use mockito::{Matcher, Mock, Server, ServerGuard}; - use crate::api::CoinGeckoRestAPIBuilder; - use super::*; + use crate::api::RestApiBuilder; - pub(crate) async fn setup() -> (ServerGuard, CoinGeckoRestAPI) { + async fn setup() -> (ServerGuard, RestApi) { let server = Server::new_async().await; - let api = CoinGeckoRestAPIBuilder::default() + let api = RestApiBuilder::default() .with_url(server.url()) .build() .unwrap(); @@ -87,7 +96,7 @@ pub(crate) mod test { (server, api) } - pub(crate) trait MockCoinGecko { + trait MockCoinGecko { fn set_successful_coin_list(&mut self, coin_list: &[Coin]) -> Mock; fn set_failed_coin_list(&mut self) -> Mock; fn set_successful_simple_price( @@ -205,7 +214,7 @@ pub(crate) mod test { let result = client.get_simple_price_usd(&["bitcoin"]).await; mocks.assert(); - assert_eq!(result, Ok(prices)); + assert_eq!(result.unwrap(), prices); } #[tokio::test] @@ -226,7 +235,7 @@ pub(crate) mod test { let result = client.get_simple_price_usd(ids).await; mocks.assert(); - assert_eq!(result, Ok(prices)); + assert_eq!(result.unwrap(), prices); } #[tokio::test] @@ -240,9 +249,7 @@ pub(crate) mod test { mock.assert(); - let expected_err = - SendError::ParseResponseFailed("error decoding response body".to_string()); - assert_eq!(result, Err(expected_err)); + assert!(result.is_err()); } #[tokio::test] diff --git a/bothan-coingecko/src/api/types.rs b/bothan-coingecko/src/api/types.rs index 0d9c1bfb..3d8f7106 100644 --- a/bothan-coingecko/src/api/types.rs +++ b/bothan-coingecko/src/api/types.rs @@ -9,7 +9,7 @@ pub(crate) const DEFAULT_PRO_URL: &str = "https://pro-api.coingecko.com/api/v3/" pub(crate) const API_KEY_HEADER: &str = "x-cg-pro-api-key"; /// Represents a coin with basic information. -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct Coin { pub id: String, pub symbol: String, @@ -17,7 +17,7 @@ pub struct Coin { } /// Represents market data for a coin. -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct Price { pub usd: f64, pub last_updated_at: i64, diff --git a/bothan-coingecko/src/lib.rs b/bothan-coingecko/src/lib.rs index 0f063b92..e31a5354 100644 --- a/bothan-coingecko/src/lib.rs +++ b/bothan-coingecko/src/lib.rs @@ -1,6 +1,5 @@ -pub use worker::builder::CoinGeckoWorkerBuilder; -pub use worker::opts::CoinGeckoWorkerBuilderOpts; -pub use worker::CoinGeckoWorker; +pub use worker::Worker; +pub use worker::opts::WorkerOpts; pub mod api; pub mod worker; diff --git a/bothan-coingecko/src/worker.rs b/bothan-coingecko/src/worker.rs index e0322261..fb9ec5e2 100644 --- a/bothan-coingecko/src/worker.rs +++ b/bothan-coingecko/src/worker.rs @@ -1,41 +1,59 @@ -use bothan_core::store::error::Error as StoreError; -use bothan_core::store::WorkerStore; -use bothan_core::worker::{AssetState, AssetWorker, SetQueryIDError}; +use std::collections::HashSet; +use std::sync::{Arc, Weak}; -use crate::api::CoinGeckoRestAPI; +use bothan_lib::store::{Store, WorkerStore}; +use bothan_lib::types::AssetState; +use bothan_lib::worker::AssetWorker; +use bothan_lib::worker::error::AssetWorkerError; +use bothan_lib::worker::rest::{AssetInfoProvider, start_polling}; + +use crate::WorkerOpts; +use crate::api::error::ProviderError; +use crate::api::{RestApi, RestApiBuilder}; -mod asset_worker; -pub mod builder; -pub mod error; pub mod opts; -pub mod types; -/// A worker that fetches and stores the asset information from CoinGecko's API. -pub struct CoinGeckoWorker { - api: CoinGeckoRestAPI, - store: WorkerStore, -} +const WORKER_NAME: &str = "coingecko"; -impl CoinGeckoWorker { - /// Create a new worker with the specified api and store. - pub fn new(api: CoinGeckoRestAPI, store: WorkerStore) -> Self { - Self { api, store } - } +pub struct Worker { + // The `api` is owned by this struct to ensure that any weak references + // are properly cleaned up when the worker is dropped. + #[allow(dead_code)] + api: Arc, + store: WorkerStore, } #[async_trait::async_trait] -impl AssetWorker for CoinGeckoWorker { - /// Fetches the AssetStatus for the given cryptocurrency ids. - async fn get_asset(&self, id: &str) -> Result { - self.store.get_asset(&id).await +impl AssetWorker for Worker { + type Opts = WorkerOpts; + + fn name(&self) -> &'static str { + WORKER_NAME + } + + async fn build(opts: Self::Opts, store: &S) -> Result { + let api = Arc::new(RestApiBuilder::new(opts.url, opts.user_agent, opts.api_key).build()?); + + let worker_store = WorkerStore::new(store, WORKER_NAME); + + tokio::spawn(start_polling( + opts.update_interval, + Arc::downgrade(&api) as Weak>, + worker_store.clone(), + )); + + Ok(Worker { + api, + store: worker_store, + }) + } + + async fn get_asset(&self, id: &str) -> Result { + Ok(self.store.get_asset(id).await?) } - /// Adds the specified cryptocurrency IDs to the query set. - async fn set_query_ids(&self, ids: Vec) -> Result<(), SetQueryIDError> { - self.store - .set_query_ids(ids) - .await - .map_err(|e| SetQueryIDError::new(e.to_string()))?; + async fn set_query_ids(&self, ids: HashSet) -> Result<(), AssetWorkerError> { + self.store.set_query_ids(ids).await?; Ok(()) } } diff --git a/bothan-coingecko/src/worker/asset_worker.rs b/bothan-coingecko/src/worker/asset_worker.rs deleted file mode 100644 index 908a2115..00000000 --- a/bothan-coingecko/src/worker/asset_worker.rs +++ /dev/null @@ -1,124 +0,0 @@ -use std::sync::Weak; -use std::time::Duration; - -use rust_decimal::prelude::FromPrimitive; -use rust_decimal::Decimal; -use tokio::time::{interval, timeout}; -use tracing::{debug, error, warn}; - -use bothan_core::store::WorkerStore; -use bothan_core::types::AssetInfo; - -use crate::api::rest::CoinGeckoRestAPI; -use crate::api::types::Price; -use crate::worker::error::ParseError; -use crate::worker::CoinGeckoWorker; - -pub(crate) fn start_asset_worker(weak_worker: Weak, update_interval: Duration) { - let mut interval = interval(update_interval); - tokio::spawn(async move { - while let Some(worker) = weak_worker.upgrade() { - interval.tick().await; - - let ids = match worker.store.get_query_ids().await { - Ok(ids) => ids.into_iter().collect::>(), - Err(e) => { - error!("failed to get query ids with error: {}", e); - Vec::new() - } - }; - - if ids.is_empty() { - debug!("no ids to update, skipping update"); - continue; - } - - let result = timeout( - interval.period(), - update_asset_info(&worker.store, &worker.api, &ids), - ) - .await; - - if result.is_err() { - warn!("updating interval exceeded timeout") - } - } - - debug!("asset worker has been dropped, stopping asset worker"); - }); -} - -async fn update_asset_info>(store: &WorkerStore, api: &CoinGeckoRestAPI, ids: &[T]) { - match api.get_simple_price_usd(ids).await { - Ok(markets) => { - // Sanity check to assure that the number of markets returned is less than the number of ids - if markets.len() <= ids.len() { - let to_set = markets - .into_iter() - .filter_map(|(id, price)| match parse_price(&id, price) { - Ok(asset_info) => Some((id, asset_info)), - Err(e) => { - warn!("failed to parse market data for {} with error {}", id, e); - None - } - }) - .collect::>(); - if let Err(e) = store.set_assets(to_set.clone()).await { - error!("failed to set asset info with error: {}", e); - } else { - debug!( - "stored data for ids: {:?}", - to_set.iter().map(|(id, _)| id).collect::>(), - ); - } - } else { - warn!( - "received more markets than ids, ids: {}, markets: {}", - ids.len(), - markets.len() - ); - } - } - Err(e) => { - warn!("failed to get market data with error: {}", e); - } - } -} - -fn parse_price>(id: T, price: Price) -> Result { - let price_value = Decimal::from_f64(price.usd).ok_or(ParseError::InvalidPrice(price.usd))?; - Ok(AssetInfo::new( - id.into(), - price_value, - price.last_updated_at, - )) -} - -#[cfg(test)] -mod test { - use super::*; - - #[test] - fn test_parse_market() { - let price = Price { - usd: 8426.69, - last_updated_at: 1609459200, - }; - let result = parse_price("bitcoin", price); - let expected = AssetInfo::new( - "bitcoin".to_string(), - Decimal::from_str_exact("8426.69").unwrap(), - 1609459200, - ); - assert_eq!(result.unwrap(), expected); - } - - #[test] - fn test_parse_market_with_failure() { - let price = Price { - usd: f64::INFINITY, - last_updated_at: 0, - }; - assert!(parse_price("bitcoin", price).is_err()); - } -} diff --git a/bothan-coingecko/src/worker/builder.rs b/bothan-coingecko/src/worker/builder.rs deleted file mode 100644 index 371494a1..00000000 --- a/bothan-coingecko/src/worker/builder.rs +++ /dev/null @@ -1,88 +0,0 @@ -use std::sync::Arc; - -use tokio::time::Duration; - -use bothan_core::store::WorkerStore; -use bothan_core::worker::AssetWorkerBuilder; - -use crate::api::error::BuildError; -use crate::api::CoinGeckoRestAPIBuilder; -use crate::worker::asset_worker::start_asset_worker; -use crate::worker::opts::CoinGeckoWorkerBuilderOpts; -use crate::worker::CoinGeckoWorker; - -/// Builds a `CoinGeckoWorker` with custom options. -/// Methods can be chained to set the configuration values and the -/// service is constructed by calling the [`build`](CoinGeckoWorker::build) method. -pub struct CoinGeckoWorkerBuilder { - store: WorkerStore, - opts: CoinGeckoWorkerBuilderOpts, -} - -impl CoinGeckoWorkerBuilder { - /// Set the URL for the `CoinGeckoWorker`. - /// The default URL is `DEFAULT_URL` when no API key is provided - /// and is `DEFAULT_PRO_URL` when an API key is provided. - pub fn with_url>(mut self, url: T) -> Self { - self.opts.url = url.into(); - self - } - - /// Sets the API key for the `CoinGeckoWorker`. - /// The default api key is `None`. - pub fn with_api_key>(mut self, api_key: T) -> Self { - self.opts.api_key = Some(api_key.into()); - self - } - - /// Sets the User-Agent header for the `CoinGeckoWorker`. - /// The default user agent is `DEFAULT_USER_AGENT`. - pub fn with_user_agent>(mut self, user_agent: T) -> Self { - self.opts.user_agent = user_agent.into(); - self - } - - /// Sets the update interval for the `CoinGeckoWorker`. - /// The default interval is `DEFAULT_UPDATE_INTERVAL`. - pub fn with_update_interval(mut self, update_interval: Duration) -> Self { - self.opts.update_interval = update_interval; - self - } - - /// Sets the store for the `CoinGeckoWorker`. - /// If not set, the store is created and owned by the worker. - pub fn with_store(mut self, store: WorkerStore) -> Self { - self.store = store; - self - } -} - -#[async_trait::async_trait] -impl<'a> AssetWorkerBuilder<'a> for CoinGeckoWorkerBuilder { - type Opts = CoinGeckoWorkerBuilderOpts; - type Worker = CoinGeckoWorker; - type Error = BuildError; - - /// Returns a new `CoinGeckoWorkerBuilder` with the given options. - fn new(store: WorkerStore, opts: Self::Opts) -> Self { - Self { store, opts } - } - - /// Returns the name of the worker. - fn worker_name() -> &'static str { - "coingecko" - } - - /// Creates the configured `CoinGeckoWorker`. - async fn build(self) -> Result, Self::Error> { - let api = - CoinGeckoRestAPIBuilder::new(self.opts.user_agent, self.opts.url, self.opts.api_key) - .build()?; - - let worker = Arc::new(CoinGeckoWorker::new(api, self.store)); - - start_asset_worker(Arc::downgrade(&worker), self.opts.update_interval); - - Ok(worker) - } -} diff --git a/bothan-coingecko/src/worker/error.rs b/bothan-coingecko/src/worker/error.rs deleted file mode 100644 index e70695d8..00000000 --- a/bothan-coingecko/src/worker/error.rs +++ /dev/null @@ -1,8 +0,0 @@ -#[derive(Debug, thiserror::Error)] -pub(crate) enum ParseError { - #[error("invalid price: {0}")] - InvalidPrice(f64), - - #[error("invalid timestamp: {0}")] - InvalidTimestamp(#[from] chrono::ParseError), -} diff --git a/bothan-coingecko/src/worker/opts.rs b/bothan-coingecko/src/worker/opts.rs index 32fc3df5..72eb2d5b 100644 --- a/bothan-coingecko/src/worker/opts.rs +++ b/bothan-coingecko/src/worker/opts.rs @@ -3,17 +3,19 @@ use std::time::Duration; use serde::{Deserialize, Deserializer, Serialize, Serializer}; use crate::api::types::{DEFAULT_URL, DEFAULT_USER_AGENT}; -use crate::worker::types::DEFAULT_UPDATE_INTERVAL; + +const DEFAULT_UPDATE_INTERVAL: Duration = Duration::from_secs(60); /// Options for configuring the `CoinGeckoWorkerBuilder`. /// -/// `CoinGeckoWorkerBuilderOpts` provides a way to specify custom settings for creating a `CoinGeckoWorker`. -/// This struct allows users to set optional parameters such as the WebSocket URL and the internal channel size, -/// which will be used during the construction of the `CoinGeckoWorker`. +/// `CoinGeckoWorkerBuilderOpts` provides a way to specify custom settings for creating a +/// `CoinGeckoWorker`. This struct allows users to set optional parameters such as the WebSocket URL +/// and the internal channel size, which will be used during the construction of the +/// `CoinGeckoWorker`. #[derive(Clone, Debug, Serialize, Deserialize)] -pub struct CoinGeckoWorkerBuilderOpts { - #[serde(default = "default_url")] - pub url: String, +pub struct WorkerOpts { + #[serde(serialize_with = "none_is_default_url")] + pub url: Option, #[serde(default)] #[serde(deserialize_with = "empty_string_is_none")] #[serde(serialize_with = "none_is_empty_string")] @@ -25,10 +27,6 @@ pub struct CoinGeckoWorkerBuilderOpts { pub update_interval: Duration, } -fn default_url() -> String { - DEFAULT_URL.to_string() -} - fn default_user_agent() -> String { DEFAULT_USER_AGENT.to_string() } @@ -37,10 +35,10 @@ fn default_update_interval() -> Duration { DEFAULT_UPDATE_INTERVAL } -impl Default for CoinGeckoWorkerBuilderOpts { +impl Default for WorkerOpts { fn default() -> Self { Self { - url: default_url(), + url: None, api_key: None, user_agent: default_user_agent(), update_interval: default_update_interval(), @@ -64,3 +62,13 @@ fn none_is_empty_string( None => serializer.serialize_str(""), } } + +fn none_is_default_url( + value: &Option, + serializer: S, +) -> Result { + match value { + Some(val) => serializer.serialize_str(val), + None => serializer.serialize_str(DEFAULT_URL), + } +} diff --git a/bothan-coingecko/src/worker/types.rs b/bothan-coingecko/src/worker/types.rs deleted file mode 100644 index 9c3f48a5..00000000 --- a/bothan-coingecko/src/worker/types.rs +++ /dev/null @@ -1,3 +0,0 @@ -use tokio::time::Duration; - -pub(crate) const DEFAULT_UPDATE_INTERVAL: Duration = Duration::from_secs(30); diff --git a/bothan-coinmarketcap/Cargo.toml b/bothan-coinmarketcap/Cargo.toml index 72291de3..92191edf 100644 --- a/bothan-coinmarketcap/Cargo.toml +++ b/bothan-coinmarketcap/Cargo.toml @@ -1,13 +1,14 @@ [package] name = "bothan-coinmarketcap" -version = "0.0.1-beta.1" +version = "0.0.1-beta.2" edition.workspace = true license.workspace = true repository.workspace = true [dependencies] +bothan-lib = { workspace = true } + async-trait = { workspace = true } -bothan-core = { workspace = true } chrono = { workspace = true } humantime-serde = { workspace = true } itertools = { workspace = true } @@ -17,8 +18,6 @@ serde = { workspace = true } serde_json = { workspace = true } thiserror = { workspace = true } tokio = { workspace = true } -tracing = { workspace = true } -tracing-subscriber = { workspace = true } url = { workspace = true } [dev-dependencies] diff --git a/bothan-coinmarketcap/examples/coinmarketcap_basic.rs b/bothan-coinmarketcap/examples/coinmarketcap_basic.rs deleted file mode 100644 index 8fb8162b..00000000 --- a/bothan-coinmarketcap/examples/coinmarketcap_basic.rs +++ /dev/null @@ -1,40 +0,0 @@ -use std::time::Duration; - -use tokio::time::sleep; -use tracing_subscriber::fmt::init; - -use bothan_coinmarketcap::{CoinMarketCapWorkerBuilder, CoinMarketCapWorkerBuilderOpts}; -use bothan_core::registry::Registry; -use bothan_core::store::SharedStore; -use bothan_core::worker::{AssetWorker, AssetWorkerBuilder}; - -#[tokio::main] -async fn main() { - init(); - let path = std::env::current_dir().unwrap(); - let store = SharedStore::new(Registry::default().validate().unwrap(), path.as_path()) - .await - .unwrap(); - let worker_store = store.create_worker_store(CoinMarketCapWorkerBuilder::worker_name()); - let opts = CoinMarketCapWorkerBuilderOpts::default(); - - let worker = CoinMarketCapWorkerBuilder::new(worker_store, opts) - .with_api_key("API_KEY_HERE") - .build() - .await - .unwrap(); - - worker - .set_query_ids(vec!["1".to_string(), "1027".to_string()]) - .await - .unwrap(); - - sleep(Duration::from_secs(2)).await; - - loop { - let btc_data = worker.get_asset("1").await; - let eth_data = worker.get_asset("1027").await; - println!("{:?} {:?}", btc_data, eth_data); - tokio::time::sleep(Duration::from_secs(5)).await; - } -} diff --git a/bothan-coinmarketcap/src/api.rs b/bothan-coinmarketcap/src/api.rs index a3294cae..6771bcac 100644 --- a/bothan-coinmarketcap/src/api.rs +++ b/bothan-coinmarketcap/src/api.rs @@ -1,5 +1,5 @@ -pub use builder::CoinMarketCapRestAPIBuilder; -pub use rest::CoinMarketCapRestAPI; +pub use builder::RestApiBuilder; +pub use rest::RestApi; pub mod builder; pub mod error; diff --git a/bothan-coinmarketcap/src/api/builder.rs b/bothan-coinmarketcap/src/api/builder.rs index b117e069..067ad0a2 100644 --- a/bothan-coinmarketcap/src/api/builder.rs +++ b/bothan-coinmarketcap/src/api/builder.rs @@ -1,27 +1,27 @@ -use reqwest::header::{HeaderMap, HeaderValue}; use reqwest::ClientBuilder; +use reqwest::header::{HeaderMap, HeaderValue}; use url::Url; +use crate::api::RestApi; use crate::api::error::BuildError; use crate::api::types::DEFAULT_URL; -use crate::api::CoinMarketCapRestAPI; /// Builds a CoinMarketCapRestAPI with custom parameters. /// Methods can be chained to set the parameters and the /// `CoinMarketCapRestAPI` is constructed -/// by calling the [`build`](CoinMarketCapRestAPIBuilder::build) method. -pub struct CoinMarketCapRestAPIBuilder { +/// by calling the [`build`](RestApiBuilder::build) method. +pub struct RestApiBuilder { url: String, api_key: Option, } -impl CoinMarketCapRestAPIBuilder { +impl RestApiBuilder { pub fn new(url: T, api_key: Option) -> Self where T: Into, U: Into, { - CoinMarketCapRestAPIBuilder { + RestApiBuilder { url: url.into(), api_key: api_key.map(Into::into), } @@ -42,31 +42,28 @@ impl CoinMarketCapRestAPIBuilder { } /// Creates the configured `CoinMarketCapRestAPI`. - pub fn build(self) -> Result { + pub fn build(self) -> Result { let mut headers = HeaderMap::new(); let parsed_url = Url::parse(&self.url)?; - let key = match &self.api_key { - Some(key) => key, - None => return Err(BuildError::MissingAPIKey()), - }; + let api_key = self.api_key.ok_or(BuildError::MissingAPIKey)?; - let mut val = HeaderValue::from_str(key)?; + let mut val = HeaderValue::from_str(&api_key)?; val.set_sensitive(true); headers.insert("X-CMC_PRO_API_KEY", val); let client = ClientBuilder::new().default_headers(headers).build()?; - Ok(CoinMarketCapRestAPI::new(parsed_url, client)) + Ok(RestApi::new(parsed_url, client)) } } -impl Default for CoinMarketCapRestAPIBuilder { +impl Default for RestApiBuilder { /// Creates a new `CoinMarketCapRestAPIBuilder` with the /// default URL and no API key. fn default() -> Self { - CoinMarketCapRestAPIBuilder { + RestApiBuilder { url: DEFAULT_URL.into(), api_key: None, } diff --git a/bothan-coinmarketcap/src/api/error.rs b/bothan-coinmarketcap/src/api/error.rs index cee64665..671e7f6d 100644 --- a/bothan-coinmarketcap/src/api/error.rs +++ b/bothan-coinmarketcap/src/api/error.rs @@ -1,59 +1,34 @@ -#[derive(Clone, Debug, PartialEq, thiserror::Error)] +use thiserror::Error; + +#[derive(Debug, Error)] pub enum BuildError { #[error("missing api key")] - MissingAPIKey(), + MissingAPIKey, #[error("invalid url")] InvalidURL(#[from] url::ParseError), #[error("invalid header value")] - InvalidHeaderValue(String), + InvalidHeaderValue(#[from] reqwest::header::InvalidHeaderValue), #[error("reqwest error: {0}")] - Reqwest(String), -} - -impl From for BuildError { - fn from(e: reqwest::header::InvalidHeaderValue) -> Self { - BuildError::InvalidHeaderValue(e.to_string()) - } + FailedToBuild(#[from] reqwest::Error), } -impl From for BuildError { - fn from(e: reqwest::Error) -> Self { - BuildError::Reqwest(e.to_string()) - } -} - -#[derive(Clone, Debug, PartialEq, thiserror::Error)] -pub enum RestAPIError { +#[derive(Debug, Error)] +pub enum Error { #[error("limit must be lower or equal to 5000")] - LimitTooHigh(), - - #[error("http error: {0}")] - Http(reqwest::StatusCode), + LimitTooHigh, - #[error("invalid id")] - InvalidID, - - #[error("failed to parse")] - Parse, - - #[error("serde error: {0}")] - Serde(String), - - #[error("reqwest error: {0}")] - Reqwest(String), + #[error("failed request: {0}")] + FailedRequest(#[from] reqwest::Error), } -impl From for RestAPIError { - fn from(e: serde_json::Error) -> Self { - RestAPIError::Serde(e.to_string()) - } -} +#[derive(Debug, Error)] +pub enum ProviderError { + #[error("ids contains non integer value")] + InvalidId, -impl From for RestAPIError { - fn from(e: reqwest::Error) -> Self { - RestAPIError::Reqwest(e.to_string()) - } + #[error("failed to fetch tickers: {0}")] + RequestError(#[from] reqwest::Error), } diff --git a/bothan-coinmarketcap/src/api/rest.rs b/bothan-coinmarketcap/src/api/rest.rs index c6a25205..45471fe2 100644 --- a/bothan-coinmarketcap/src/api/rest.rs +++ b/bothan-coinmarketcap/src/api/rest.rs @@ -1,18 +1,21 @@ use std::collections::HashMap; +use bothan_lib::types::AssetInfo; +use bothan_lib::worker::rest::AssetInfoProvider; use itertools::Itertools; -use reqwest::{Client, RequestBuilder, Response, Url}; +use reqwest::{Client, Url}; +use rust_decimal::Decimal; -use crate::api::error::RestAPIError; use crate::api::types::{Quote, Response as CmcResponse}; +use crate::worker::error::ProviderError; /// CoinMarketCap REST API client. -pub struct CoinMarketCapRestAPI { +pub struct RestApi { url: Url, client: Client, } -impl CoinMarketCapRestAPI { +impl RestApi { /// Creates a new CoinMarketCap REST API client. pub fn new(url: Url, client: Client) -> Self { Self { url, client } @@ -23,13 +26,14 @@ impl CoinMarketCapRestAPI { pub async fn get_latest_quotes( &self, ids: &[usize], - ) -> Result>, RestAPIError> { + ) -> Result>, reqwest::Error> { let url = format!("{}v2/cryptocurrency/quotes/latest", self.url); let ids_string = ids.iter().map(|id| id.to_string()).join(","); let params = vec![("id", ids_string)]; - let builder_with_query = self.client.get(&url).query(¶ms); - let response = send_request(builder_with_query).await?; + let request_builder = self.client.get(&url).query(¶ms); + let response = request_builder.send().await?.error_for_status()?; + let cmc_response = response .json::>>() .await?; @@ -43,30 +47,59 @@ impl CoinMarketCapRestAPI { } } -async fn send_request(request_builder: RequestBuilder) -> Result { - let response = request_builder.send().await?; +#[async_trait::async_trait] +impl AssetInfoProvider for RestApi { + type Error = ProviderError; - let status = response.status(); - if status.is_client_error() || status.is_server_error() { - return Err(RestAPIError::Http(status)); + async fn get_asset_info(&self, ids: &[String]) -> Result, Self::Error> { + let int_ids = ids + .iter() + .map(|id| { + id.parse::() + .map_err(|_| ProviderError::InvalidId(id.clone())) + }) + .collect::, _>>()?; + + self.get_latest_quotes(&int_ids) + .await? + .into_iter() + .map(|q| match q { + None => Err(ProviderError::MissingValue), + Some(quote) => { + let price_float = quote + .price_quotes + .usd + .price + .ok_or_else(|| ProviderError::MissingValue)?; + let price_dec = Decimal::from_f64_retain(price_float) + .ok_or_else(|| ProviderError::InvalidValue)?; + let ts = quote + .price_quotes + .usd + .last_updated + .parse::>() + .map_err(|_| ProviderError::InvalidValue)? + .timestamp(); + let ai = AssetInfo::new(quote.symbol, price_dec, ts); + Ok(ai) + } + }) + .collect() } - - Ok(response) } #[cfg(test)] pub(crate) mod test { use mockito::{Matcher, Mock, Server, ServerGuard}; - use crate::api::types::{PriceQuote, PriceQuotes, Status}; - use crate::api::CoinMarketCapRestAPIBuilder; - use super::*; + use crate::api::RestApiBuilder; + use crate::api::types::{PriceQuote, PriceQuotes, Status}; - pub(crate) async fn setup() -> (ServerGuard, CoinMarketCapRestAPI) { + pub(crate) async fn setup() -> (ServerGuard, RestApi) { let server = Server::new_async().await; - let builder = CoinMarketCapRestAPIBuilder::default() + let builder = RestApiBuilder::default() .with_url(&server.url()) .with_api_key("test"); let api = builder.build().unwrap(); @@ -159,9 +192,9 @@ pub(crate) mod test { let mock = server.set_successful_quotes(&["1"], "es); let result = client.get_latest_quotes(&[1]).await; - let expected_result = quotes.into_iter().map(Some).collect(); + let expected_result = quotes.into_iter().map(Some).collect::>>(); mock.assert(); - assert_eq!(result, Ok(expected_result)); + assert_eq!(result.unwrap(), expected_result); } #[tokio::test] @@ -176,7 +209,7 @@ pub(crate) mod test { mock.assert(); let expected_result = vec![Some(quotes[0].clone()), None]; - assert_eq!(result, Ok(expected_result)); + assert_eq!(result.unwrap(), expected_result); } #[tokio::test] @@ -189,8 +222,7 @@ pub(crate) mod test { mock.assert(); - let expected_err = RestAPIError::Reqwest("error decoding response body".to_string()); - assert_eq!(result, Err(expected_err)); + assert!(result.is_err()); } #[tokio::test] diff --git a/bothan-coinmarketcap/src/lib.rs b/bothan-coinmarketcap/src/lib.rs index 588f0ba0..e31a5354 100644 --- a/bothan-coinmarketcap/src/lib.rs +++ b/bothan-coinmarketcap/src/lib.rs @@ -1,6 +1,5 @@ -pub use worker::builder::CoinMarketCapWorkerBuilder; -pub use worker::opts::CoinMarketCapWorkerBuilderOpts; -pub use worker::CoinMarketCapWorker; +pub use worker::Worker; +pub use worker::opts::WorkerOpts; pub mod api; pub mod worker; diff --git a/bothan-coinmarketcap/src/service.rs b/bothan-coinmarketcap/src/service.rs deleted file mode 100644 index 06ba3bbb..00000000 --- a/bothan-coinmarketcap/src/service.rs +++ /dev/null @@ -1,176 +0,0 @@ -use std::sync::Arc; - -use tokio::task::JoinSet; -use tokio::time::{interval, Duration, Interval}; -use tracing::{info, warn}; - -use bothan_core::cache::{Cache, Error as CacheError}; -use bothan_core::service::{Error as ServiceError, Service, ServiceResult}; -use bothan_core::types::AssetInfo; - -use crate::api::types::Quote; -use crate::api::CoinMarketCapRestAPI; -use crate::service::parser::{parse_quote, QuoteParserError}; - -pub mod builder; -mod parser; - -/// A service that fetches and caches cryptocurrency prices from CoinMarketCap. -pub struct CoinMarketCapService { - cache: Arc>, -} - -impl CoinMarketCapService { - /// Creates a new CoinMarketCap service with the given REST API and update interval. - pub async fn new(rest_api: CoinMarketCapRestAPI, update_interval: Duration) -> Self { - let cache = Arc::new(Cache::new(None)); - let update_price_interval = interval(update_interval); - - start_service(Arc::new(rest_api), cache.clone(), update_price_interval).await; - - Self { cache } - } -} - -#[async_trait::async_trait] -impl Service for CoinMarketCapService { - /// Fetches the price data for the given cryptocurrency IDs. - async fn get_price_data(&mut self, ids: &[&str]) -> Vec> { - let mut to_set_pending = Vec::::new(); - - let result = self - .cache - .get_batch(ids) - .await - .into_iter() - .enumerate() - .map(|(idx, result)| match result { - Ok(price_data) => Ok(price_data), - Err(CacheError::DoesNotExist) => { - to_set_pending.push(ids[idx].to_string()); - Err(ServiceError::PendingResult) - } - Err(CacheError::Invalid) => Err(ServiceError::InvalidSymbol), - Err(e) => panic!("unexpected error: {}", e), // This should never happen - }) - .collect(); - - if !to_set_pending.is_empty() { - self.cache.set_batch_pending(to_set_pending).await - } - - result - } -} - -async fn start_service( - rest_api: Arc, - cache: Arc>, - mut update_price_interval: Interval, -) { - tokio::spawn(async move { - loop { - update_price_interval.tick().await; - update_price_data(rest_api.clone(), cache.clone()).await; - } - }); -} - -async fn update_price_data(rest_api: Arc, cache: Arc>) { - let keys = cache.keys().await; - - if !keys.is_empty() { - let parsed_ids = keys - .iter() - .filter_map(|s| s.parse().ok()) - .collect::>(); - - if let Ok(quote) = rest_api.get_latest_quotes(parsed_ids.as_slice()).await { - let mut set = JoinSet::new(); - for (id, quote) in parsed_ids.iter().zip(quote.into_iter()) { - if let Some(q) = quote { - let cloned_cache = cache.clone(); - set.spawn(async move { - process_price_quote(&q, &cloned_cache).await; - }); - } else { - warn!("id {} is missing market data", id); - } - } - while set.join_next().await.is_some() {} - } else { - warn!("failed to get market data"); - } - } -} - -async fn process_price_quote(quote: &Quote, cache: &Cache) { - match parse_quote(quote) { - Ok(price_data) => { - let id = price_data.id.clone(); - if cache.set_data(id.clone(), price_data).await.is_err() { - warn!("unexpected request to set data for id: {}", id); - } else { - info!("set price for id {}", id); - } - } - Err(QuoteParserError::InvalidTimestamp) => warn!("failed to parse date time"), - Err(QuoteParserError::InvalidPrice) => warn!("invalid price given"), - }; -} - -#[cfg(test)] -mod test { - use mockito::ServerGuard; - - use crate::api::rest::test::{mock_quote, setup as api_setup, MockCoinMarketCap}; - - use super::*; - - async fn setup() -> ( - Arc, - Arc>, - ServerGuard, - ) { - let cache = Arc::new(Cache::::new(None)); - let (server, rest_api) = api_setup().await; - (Arc::new(rest_api), cache, server) - } - - #[tokio::test] - async fn test_update_price_data() { - let (rest_api, cache, mut server) = setup().await; - let mock_quote = mock_quote(); - let coin_market = vec![mock_quote.clone()]; - server.set_successful_quotes(&["1"], &coin_market); - cache.set_batch_pending(vec!["1".to_string()]).await; - - update_price_data(rest_api, cache.clone()).await; - - let result = cache.get("1").await; - let expected = parse_quote(&mock_quote).unwrap(); - assert_eq!(result.unwrap(), expected); - } - - #[tokio::test] - async fn test_process_market_data() { - let cache = Arc::new(Cache::::new(None)); - let quote = mock_quote(); - - cache.set_batch_pending(vec!["1".to_string()]).await; - process_price_quote("e, &cache).await; - let result = cache.get("1").await; - let expected = parse_quote("e).unwrap(); - assert_eq!(result.unwrap(), expected); - } - - #[tokio::test] - async fn test_process_market_data_without_set_pending() { - let cache = Arc::new(Cache::::new(None)); - let quote = mock_quote(); - - process_price_quote("e, &cache).await; - let result = cache.get("1").await; - assert!(result.is_err()); - } -} diff --git a/bothan-coinmarketcap/src/worker.rs b/bothan-coinmarketcap/src/worker.rs index 869c5f15..3abd70be 100644 --- a/bothan-coinmarketcap/src/worker.rs +++ b/bothan-coinmarketcap/src/worker.rs @@ -1,41 +1,63 @@ -use bothan_core::store::error::Error as StoreError; -use bothan_core::store::WorkerStore; -use bothan_core::worker::{AssetState, AssetWorker, SetQueryIDError}; +use std::collections::HashSet; +use std::sync::{Arc, Weak}; -use crate::api::CoinMarketCapRestAPI; +use bothan_lib::store::{Store, WorkerStore}; +use bothan_lib::types::AssetState; +use bothan_lib::worker::AssetWorker; +use bothan_lib::worker::error::AssetWorkerError; +use bothan_lib::worker::rest::{AssetInfoProvider, start_polling}; + +use crate::WorkerOpts; +use crate::api::{RestApi, RestApiBuilder}; +use crate::worker::error::ProviderError; -mod asset_worker; -pub mod builder; pub mod error; pub mod opts; pub mod types; -/// A worker that fetches and stores the asset information from CoinMarketCap's API. -pub struct CoinMarketCapWorker { - api: CoinMarketCapRestAPI, - store: WorkerStore, -} +const WORKER_NAME: &str = "coinmarketcap"; -impl CoinMarketCapWorker { - /// Create a new worker with the specified api and store. - pub fn new(api: CoinMarketCapRestAPI, store: WorkerStore) -> Self { - Self { api, store } - } +pub struct Worker { + // The `api` is owned by this struct to ensure that any weak references + // are properly cleaned up when the worker is dropped. + #[allow(dead_code)] + api: Arc, + store: WorkerStore, } #[async_trait::async_trait] -impl AssetWorker for CoinMarketCapWorker { +impl AssetWorker for Worker { + type Opts = WorkerOpts; + + fn name(&self) -> &'static str { + WORKER_NAME + } + + async fn build(opts: Self::Opts, store: &S) -> Result { + let api = Arc::new(RestApiBuilder::new(opts.url, opts.api_key).build()?); + + let worker_store = WorkerStore::new(store, WORKER_NAME); + + tokio::spawn(start_polling( + opts.update_interval, + Arc::downgrade(&api) as Weak>, + worker_store.clone(), + )); + + Ok(Worker { + api, + store: worker_store, + }) + } + /// Fetches the AssetStatus for the given cryptocurrency ids. - async fn get_asset(&self, id: &str) -> Result { - self.store.get_asset(&id).await + async fn get_asset(&self, id: &str) -> Result { + Ok(self.store.get_asset(id).await?) } /// Adds the specified cryptocurrency IDs to the query set. - async fn set_query_ids(&self, ids: Vec) -> Result<(), SetQueryIDError> { - self.store - .set_query_ids(ids) - .await - .map_err(|e| SetQueryIDError::new(e.to_string()))?; + async fn set_query_ids(&self, ids: HashSet) -> Result<(), AssetWorkerError> { + self.store.set_query_ids(ids).await?; Ok(()) } } diff --git a/bothan-coinmarketcap/src/worker/asset_worker.rs b/bothan-coinmarketcap/src/worker/asset_worker.rs deleted file mode 100644 index a5f96e46..00000000 --- a/bothan-coinmarketcap/src/worker/asset_worker.rs +++ /dev/null @@ -1,190 +0,0 @@ -use std::sync::Weak; -use std::time::Duration; - -use chrono::NaiveDateTime; -use rust_decimal::prelude::FromPrimitive; -use rust_decimal::Decimal; -use tokio::time::{interval, timeout}; -use tracing::{debug, error, warn}; - -use bothan_core::store::WorkerStore; -use bothan_core::types::AssetInfo; - -use crate::api::rest::CoinMarketCapRestAPI; -use crate::api::types::Quote; -use crate::worker::error::ParseError; -use crate::worker::CoinMarketCapWorker; - -pub(crate) fn start_asset_worker( - weak_worker: Weak, - update_interval: Duration, -) { - let mut interval = interval(update_interval); - tokio::spawn(async move { - while let Some(worker) = weak_worker.upgrade() { - interval.tick().await; - - let ids = match worker.store.get_query_ids().await { - Ok(ids) => ids.into_iter().collect::>(), - Err(e) => { - error!("failed to get query ids with error: {}", e); - Vec::new() - } - }; - - if ids.is_empty() { - debug!("no ids to update, skipping update"); - continue; - } - - let result = timeout( - interval.period(), - update_asset_info(&worker.store, &worker.api, &ids), - ) - .await; - - if result.is_err() { - warn!("updating interval exceeded timeout") - } - } - - debug!("asset worker has been dropped, stopping asset worker"); - }); -} - -async fn update_asset_info>( - store: &WorkerStore, - api: &CoinMarketCapRestAPI, - ids: &[T], -) { - // Convert ids to a slice of &str - // let ids_str: Vec<&str> = ids.iter().map(|id| id.as_ref()).collect(); - let parsed_ids = ids - .iter() - .filter_map(|s| s.as_ref().parse().ok()) - .collect::>(); - - match api.get_latest_quotes(parsed_ids.as_slice()).await { - Ok(quotes) => { - let mut to_set = Vec::new(); - - for (id, quote) in parsed_ids.iter().zip(quotes.iter()) { - let quote = match quote { - Some(price) => price, - None => { - warn!("missing price data for id: {}", id); - continue; - } - }; - - match parse_quote(quote) { - Ok(asset_info) => to_set.push((id.to_string(), asset_info)), - Err(_) => warn!("failed to parse price data for id: {}", id), - } - } - - // Set multiple assets at once - if let Err(e) = store.set_assets(to_set.clone()).await { - error!("failed to set multiple assets with error: {}", e); - } else { - debug!( - "stored data for ids: {:?}", - to_set.iter().map(|(id, _)| id).collect::>(), - ); - } - } - Err(e) => warn!("failed to get price data with error: {}", e), - } -} - -pub(crate) fn parse_quote(quote: &Quote) -> Result { - let id = quote.id.to_string(); - let price = quote - .price_quotes - .usd - .price - .ok_or(ParseError::MissingPriceData)?; - let price_value = - Decimal::from_f64(price.to_owned()).ok_or(ParseError::InvalidPrice(price.to_owned()))?; - let last_updated = quote.price_quotes.usd.last_updated.as_str(); - let naive_date_time = NaiveDateTime::parse_from_str(last_updated, "%Y-%m-%dT%H:%M:%S.%fZ")?; - let timestamp = naive_date_time.and_utc().timestamp(); - - Ok(AssetInfo::new( - id.to_owned(), - price_value, - timestamp.to_owned(), - )) -} - -#[cfg(test)] -mod test { - use super::*; - use rust_decimal::Decimal; - use std::str::FromStr; - - use crate::api::types::{PriceQuote, PriceQuotes}; - - #[test] - fn test_parse_quote() { - let price = 8426.69; - let timestamp = "2021-01-01T00:00:00.000Z"; - - let quote = Quote { - id: 1_usize, - symbol: "BTC".to_string(), - slug: "bitcoin".to_string(), - name: "Bitcoin".to_string(), - price_quotes: PriceQuotes { - usd: PriceQuote { - price: Some(price), - volume_24h: 123.0, - volume_change_24h: 456.0, - market_cap: 789.0, - market_cap_dominance: 0.0, - fully_diluted_market_cap: 0.0, - percent_change_1h: 0.0, - percent_change_24h: 0.0, - percent_change_7d: 0.0, - percent_change_30d: 0.0, - last_updated: timestamp.to_string(), - }, - }, - }; - - let asset_info = parse_quote("e).unwrap(); - assert_eq!(asset_info.id, quote.id.to_string()); - assert_eq!(asset_info.price, Decimal::from_str("8426.69").unwrap()); - assert_eq!(asset_info.timestamp, 1609459200); - } - - #[test] - fn test_parse_quote_with_failure() { - let price = f64::INFINITY; - let timestamp = "2021-01-01T00:00:00.000Z"; - - let quote = Quote { - id: 1_usize, - symbol: "BTC".to_string(), - slug: "bitcoin".to_string(), - name: "Bitcoin".to_string(), - price_quotes: PriceQuotes { - usd: PriceQuote { - price: Some(price), - volume_24h: 123.0, - volume_change_24h: 456.0, - market_cap: 789.0, - market_cap_dominance: 0.0, - fully_diluted_market_cap: 0.0, - percent_change_1h: 0.0, - percent_change_24h: 0.0, - percent_change_7d: 0.0, - percent_change_30d: 0.0, - last_updated: timestamp.to_string(), - }, - }, - }; - - assert!(parse_quote("e).is_err()); - } -} diff --git a/bothan-coinmarketcap/src/worker/builder.rs b/bothan-coinmarketcap/src/worker/builder.rs deleted file mode 100644 index 362fb962..00000000 --- a/bothan-coinmarketcap/src/worker/builder.rs +++ /dev/null @@ -1,78 +0,0 @@ -use std::sync::Arc; - -use tokio::time::Duration; - -use bothan_core::store::WorkerStore; -use bothan_core::worker::AssetWorkerBuilder; - -use crate::api::error::BuildError; -use crate::api::CoinMarketCapRestAPIBuilder; -use crate::worker::asset_worker::start_asset_worker; -use crate::worker::opts::CoinMarketCapWorkerBuilderOpts; -use crate::worker::CoinMarketCapWorker; - -/// Builds a `CoinMarketCapWorker` with custom options. -/// Methods can be chained to set the configuration values and the -/// service is constructed by calling the [`build`](CoinMarketCapWorker::build) method. -pub struct CoinMarketCapWorkerBuilder { - store: WorkerStore, - opts: CoinMarketCapWorkerBuilderOpts, -} - -impl CoinMarketCapWorkerBuilder { - /// Set the URL for the `CoinMarketCapWorker`. - /// The default URL is `DEFAULT_URL` - pub fn with_url>(mut self, url: T) -> Self { - self.opts.url = url.into(); - self - } - - /// Sets the API key for the `CoinMarketCapWorker`. - /// The default api key is `None`. - pub fn with_api_key>(mut self, api_key: T) -> Self { - self.opts.api_key = Some(api_key.into()); - self - } - - /// Sets the update interval for the `CoinMarketCapWorker`. - /// The default interval is `DEFAULT_UPDATE_INTERVAL`. - pub fn with_update_interval(mut self, update_interval: Duration) -> Self { - self.opts.update_interval = update_interval; - self - } - - /// Sets the store for the `CoinMarketCapWorker`. - /// If not set, the store is created and owned by the worker. - pub fn with_store(mut self, store: WorkerStore) -> Self { - self.store = store; - self - } -} - -#[async_trait::async_trait] -impl<'a> AssetWorkerBuilder<'a> for CoinMarketCapWorkerBuilder { - type Opts = CoinMarketCapWorkerBuilderOpts; - type Worker = CoinMarketCapWorker; - type Error = BuildError; - - /// Returns a new `CoinMarketCapWorkerBuilder` with the given options. - fn new(store: WorkerStore, opts: Self::Opts) -> Self { - Self { store, opts } - } - - /// Returns the name of the worker. - fn worker_name() -> &'static str { - "coinmarketcap" - } - - /// Creates the configured `CoinMarketCapWorker`. - async fn build(self) -> Result, Self::Error> { - let api = CoinMarketCapRestAPIBuilder::new(self.opts.url, self.opts.api_key).build()?; - - let worker = Arc::new(CoinMarketCapWorker::new(api, self.store)); - - start_asset_worker(Arc::downgrade(&worker), self.opts.update_interval); - - Ok(worker) - } -} diff --git a/bothan-coinmarketcap/src/worker/error.rs b/bothan-coinmarketcap/src/worker/error.rs index d76ac1dc..3ad0dedb 100644 --- a/bothan-coinmarketcap/src/worker/error.rs +++ b/bothan-coinmarketcap/src/worker/error.rs @@ -1,11 +1,16 @@ -#[derive(Debug, thiserror::Error)] -pub(crate) enum ParseError { - #[error("missing price data")] - MissingPriceData, +use thiserror::Error; - #[error("invalid price: {0}")] - InvalidPrice(f64), +#[derive(Debug, Error)] +pub enum ProviderError { + #[error("ids contains non integer value: {0}")] + InvalidId(String), - #[error("invalid timestamp: {0}")] - InvalidTimestamp(#[from] chrono::ParseError), + #[error("failed to fetch tickers: {0}")] + RequestError(#[from] reqwest::Error), + + #[error("missing value")] + MissingValue, + + #[error("value contains nan")] + InvalidValue, } diff --git a/bothan-coinmarketcap/src/worker/opts.rs b/bothan-coinmarketcap/src/worker/opts.rs index 913ef512..7ad996dd 100644 --- a/bothan-coinmarketcap/src/worker/opts.rs +++ b/bothan-coinmarketcap/src/worker/opts.rs @@ -7,11 +7,12 @@ use crate::worker::types::DEFAULT_UPDATE_INTERVAL; /// Options for configuring the `CoinMarketCapWorkerBuilder`. /// -/// `CoinMarketCapWorkerBuilderOpts` provides a way to specify custom settings for creating a `CoinMarketCapWorker`. -/// This struct allows users to set optional parameters such as the WebSocket URL and the internal channel size, -/// which will be used during the construction of the `CoinMarketCapWorker`. +/// `CoinMarketCapWorkerBuilderOpts` provides a way to specify custom settings for creating a +/// `CoinMarketCapWorker`. This struct allows users to set optional parameters such as the WebSocket +/// URL and the internal channel size, which will be used during the construction of the +/// `CoinMarketCapWorker`. #[derive(Clone, Debug, Serialize, Deserialize)] -pub struct CoinMarketCapWorkerBuilderOpts { +pub struct WorkerOpts { #[serde(default = "default_url")] pub url: String, #[serde(default)] @@ -31,7 +32,7 @@ fn default_update_interval() -> Duration { DEFAULT_UPDATE_INTERVAL } -impl Default for CoinMarketCapWorkerBuilderOpts { +impl Default for WorkerOpts { fn default() -> Self { Self { url: default_url(), diff --git a/bothan-core/Cargo.toml b/bothan-core/Cargo.toml index 426142b2..09004bc4 100644 --- a/bothan-core/Cargo.toml +++ b/bothan-core/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "bothan-core" -version = "0.0.1-beta.1" +version = "0.0.1-beta.2" description = "Core library for Bothan" authors.workspace = true edition.workspace = true @@ -8,23 +8,40 @@ license.workspace = true repository.workspace = true [dependencies] +bothan-lib = { workspace = true } + +bothan-binance = { workspace = true } +bothan-bitfinex = { workspace = true } +bothan-bybit = { workspace = true } +bothan-coinbase = { workspace = true } +bothan-coingecko = { workspace = true } +bothan-coinmarketcap = { workspace = true } +bothan-htx = { workspace = true } +bothan-kraken = { workspace = true } +bothan-okx = { workspace = true } + async-trait = { workspace = true } +bincode = { workspace = true } +chrono = { workspace = true } derive_more = { workspace = true } +num-traits = { workspace = true } semver = { workspace = true } serde = { workspace = true, features = ["rc"] } +serde_json = { workspace = true } thiserror = { workspace = true } tokio = { workspace = true } tracing = { workspace = true } +rand = { workspace = true } reqwest = { workspace = true } rust_decimal = { workspace = true, features = ["maths", "serde-str"] } -bincode = "2.0.0-rc.3" -chrono = "0.4.39" ed25519 = "2.2.3" ed25519-dalek = { version = "2.1.1", features = ["std", "rand_core"] } hex = "0.4.3" mini-moka = "0.10.3" -num-traits = "0.2.19" -rand = "0.8.5" -rust-rocksdb = "0.34.0" -serde_json = "1.0.133" +rust-rocksdb = { version = "0.36.0", optional = true } + +[features] +default = [] +rocksdb = ["dep:rust-rocksdb"] +all = ["rocksdb"] diff --git a/bothan-core/src/ipfs/builder.rs b/bothan-core/src/ipfs/builder.rs index 82262635..35cad82e 100644 --- a/bothan-core/src/ipfs/builder.rs +++ b/bothan-core/src/ipfs/builder.rs @@ -1,7 +1,7 @@ use std::time::Duration; -use reqwest::header::{HeaderMap, HeaderName, HeaderValue}; use reqwest::ClientBuilder; +use reqwest::header::{HeaderMap, HeaderName, HeaderValue}; use crate::ipfs::client::IpfsClient; diff --git a/bothan-core/src/lib.rs b/bothan-core/src/lib.rs index 8eba5b13..0d9fd7ca 100644 --- a/bothan-core/src/lib.rs +++ b/bothan-core/src/lib.rs @@ -1,7 +1,4 @@ pub mod ipfs; pub mod manager; pub mod monitoring; -pub mod registry; pub mod store; -pub mod types; -pub mod worker; diff --git a/bothan-core/src/manager/crypto_asset_info.rs b/bothan-core/src/manager/crypto_asset_info.rs index f0e93fcc..40e7e3c7 100644 --- a/bothan-core/src/manager/crypto_asset_info.rs +++ b/bothan-core/src/manager/crypto_asset_info.rs @@ -1,7 +1,8 @@ pub use manager::CryptoAssetInfoManager; pub mod error; -mod manager; -mod price; -mod signal_ids; +pub mod manager; +pub mod price; +pub mod signal_ids; pub mod types; +pub mod worker; diff --git a/bothan-core/src/manager/crypto_asset_info/error.rs b/bothan-core/src/manager/crypto_asset_info/error.rs index 99e9593e..d941bbca 100644 --- a/bothan-core/src/manager/crypto_asset_info/error.rs +++ b/bothan-core/src/manager/crypto_asset_info/error.rs @@ -1,12 +1,11 @@ use thiserror::Error; use crate::monitoring::error::Error as MonitoringError; -use crate::store::error::Error as StoreError; #[derive(Debug, Error, PartialEq)] pub enum SetRegistryError { - #[error("Failed to set registry: {0}")] - FailedToSetRegistry(#[from] StoreError), + #[error("Failed to set registry")] + FailedToSetRegistry, #[error("Failed to get registry from IPFS")] FailedToRetrieve(String), diff --git a/bothan-core/src/manager/crypto_asset_info/manager.rs b/bothan-core/src/manager/crypto_asset_info/manager.rs index e2170ff8..aab207bc 100644 --- a/bothan-core/src/manager/crypto_asset_info/manager.rs +++ b/bothan-core/src/manager/crypto_asset_info/manager.rs @@ -1,28 +1,28 @@ use std::collections::HashMap; use std::sync::Arc; +use bothan_lib::registry::{Invalid, Registry}; +use bothan_lib::store::{RegistryStore, Store}; use mini_moka::sync::Cache; use semver::{Version, VersionReq}; use serde_json::from_str; -use crate::ipfs::{error::Error as IpfsError, IpfsClient}; +use crate::ipfs::IpfsClient; +use crate::ipfs::error::Error as IpfsError; use crate::manager::crypto_asset_info::error::{ PostHeartbeatError, PushMonitoringRecordError, SetRegistryError, }; use crate::manager::crypto_asset_info::price::tasks::get_signal_price_states; use crate::manager::crypto_asset_info::signal_ids::set_workers_query_ids; use crate::manager::crypto_asset_info::types::{ - CryptoAssetManagerInfo, PriceSignalComputationRecord, PriceState, MONITORING_TTL, + CryptoAssetManagerInfo, MONITORING_TTL, PriceSignalComputationRecord, PriceState, }; -use crate::monitoring::{create_uuid, Client as MonitoringClient}; -use crate::registry::{Invalid, Registry}; -use crate::store::error::Error as StoreError; -use crate::store::ManagerStore; -use crate::worker::AssetWorker; - -pub struct CryptoAssetInfoManager<'a> { - workers: HashMap>, - store: ManagerStore, +use crate::manager::crypto_asset_info::worker::CryptoAssetWorker; +use crate::monitoring::{Client as MonitoringClient, create_uuid}; + +pub struct CryptoAssetInfoManager { + workers: HashMap>, + store: RegistryStore, stale_threshold: i64, ipfs_client: IpfsClient, bothan_version: Version, @@ -31,10 +31,11 @@ pub struct CryptoAssetInfoManager<'a> { monitoring_cache: Option>>>, } -impl<'a> CryptoAssetInfoManager<'a> { +impl CryptoAssetInfoManager { + /// Creates a new `CryptoAssetInfoManager`. pub fn new( - workers: HashMap>, - store: ManagerStore, + workers: HashMap>, + store: RegistryStore, ipfs_client: IpfsClient, stale_threshold: i64, bothan_version: Version, @@ -57,11 +58,12 @@ impl<'a> CryptoAssetInfoManager<'a> { } } - pub async fn get_info(&self) -> Result { + /// Gets the `CryptoAssetManagerInfo`. + pub async fn get_info(&self) -> Result { let bothan_version = self.bothan_version.to_string(); let registry_hash = self .store - .get_registry_hash() + .get_registry_ipfs_hash() .await? .unwrap_or(String::new()); // If value doesn't exist, return an empty string let registry_version_requirement = self.registry_version_requirement.to_string(); @@ -76,6 +78,7 @@ impl<'a> CryptoAssetInfoManager<'a> { )) } + /// Posts a heartbeat to the monitoring service. pub async fn post_heartbeat(&self) -> Result { let client = self .monitoring_client @@ -88,7 +91,7 @@ impl<'a> CryptoAssetInfoManager<'a> { let bothan_version = self.bothan_version.clone(); let registry_hash = self .store - .get_registry_hash() + .get_registry_ipfs_hash() .await .map_err(|_| PostHeartbeatError::FailedToGetRegistryHash)? .unwrap_or_else(|| "".to_string()); @@ -105,7 +108,7 @@ impl<'a> CryptoAssetInfoManager<'a> { pub async fn get_prices( &self, ids: Vec, - ) -> Result<(String, Vec), StoreError> { + ) -> Result<(String, Vec), S::Error> { let registry = self.store.get_registry().await; let current_time = chrono::Utc::now().timestamp(); @@ -129,6 +132,7 @@ impl<'a> CryptoAssetInfoManager<'a> { Ok((uuid, price_states)) } + /// Pushes a monitoring record to the monitoring service. pub async fn push_monitoring_record( &self, uuid: String, @@ -153,6 +157,7 @@ impl<'a> CryptoAssetInfoManager<'a> { Ok(()) } + /// Sets the registry from an IPFS hash. pub async fn set_registry_from_ipfs( &self, hash: &str, @@ -182,7 +187,10 @@ impl<'a> CryptoAssetInfoManager<'a> { .validate() .map_err(|e| SetRegistryError::InvalidRegistry(e.to_string()))?; - self.store.set_registry(registry, hash.to_string()).await?; + self.store + .set_registry(registry, hash.to_string()) + .await + .map_err(|_| SetRegistryError::FailedToSetRegistry)?; set_workers_query_ids(&self.workers, &self.store.get_registry().await).await; diff --git a/bothan-core/src/manager/crypto_asset_info/price/cache.rs b/bothan-core/src/manager/crypto_asset_info/price/cache.rs index 8e0bc055..a4aea26a 100644 --- a/bothan-core/src/manager/crypto_asset_info/price/cache.rs +++ b/bothan-core/src/manager/crypto_asset_info/price/cache.rs @@ -14,12 +14,14 @@ impl PriceCache where K: Hash + Eq, { + /// Creates an empty `PriceCache`. pub fn new() -> Self { PriceCache { cache: HashMap::new(), } } + /// Returns a reference to the `PriceState` corresponding to the given `id`. pub fn get(&self, id: &Q) -> Option<&PriceState> where K: Borrow, @@ -28,18 +30,22 @@ where self.cache.get(id) } + /// Inserts a new `PriceState::Available` with the given `id` and `value`. pub fn set_available(&mut self, id: K, value: Decimal) -> Option { self.cache.insert(id, PriceState::Available(value)) } + /// Inserts a new `PriceState::Unavailable` with the given `id`. pub fn set_unavailable(&mut self, id: K) -> Option { self.cache.insert(id, PriceState::Unavailable) } + /// Inserts a new `PriceState::Unsupported` with the given `id`. pub fn set_unsupported(&mut self, id: K) -> Option { self.cache.insert(id, PriceState::Unsupported) } + /// Returns `true` if the cache contains the given `id`. pub fn contains(&self, id: &Q) -> bool where K: Borrow, diff --git a/bothan-core/src/manager/crypto_asset_info/price/error.rs b/bothan-core/src/manager/crypto_asset_info/price/error.rs index 33fce3d2..2ad0a3f5 100644 --- a/bothan-core/src/manager/crypto_asset_info/price/error.rs +++ b/bothan-core/src/manager/crypto_asset_info/price/error.rs @@ -1,10 +1,9 @@ use std::collections::HashSet; +use bothan_lib::registry::post_processor::PostProcessError; +use bothan_lib::registry::processor::ProcessError; use thiserror::Error; -use crate::registry::post_processor::PostProcessError; -use crate::registry::processor::ProcessError; - #[derive(Debug, Error)] pub enum Error { #[error("Signal does not exist")] diff --git a/bothan-core/src/manager/crypto_asset_info/price/tasks.rs b/bothan-core/src/manager/crypto_asset_info/price/tasks.rs index 077d1ecf..0cd59778 100644 --- a/bothan-core/src/manager/crypto_asset_info/price/tasks.rs +++ b/bothan-core/src/manager/crypto_asset_info/price/tasks.rs @@ -1,30 +1,35 @@ -use std::collections::{HashSet, VecDeque}; - +use std::collections::{HashMap, HashSet, VecDeque}; + +use bothan_lib::registry::signal::Signal; +use bothan_lib::registry::source::{OperationRoute, SourceQuery}; +use bothan_lib::registry::{Registry, Valid}; +use bothan_lib::store::Store; +use bothan_lib::types::AssetState; +use bothan_lib::worker::AssetWorker; use num_traits::Zero; use rust_decimal::Decimal; use tracing::{debug, info, warn}; use crate::manager::crypto_asset_info::price::cache::PriceCache; use crate::manager::crypto_asset_info::price::error::{Error, MissingPrerequisiteError}; -use crate::manager::crypto_asset_info::types::{ - PriceSignalComputationRecord, PriceState, WorkerMap, -}; -use crate::monitoring::records::{ +use crate::manager::crypto_asset_info::types::{PriceSignalComputationRecord, PriceState}; +use crate::monitoring::types::{ OperationRecord, ProcessRecord, SignalComputationRecord, SourceRecord, }; -use crate::registry::signal::Signal; -use crate::registry::source::{OperationRoute, SourceQuery}; -use crate::registry::{Registry, Valid}; -use crate::worker::{AssetState, AssetWorker}; // TODO: Allow records to be Option -pub async fn get_signal_price_states<'a>( +/// Computes the price states for a list of signal ids. +pub async fn get_signal_price_states( ids: Vec, - workers: &WorkerMap<'a>, + workers: &HashMap, registry: &Registry, stale_cutoff: i64, records: &mut Vec, -) -> Vec { +) -> Vec +where + S: Store + 'static, + W: AssetWorker, +{ let mut cache = PriceCache::new(); let mut queue = VecDeque::from(ids.clone()); @@ -70,14 +75,18 @@ pub async fn get_signal_price_states<'a>( .collect() } -async fn compute_signal_result<'a>( +async fn compute_signal_result( id: &str, - workers: &WorkerMap<'a>, + workers: &HashMap, registry: &Registry, stale_cutoff: i64, cache: &PriceCache, records: &mut Vec, -) -> Result { +) -> Result +where + S: Store + 'static, + W: AssetWorker, +{ match registry.get(id) { Some(signal) => { let mut record = SignalComputationRecord::new(id.to_string()); @@ -89,7 +98,6 @@ async fn compute_signal_result<'a>( // We can unwrap here because we just pushed the record, so it's guaranteed to be there let record_ref = records.last_mut().unwrap(); - // let processor_ref: Processor = &signal.processor; let process_signal_result = signal.processor.process(source_results); record_ref.process_result = Some(ProcessRecord::new( signal.processor.name().to_string(), @@ -125,13 +133,17 @@ async fn compute_signal_result<'a>( } } -async fn compute_source_result<'a>( +async fn compute_source_result( signal: &Signal, - workers: &WorkerMap<'a>, + workers: &HashMap, cache: &PriceCache, stale_cutoff: i64, record: &mut PriceSignalComputationRecord, -) -> Result, MissingPrerequisiteError> { +) -> Result, MissingPrerequisiteError> +where + S: Store + 'static, + W: AssetWorker, +{ // Create a temporary cache here as we don't want to write to the main record until we can // confirm that all prerequisites are settled let mut records_cache = Vec::new(); @@ -141,7 +153,7 @@ async fn compute_source_result<'a>( for source_query in &signal.source_queries { if let Some(worker) = workers.get(&source_query.source_id) { let source_value_opt = process_source_query( - worker.as_ref(), + worker, source_query, stale_cutoff, cache, @@ -165,13 +177,17 @@ async fn compute_source_result<'a>( Ok(source_values) } -async fn process_source_query<'a>( - worker: &'a dyn AssetWorker, +async fn process_source_query( + worker: &W, source_query: &SourceQuery, stale_cutoff: i64, cache: &PriceCache, source_records: &mut Vec>, -) -> Result, MissingPrerequisiteError> { +) -> Result, MissingPrerequisiteError> +where + S: Store + 'static, + W: AssetWorker, +{ let source_id = &source_query.source_id; let query_id = &source_query.query_id; @@ -217,7 +233,7 @@ fn compute_source_routes( cache: &PriceCache, record: &mut SourceRecord, ) -> Result, MissingPrerequisiteError> { - // Get all pre requisites + // Get all prerequisites let mut missing = Vec::with_capacity(routes.len()); let mut values = Vec::with_capacity(routes.len()); for route in routes { @@ -252,34 +268,100 @@ fn compute_source_routes( #[cfg(test)] mod tests { use std::collections::HashMap; - use std::sync::Arc; - + use std::fmt; + use std::marker::PhantomData; + + use bothan_lib::registry::processor::Processor; + use bothan_lib::registry::processor::median::MedianProcessor; + use bothan_lib::registry::source::Operation; + use bothan_lib::types::AssetInfo; + use bothan_lib::worker::error::AssetWorkerError; + use derive_more::Error; use num_traits::One; - use crate::registry::processor::median::MedianProcessor; - use crate::registry::processor::Processor; - use crate::registry::source::Operation; - use crate::registry::tests::valid_mock_registry; - use crate::store::error::Error as StoreError; - use crate::types::AssetInfo; - use crate::worker::SetQueryIDError; - use super::*; + #[derive(Debug, Error)] + struct MockError {} + + impl fmt::Display for MockError { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "mock error") + } + } + + #[derive(Clone)] + struct MockStore {} + + #[async_trait::async_trait] + impl Store for MockStore { + type Error = MockError; + + async fn set_registry(&self, _: Registry, _: String) -> Result<(), Self::Error> { + Ok(()) + } + + async fn get_registry(&self) -> Registry { + Registry::default() + } + + async fn get_registry_ipfs_hash(&self) -> Result, Self::Error> { + Ok(None) + } + + async fn set_query_ids(&self, _: &str, _: HashSet) -> Result<(), Self::Error> { + Ok(()) + } + + async fn get_query_ids(&self, _: &str) -> Result>, Self::Error> { + Ok(None) + } + + async fn contains_query_id(&self, _: &str, _: &str) -> Result { + Ok(false) + } + + async fn get_asset_info(&self, _: &str, _: &str) -> Result, Self::Error> { + Ok(None) + } + + async fn insert_asset_info(&self, _: &str, _: AssetInfo) -> Result<(), Self::Error> { + Ok(()) + } + + async fn insert_asset_infos(&self, _: &str, _: Vec) -> Result<(), Self::Error> { + Ok(()) + } + } + #[derive(Default)] - struct MockWorker { + struct MockWorker { expected_results: HashMap, + phantom_data: PhantomData, } - impl MockWorker { + impl MockWorker { fn add_expected_query(&mut self, id: String, asset_state: AssetState) { self.expected_results.insert(id, asset_state); } } #[async_trait::async_trait] - impl AssetWorker for MockWorker { - async fn get_asset(&self, id: &str) -> Result { + impl AssetWorker for MockWorker { + type Opts = (); + + fn name(&self) -> &'static str { + "mock" + } + + async fn build(_: Self::Opts, _: &S) -> Result { + Ok(MockWorker { + expected_results: HashMap::new(), + phantom_data: PhantomData, + }) + } + + async fn get_asset(&self, id: &str) -> Result { Ok(self .expected_results .get(id) @@ -287,23 +369,22 @@ mod tests { .clone()) } - async fn set_query_ids(&self, _: Vec) -> Result<(), SetQueryIDError> { + async fn set_query_ids(&self, _: HashSet) -> Result<(), AssetWorkerError> { Ok(()) } } - fn mock_workers<'a, K: Into>(workers: Vec<(K, MockWorker)>) -> WorkerMap<'a> { - workers - .into_iter() - .map(|(id, worker)| (id.into(), Arc::new(worker) as Arc)) - .collect() + pub fn mock_registry() -> Registry { + let json_string = "{\"CS:USDT-USD\":{\"sources\":[{\"source_id\":\"coingecko\",\"id\":\"tether\",\"routes\":[]}],\"processor\":{\"function\":\"median\",\"params\":{\"min_source_count\":1}},\"post_processors\":[]},\"CS:BTC-USD\":{\"sources\":[{\"source_id\":\"binance\",\"id\":\"btcusdt\",\"routes\":[{\"signal_id\":\"CS:USDT-USD\",\"operation\":\"*\"}]},{\"source_id\":\"coingecko\",\"id\":\"bitcoin\",\"routes\":[]}],\"processor\":{\"function\":\"median\",\"params\":{\"min_source_count\":1}},\"post_processors\":[]}}"; + let registry = serde_json::from_str::(json_string).unwrap(); + registry.validate().unwrap() } #[tokio::test] async fn test_get_signal_price_states() { let ids = vec!["CS:BTC-USD".to_string(), "CS:USDT-USD".to_string()]; - let mut binance_worker = MockWorker::default(); + let mut binance_worker = MockWorker::build((), &MockStore {}).await.unwrap(); binance_worker.add_expected_query( "btcusdt".to_string(), AssetState::Available(AssetInfo::new( @@ -313,7 +394,7 @@ mod tests { )), ); - let mut coingecko_worker = MockWorker::default(); + let mut coingecko_worker = MockWorker::build((), &MockStore {}).await.unwrap(); coingecko_worker.add_expected_query( "bitcoin".to_string(), AssetState::Available(AssetInfo::new( @@ -327,12 +408,11 @@ mod tests { AssetState::Available(AssetInfo::new("tether".to_string(), Decimal::one(), 11002)), ); - let workers = mock_workers(vec![ - ("binance", binance_worker), - ("coingecko", coingecko_worker), - ]); + let mut workers = HashMap::new(); + workers.insert("binance".to_string(), binance_worker); + workers.insert("coingecko".to_string(), coingecko_worker); - let registry = valid_mock_registry().validate().unwrap(); + let registry = mock_registry(); let stale_cutoff = 0; let mut records = Vec::new(); @@ -349,19 +429,18 @@ mod tests { #[tokio::test] async fn test_get_signal_price_states_with_unavailable() { let ids = vec!["CS:BTC-USD".to_string(), "CS:USDT-USD".to_string()]; - let mut binance_worker = MockWorker::default(); + let mut binance_worker = MockWorker::build((), &MockStore {}).await.unwrap(); binance_worker.add_expected_query("btcusdt".to_string(), AssetState::Unsupported); - let mut coingecko_worker = MockWorker::default(); + let mut coingecko_worker = MockWorker::build((), &MockStore {}).await.unwrap(); coingecko_worker.add_expected_query("bitcoin".to_string(), AssetState::Unsupported); coingecko_worker.add_expected_query("tether".to_string(), AssetState::Unsupported); - let workers = mock_workers(vec![ - ("binance", binance_worker), - ("coingecko", coingecko_worker), - ]); + let mut workers = HashMap::new(); + workers.insert("binance".to_string(), binance_worker); + workers.insert("coingecko".to_string(), coingecko_worker); - let registry = valid_mock_registry().validate().unwrap(); + let registry = mock_registry(); let stale_cutoff = 0; let mut records = Vec::new(); @@ -380,7 +459,7 @@ mod tests { "CS:DNE-USD".to_string(), ]; - let mut binance_worker = MockWorker::default(); + let mut binance_worker = MockWorker::build((), &MockStore {}).await.unwrap(); binance_worker.add_expected_query( "btcusdt".to_string(), AssetState::Available(AssetInfo::new( @@ -390,7 +469,7 @@ mod tests { )), ); - let mut coingecko_worker = MockWorker::default(); + let mut coingecko_worker = MockWorker::build((), &MockStore {}).await.unwrap(); coingecko_worker.add_expected_query( "bitcoin".to_string(), AssetState::Available(AssetInfo::new( @@ -404,12 +483,11 @@ mod tests { AssetState::Available(AssetInfo::new("tether".to_string(), Decimal::one(), 11002)), ); - let workers = mock_workers(vec![ - ("binance", binance_worker), - ("coingecko", coingecko_worker), - ]); + let mut workers = HashMap::new(); + workers.insert("binance".to_string(), binance_worker); + workers.insert("coingecko".to_string(), coingecko_worker); - let registry = valid_mock_registry().validate().unwrap(); + let registry = mock_registry(); let stale_cutoff = 10000; let mut records = Vec::new(); @@ -434,12 +512,13 @@ mod tests { let processor = Processor::Median(MedianProcessor::new(1)); let signal = Signal::new(source_queries, processor, vec![]); - let mut test_source_worker = MockWorker::default(); + let mut test_source_worker = MockWorker::build((), &MockStore {}).await.unwrap(); let asset_state = AssetState::Available(AssetInfo::new("testusd".to_string(), Decimal::default(), 0)); test_source_worker.add_expected_query("testusd".to_string(), asset_state.clone()); - let workers = mock_workers(vec![("test-source", test_source_worker)]); + let mut workers = HashMap::new(); + workers.insert("test-source".to_string(), test_source_worker); let cache = PriceCache::new(); let stale_cutoff = 0; @@ -477,13 +556,14 @@ mod tests { let processor = Processor::Median(MedianProcessor::new(1)); let signal = Signal::new(source_queries, processor, vec![]); - let mut test_source_worker = MockWorker::default(); + let mut test_source_worker = MockWorker::build((), &MockStore {}).await.unwrap(); test_source_worker.add_expected_query( "testusd".to_string(), AssetState::Available(AssetInfo::new("testusd".to_string(), Decimal::default(), 0)), ); - let workers = mock_workers(vec![("test-source", test_source_worker)]); + let mut workers = HashMap::new(); + workers.insert("test-source".to_string(), test_source_worker); let cache = PriceCache::new(); let stale_cutoff = 0; @@ -500,7 +580,7 @@ mod tests { #[tokio::test] async fn test_process_source_query() { - let mut worker = MockWorker::default(); + let mut worker = MockWorker::build((), &MockStore {}).await.unwrap(); let id = "testusd".to_string(); let asset_state = AssetState::Available(AssetInfo::new(id.clone(), Decimal::new(1000, 0), 10)); @@ -529,7 +609,7 @@ mod tests { #[tokio::test] async fn test_process_source_query_with_timeout() { - let mut worker = MockWorker::default(); + let mut worker = MockWorker::build((), &MockStore {}).await.unwrap(); let id = "testusd".to_string(); let asset_info = AssetInfo::new(id.clone(), Decimal::default(), 0); worker.add_expected_query( @@ -560,7 +640,7 @@ mod tests { #[tokio::test] async fn test_process_source_query_with_unsupported_asset_state() { - let mut worker = MockWorker::default(); + let mut worker = MockWorker::build((), &MockStore {}).await.unwrap(); worker.add_expected_query("testusd".to_string(), AssetState::Unsupported); let source_query = diff --git a/bothan-core/src/manager/crypto_asset_info/signal_ids.rs b/bothan-core/src/manager/crypto_asset_info/signal_ids.rs index 81c1e4f3..bfb9e7ca 100644 --- a/bothan-core/src/manager/crypto_asset_info/signal_ids.rs +++ b/bothan-core/src/manager/crypto_asset_info/signal_ids.rs @@ -1,18 +1,21 @@ use std::collections::{HashMap, HashSet, VecDeque}; -use std::sync::Arc; +use bothan_lib::registry::{Registry, Valid}; +use bothan_lib::store::Store; +use bothan_lib::worker::AssetWorker; use tracing::{error, info}; -use crate::registry::{Registry, Valid}; -use crate::worker::AssetWorker; +use crate::manager::crypto_asset_info::worker::CryptoAssetWorker; -pub async fn set_workers_query_ids<'a>( - workers: &HashMap>, +/// Sets the query ids for each worker based on the registry. If the registry contains a worker that +/// is not in the worker map, it will be ignored and logged. +pub async fn set_workers_query_ids( + workers: &HashMap>, registry: &Registry, ) { - for (source, mut query_ids) in get_source_batched_query_ids(registry).drain() { + for (source, query_ids) in get_source_batched_query_ids(registry).drain() { match workers.get(&source) { - Some(worker) => match worker.set_query_ids(query_ids.drain().collect()).await { + Some(worker) => match worker.set_query_ids(query_ids).await { Ok(_) => info!("set query ids for {} worker", source), Err(e) => error!("failed to set query ids for {} worker: {}", source, e), }, @@ -26,7 +29,7 @@ fn get_source_batched_query_ids(registry: &Registry) -> HashMap::new(); - let mut queue = VecDeque::from_iter(registry.keys()); + let mut queue = VecDeque::from_iter(registry.signal_ids()); while let Some(signal_id) = queue.pop_front() { if seen.contains(signal_id) { continue; @@ -56,12 +59,18 @@ fn get_source_batched_query_ids(registry: &Registry) -> HashMap Registry { + let json_string = "{\"CS:USDT-USD\":{\"sources\":[{\"source_id\":\"coingecko\",\"id\":\"tether\",\"routes\":[]}],\"processor\":{\"function\":\"median\",\"params\":{\"min_source_count\":1}},\"post_processors\":[]},\"CS:BTC-USD\":{\"sources\":[{\"source_id\":\"binance\",\"id\":\"btcusdt\",\"routes\":[{\"signal_id\":\"CS:USDT-USD\",\"operation\":\"*\"}]},{\"source_id\":\"coingecko\",\"id\":\"bitcoin\",\"routes\":[]}],\"processor\":{\"function\":\"median\",\"params\":{\"min_source_count\":1}},\"post_processors\":[]}}"; + serde_json::from_str::(json_string).unwrap() + } #[test] fn test_get_source_batched_query_ids() { - let registry = valid_mock_registry().validate().unwrap(); + let registry = mock_registry().validate().unwrap(); let diff = get_source_batched_query_ids(®istry); let expected = HashMap::from_iter([ diff --git a/bothan-core/src/manager/crypto_asset_info/types.rs b/bothan-core/src/manager/crypto_asset_info/types.rs index 1434f043..b32be9e0 100644 --- a/bothan-core/src/manager/crypto_asset_info/types.rs +++ b/bothan-core/src/manager/crypto_asset_info/types.rs @@ -1,16 +1,13 @@ -use std::collections::HashMap; -use std::sync::Arc; use std::time::Duration; +use bothan_lib::types::AssetState; use rust_decimal::Decimal; -use crate::monitoring::records::SignalComputationRecord; -use crate::worker::{AssetState, AssetWorker}; +use crate::monitoring::types::SignalComputationRecord; pub const MONITORING_TTL: Duration = Duration::from_secs(60); pub const HEARTBEAT: Duration = Duration::from_secs(60); -pub type WorkerMap<'a> = HashMap>; pub type PriceSignalComputationRecord = SignalComputationRecord; #[derive(Debug, Clone, PartialEq)] diff --git a/bothan-core/src/manager/crypto_asset_info/worker.rs b/bothan-core/src/manager/crypto_asset_info/worker.rs new file mode 100644 index 00000000..c22d3f44 --- /dev/null +++ b/bothan-core/src/manager/crypto_asset_info/worker.rs @@ -0,0 +1,103 @@ +pub mod opts; + +use std::collections::HashSet; + +use bothan_lib::store::Store; +use bothan_lib::types::AssetState; +use bothan_lib::worker::AssetWorker; +use bothan_lib::worker::error::AssetWorkerError; +use derive_more::From; + +use crate::manager::crypto_asset_info::worker::opts::CryptoAssetWorkerOpts; + +#[derive(From)] +pub enum CryptoAssetWorker { + Binance(bothan_binance::Worker), + Bitfinex(bothan_bitfinex::Worker), + Bybit(bothan_bybit::Worker), + Coinbase(bothan_coinbase::Worker), + CoinGecko(bothan_coingecko::Worker), + CoinMarketCap(bothan_coinmarketcap::Worker), + Htx(bothan_htx::Worker), + Kraken(bothan_kraken::Worker), + Okx(bothan_okx::Worker), +} + +#[async_trait::async_trait] +impl AssetWorker for CryptoAssetWorker { + type Opts = CryptoAssetWorkerOpts; + + fn name(&self) -> &'static str { + match self { + CryptoAssetWorker::Binance(w) => w.name(), + CryptoAssetWorker::Bitfinex(w) => w.name(), + CryptoAssetWorker::Bybit(w) => w.name(), + CryptoAssetWorker::Coinbase(w) => w.name(), + CryptoAssetWorker::CoinGecko(w) => w.name(), + CryptoAssetWorker::CoinMarketCap(w) => w.name(), + CryptoAssetWorker::Htx(w) => w.name(), + CryptoAssetWorker::Kraken(w) => w.name(), + CryptoAssetWorker::Okx(w) => w.name(), + } + } + + async fn build(opts: Self::Opts, store: &S) -> Result { + Ok(match opts { + CryptoAssetWorkerOpts::Binance(opts) => { + CryptoAssetWorker::from(bothan_binance::Worker::build(opts, store).await?) + } + CryptoAssetWorkerOpts::Bitfinex(opts) => { + CryptoAssetWorker::from(bothan_bitfinex::Worker::build(opts, store).await?) + } + CryptoAssetWorkerOpts::Bybit(opts) => { + CryptoAssetWorker::from(bothan_bybit::Worker::build(opts, store).await?) + } + CryptoAssetWorkerOpts::Coinbase(opts) => { + CryptoAssetWorker::from(bothan_coinbase::Worker::build(opts, store).await?) + } + CryptoAssetWorkerOpts::CoinGecko(opts) => { + CryptoAssetWorker::from(bothan_coingecko::Worker::build(opts, store).await?) + } + CryptoAssetWorkerOpts::CoinMarketCap(opts) => { + CryptoAssetWorker::from(bothan_coinmarketcap::Worker::build(opts, store).await?) + } + CryptoAssetWorkerOpts::Htx(opts) => { + CryptoAssetWorker::from(bothan_htx::Worker::build(opts, store).await?) + } + CryptoAssetWorkerOpts::Kraken(opts) => { + CryptoAssetWorker::from(bothan_kraken::Worker::build(opts, store).await?) + } + CryptoAssetWorkerOpts::Okx(opts) => { + CryptoAssetWorker::from(bothan_okx::Worker::build(opts, store).await?) + } + }) + } + + async fn get_asset(&self, id: &str) -> Result { + match self { + CryptoAssetWorker::Binance(worker) => worker.get_asset(id).await, + CryptoAssetWorker::Bitfinex(worker) => worker.get_asset(id).await, + CryptoAssetWorker::Bybit(worker) => worker.get_asset(id).await, + CryptoAssetWorker::Coinbase(worker) => worker.get_asset(id).await, + CryptoAssetWorker::CoinGecko(worker) => worker.get_asset(id).await, + CryptoAssetWorker::CoinMarketCap(worker) => worker.get_asset(id).await, + CryptoAssetWorker::Htx(worker) => worker.get_asset(id).await, + CryptoAssetWorker::Kraken(worker) => worker.get_asset(id).await, + CryptoAssetWorker::Okx(worker) => worker.get_asset(id).await, + } + } + + async fn set_query_ids(&self, ids: HashSet) -> Result<(), AssetWorkerError> { + match self { + CryptoAssetWorker::Binance(worker) => worker.set_query_ids(ids).await, + CryptoAssetWorker::Bitfinex(worker) => worker.set_query_ids(ids).await, + CryptoAssetWorker::Bybit(worker) => worker.set_query_ids(ids).await, + CryptoAssetWorker::Coinbase(worker) => worker.set_query_ids(ids).await, + CryptoAssetWorker::CoinGecko(worker) => worker.set_query_ids(ids).await, + CryptoAssetWorker::CoinMarketCap(worker) => worker.set_query_ids(ids).await, + CryptoAssetWorker::Htx(worker) => worker.set_query_ids(ids).await, + CryptoAssetWorker::Kraken(worker) => worker.set_query_ids(ids).await, + CryptoAssetWorker::Okx(worker) => worker.set_query_ids(ids).await, + } + } +} diff --git a/bothan-core/src/manager/crypto_asset_info/worker/opts.rs b/bothan-core/src/manager/crypto_asset_info/worker/opts.rs new file mode 100644 index 00000000..d54e0402 --- /dev/null +++ b/bothan-core/src/manager/crypto_asset_info/worker/opts.rs @@ -0,0 +1,65 @@ +pub enum CryptoAssetWorkerOpts { + Binance(bothan_binance::WorkerOpts), + Bitfinex(bothan_bitfinex::WorkerOpts), + Bybit(bothan_bybit::WorkerOpts), + Coinbase(bothan_coinbase::WorkerOpts), + CoinGecko(bothan_coingecko::WorkerOpts), + CoinMarketCap(bothan_coinmarketcap::WorkerOpts), + Htx(bothan_htx::WorkerOpts), + Kraken(bothan_kraken::WorkerOpts), + Okx(bothan_okx::WorkerOpts), +} + +impl From for CryptoAssetWorkerOpts { + fn from(value: bothan_binance::WorkerOpts) -> Self { + CryptoAssetWorkerOpts::Binance(value) + } +} + +impl From for CryptoAssetWorkerOpts { + fn from(value: bothan_bitfinex::WorkerOpts) -> Self { + CryptoAssetWorkerOpts::Bitfinex(value) + } +} + +impl From for CryptoAssetWorkerOpts { + fn from(value: bothan_bybit::WorkerOpts) -> Self { + CryptoAssetWorkerOpts::Bybit(value) + } +} + +impl From for CryptoAssetWorkerOpts { + fn from(value: bothan_coinbase::WorkerOpts) -> Self { + CryptoAssetWorkerOpts::Coinbase(value) + } +} + +impl From for CryptoAssetWorkerOpts { + fn from(value: bothan_coingecko::WorkerOpts) -> Self { + CryptoAssetWorkerOpts::CoinGecko(value) + } +} + +impl From for CryptoAssetWorkerOpts { + fn from(value: bothan_coinmarketcap::WorkerOpts) -> Self { + CryptoAssetWorkerOpts::CoinMarketCap(value) + } +} + +impl From for CryptoAssetWorkerOpts { + fn from(value: bothan_htx::WorkerOpts) -> Self { + CryptoAssetWorkerOpts::Htx(value) + } +} + +impl From for CryptoAssetWorkerOpts { + fn from(value: bothan_kraken::WorkerOpts) -> Self { + CryptoAssetWorkerOpts::Kraken(value) + } +} + +impl From for CryptoAssetWorkerOpts { + fn from(value: bothan_okx::WorkerOpts) -> Self { + CryptoAssetWorkerOpts::Okx(value) + } +} diff --git a/bothan-core/src/monitoring.rs b/bothan-core/src/monitoring.rs index 70b64211..f0b8d1ed 100644 --- a/bothan-core/src/monitoring.rs +++ b/bothan-core/src/monitoring.rs @@ -1,11 +1,10 @@ pub use client::Client; pub use signer::Signer; -pub use types::{BothanInfo, Entry, Topic, HEARTBEAT_INTERVAL}; +pub use types::{BothanInfo, Entry, HEARTBEAT_INTERVAL, Topic}; pub use utils::create_uuid; -mod client; +pub mod client; pub mod error; -pub mod records; mod signer; -mod types; +pub mod types; mod utils; diff --git a/bothan-core/src/monitoring/client.rs b/bothan-core/src/monitoring/client.rs index 4cba67bd..2bedb093 100644 --- a/bothan-core/src/monitoring/client.rs +++ b/bothan-core/src/monitoring/client.rs @@ -1,14 +1,15 @@ use std::sync::Arc; -use reqwest::header::HeaderMap; use reqwest::Response; +use reqwest::header::HeaderMap; use semver::Version; use serde::Serialize; use crate::monitoring::error::Error; -use crate::monitoring::records::{SignalComputationRecord, SignalRecordsWithTxHash}; use crate::monitoring::signer::Signer; -use crate::monitoring::types::{BothanInfo, Entry, Topic}; +use crate::monitoring::types::{ + BothanInfo, Entry, SignalComputationRecord, SignalTransactionRecord, Topic, +}; pub struct Client { url: String, @@ -38,7 +39,7 @@ impl Client { self.post( uuid, Topic::Record, - SignalRecordsWithTxHash { tx_hash, records }, + SignalTransactionRecord { tx_hash, records }, ) .await } diff --git a/bothan-core/src/monitoring/types.rs b/bothan-core/src/monitoring/types.rs index d10886c0..2ee8433b 100644 --- a/bothan-core/src/monitoring/types.rs +++ b/bothan-core/src/monitoring/types.rs @@ -1,55 +1,14 @@ -use semver::Version; -use serde::ser::SerializeStruct; -use serde::Serialize; +pub use entry::Entry; +pub use info::BothanInfo; +pub use record::{ + OperationRecord, ProcessRecord, SignalComputationRecord, SignalTransactionRecord, SourceRecord, +}; use tokio::time::Duration; +pub use topic::Topic; -pub const HEARTBEAT_INTERVAL: Duration = Duration::new(60, 0); - -#[derive(Serialize)] -#[serde(rename_all = "lowercase")] -pub enum Topic { - Record, - Heartbeat, -} - -pub struct BothanInfo { - pub active_sources: Vec, - pub version: Version, - pub registry_hash: String, -} - -impl BothanInfo { - pub fn new(active_sources: Vec, version: Version, registry_hash: String) -> BothanInfo { - BothanInfo { - active_sources, - version, - registry_hash, - } - } -} +mod entry; +mod info; +mod record; +mod topic; -impl Serialize for BothanInfo { - fn serialize(&self, serializer: S) -> Result - where - S: serde::Serializer, - { - let mut state = serializer.serialize_struct("BothanInfo", 4)?; - state.serialize_field("active_sources", &self.active_sources)?; - state.serialize_field("version", &self.version.to_string())?; - state.serialize_field("registry_hash", &self.registry_hash)?; - state.end() - } -} - -#[derive(Serialize)] -pub struct Entry { - pub uuid: String, - pub topic: Topic, - pub data: T, -} - -impl Entry { - pub fn new(uuid: String, topic: Topic, data: T) -> Entry { - Entry { uuid, topic, data } - } -} +pub const HEARTBEAT_INTERVAL: Duration = Duration::new(60, 0); diff --git a/bothan-core/src/monitoring/types/entry.rs b/bothan-core/src/monitoring/types/entry.rs new file mode 100644 index 00000000..968a592d --- /dev/null +++ b/bothan-core/src/monitoring/types/entry.rs @@ -0,0 +1,16 @@ +use serde::Serialize; + +use super::topic::Topic; + +#[derive(Serialize)] +pub struct Entry { + pub uuid: String, + pub topic: Topic, + pub data: T, +} + +impl Entry { + pub fn new(uuid: String, topic: Topic, data: T) -> Entry { + Entry { uuid, topic, data } + } +} diff --git a/bothan-core/src/monitoring/types/info.rs b/bothan-core/src/monitoring/types/info.rs new file mode 100644 index 00000000..5cce4e09 --- /dev/null +++ b/bothan-core/src/monitoring/types/info.rs @@ -0,0 +1,32 @@ +use semver::Version; +use serde::Serialize; +use serde::ser::SerializeStruct; + +pub struct BothanInfo { + pub active_sources: Vec, + pub version: Version, + pub registry_hash: String, +} + +impl BothanInfo { + pub fn new(active_sources: Vec, version: Version, registry_hash: String) -> BothanInfo { + BothanInfo { + active_sources, + version, + registry_hash, + } + } +} + +impl Serialize for BothanInfo { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + let mut state = serializer.serialize_struct("BothanInfo", 4)?; + state.serialize_field("active_sources", &self.active_sources)?; + state.serialize_field("version", &self.version.to_string())?; + state.serialize_field("registry_hash", &self.registry_hash)?; + state.end() + } +} diff --git a/bothan-core/src/monitoring/records.rs b/bothan-core/src/monitoring/types/record.rs similarity index 91% rename from bothan-core/src/monitoring/records.rs rename to bothan-core/src/monitoring/types/record.rs index 142fada0..8b6a40c8 100644 --- a/bothan-core/src/monitoring/records.rs +++ b/bothan-core/src/monitoring/types/record.rs @@ -1,14 +1,13 @@ use std::sync::Arc; +use bothan_lib::registry::post_processor::PostProcessError; +use bothan_lib::registry::processor::ProcessError; +use bothan_lib::registry::source::Operation; use rust_decimal::Decimal; use serde::{Deserialize, Serialize}; -use crate::registry::post_processor::PostProcessError; -use crate::registry::processor::ProcessError; -use crate::registry::source::Operation; - #[derive(Debug, Serialize, Deserialize)] -pub struct SignalRecordsWithTxHash { +pub struct SignalTransactionRecord { pub tx_hash: String, pub records: Arc>>, } diff --git a/bothan-core/src/monitoring/types/topic.rs b/bothan-core/src/monitoring/types/topic.rs new file mode 100644 index 00000000..dfe78a6e --- /dev/null +++ b/bothan-core/src/monitoring/types/topic.rs @@ -0,0 +1,8 @@ +use serde::Serialize; + +#[derive(Serialize)] +#[serde(rename_all = "lowercase")] +pub enum Topic { + Record, + Heartbeat, +} diff --git a/bothan-core/src/monitoring/utils.rs b/bothan-core/src/monitoring/utils.rs index a294e6a7..b857e509 100644 --- a/bothan-core/src/monitoring/utils.rs +++ b/bothan-core/src/monitoring/utils.rs @@ -1,5 +1,5 @@ -use rand::rngs::OsRng; use rand::RngCore; +use rand::rngs::OsRng; pub fn create_uuid() -> String { let mut uuid_bytes = [0u8; 16]; diff --git a/bothan-core/src/store.rs b/bothan-core/src/store.rs index 085f276b..6bdceb3f 100644 --- a/bothan-core/src/store.rs +++ b/bothan-core/src/store.rs @@ -1,257 +1,2 @@ -use std::collections::{HashMap, HashSet}; -use std::path::Path; -use std::sync::Arc; - -use bincode::{config, decode_from_slice, encode_to_vec}; -use rust_rocksdb::{Options, DB}; -use tokio::sync::RwLock; -use tracing::debug; - -pub use manager::ManagerStore; -pub use worker::WorkerStore; - -use crate::registry::{Registry, Valid}; -use crate::store::error::Error; -use crate::store::types::Key; -use crate::types::AssetInfo; - -pub mod error; -mod manager; -mod types; -mod worker; - -#[derive(Clone)] -pub struct SharedStore { - inner: Arc>, -} - -pub type AssetStore = HashMap; -pub type QueryIDs = HashSet; - -struct Inner { - registry: Registry, - db: DB, -} - -impl SharedStore { - /// Create a new shared store with the given registry and flush path. - pub async fn new(registry: Registry, flush_path: &Path) -> Result { - let mut opts = Options::default(); - opts.create_if_missing(true); - - let inner = Inner { - registry, - db: DB::open(&opts, flush_path)?, - }; - - let store = Self { - inner: Arc::new(RwLock::new(inner)), - }; - - Ok(store) - } - - /// Restore the store's registry from the database - pub async fn restore(&mut self) -> Result<(), Error> { - let mut inner = self.inner.write().await; - - if let Some(unvalidated_registry) = inner - .db - .get(Key::Registry.to_prefixed_bytes())? - .map(|b| decode_from_slice::(&b, config::standard())) - .transpose()? - .map(|(r, _)| r) - { - inner.registry = unvalidated_registry.validate()?; - debug!("loaded registry"); - } - - Ok(()) - } - - pub fn create_manager_store(&self) -> ManagerStore { - ManagerStore::new(self.clone()) - } - - pub fn create_worker_store>(&self, prefix: T) -> WorkerStore { - WorkerStore::new(self.clone(), prefix.into()) - } - - async fn get_registry(&self) -> Registry { - self.inner.read().await.registry.clone() - } - - async fn set_registry(&self, registry: Registry) -> Result<(), Error> { - let mut inner = self.inner.write().await; - let encoded = encode_to_vec(®istry, config::standard())?; - inner.db.put(Key::Registry.to_prefixed_bytes(), encoded)?; - inner.registry = registry; - Ok(()) - } - - async fn get_registry_hash(&self) -> Result, Error> { - let encoded = self - .inner - .read() - .await - .db - .get(Key::RegistryHash.to_prefixed_bytes())?; - let hash = encoded - .map(|b| decode_from_slice(&b, config::standard())) - .transpose()? - .map(|(hash, _)| hash); - Ok(hash) - } - - async fn set_registry_hash(&self, hash: String) -> Result<(), Error> { - let encoded = encode_to_vec(&hash, config::standard())?; - self.inner - .write() - .await - .db - .put(Key::RegistryHash.to_prefixed_bytes(), encoded)?; - Ok(()) - } - - async fn get_query_ids>(&self, source_id: &S) -> Result, Error> { - let key = Key::QueryIDs { - source_id: source_id.as_ref(), - }; - - let encoded = self.inner.read().await.db.get(key.to_prefixed_bytes())?; - let query_ids = encoded - .map(|b| decode_from_slice(&b, config::standard())) - .transpose()? - .map(|(ids, _)| ids); - Ok(query_ids) - } - - async fn contains_query_id(&self, source_id: &S, id: &I) -> Result - where - S: AsRef, - I: AsRef, - { - match self.get_query_ids(source_id).await { - Ok(Some(query_ids)) => Ok(query_ids.contains(id.as_ref())), - Ok(None) => Ok(false), - Err(e) => Err(e), - } - } - - async fn set_query_ids>( - &self, - source_id: &S, - query_ids: QueryIDs, - ) -> Result<(), Error> { - let key = Key::QueryIDs { - source_id: source_id.as_ref(), - }; - - let encoded = encode_to_vec(&query_ids, config::standard())?; - self.inner - .write() - .await - .db - .put(key.to_prefixed_bytes(), encoded)?; - Ok(()) - } - - async fn insert_query_ids(&self, source_id: &S, ids: Vec) -> Result, Error> - where - S: AsRef, - I: Into, - { - let mut query_ids = self.get_query_ids(&source_id).await?.unwrap_or_default(); - let inserted = ids - .into_iter() - .map(|id| query_ids.insert(id.into())) - .collect(); - - self.set_query_ids(&source_id, query_ids).await?; - Ok(inserted) - } - - async fn remove_query_ids(&self, source_id: &S, ids: &[I]) -> Result, Error> - where - S: AsRef, - I: AsRef, - { - let mut current_set = self.get_query_ids(&source_id).await?.unwrap_or_default(); - let removed = ids - .iter() - .map(|id| current_set.remove(id.as_ref())) - .collect(); - - self.set_query_ids(&source_id, current_set).await?; - Ok(removed) - } - - async fn get_asset_info(&self, source_id: &S, id: &I) -> Result, Error> - where - S: AsRef, - I: AsRef, - { - let key = Key::AssetStore { - source_id: source_id.as_ref(), - id: id.as_ref(), - }; - - let encoded = self.inner.read().await.db.get(key.to_prefixed_bytes())?; - let asset_info = encoded - .map(|b| decode_from_slice(&b, config::standard())) - .transpose()? - .map(|(info, _)| info); - Ok(asset_info) - } - - async fn insert_asset_info( - &self, - source_id: &S, - id: &I, - asset_info: AssetInfo, - ) -> Result<(), Error> - where - S: AsRef, - I: AsRef, - { - let key = Key::AssetStore { - source_id: source_id.as_ref(), - id: id.as_ref(), - }; - - let encoded = encode_to_vec(&asset_info, config::standard())?; - self.inner - .write() - .await - .db - .put(key.to_prefixed_bytes(), encoded)?; - Ok(()) - } - - async fn insert_asset_infos( - &self, - source_id: &S, - assets: Vec<(I, AssetInfo)>, - ) -> Result<(), Error> - where - S: AsRef, - I: AsRef, - { - let inner = self.inner.write().await; - for (id, asset_info) in assets { - let key = Key::AssetStore { - source_id: source_id.as_ref(), - id: id.as_ref(), - }; - let encoded = encode_to_vec(&asset_info, config::standard())?; - inner.db.put(key.to_prefixed_bytes(), encoded)?; - } - Ok(()) - } -} - -impl PartialEq for SharedStore { - fn eq(&self, other: &Self) -> bool { - Arc::ptr_eq(&self.inner, &other.inner) - } -} +#[cfg(feature = "rocksdb")] +pub mod rocksdb; diff --git a/bothan-core/src/store/error.rs b/bothan-core/src/store/error.rs deleted file mode 100644 index 3bad9193..00000000 --- a/bothan-core/src/store/error.rs +++ /dev/null @@ -1,39 +0,0 @@ -use crate::registry::validate::ValidationError; - -#[derive(Clone, Debug, thiserror::Error, PartialEq)] -#[error("An error occurred while storing the data: {message}")] -pub struct Error { - message: String, -} - -impl From for Error { - fn from(error: ValidationError) -> Self { - Self { - message: error.to_string(), - } - } -} - -impl From for Error { - fn from(error: rust_rocksdb::Error) -> Self { - Self { - message: error.to_string(), - } - } -} - -impl From for Error { - fn from(error: bincode::error::EncodeError) -> Self { - Self { - message: error.to_string(), - } - } -} - -impl From for Error { - fn from(error: bincode::error::DecodeError) -> Self { - Self { - message: error.to_string(), - } - } -} diff --git a/bothan-core/src/store/manager.rs b/bothan-core/src/store/manager.rs deleted file mode 100644 index f6b0a7d4..00000000 --- a/bothan-core/src/store/manager.rs +++ /dev/null @@ -1,26 +0,0 @@ -use crate::registry::{Registry, Valid}; -use crate::store::error::Error; -use crate::store::SharedStore; - -pub struct ManagerStore { - store: SharedStore, -} - -impl ManagerStore { - pub fn new(store: SharedStore) -> Self { - Self { store } - } - - pub async fn set_registry(&self, registry: Registry, hash: String) -> Result<(), Error> { - self.store.set_registry(registry).await?; - self.store.set_registry_hash(hash).await - } - - pub async fn get_registry(&self) -> Registry { - self.store.get_registry().await - } - - pub async fn get_registry_hash(&self) -> Result, Error> { - self.store.get_registry_hash().await - } -} diff --git a/bothan-core/src/store/rocksdb.rs b/bothan-core/src/store/rocksdb.rs new file mode 100644 index 00000000..8480a539 --- /dev/null +++ b/bothan-core/src/store/rocksdb.rs @@ -0,0 +1,167 @@ +pub mod error; +mod key; + +use std::collections::HashSet; +use std::path::Path; +use std::sync::Arc; + +use bincode::{Decode, Encode, config, decode_from_slice, encode_to_vec}; +use bothan_lib::registry::{Registry, Valid}; +use bothan_lib::store::Store; +use bothan_lib::types::AssetInfo; +use rust_rocksdb::{DB, Options, WriteBatch}; +use tokio::sync::RwLock; + +use crate::store::rocksdb::error::{LoadError, RocksDbError}; +use crate::store::rocksdb::key::Key; + +#[derive(Clone)] +pub struct RocksDbStore { + db: Arc, + registry: Arc>>, +} + +impl RocksDbStore { + pub fn new>(flush_path: P) -> Result { + let mut opts = Options::default(); + opts.create_if_missing(true); + DB::destroy(&opts, &flush_path)?; + + let db = Arc::new(DB::open(&opts, &flush_path)?); + Ok(RocksDbStore { + db, + registry: Arc::new(RwLock::new(Registry::default())), + }) + } + + pub fn load>(flush_path: P) -> Result { + let mut opts = Options::default(); + opts.create_if_missing(true); + + let db = Arc::new(DB::open(&opts, &flush_path)?); + let unvalidated_registry = db + .get(Key::Registry.to_prefixed_bytes())? + .map(|b| decode_from_slice::(&b, config::standard())) + .transpose()? + .map(|(r, _)| r) + .ok_or(LoadError::NoExistingRegistry)?; + + let registry = Registry::validate(unvalidated_registry)?; + + Ok(RocksDbStore { + db, + registry: Arc::new(RwLock::new(registry)), + }) + } + + fn set(&self, key: &Key, value: &V) -> Result<(), RocksDbError> { + let encoded = encode_to_vec(value, config::standard())?; + self.db.put(key.to_prefixed_bytes(), encoded)?; + Ok(()) + } + + fn get(&self, key: &Key) -> Result, RocksDbError> { + let value = self + .db + .get(key.to_prefixed_bytes())? + .map(|b| decode_from_slice(&b, config::standard())) + .transpose()? + .map(|(v, _)| v); + Ok(value) + } +} + +#[async_trait::async_trait] +impl Store for RocksDbStore { + type Error = RocksDbError; + + async fn set_registry( + &self, + registry: Registry, + ipfs_hash: String, + ) -> Result<(), Self::Error> { + let encoded_registry = encode_to_vec(®istry, config::standard())?; + let encoded_hash = encode_to_vec(&ipfs_hash, config::standard())?; + + // if the registry can be encoded, lock first to prevent race conditions + let mut curr_reg = self.registry.write().await; + + // save to db + let mut write_batch = WriteBatch::default(); + write_batch.put(Key::Registry.to_prefixed_bytes(), encoded_registry); + write_batch.put(Key::RegistryIpfsHash.to_prefixed_bytes(), encoded_hash); + + self.db.write(write_batch)?; + + // save to local + *curr_reg = registry; + + Ok(()) + } + + async fn get_registry(&self) -> Registry { + self.registry.read().await.clone() + } + + async fn get_registry_ipfs_hash(&self) -> Result, Self::Error> { + self.get(&Key::RegistryIpfsHash) + } + + async fn set_query_ids(&self, prefix: &str, ids: HashSet) -> Result<(), Self::Error> { + self.set(&Key::QueryIDs { source_id: prefix }, &ids) + } + + async fn get_query_ids(&self, prefix: &str) -> Result>, Self::Error> { + self.get(&Key::QueryIDs { source_id: prefix }) + } + + async fn contains_query_id(&self, prefix: &str, id: &str) -> Result { + let bool = self + .get::>(&Key::QueryIDs { source_id: prefix })? + .unwrap_or_default() + .contains(id); + Ok(bool) + } + + async fn get_asset_info( + &self, + prefix: &str, + id: &str, + ) -> Result, Self::Error> { + self.get(&Key::AssetStore { + source_id: prefix, + id, + }) + } + + async fn insert_asset_info( + &self, + prefix: &str, + asset_info: AssetInfo, + ) -> Result<(), Self::Error> { + let key = Key::AssetStore { + source_id: prefix, + id: &asset_info.id, + }; + self.set(&key, &asset_info) + } + + async fn insert_asset_infos( + &self, + prefix: &str, + asset_infos: Vec, + ) -> Result<(), Self::Error> { + let mut write_batch = WriteBatch::default(); + for asset_info in asset_infos { + let key = Key::AssetStore { + source_id: prefix, + id: &asset_info.id, + }; + let encoded = encode_to_vec(&asset_info, config::standard())?; + write_batch.put(key.to_prefixed_bytes(), encoded); + } + + self.db.write(write_batch)?; + Ok(()) + } +} diff --git a/bothan-core/src/store/rocksdb/error.rs b/bothan-core/src/store/rocksdb/error.rs new file mode 100644 index 00000000..e50ebd0c --- /dev/null +++ b/bothan-core/src/store/rocksdb/error.rs @@ -0,0 +1,26 @@ +use bincode::error::{DecodeError, EncodeError}; +use bothan_lib::registry::ValidationError; + +#[derive(Debug, thiserror::Error)] +pub enum LoadError { + #[error("No existing registry found")] + NoExistingRegistry, + #[error("Failed to decode registry: {0}")] + FailedToDecodeRegistry(#[from] DecodeError), + #[error("{0}")] + RocksDBError(#[from] rust_rocksdb::Error), + #[error("registry is not valid: {0}")] + InvalidRegistry(#[from] ValidationError), +} + +#[derive(Debug, thiserror::Error)] +pub enum RocksDbError { + #[error("Failed to encode data: {0}")] + FailedToeEncode(#[from] EncodeError), + + #[error("Failed to decode data: {0}")] + FailedToDecode(#[from] DecodeError), + + #[error("{0}")] + RocksDBError(#[from] rust_rocksdb::Error), +} diff --git a/bothan-core/src/store/types.rs b/bothan-core/src/store/rocksdb/key.rs similarity index 81% rename from bothan-core/src/store/types.rs rename to bothan-core/src/store/rocksdb/key.rs index 83c90f94..c7bf032d 100644 --- a/bothan-core/src/store/types.rs +++ b/bothan-core/src/store/rocksdb/key.rs @@ -1,25 +1,25 @@ use std::fmt::Display; -pub(crate) enum Key<'a> { +pub enum Key<'a> { AssetStore { source_id: &'a str, id: &'a str }, QueryIDs { source_id: &'a str }, Registry, - RegistryHash, + RegistryIpfsHash, } -impl<'a> Display for Key<'a> { +impl Display for Key<'_> { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { let s = match self { Key::AssetStore { source_id, id } => format!("asset_store::{}::{}", source_id, id), Key::QueryIDs { source_id } => format!("query_id::{}", source_id), Key::Registry => "registry".to_string(), - Key::RegistryHash => "registry_hash".to_string(), + Key::RegistryIpfsHash => "registry_ipfs_hash".to_string(), }; write!(f, "{}", s) } } -impl<'a> Key<'a> { +impl Key<'_> { pub fn to_prefixed_bytes(&self) -> Vec { let prefix = "bothan::".as_bytes(); let content = self.to_string().into_bytes(); diff --git a/bothan-core/src/store/worker.rs b/bothan-core/src/store/worker.rs deleted file mode 100644 index e472eef3..00000000 --- a/bothan-core/src/store/worker.rs +++ /dev/null @@ -1,139 +0,0 @@ -use std::collections::HashSet; - -use crate::store::error::Error; -use crate::store::{QueryIDs, SharedStore}; -use crate::types::AssetInfo; -use crate::worker::AssetState; - -pub struct WorkerStore { - store: SharedStore, - prefix: String, -} - -impl WorkerStore { - pub fn new>(store: SharedStore, prefix: T) -> Self { - Self { - store, - prefix: prefix.into(), - } - } - - pub async fn get_asset>(&self, id: &K) -> Result { - if !self.store.contains_query_id(&self.prefix, id).await? { - return Ok(AssetState::Unsupported); - } - - match self.store.get_asset_info(&self.prefix, id).await? { - Some(asset) => Ok(AssetState::Available(asset)), - None => Ok(AssetState::Pending), - } - } - - pub async fn set_asset>( - &self, - id: K, - asset_info: AssetInfo, - ) -> Result<(), Error> { - self.store - .insert_asset_info(&self.prefix, &id, asset_info) - .await - } - - pub async fn set_assets>( - &self, - assets: Vec<(K, AssetInfo)>, - ) -> Result<(), Error> { - self.store.insert_asset_infos(&self.prefix, assets).await - } - - // TODO: Deprecate when the new query_id system is in place - pub async fn set_query_ids(&self, ids: Vec) -> Result<(Vec, Vec), Error> - where - K: Into + Clone, - { - let current_ids = self.get_query_ids().await?; - let new_ids: QueryIDs = HashSet::from_iter(ids.into_iter().map(Into::into)); - - let added = new_ids - .difference(¤t_ids) - .cloned() - .collect::>(); - let removed = current_ids - .difference(&new_ids) - .cloned() - .collect::>(); - - self.store.set_query_ids(&self.prefix, new_ids).await?; - - Ok((added, removed)) - } - - // Computes the signals to add and remove from the query set if the given IDs is to replace - // the current query_id set - pub async fn compute_query_id_differences( - &self, - ids: Vec, - ) -> Result<(Vec, Vec), Error> - where - K: Into + Clone, - { - let current_ids = self.get_query_ids().await?; - let new_ids: QueryIDs = HashSet::from_iter(ids.into_iter().map(Into::into)); - - let to_add = new_ids - .difference(¤t_ids) - .cloned() - .collect::>(); - let to_remove = current_ids - .difference(&new_ids) - .cloned() - .collect::>(); - - Ok((to_add, to_remove)) - } - - pub async fn add_query_ids(&self, ids: Vec) -> Result, Error> - where - K: Into + Clone, - { - let changes = self - .store - .insert_query_ids(&self.prefix, ids.clone()) - .await?; - - let added = ids - .into_iter() - .zip(changes.into_iter()) - .filter(|(_, changed)| *changed) - .map(|(id, _)| id) - .collect(); - Ok(added) - } - - pub async fn remove_query_ids(&self, ids: Vec) -> Result, Error> - where - K: Into + AsRef, - { - let changes = self - .store - .remove_query_ids(&self.prefix, ids.as_slice()) - .await?; - - let removed = ids - .into_iter() - .zip(changes.into_iter()) - .filter(|(_, changed)| *changed) - .map(|(id, _)| id) - .collect(); - Ok(removed) - } - - pub async fn get_query_ids(&self) -> Result { - let query_ids = self - .store - .get_query_ids(&self.prefix) - .await? - .unwrap_or_default(); - Ok(query_ids) - } -} diff --git a/bothan-core/src/worker.rs b/bothan-core/src/worker.rs deleted file mode 100644 index 40fb4bbf..00000000 --- a/bothan-core/src/worker.rs +++ /dev/null @@ -1,44 +0,0 @@ -use crate::store::error::Error as StoreError; -use crate::store::WorkerStore; -use crate::types::AssetInfo; -use serde::{Deserialize, Serialize}; -use std::sync::Arc; - -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -pub enum AssetState { - Unsupported, - Pending, - Available(AssetInfo), -} - -#[derive(Clone, Debug, thiserror::Error, PartialEq)] -#[error("failed to modify query IDs: {msg}")] -pub struct SetQueryIDError { - msg: String, -} - -impl SetQueryIDError { - pub fn new(msg: String) -> Self { - Self { msg } - } -} - -/// The universal trait for all workers that provide asset info. -#[async_trait::async_trait] -pub trait AssetWorker: Send + Sync { - async fn get_asset(&self, id: &str) -> Result; - async fn set_query_ids(&self, ids: Vec) -> Result<(), SetQueryIDError>; -} - -#[async_trait::async_trait] -pub trait AssetWorkerBuilder<'a> { - type Opts; - type Worker: AssetWorker + 'a; - type Error: std::error::Error; - - fn new(store: WorkerStore, opts: Self::Opts) -> Self; - - fn worker_name() -> &'static str; - - async fn build(self) -> Result, Self::Error>; -} diff --git a/bothan-cryptocompare/Cargo.toml b/bothan-cryptocompare/Cargo.toml deleted file mode 100644 index a0e66775..00000000 --- a/bothan-cryptocompare/Cargo.toml +++ /dev/null @@ -1,23 +0,0 @@ -[package] -name = "bothan-cryptocompare" -version = "0.0.1-beta.1" -edition.workspace = true -license.workspace = true -repository.workspace = true - -[dependencies] -async-trait = { workspace = true } -bothan-core = { workspace = true } -futures-util = { workspace = true, features = ["sink", "std"] } -rust_decimal = { workspace = true } -serde = { workspace = true } -serde_json = { workspace = true } -thiserror = { workspace = true } -tokio = { workspace = true } -tokio-tungstenite = { workspace = true } -tracing = { workspace = true } -tracing-subscriber = { workspace = true } -url = { workspace = true } - -[dev-dependencies] -serde_json = { workspace = true } diff --git a/bothan-cryptocompare/examples/cryptocompare_basic.rs b/bothan-cryptocompare/examples/cryptocompare_basic.rs deleted file mode 100644 index c9266c05..00000000 --- a/bothan-cryptocompare/examples/cryptocompare_basic.rs +++ /dev/null @@ -1,38 +0,0 @@ -use std::time::Duration; - -use tokio::time::sleep; -use tracing_subscriber::fmt::init; - -use bothan_core::registry::Registry; -use bothan_core::store::SharedStore; -use bothan_core::worker::{AssetWorker, AssetWorkerBuilder}; -use bothan_cryptocompare::{CryptoCompareWorkerBuilder, CryptoCompareWorkerBuilderOpts}; - -#[tokio::main] -async fn main() { - init(); - let path = std::env::current_dir().unwrap().join("store"); - let registry = Registry::default().validate().unwrap(); - let store = SharedStore::new(registry, &path).await.unwrap(); - let worker_store = store.create_worker_store(CryptoCompareWorkerBuilder::worker_name()); - let opts = CryptoCompareWorkerBuilderOpts::default(); - let worker = CryptoCompareWorkerBuilder::new(worker_store, opts) - .with_api_key("YOUR_API_KEY") - .build() - .await - .unwrap(); - - worker - .set_query_ids(vec!["BTC-USD".to_string(), "ETH-USD".to_string()]) - .await - .unwrap(); - - sleep(Duration::from_secs(2)).await; - - loop { - let btc_data = worker.get_asset("BTC-USD").await; - let eth_data = worker.get_asset("ETH-USD").await; - println!("{:?} {:?}", btc_data, eth_data); - tokio::time::sleep(Duration::from_secs(5)).await; - } -} diff --git a/bothan-cryptocompare/src/api.rs b/bothan-cryptocompare/src/api.rs deleted file mode 100644 index 08e37897..00000000 --- a/bothan-cryptocompare/src/api.rs +++ /dev/null @@ -1,7 +0,0 @@ -pub use websocket::{ - CryptoCompareWebSocketConnection, CryptoCompareWebSocketConnector, DEFAULT_URL, -}; - -pub mod errors; -pub mod msgs; -mod websocket; diff --git a/bothan-cryptocompare/src/api/errors.rs b/bothan-cryptocompare/src/api/errors.rs deleted file mode 100644 index 5623a79f..00000000 --- a/bothan-cryptocompare/src/api/errors.rs +++ /dev/null @@ -1,26 +0,0 @@ -use thiserror::Error; -use tokio_tungstenite::tungstenite; - -#[derive(Debug, Error)] -pub enum ConnectionError { - #[error("invalid url")] - InvalidURL(#[from] url::ParseError), - - #[error("failed to connect to endpoint {0}")] - ConnectionFailure(#[from] tungstenite::Error), - - #[error("received unsuccessful WebSocket response: {0}")] - UnsuccessfulWebSocketResponse(tungstenite::http::StatusCode), -} - -#[derive(Debug, Error)] -pub enum MessageError { - #[error("failed to parse message")] - Parse(#[from] serde_json::Error), - - #[error("channel closed")] - ChannelClosed, - - #[error("unsupported message")] - UnsupportedMessage, -} diff --git a/bothan-cryptocompare/src/api/msgs.rs b/bothan-cryptocompare/src/api/msgs.rs deleted file mode 100644 index 81e2f40b..00000000 --- a/bothan-cryptocompare/src/api/msgs.rs +++ /dev/null @@ -1,81 +0,0 @@ -use std::fmt::Display; - -use serde::{Deserialize, Serialize}; - -#[derive(Debug, Serialize, Deserialize)] -#[serde(tag = "TYPE")] -pub enum Packet { - #[serde(rename = "4000")] - SessionWelcome(Message), - - #[serde(rename = "4001")] - StreamerError(Message), - - #[serde(rename = "4002")] - RateLimitError(Message), - - #[serde(rename = "4003")] - SubscriptionError(Message), - - #[serde(rename = "4004")] - SubscriptionValidationError(Message), - - #[serde(rename = "4005")] - SubscriptionAccepted(Message), - - #[serde(rename = "4006")] - SubscriptionRejected(Message), - - #[serde(rename = "4007")] - SubscriptionAddComplete(Message), - - #[serde(rename = "4008")] - SubscriptionRemoveComplete(Message), - - #[serde(rename = "4009")] - SubscriptionRemoveAllComplete(Message), - - #[serde(rename = "4010")] - SubscriptionWarning(Message), - - #[serde(rename = "4011")] - AuthenticationWarning(Message), - - #[serde(rename = "4012")] - MessageValidationError(Message), - - #[serde(rename = "4013")] - Heartbeat(Message), - - #[serde(rename = "1101")] - RefTickAdaptive(ReferenceTick), - - #[serde(rename = "985")] - RefTickAdaptiveWithConversion(ReferenceTick), - - #[serde(rename = "987")] - RefTickAdaptiveWithInversion(ReferenceTick), - - Ping, -} - -#[derive(Debug, Serialize, Deserialize)] -#[serde(rename_all = "SCREAMING_SNAKE_CASE")] -pub struct ReferenceTick { - pub instrument: String, - pub value: f64, - #[serde(rename = "VALUE_LAST_UPDATE_TS")] - pub last_update: i64, -} - -#[derive(Debug, Serialize, Deserialize)] -#[serde(rename_all = "SCREAMING_SNAKE_CASE")] -pub struct Message { - message: String, -} - -impl Display for Message { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{}", self.message) - } -} diff --git a/bothan-cryptocompare/src/api/websocket.rs b/bothan-cryptocompare/src/api/websocket.rs deleted file mode 100644 index 851a2b8f..00000000 --- a/bothan-cryptocompare/src/api/websocket.rs +++ /dev/null @@ -1,98 +0,0 @@ -use futures_util::{SinkExt, StreamExt}; -use serde_json::json; -use tokio::net::TcpStream; -use tokio_tungstenite::tungstenite::Message; -use tokio_tungstenite::{connect_async, tungstenite, MaybeTlsStream, WebSocketStream}; -use url::Url; - -use crate::api::errors::{ConnectionError, MessageError}; -use crate::api::msgs::Packet; - -/// The default URL for the CryptoCompare API. -pub const DEFAULT_URL: &str = "wss://data-streamer.cryptocompare.com"; - -pub struct CryptoCompareWebSocketConnector { - url: String, - api_key: String, -} - -impl CryptoCompareWebSocketConnector { - pub fn new(url: String, api_key: String) -> Self { - Self { url, api_key } - } - - pub async fn connect(&self) -> Result { - let url = Url::parse_with_params(&self.url, &[("api_key", &self.api_key)])?; - let (wss, resp) = connect_async(url.as_str()).await?; - - let status = resp.status(); - if status.as_u16() >= 400 { - return Err(ConnectionError::UnsuccessfulWebSocketResponse(status)); - } - - Ok(CryptoCompareWebSocketConnection::new(wss)) - } -} - -pub struct CryptoCompareWebSocketConnection { - ws_stream: WebSocketStream>, -} - -impl CryptoCompareWebSocketConnection { - pub fn new(ws_stream: WebSocketStream>) -> Self { - Self { ws_stream } - } - - pub async fn subscribe_latest_tick_adaptive_inclusion>( - &mut self, - instruments: &[T], - ) -> Result<(), tungstenite::Error> { - let payload = json!({ - "action": "SUBSCRIBE", - "type": "index_cc_v1_latest_tick", - "groups": ["VALUE"], - "market": "cadli", - "instruments": instruments.iter().map(|s| s.as_ref()).collect::>() - }); - - let message = Message::Text(payload.to_string()); - self.ws_stream.send(message).await?; - Ok(()) - } - - pub async fn unsubscribe_latest_tick_adaptive_inclusion>( - &mut self, - instruments: &[T], - ) -> Result<(), tungstenite::Error> { - let payload = json!({ - "action": "UNSUBSCRIBE", - "type": "index_cc_v1_latest_tick", - "groups": ["VALUE"], - "market": "cadli", - "instruments": instruments.iter().map(|s| s.as_ref()).collect::>() - }); - - let message = Message::Text(payload.to_string()); - self.ws_stream.send(message).await?; - Ok(()) - } - - pub async fn next(&mut self) -> Result { - if let Some(Ok(result_msg)) = self.ws_stream.next().await { - match result_msg { - Message::Text(msg) => Ok(serde_json::from_str::(&msg)?), - Message::Close(_) => Err(MessageError::ChannelClosed), - // We skip deserializing ping as it's the only packet that we get from this message type - Message::Ping(_) => Ok(Packet::Ping), - _ => Err(MessageError::UnsupportedMessage), - } - } else { - Err(MessageError::ChannelClosed) - } - } - - pub async fn close(&mut self) -> Result<(), tungstenite::Error> { - self.ws_stream.close(None).await?; - Ok(()) - } -} diff --git a/bothan-cryptocompare/src/lib.rs b/bothan-cryptocompare/src/lib.rs deleted file mode 100644 index e159c8c9..00000000 --- a/bothan-cryptocompare/src/lib.rs +++ /dev/null @@ -1,6 +0,0 @@ -pub use worker::builder::CryptoCompareWorkerBuilder; -pub use worker::opts::CryptoCompareWorkerBuilderOpts; -pub use worker::CryptoCompareWorker; - -pub mod api; -pub mod worker; diff --git a/bothan-cryptocompare/src/worker.rs b/bothan-cryptocompare/src/worker.rs deleted file mode 100644 index 9df4b467..00000000 --- a/bothan-cryptocompare/src/worker.rs +++ /dev/null @@ -1,68 +0,0 @@ -use tokio::sync::mpsc::Sender; - -use bothan_core::store::error::Error as StoreError; -use bothan_core::store::WorkerStore; -use bothan_core::worker::{AssetState, AssetWorker, SetQueryIDError}; - -use crate::api::CryptoCompareWebSocketConnector; - -mod asset_worker; -pub mod builder; -pub mod errors; -pub mod opts; -pub mod types; - -/// A worker that fetches and stores the asset information from CryptoCompare's API. -pub struct CryptoCompareWorker { - connector: CryptoCompareWebSocketConnector, - store: WorkerStore, - subscribe_tx: Sender>, - unsubscribe_tx: Sender>, -} - -impl CryptoCompareWorker { - /// Create a new worker with the specified api and store. - pub fn new( - connector: CryptoCompareWebSocketConnector, - store: WorkerStore, - subscribe_tx: Sender>, - unsubscribe_tx: Sender>, - ) -> Self { - Self { - connector, - store, - subscribe_tx, - unsubscribe_tx, - } - } -} - -#[async_trait::async_trait] -impl AssetWorker for CryptoCompareWorker { - /// Fetches the AssetStatus for the given cryptocurrency ids. - async fn get_asset(&self, id: &str) -> Result { - self.store.get_asset(&id).await - } - - /// Sets the specified cryptocurrency IDs to the query. If the ids are already in the query set, - /// it will not be resubscribed. - async fn set_query_ids(&self, ids: Vec) -> Result<(), SetQueryIDError> { - let (to_sub, to_unsub) = self - .store - .set_query_ids(ids) - .await - .map_err(|e| SetQueryIDError::new(e.to_string()))?; - - self.subscribe_tx - .send(to_sub) - .await - .map_err(|e| SetQueryIDError::new(e.to_string()))?; - - self.unsubscribe_tx - .send(to_unsub) - .await - .map_err(|e| SetQueryIDError::new(e.to_string()))?; - - Ok(()) - } -} diff --git a/bothan-cryptocompare/src/worker/asset_worker.rs b/bothan-cryptocompare/src/worker/asset_worker.rs deleted file mode 100644 index 2bf99bbd..00000000 --- a/bothan-cryptocompare/src/worker/asset_worker.rs +++ /dev/null @@ -1,184 +0,0 @@ -use std::sync::Weak; - -use rust_decimal::prelude::FromPrimitive; -use rust_decimal::Decimal; -use tokio::select; -use tokio::sync::mpsc::Receiver; -use tokio::time::{sleep, timeout}; -use tokio_tungstenite::tungstenite; -use tracing::{debug, error, info, warn}; - -use bothan_core::store::WorkerStore; -use bothan_core::types::AssetInfo; - -use crate::api::errors::MessageError; -use crate::api::msgs::{Packet, ReferenceTick}; -use crate::api::{CryptoCompareWebSocketConnection, CryptoCompareWebSocketConnector}; -use crate::worker::errors::WorkerError; -use crate::worker::types::{DEFAULT_TIMEOUT, RECONNECT_BUFFER}; -use crate::worker::CryptoCompareWorker; - -pub(crate) async fn start_asset_worker( - weak_worker: Weak, - mut connection: CryptoCompareWebSocketConnection, - mut subscribe_rx: Receiver>, - mut unsubscribe_rx: Receiver>, -) { - while let Some(worker) = weak_worker.upgrade() { - select! { - Some(ids) = subscribe_rx.recv() => handle_subscribe_recv(ids, &worker.store, &mut connection).await, - Some(ids) = unsubscribe_rx.recv() => handle_unsubscribe_recv(ids, &worker.store, &mut connection).await, - result = timeout(DEFAULT_TIMEOUT, connection.next()) => { - match result { - Err(_) => handle_reconnect(&worker.connector, &mut connection, &worker.store).await, - Ok(packet_result) => handle_connection_recv(packet_result, &worker.connector, &mut connection, &worker.store).await, - } - } - } - } - - // Close the connection upon exiting - if let Err(e) = connection.close().await { - error!("asset worker failed to send close frame: {}", e) - } else { - debug!("asset worker successfully sent close frame") - } - - debug!("asset worker has been dropped, stopping asset worker"); -} - -async fn subscribe>( - ids: &[T], - connection: &mut CryptoCompareWebSocketConnection, -) -> Result<(), tungstenite::Error> { - if !ids.is_empty() { - connection - .subscribe_latest_tick_adaptive_inclusion( - &ids.iter().map(|s| s.as_ref()).collect::>(), - ) - .await? - } - - Ok(()) -} - -async fn handle_subscribe_recv( - ids: Vec, - worker_store: &WorkerStore, - connection: &mut CryptoCompareWebSocketConnection, -) { - match subscribe(&ids, connection).await { - Ok(_) => info!("subscribed to ids {:?}", ids), - Err(e) => { - error!("failed to subscribe to ids {:?}: {}", ids, e); - if worker_store.remove_query_ids(ids).await.is_err() { - error!("failed to remove query ids from store") - } - } - } -} - -async fn unsubscribe>( - ids: &[T], - connection: &mut CryptoCompareWebSocketConnection, -) -> Result<(), tungstenite::Error> { - if !ids.is_empty() { - connection - .unsubscribe_latest_tick_adaptive_inclusion( - &ids.iter().map(|s| s.as_ref()).collect::>(), - ) - .await? - } - - Ok(()) -} - -async fn handle_unsubscribe_recv( - ids: Vec, - worker_store: &WorkerStore, - connection: &mut CryptoCompareWebSocketConnection, -) { - match unsubscribe(&ids, connection).await { - Ok(_) => info!("unsubscribed to ids {:?}", ids), - Err(e) => { - error!("failed to unsubscribe to ids {:?}: {}", ids, e); - if worker_store.add_query_ids(ids).await.is_err() { - error!("failed to add query ids to store") - } - } - } -} - -async fn handle_reconnect( - connector: &CryptoCompareWebSocketConnector, - connection: &mut CryptoCompareWebSocketConnection, - query_ids: &WorkerStore, -) { - let mut retry_count: usize = 1; - loop { - warn!("reconnecting: attempt {}", retry_count); - - if let Ok(new_connection) = connector.connect().await { - *connection = new_connection; - - // Resubscribe to all ids - let Ok(ids) = query_ids.get_query_ids().await else { - error!("failed to get query ids from store"); - return; - }; - - let ids_vec = ids.into_iter().collect::>(); - - if subscribe(&ids_vec, connection).await.is_ok() { - info!("resubscribed to all ids"); - } else { - error!("failed to resubscribe to all ids"); - } - } else { - error!("failed to reconnect to binance"); - } - - retry_count += 1; - sleep(RECONNECT_BUFFER).await; - } -} - -async fn store_ref_tick(store: &WorkerStore, ref_tick: ReferenceTick) -> Result<(), WorkerError> { - let id = ref_tick.instrument.clone(); - let price = - Decimal::from_f64(ref_tick.value).ok_or(WorkerError::InvalidDecimal(ref_tick.value))?; - - let asset_info = AssetInfo::new(ref_tick.instrument, price, ref_tick.last_update); - store.set_asset(id, asset_info).await?; - - Ok(()) -} - -async fn process_packet(store: &WorkerStore, packet: Packet) { - match packet { - Packet::RefTickAdaptive(ref_tick) => match store_ref_tick(store, ref_tick).await { - Ok(_) => info!("stored data"), - Err(e) => error!("failed to store ref tick: {}", e), - }, - Packet::SubscriptionError(msg) => error!("subscription error: {}", msg), - Packet::SubscriptionRejected(msg) => error!("subscription rejected: {}", msg), - Packet::SubscriptionWarning(msg) => warn!("subscription warning: {}", msg), - _ => (), - } -} - -async fn handle_connection_recv( - result: Result, - connector: &CryptoCompareWebSocketConnector, - connection: &mut CryptoCompareWebSocketConnection, - store: &WorkerStore, -) { - match result { - Ok(resp) => process_packet(store, resp).await, - Err(MessageError::ChannelClosed) => handle_reconnect(connector, connection, store).await, - Err(MessageError::UnsupportedMessage) => { - error!("unsupported message received from cryptocompare") - } - Err(MessageError::Parse(_)) => error!("unable to parse message from cryptocompare"), - } -} diff --git a/bothan-cryptocompare/src/worker/builder.rs b/bothan-cryptocompare/src/worker/builder.rs deleted file mode 100644 index 3e316693..00000000 --- a/bothan-cryptocompare/src/worker/builder.rs +++ /dev/null @@ -1,100 +0,0 @@ -use std::sync::Arc; - -use tokio::sync::mpsc::channel; - -use bothan_core::store::WorkerStore; -use bothan_core::worker::AssetWorkerBuilder; - -use crate::api::CryptoCompareWebSocketConnector; -use crate::worker::asset_worker::start_asset_worker; -use crate::worker::errors::BuildError; -use crate::worker::opts::CryptoCompareWorkerBuilderOpts; -use crate::worker::CryptoCompareWorker; - -/// Builds a `CryptoCompareWorker` with custom options. -/// Methods can be chained to set the configuration values and the -/// service is constructed by calling the [`build`](CryptoCompareWorker::build) method. -pub struct CryptoCompareWorkerBuilder { - store: WorkerStore, - opts: CryptoCompareWorkerBuilderOpts, -} - -impl CryptoCompareWorkerBuilder { - /// Set the URL for the `CryptoCompareWorker`. - /// The default URL is `DEFAULT_URL` when no API key is provided - /// and is `DEFAULT_PRO_URL` when an API key is provided. - pub fn with_url>(mut self, url: T) -> Self { - self.opts.url = url.into(); - self - } - - /// Set the api key for the `CryptoCompareWorker`. - pub fn with_api_key>(mut self, api_key: T) -> Self { - self.opts.api_key = Some(api_key.into()); - self - } - - /// Set the internal channel size for the `CryptoCompareWorker`. - /// The default size is `DEFAULT_CHANNEL_SIZE`. - pub fn with_internal_ch_size(mut self, size: usize) -> Self { - self.opts.internal_ch_size = size; - self - } - - /// Sets the store for the `CryptoCompareWorker`. - /// If not set, the store is created and owned by the worker. - pub fn with_store(mut self, store: WorkerStore) -> Self { - self.store = store; - self - } -} - -#[async_trait::async_trait] -impl<'a> AssetWorkerBuilder<'a> for CryptoCompareWorkerBuilder { - type Opts = CryptoCompareWorkerBuilderOpts; - type Worker = CryptoCompareWorker; - type Error = BuildError; - - /// Returns a new `CryptoCompareWorkerBuilder` with the given options. - fn new(store: WorkerStore, opts: Self::Opts) -> Self { - Self { store, opts } - } - - /// Returns the name of the worker. - fn worker_name() -> &'static str { - "cryptocompare" - } - - /// Creates the configured `CryptoCompareWorker`. - async fn build(self) -> Result, Self::Error> { - let api_key = self.opts.api_key.ok_or(BuildError::MissingApiKey)?; - let connector = CryptoCompareWebSocketConnector::new(self.opts.url, api_key); - let connection = connector.connect().await?; - - let (sub_tx, sub_rx) = channel(self.opts.internal_ch_size); - let (unsub_tx, unsub_rx) = channel(self.opts.internal_ch_size); - let to_sub = self - .store - .get_query_ids() - .await? - .into_iter() - .collect::>(); - - if !to_sub.is_empty() { - sub_tx.send(to_sub).await?; - } - - let worker = Arc::new(CryptoCompareWorker::new( - connector, self.store, sub_tx, unsub_tx, - )); - - tokio::spawn(start_asset_worker( - Arc::downgrade(&worker), - connection, - sub_rx, - unsub_rx, - )); - - Ok(worker) - } -} diff --git a/bothan-cryptocompare/src/worker/errors.rs b/bothan-cryptocompare/src/worker/errors.rs deleted file mode 100644 index 7c47445f..00000000 --- a/bothan-cryptocompare/src/worker/errors.rs +++ /dev/null @@ -1,29 +0,0 @@ -use bothan_core::store; -use thiserror::Error; -use tokio::sync::mpsc::error::SendError; - -use crate::api::errors::ConnectionError; - -#[derive(Debug, Error)] -pub enum BuildError { - #[error("missing api key")] - MissingApiKey, - - #[error("failed to connect: {0}")] - FailedToConnect(#[from] ConnectionError), - - #[error("internal channel is closed: {0}")] - InternalChannelError(#[from] SendError>), - - #[error("store error: {0}")] - StoreError(#[from] store::error::Error), -} - -#[derive(Debug, Error)] -pub(crate) enum WorkerError { - #[error("value is not a valid decimal: {0}")] - InvalidDecimal(f64), - - #[error("store error: {0}")] - StoreError(#[from] store::error::Error), -} diff --git a/bothan-cryptocompare/src/worker/opts.rs b/bothan-cryptocompare/src/worker/opts.rs deleted file mode 100644 index 44f260d5..00000000 --- a/bothan-cryptocompare/src/worker/opts.rs +++ /dev/null @@ -1,56 +0,0 @@ -use serde::{Deserialize, Deserializer, Serialize, Serializer}; - -use crate::api::DEFAULT_URL; -use crate::worker::types::DEFAULT_CHANNEL_SIZE; - -/// Options for configuring the `CryptoCompareWorkerBuilder`. -/// -/// `CryptoCompareWorkerBuilderOpts` provides a way to specify custom settings for creating a `CryptoCompareWorker`. -/// This struct allows users to set optional parameters such as the WebSocket URL and the internal channel size, -/// which will be used during the construction of the `CryptoCompareWorker`. -#[derive(Clone, Debug, Serialize, Deserialize)] -pub struct CryptoCompareWorkerBuilderOpts { - #[serde(default = "default_url")] - pub url: String, - #[serde(default)] - #[serde(deserialize_with = "empty_string_is_none")] - #[serde(serialize_with = "none_is_empty_string")] - pub api_key: Option, - #[serde(default = "default_internal_ch_size")] - pub internal_ch_size: usize, -} - -fn default_url() -> String { - DEFAULT_URL.to_string() -} - -fn default_internal_ch_size() -> usize { - DEFAULT_CHANNEL_SIZE -} - -impl Default for CryptoCompareWorkerBuilderOpts { - fn default() -> Self { - Self { - url: default_url(), - api_key: None, - internal_ch_size: default_internal_ch_size(), - } - } -} - -fn empty_string_is_none<'de, D: Deserializer<'de>>( - deserializer: D, -) -> Result, D::Error> { - let s: Option = Option::deserialize(deserializer)?; - Ok(s.filter(|s| !s.is_empty())) -} - -fn none_is_empty_string( - value: &Option, - serializer: S, -) -> Result { - match value { - Some(val) => serializer.serialize_str(val), - None => serializer.serialize_str(""), - } -} diff --git a/bothan-cryptocompare/src/worker/types.rs b/bothan-cryptocompare/src/worker/types.rs deleted file mode 100644 index 1e216542..00000000 --- a/bothan-cryptocompare/src/worker/types.rs +++ /dev/null @@ -1,5 +0,0 @@ -use tokio::time::Duration; - -pub const DEFAULT_CHANNEL_SIZE: usize = 1000; -pub const DEFAULT_TIMEOUT: Duration = Duration::from_secs(60); -pub const RECONNECT_BUFFER: Duration = Duration::from_secs(5); diff --git a/bothan-htx/Cargo.toml b/bothan-htx/Cargo.toml index cad008d8..c3637187 100644 --- a/bothan-htx/Cargo.toml +++ b/bothan-htx/Cargo.toml @@ -1,15 +1,14 @@ [package] name = "bothan-htx" -version = "0.0.1-beta.1" +version = "0.0.1-beta.2" edition.workspace = true license.workspace = true repository.workspace = true [dependencies] -flate2 = "1.0.35" +bothan-lib = { workspace = true } async-trait = { workspace = true } -bothan-core = { workspace = true } futures-util = { workspace = true } rust_decimal = { workspace = true } serde = { workspace = true } @@ -18,7 +17,8 @@ thiserror = { workspace = true } tokio = { workspace = true } tokio-tungstenite = { workspace = true } tracing = { workspace = true } -tracing-subscriber = { workspace = true } + +flate2 = "1.0.35" [dev-dependencies] ws-mock = { git = "https://github.com/bandprotocol/ws-mock.git", branch = "master" } diff --git a/bothan-htx/examples/htx_basic.rs b/bothan-htx/examples/htx_basic.rs deleted file mode 100644 index b3835368..00000000 --- a/bothan-htx/examples/htx_basic.rs +++ /dev/null @@ -1,39 +0,0 @@ -use std::time::Duration; - -use tokio::time::sleep; -use tracing_subscriber::fmt::init; - -use bothan_core::registry::Registry; -use bothan_core::store::SharedStore; -use bothan_core::worker::{AssetWorker, AssetWorkerBuilder}; -use bothan_htx::{HtxWorkerBuilder, HtxWorkerBuilderOpts}; - -#[tokio::main] -async fn main() { - init(); - let path = std::env::current_dir().unwrap(); - let registry = Registry::default().validate().unwrap(); - let store = SharedStore::new(registry, path.as_path()).await.unwrap(); - - let worker_store = store.create_worker_store(HtxWorkerBuilder::worker_name()); - let opts = HtxWorkerBuilderOpts::default(); - - let worker = HtxWorkerBuilder::new(worker_store, opts) - .build() - .await - .unwrap(); - - worker - .set_query_ids(vec!["btcusdt".to_string(), "ethusdt".to_string()]) - .await - .unwrap(); - - sleep(Duration::from_secs(2)).await; - - loop { - let btc_data = worker.get_asset("btcusdt").await; - let eth_data = worker.get_asset("ethusdt").await; - println!("{:?} {:?}", btc_data, eth_data); - sleep(Duration::from_secs(5)).await; - } -} diff --git a/bothan-htx/src/api/websocket.rs b/bothan-htx/src/api/websocket.rs index 7a702705..94d4d0a5 100644 --- a/bothan-htx/src/api/websocket.rs +++ b/bothan-htx/src/api/websocket.rs @@ -5,10 +5,10 @@ use futures_util::stream::{SplitSink, SplitStream}; use futures_util::{SinkExt, StreamExt}; use serde_json::json; use tokio::net::TcpStream; +use tokio_tungstenite::tungstenite::Message; use tokio_tungstenite::tungstenite::error::Error as TungsteniteError; use tokio_tungstenite::tungstenite::http::StatusCode; -use tokio_tungstenite::tungstenite::Message; -use tokio_tungstenite::{connect_async, MaybeTlsStream, WebSocketStream}; +use tokio_tungstenite::{MaybeTlsStream, WebSocketStream, connect_async}; use tracing::warn; use crate::api::error::{ConnectionError, MessageError, SendError}; @@ -127,11 +127,12 @@ impl HtxWebSocketConnection { #[cfg(test)] pub(crate) mod test { - use super::*; - use crate::api::types::{DataUpdate, HtxResponse, Ping, SubResponse, Tick, UnsubResponse}; use tokio::sync::mpsc; use ws_mock::ws_mock_server::{WsMock, WsMockServer}; + use super::*; + use crate::api::types::{DataUpdate, HtxResponse, Ping, SubResponse, Tick, UnsubResponse}; + pub(crate) async fn setup_mock_server() -> WsMockServer { WsMockServer::start().await } diff --git a/bothan-htx/src/lib.rs b/bothan-htx/src/lib.rs index 68b448e3..e31a5354 100644 --- a/bothan-htx/src/lib.rs +++ b/bothan-htx/src/lib.rs @@ -1,6 +1,5 @@ -pub use worker::builder::HtxWorkerBuilder; -pub use worker::opts::HtxWorkerBuilderOpts; -pub use worker::HtxWorker; +pub use worker::Worker; +pub use worker::opts::WorkerOpts; pub mod api; pub mod worker; diff --git a/bothan-htx/src/worker.rs b/bothan-htx/src/worker.rs index 767fc0f6..85efe70f 100644 --- a/bothan-htx/src/worker.rs +++ b/bothan-htx/src/worker.rs @@ -1,68 +1,89 @@ -use tokio::sync::mpsc::Sender; +use std::collections::HashSet; +use std::sync::Arc; -use bothan_core::store::error::Error as StoreError; -use bothan_core::store::WorkerStore; -use bothan_core::worker::{AssetState, AssetWorker, SetQueryIDError}; +use bothan_lib::store::{Store, WorkerStore}; +use bothan_lib::types::AssetState; +use bothan_lib::worker::AssetWorker; +use bothan_lib::worker::error::AssetWorkerError; +use tokio::sync::mpsc::{Sender, channel}; +use crate::WorkerOpts; use crate::api::websocket::HtxWebSocketConnector; +use crate::worker::asset_worker::start_asset_worker; mod asset_worker; -pub mod builder; -pub(crate) mod error; pub mod opts; mod types; -/// A worker that fetches and stores the asset information from Htx's API. -pub struct HtxWorker { - connector: HtxWebSocketConnector, - store: WorkerStore, - subscribe_tx: Sender>, - unsubscribe_tx: Sender>, +const WORKER_NAME: &str = "htx"; + +pub struct Worker { + inner: Arc>, } -impl HtxWorker { - /// Create a new worker with the specified connector, store and channels. - pub fn new( - connector: HtxWebSocketConnector, - store: WorkerStore, - subscribe_tx: Sender>, - unsubscribe_tx: Sender>, - ) -> Self { - Self { - connector, - store, - subscribe_tx, - unsubscribe_tx, +#[async_trait::async_trait] +impl AssetWorker for Worker { + type Opts = WorkerOpts; + + fn name(&self) -> &'static str { + WORKER_NAME + } + + async fn build(opts: Self::Opts, store: &S) -> Result { + let url = opts.url; + let ch_size = opts.internal_ch_size; + + let connector = HtxWebSocketConnector::new(url); + let connection = connector.connect().await?; + + let (sub_tx, sub_rx) = channel(ch_size); + let (unsub_tx, unsub_rx) = channel(ch_size); + + let worker_store = WorkerStore::new(store, WORKER_NAME); + let to_sub = worker_store + .get_query_ids() + .await? + .into_iter() + .collect::>(); + + if !to_sub.is_empty() { + sub_tx.send(to_sub).await?; } + + let inner = Arc::new(InnerWorker { + connector, + store: worker_store, + subscribe_tx: sub_tx, + unsubscribe_tx: unsub_tx, + }); + + tokio::spawn(start_asset_worker( + Arc::downgrade(&inner), + connection, + sub_rx, + unsub_rx, + )); + + Ok(Worker { inner }) } -} -#[async_trait::async_trait] -impl AssetWorker for HtxWorker { - /// Fetches the AssetStatus for the given cryptocurrency id. - async fn get_asset(&self, id: &str) -> Result { - self.store.get_asset(&id).await + async fn get_asset(&self, id: &str) -> Result { + Ok(self.inner.store.get_asset(id).await?) } - /// Sets the specified cryptocurrency IDs to the query. If the ids are already in the query set, - /// it will not be resubscribed. - async fn set_query_ids(&self, ids: Vec) -> Result<(), SetQueryIDError> { - let (to_sub, to_unsub) = self - .store - .set_query_ids(ids) - .await - .map_err(|e| SetQueryIDError::new(e.to_string()))?; - - self.subscribe_tx - .send(to_sub) - .await - .map_err(|e| SetQueryIDError::new(e.to_string()))?; - - self.unsubscribe_tx - .send(to_unsub) - .await - .map_err(|e| SetQueryIDError::new(e.to_string()))?; + async fn set_query_ids(&self, ids: HashSet) -> Result<(), AssetWorkerError> { + let diff = self.inner.store.compute_query_id_difference(ids).await?; + + self.inner.subscribe_tx.send(diff.added).await?; + self.inner.unsubscribe_tx.send(diff.removed).await?; Ok(()) } } + +struct InnerWorker { + connector: HtxWebSocketConnector, + store: WorkerStore, + subscribe_tx: Sender>, + unsubscribe_tx: Sender>, +} diff --git a/bothan-htx/src/worker/asset_worker.rs b/bothan-htx/src/worker/asset_worker.rs index 3ceda7bb..5bdeb40e 100644 --- a/bothan-htx/src/worker/asset_worker.rs +++ b/bothan-htx/src/worker/asset_worker.rs @@ -1,24 +1,22 @@ use std::sync::Weak; -use rust_decimal::prelude::FromPrimitive; +use bothan_lib::store::{Store, WorkerStore}; +use bothan_lib::types::AssetInfo; use rust_decimal::Decimal; +use rust_decimal::prelude::FromPrimitive; use tokio::select; use tokio::sync::mpsc::Receiver; use tokio::time::{sleep, timeout}; use tracing::{debug, error, info, warn}; -use bothan_core::store::WorkerStore; -use bothan_core::types::AssetInfo; - use crate::api::error::{MessageError, SendError}; use crate::api::types::{HtxResponse, Pong, Tick}; use crate::api::{HtxWebSocketConnection, HtxWebSocketConnector}; -use crate::worker::error::WorkerError; +use crate::worker::InnerWorker; use crate::worker::types::{DEFAULT_TIMEOUT, RECONNECT_BUFFER}; -use crate::worker::HtxWorker; -pub(crate) async fn start_asset_worker( - worker: Weak, +pub(crate) async fn start_asset_worker( + worker: Weak>, mut connection: HtxWebSocketConnection, mut subscribe_rx: Receiver>, mut unsubscribe_rx: Receiver>, @@ -83,10 +81,10 @@ async fn unsubscribe(id: &str, connection: &mut HtxWebSocketConnection) -> Resul } /// Handles reconnection to the WebSocket and re-subscribes to all previously subscribed ids. -async fn handle_reconnect( +async fn handle_reconnect( connector: &HtxWebSocketConnector, connection: &mut HtxWebSocketConnection, - store: &WorkerStore, + store: &WorkerStore, ) { let mut retry_count: usize = 1; loop { @@ -118,31 +116,27 @@ async fn handle_reconnect( } } -/// Parses a tick response into an AssetInfo structure. -fn parse_tick(id: &str, tick: Tick, timestamp: i64) -> Result { - let price_value = - Decimal::from_f64(tick.last_price).ok_or(WorkerError::InvalidPrice(tick.last_price))?; - Ok(AssetInfo::new(id.to_string(), price_value, timestamp)) -} - /// Stores tick information into the worker store. -async fn store_tick( - store: &WorkerStore, - id: &str, - tick: Tick, - timestamp: i64, -) -> Result<(), WorkerError> { - store - .set_asset(id, parse_tick(id, tick, timestamp)?) - .await?; - debug!("stored data for id {}", id); - Ok(()) +async fn store_tick>(store: &WorkerStore, id: T, tick: Tick) { + let id = id.into(); + match Decimal::from_f64(tick.last_price) { + Some(price) => { + let asset_info = AssetInfo::new(id.clone(), price, 0); + if let Err(e) = store.set_asset_info(asset_info).await { + error!("failed to store data for id {}: {}", id, e); + } + debug!("stored data for id {}", id); + } + None => { + error!("data for id {} has a nan price", id); + } + } } /// Processes the response from the Htx API and handles each type accordingly. -async fn process_response( +async fn process_response( resp: HtxResponse, - store: &WorkerStore, + store: &WorkerStore, connection: &mut HtxWebSocketConnection, ) { match resp { @@ -156,10 +150,7 @@ async fn process_response( debug!("received data update from channel {}", data.ch); if let Some(id) = data.ch.split('.').nth(1) { // Handle processing of data update, e.g., storing tick data - match store_tick(store, id, data.tick, data.timestamp).await { - Ok(_) => debug!("saved data"), - Err(e) => error!("failed to save data: {}", e), - } + store_tick(store, id, data.tick).await } } HtxResponse::Ping(ping) => { @@ -177,15 +168,16 @@ async fn process_response( /// Sends a pong response back to the WebSocket connection. async fn send_pong(ping: u64, connection: &mut HtxWebSocketConnection) -> Result<(), SendError> { let pong_payload = Pong { pong: ping }; // Create the Pong struct with the ping timestamp - connection.send_pong(pong_payload).await // Assuming send_pong is implemented to send Pong struct + connection.send_pong(pong_payload).await // Assuming send_pong is implemented to send Pong + // struct } /// Handles received messages from the WebSocket connection. -async fn handle_connection_recv( +async fn handle_connection_recv( recv_result: Result, connector: &HtxWebSocketConnector, connection: &mut HtxWebSocketConnection, - store: &WorkerStore, + store: &WorkerStore, ) { match recv_result { Ok(resp) => { @@ -202,67 +194,3 @@ async fn handle_connection_recv( } } } - -#[cfg(test)] -mod test { - use super::*; - use rust_decimal::Decimal; - use std::str::FromStr; - - #[test] - fn test_parse_tick() { - // Create a mock Tick struct with valid data - let tick = Tick { - open: 51732.0, - high: 52785.64, - low: 51000.0, - close: 52735.63, - amount: 13259.24137056181, - vol: 687640987.4125315, - count: 448737, - bid: 52732.88, - bid_size: 0.036, - ask: 52732.89, - ask_size: 0.583653, - last_price: 52735.63, - last_size: 0.03, - }; - - // Parse the tick into AssetInfo - let result = parse_tick("btcusdt", tick, 1000); - - // Expected AssetInfo object - let expected = AssetInfo::new( - "btcusdt".to_string(), - Decimal::from_str("52735.63").unwrap(), - 1000, - ); - - // Assert that the parsed result matches the expected output - assert_eq!(result.as_ref().unwrap().id, expected.id); - assert_eq!(result.unwrap().price, expected.price); - } - - #[test] - fn test_parse_tick_with_failure() { - // Create a mock Tick struct with an invalid price - let tick = Tick { - open: 51732.0, - high: 52785.64, - low: 51000.0, - close: 52735.63, - amount: 13259.24137056181, - vol: 687640987.4125315, - count: 448737, - bid: 52732.88, - bid_size: 0.036, - ask: 52732.89, - ask_size: 0.583653, - last_price: f64::INFINITY, // Invalid price to trigger error - last_size: 0.03, - }; - - // Assert that parsing the tick with an invalid price results in an error - assert!(parse_tick("btcusdt", tick, 1000).is_err()); - } -} diff --git a/bothan-htx/src/worker/builder.rs b/bothan-htx/src/worker/builder.rs deleted file mode 100644 index 165f6171..00000000 --- a/bothan-htx/src/worker/builder.rs +++ /dev/null @@ -1,99 +0,0 @@ -use std::sync::Arc; - -use tokio::sync::mpsc::channel; - -use crate::api::HtxWebSocketConnector; -use crate::worker::asset_worker::start_asset_worker; -use crate::worker::error::BuildError; -use crate::worker::opts::HtxWorkerBuilderOpts; -use crate::worker::HtxWorker; -use bothan_core::store::WorkerStore; -use bothan_core::worker::AssetWorkerBuilder; - -/// Builds a `HtxWorker` with custom options. -/// Methods can be chained to set the configuration values and the -/// service is constructed by calling the [`build`](HtxWorkerBuilder::build) method. -pub struct HtxWorkerBuilder { - store: WorkerStore, - opts: HtxWorkerBuilderOpts, -} - -impl HtxWorkerBuilder { - /// Returns a new `HtxWorkerBuilder` with the given options. - pub fn new(store: WorkerStore, opts: HtxWorkerBuilderOpts) -> Self { - Self { store, opts } - } - - /// Set the URL for the `HtxWorker`. - /// The default URL is `DEFAULT_URL`. - pub fn with_url>(mut self, url: T) -> Self { - self.opts.url = url.into(); - self - } - - /// Set the internal channel size for the `HtxWorker`. - /// The default size is `DEFAULT_CHANNEL_SIZE`. - pub fn with_internal_ch_size(mut self, size: usize) -> Self { - self.opts.internal_ch_size = size; - self - } - - /// Sets the store for the `HtxWorker`. - /// If not set, the store is created and owned by the worker. - pub fn with_store(mut self, store: WorkerStore) -> Self { - self.store = store; - self - } -} - -#[async_trait::async_trait] -impl<'a> AssetWorkerBuilder<'a> for HtxWorkerBuilder { - type Opts = HtxWorkerBuilderOpts; - type Worker = HtxWorker; - type Error = BuildError; - - /// Returns a new `HtxWorkerBuilder` with the given options. - fn new(store: WorkerStore, opts: Self::Opts) -> Self { - Self { store, opts } - } - - /// Returns the name of the worker. - fn worker_name() -> &'static str { - "htx" - } - - /// Creates the configured `HtxWorker`. - async fn build(self) -> Result, BuildError> { - let url = self.opts.url; - let ch_size = self.opts.internal_ch_size; - - let connector = HtxWebSocketConnector::new(url); - let connection = connector.connect().await?; - - let (sub_tx, sub_rx) = channel(ch_size); - let (unsub_tx, unsub_rx) = channel(ch_size); - - let to_sub = self - .store - .get_query_ids() - .await? - .into_iter() - .collect::>(); - - if !to_sub.is_empty() { - // Unwrap here as the channel is guaranteed to be open - sub_tx.send(to_sub).await.unwrap(); - } - - let worker = Arc::new(HtxWorker::new(connector, self.store, sub_tx, unsub_tx)); - - tokio::spawn(start_asset_worker( - Arc::downgrade(&worker), - connection, - sub_rx, - unsub_rx, - )); - - Ok(worker) - } -} diff --git a/bothan-htx/src/worker/error.rs b/bothan-htx/src/worker/error.rs deleted file mode 100644 index c2ef356f..00000000 --- a/bothan-htx/src/worker/error.rs +++ /dev/null @@ -1,21 +0,0 @@ -use crate::api; -use bothan_core::store; -use thiserror::Error; - -#[derive(Error, Debug)] -pub(crate) enum WorkerError { - #[error("invalid price: {0}")] - InvalidPrice(f64), - - #[error("failed to set data to the store: {0}")] - SetFailed(#[from] store::error::Error), -} - -#[derive(Debug, Error)] -pub enum BuildError { - #[error("failed to connect: {0}")] - FailedToConnect(#[from] api::ConnectionError), - - #[error("store error: {0}")] - StoreError(#[from] store::error::Error), -} diff --git a/bothan-htx/src/worker/opts.rs b/bothan-htx/src/worker/opts.rs index a8ad4032..1083379b 100644 --- a/bothan-htx/src/worker/opts.rs +++ b/bothan-htx/src/worker/opts.rs @@ -3,13 +3,8 @@ use serde::{Deserialize, Serialize}; use crate::api::types::DEFAULT_URL; use crate::worker::types::DEFAULT_CHANNEL_SIZE; -/// Options for configuring the `HtxWorkerBuilder`. -/// -/// `HtxWorkerBuilderOpts` provides a way to specify custom settings for creating a `HtxWorker`. -/// This struct allows users to set optional parameters such as the WebSocket URL and the internal channel size, -/// which will be used during the construction of the `HtxWorker`. #[derive(Clone, Debug, Serialize, Deserialize)] -pub struct HtxWorkerBuilderOpts { +pub struct WorkerOpts { #[serde(default = "default_url")] pub url: String, #[serde(default = "default_internal_ch_size")] @@ -24,7 +19,7 @@ fn default_internal_ch_size() -> usize { DEFAULT_CHANNEL_SIZE } -impl Default for HtxWorkerBuilderOpts { +impl Default for WorkerOpts { fn default() -> Self { Self { url: default_url(), diff --git a/bothan-kraken/Cargo.toml b/bothan-kraken/Cargo.toml index be03af67..6306e00f 100644 --- a/bothan-kraken/Cargo.toml +++ b/bothan-kraken/Cargo.toml @@ -1,17 +1,16 @@ [package] name = "bothan-kraken" -version = "0.0.1-beta.1" +version = "0.0.1-beta.2" edition.workspace = true license.workspace = true repository.workspace = true [dependencies] +bothan-lib = { workspace = true } async-trait = { workspace = true } -bothan-core = { workspace = true } chrono = { workspace = true } futures-util = { workspace = true } -humantime-serde = { workspace = true } rust_decimal = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } @@ -19,10 +18,6 @@ thiserror = { workspace = true } tokio = { workspace = true } tokio-tungstenite = { workspace = true } tracing = { workspace = true } -tracing-subscriber = { workspace = true } [dev-dependencies] ws-mock = { git = "https://github.com/bandprotocol/ws-mock.git", branch = "master" } - -[package.metadata.cargo-machete] -ignored = ["humantime-serde"] diff --git a/bothan-kraken/examples/kraken_basic.rs b/bothan-kraken/examples/kraken_basic.rs deleted file mode 100644 index ca699a91..00000000 --- a/bothan-kraken/examples/kraken_basic.rs +++ /dev/null @@ -1,39 +0,0 @@ -use std::time::Duration; - -use tokio::time::sleep; -use tracing_subscriber::fmt::init; - -use bothan_core::registry::Registry; -use bothan_core::store::SharedStore; -use bothan_core::worker::{AssetWorker, AssetWorkerBuilder}; -use bothan_kraken::{KrakenWorkerBuilder, KrakenWorkerBuilderOpts}; - -#[tokio::main] -async fn main() { - init(); - let path = std::env::current_dir().unwrap(); - let registry = Registry::default().validate().unwrap(); - let store = SharedStore::new(registry, path.as_path()).await.unwrap(); - - let worker_store = store.create_worker_store(KrakenWorkerBuilder::worker_name()); - let opts = KrakenWorkerBuilderOpts::default(); - - let worker = KrakenWorkerBuilder::new(worker_store, opts) - .build() - .await - .unwrap(); - - worker - .set_query_ids(vec!["BTC/USD".to_string(), "ETH/USD".to_string()]) - .await - .unwrap(); - - sleep(Duration::from_secs(2)).await; - - loop { - let btc_data = worker.get_asset("BTC/USD").await; - let eth_data = worker.get_asset("ETH/USD").await; - println!("{:?}, {:?}", btc_data, eth_data); - sleep(Duration::from_secs(5)).await; - } -} diff --git a/bothan-kraken/src/api/types.rs b/bothan-kraken/src/api/types.rs index 1ec5acaa..47618449 100644 --- a/bothan-kraken/src/api/types.rs +++ b/bothan-kraken/src/api/types.rs @@ -1,10 +1,9 @@ +pub use channel::ChannelResponse; +pub use channel::ticker::TickerResponse; use serde::{Deserialize, Serialize}; use crate::api::types::message::PublicMessageResponse; -pub use channel::ticker::TickerResponse; -pub use channel::ChannelResponse; - pub mod channel; pub mod message; diff --git a/bothan-kraken/src/api/websocket.rs b/bothan-kraken/src/api/websocket.rs index 7ad6f77f..8bfd748f 100644 --- a/bothan-kraken/src/api/websocket.rs +++ b/bothan-kraken/src/api/websocket.rs @@ -1,16 +1,16 @@ use futures_util::stream::{SplitSink, SplitStream}; use futures_util::{SinkExt, StreamExt}; use tokio::net::TcpStream; +use tokio_tungstenite::tungstenite::Message; use tokio_tungstenite::tungstenite::error::Error as TungsteniteError; use tokio_tungstenite::tungstenite::http::StatusCode; -use tokio_tungstenite::tungstenite::Message; -use tokio_tungstenite::{connect_async, MaybeTlsStream, WebSocketStream}; +use tokio_tungstenite::{MaybeTlsStream, WebSocketStream, connect_async}; use tracing::warn; use crate::api::error::{ConnectionError, MessageError, SendError}; +use crate::api::types::KrakenResponse; use crate::api::types::channel::ticker::{EventTrigger, TickerRequestParameters}; use crate::api::types::message::{Method, PublicMessage}; -use crate::api::types::KrakenResponse; /// A connector for establishing a WebSocket connection to the Kraken API. pub struct KrakenWebSocketConnector { @@ -139,9 +139,8 @@ pub(crate) mod test { use tokio::sync::mpsc; use ws_mock::ws_mock_server::{WsMock, WsMockServer}; - use crate::api::types::{ChannelResponse, KrakenResponse, TickerResponse}; - use super::*; + use crate::api::types::{ChannelResponse, KrakenResponse, TickerResponse}; pub(crate) async fn setup_mock_server() -> WsMockServer { WsMockServer::start().await diff --git a/bothan-kraken/src/lib.rs b/bothan-kraken/src/lib.rs index 29379189..e644efe2 100644 --- a/bothan-kraken/src/lib.rs +++ b/bothan-kraken/src/lib.rs @@ -1,8 +1,6 @@ pub use api::websocket::{KrakenWebSocketConnection, KrakenWebSocketConnector}; -pub use worker::builder::KrakenWorkerBuilder; -pub use worker::error::BuildError; -pub use worker::opts::KrakenWorkerBuilderOpts; -pub use worker::KrakenWorker; +pub use worker::Worker; +pub use worker::opts::WorkerOpts; pub mod api; pub mod worker; diff --git a/bothan-kraken/src/worker.rs b/bothan-kraken/src/worker.rs index a12a04dd..ffbd12d7 100644 --- a/bothan-kraken/src/worker.rs +++ b/bothan-kraken/src/worker.rs @@ -1,68 +1,88 @@ -use tokio::sync::mpsc::Sender; +use std::collections::HashSet; +use std::sync::Arc; -use bothan_core::store::error::Error as StoreError; -use bothan_core::store::WorkerStore; -use bothan_core::worker::{AssetState, AssetWorker, SetQueryIDError}; +use bothan_lib::store::{Store, WorkerStore}; +use bothan_lib::types::AssetState; +use bothan_lib::worker::AssetWorker; +use bothan_lib::worker::error::AssetWorkerError; +use tokio::sync::mpsc::{Sender, channel}; +use crate::WorkerOpts; use crate::api::websocket::KrakenWebSocketConnector; +use crate::worker::asset_worker::start_asset_worker; mod asset_worker; -pub mod builder; -pub(crate) mod error; pub mod opts; -mod types; -/// A worker that fetches and stores the asset information from Kraken's API. -pub struct KrakenWorker { - connector: KrakenWebSocketConnector, - store: WorkerStore, - subscribe_tx: Sender>, - unsubscribe_tx: Sender>, +const WORKER_NAME: &str = "kraken"; + +pub struct Worker { + inner: Arc>, } -impl KrakenWorker { - /// Create a new worker with the specified connector, store and channels. - pub fn new( - connector: KrakenWebSocketConnector, - store: WorkerStore, - subscribe_tx: Sender>, - unsubscribe_tx: Sender>, - ) -> Self { - Self { - connector, - store, - subscribe_tx, - unsubscribe_tx, +#[async_trait::async_trait] +impl AssetWorker for Worker { + type Opts = WorkerOpts; + + fn name(&self) -> &'static str { + WORKER_NAME + } + + async fn build(opts: Self::Opts, store: &S) -> Result, AssetWorkerError> { + let url = opts.url; + let ch_size = opts.internal_ch_size; + + let connector = KrakenWebSocketConnector::new(url); + let connection = connector.connect().await?; + + let (sub_tx, sub_rx) = channel(ch_size); + let (unsub_tx, unsub_rx) = channel(ch_size); + + let worker_store = WorkerStore::new(store, WORKER_NAME); + let to_sub = worker_store + .get_query_ids() + .await? + .into_iter() + .collect::>(); + + if !to_sub.is_empty() { + sub_tx.send(to_sub).await?; } + + let inner = Arc::new(InnerWorker { + connector, + store: worker_store, + subscribe_tx: sub_tx, + unsubscribe_tx: unsub_tx, + }); + + tokio::spawn(start_asset_worker( + Arc::downgrade(&inner), + connection, + sub_rx, + unsub_rx, + )); + + Ok(Worker { inner }) } -} -#[async_trait::async_trait] -impl AssetWorker for KrakenWorker { - /// Fetches the AssetStatus for the given cryptocurrency id. - async fn get_asset(&self, id: &str) -> Result { - self.store.get_asset(&id).await + async fn get_asset(&self, id: &str) -> Result { + Ok(self.inner.store.get_asset(id).await?) } - /// Sets the specified cryptocurrency IDs to the query. If the ids are already in the query set, - /// it will not be resubscribed. - async fn set_query_ids(&self, ids: Vec) -> Result<(), SetQueryIDError> { - let (to_sub, to_unsub) = self - .store - .set_query_ids(ids) - .await - .map_err(|e| SetQueryIDError::new(e.to_string()))?; - - self.subscribe_tx - .send(to_sub) - .await - .map_err(|e| SetQueryIDError::new(e.to_string()))?; - - self.unsubscribe_tx - .send(to_unsub) - .await - .map_err(|e| SetQueryIDError::new(e.to_string()))?; + async fn set_query_ids(&self, ids: HashSet) -> Result<(), AssetWorkerError> { + let diff = self.inner.store.compute_query_id_difference(ids).await?; + + self.inner.subscribe_tx.send(diff.added).await?; + self.inner.unsubscribe_tx.send(diff.removed).await?; Ok(()) } } + +pub struct InnerWorker { + connector: KrakenWebSocketConnector, + store: WorkerStore, + subscribe_tx: Sender>, + unsubscribe_tx: Sender>, +} diff --git a/bothan-kraken/src/worker/asset_worker.rs b/bothan-kraken/src/worker/asset_worker.rs index 63605c83..42a5eaef 100644 --- a/bothan-kraken/src/worker/asset_worker.rs +++ b/bothan-kraken/src/worker/asset_worker.rs @@ -1,24 +1,24 @@ use std::sync::Weak; +use std::time::Duration; -use rust_decimal::prelude::FromPrimitive; +use bothan_lib::store::{Store, WorkerStore}; +use bothan_lib::types::AssetInfo; use rust_decimal::Decimal; use tokio::select; use tokio::sync::mpsc::Receiver; use tokio::time::{sleep, timeout}; use tracing::{debug, error, info, warn}; -use bothan_core::store::WorkerStore; -use bothan_core::types::AssetInfo; - use crate::api::error::{MessageError, SendError}; use crate::api::types::{ChannelResponse, KrakenResponse, TickerResponse}; use crate::api::{KrakenWebSocketConnection, KrakenWebSocketConnector}; -use crate::worker::error::WorkerError; -use crate::worker::types::{DEFAULT_TIMEOUT, RECONNECT_BUFFER}; -use crate::worker::KrakenWorker; +use crate::worker::InnerWorker; + +pub const DEFAULT_TIMEOUT: Duration = Duration::from_secs(60); +pub const RECONNECT_BUFFER: Duration = Duration::from_secs(5); -pub(crate) async fn start_asset_worker( - worker: Weak, +pub(crate) async fn start_asset_worker( + worker: Weak>, mut connection: KrakenWebSocketConnection, mut subscribe_rx: Receiver>, mut unsubscribe_rx: Receiver>, @@ -91,10 +91,10 @@ async fn handle_unsubscribe_recv(ids: Vec, connection: &mut KrakenWebSoc } } -async fn handle_reconnect( +async fn handle_reconnect( connector: &KrakenWebSocketConnector, connection: &mut KrakenWebSocketConnection, - query_ids: &WorkerStore, + query_ids: &WorkerStore, ) { let mut retry_count: usize = 1; loop { @@ -129,34 +129,31 @@ async fn handle_reconnect( } } -fn parse_ticker(ticker: TickerResponse) -> Result { - let id = ticker.symbol.clone(); - let price_value = - Decimal::from_f64(ticker.last).ok_or(WorkerError::InvalidPrice(ticker.last))?; - Ok(AssetInfo::new( - id, - price_value, - chrono::Utc::now().timestamp(), - )) -} - -async fn store_ticker(store: &WorkerStore, ticker: TickerResponse) -> Result<(), WorkerError> { +async fn store_ticker(store: &WorkerStore, ticker: TickerResponse, timestamp: i64) { let id = ticker.symbol.clone(); - store.set_asset(id.clone(), parse_ticker(ticker)?).await?; - debug!("stored data for id {}", id); - Ok(()) + match Decimal::from_f64_retain(ticker.last) { + Some(price) => { + let asset_info = AssetInfo::new(id.clone(), price, timestamp); + if let Err(e) = store.set_asset_info(asset_info).await { + error!("failed to store data for id {}: {}", id, e); + } else { + debug!("stored data for id {}", id); + } + } + None => { + error!("failed to parse price for id {}", id); + } + } } /// Processes the response from the Kraken API. -async fn process_response(resp: KrakenResponse, store: &WorkerStore) { +async fn process_response(resp: KrakenResponse, store: &WorkerStore) { match resp { KrakenResponse::Channel(resp) => match resp { ChannelResponse::Ticker(tickers) => { + let timestamp = chrono::Utc::now().timestamp(); for ticker in tickers { - match store_ticker(store, ticker).await { - Ok(_) => debug!("saved data"), - Err(e) => error!("failed to save data: {}", e), - } + store_ticker(store, ticker, timestamp).await } } ChannelResponse::Heartbeat => { @@ -175,11 +172,11 @@ async fn process_response(resp: KrakenResponse, store: &WorkerStore) { } } -async fn handle_connection_recv( +async fn handle_connection_recv( recv_result: Result, connector: &KrakenWebSocketConnector, connection: &mut KrakenWebSocketConnection, - store: &WorkerStore, + store: &WorkerStore, ) { match recv_result { Ok(resp) => { @@ -196,53 +193,3 @@ async fn handle_connection_recv( } } } - -#[cfg(test)] -mod test { - use super::*; - - #[test] - fn test_parse_market() { - let ticker = TickerResponse { - symbol: "BTC".to_string(), - bid: 42000.00, - bid_qty: 50000.00, - ask: 42001.00, - ask_qty: 50000.00, - last: 42000.99, - volume: 100000.00, - vwap: 42000.00, - low: 40000.00, - high: 44000.00, - change: 2000.00, - change_pct: 0.05, - }; - let result = parse_ticker(ticker); - let expected = AssetInfo::new( - "BTC".to_string(), - Decimal::from_str_exact("42000.99").unwrap(), - 0, - ); - assert_eq!(result.as_ref().unwrap().id, expected.id); - assert_eq!(result.unwrap().price, expected.price); - } - - #[test] - fn test_parse_market_with_failure() { - let ticker = TickerResponse { - symbol: "BTC".to_string(), - bid: 42000.00, - bid_qty: 50000.00, - ask: 42001.00, - ask_qty: 50000.00, - last: f64::INFINITY, - volume: 100000.00, - vwap: 42000.00, - low: 40000.00, - high: 44000.00, - change: 2000.00, - change_pct: 0.05, - }; - assert!(parse_ticker(ticker).is_err()); - } -} diff --git a/bothan-kraken/src/worker/builder.rs b/bothan-kraken/src/worker/builder.rs deleted file mode 100644 index 5c7a73d7..00000000 --- a/bothan-kraken/src/worker/builder.rs +++ /dev/null @@ -1,99 +0,0 @@ -use std::sync::Arc; - -use tokio::sync::mpsc::channel; - -use crate::api::KrakenWebSocketConnector; -use crate::worker::asset_worker::start_asset_worker; -use crate::worker::error::BuildError; -use crate::worker::opts::KrakenWorkerBuilderOpts; -use crate::worker::KrakenWorker; -use bothan_core::store::WorkerStore; -use bothan_core::worker::AssetWorkerBuilder; - -/// Builds a `KrakenWorker` with custom options. -/// Methods can be chained to set the configuration values and the -/// service is constructed by calling the [`build`](KrakenWorkerBuilder::build) method. -pub struct KrakenWorkerBuilder { - store: WorkerStore, - opts: KrakenWorkerBuilderOpts, -} - -impl KrakenWorkerBuilder { - /// Returns a new `KrakenWorkerBuilder` with the given options. - pub fn new(store: WorkerStore, opts: KrakenWorkerBuilderOpts) -> Self { - Self { store, opts } - } - - /// Set the URL for the `KrakenWorker`. - /// The default URL is `DEFAULT_URL`. - pub fn with_url>(mut self, url: T) -> Self { - self.opts.url = url.into(); - self - } - - /// Set the internal channel size for the `KrakenWorker`. - /// The default size is `DEFAULT_CHANNEL_SIZE`. - pub fn with_internal_ch_size(mut self, size: usize) -> Self { - self.opts.internal_ch_size = size; - self - } - - /// Sets the store for the `KrakenWorker`. - /// If not set, the store is created and owned by the worker. - pub fn with_store(mut self, store: WorkerStore) -> Self { - self.store = store; - self - } -} - -#[async_trait::async_trait] -impl<'a> AssetWorkerBuilder<'a> for KrakenWorkerBuilder { - type Opts = KrakenWorkerBuilderOpts; - type Worker = KrakenWorker; - type Error = BuildError; - - /// Returns a new `KrakenWorkerBuilder` with the given options. - fn new(store: WorkerStore, opts: Self::Opts) -> Self { - Self { store, opts } - } - - /// Returns the name of the worker. - fn worker_name() -> &'static str { - "kraken" - } - - /// Creates the configured `KrakenWorker`. - async fn build(self) -> Result, BuildError> { - let url = self.opts.url; - let ch_size = self.opts.internal_ch_size; - - let connector = KrakenWebSocketConnector::new(url); - let connection = connector.connect().await?; - - let (sub_tx, sub_rx) = channel(ch_size); - let (unsub_tx, unsub_rx) = channel(ch_size); - - let to_sub = self - .store - .get_query_ids() - .await? - .into_iter() - .collect::>(); - - if !to_sub.is_empty() { - // Unwrap here as the channel is guaranteed to be open - sub_tx.send(to_sub).await.unwrap(); - } - - let worker = Arc::new(KrakenWorker::new(connector, self.store, sub_tx, unsub_tx)); - - tokio::spawn(start_asset_worker( - Arc::downgrade(&worker), - connection, - sub_rx, - unsub_rx, - )); - - Ok(worker) - } -} diff --git a/bothan-kraken/src/worker/error.rs b/bothan-kraken/src/worker/error.rs deleted file mode 100644 index c2ef356f..00000000 --- a/bothan-kraken/src/worker/error.rs +++ /dev/null @@ -1,21 +0,0 @@ -use crate::api; -use bothan_core::store; -use thiserror::Error; - -#[derive(Error, Debug)] -pub(crate) enum WorkerError { - #[error("invalid price: {0}")] - InvalidPrice(f64), - - #[error("failed to set data to the store: {0}")] - SetFailed(#[from] store::error::Error), -} - -#[derive(Debug, Error)] -pub enum BuildError { - #[error("failed to connect: {0}")] - FailedToConnect(#[from] api::ConnectionError), - - #[error("store error: {0}")] - StoreError(#[from] store::error::Error), -} diff --git a/bothan-kraken/src/worker/opts.rs b/bothan-kraken/src/worker/opts.rs index de35068e..e6e4352a 100644 --- a/bothan-kraken/src/worker/opts.rs +++ b/bothan-kraken/src/worker/opts.rs @@ -1,15 +1,16 @@ use serde::{Deserialize, Serialize}; use crate::api::types::DEFAULT_URL; -use crate::worker::types::DEFAULT_CHANNEL_SIZE; + +pub const DEFAULT_CHANNEL_SIZE: usize = 1000; /// Options for configuring the `KrakenWorkerBuilder`. /// -/// `KrakenWorkerBuilderOpts` provides a way to specify custom settings for creating a `KrakenWorker`. -/// This struct allows users to set optional parameters such as the WebSocket URL and the internal channel size, -/// which will be used during the construction of the `KrakenWorker`. +/// `KrakenWorkerBuilderOpts` provides a way to specify custom settings for creating a +/// `KrakenWorker`. This struct allows users to set optional parameters such as the WebSocket URL +/// and the internal channel size, which will be used during the construction of the `KrakenWorker`. #[derive(Clone, Debug, Serialize, Deserialize)] -pub struct KrakenWorkerBuilderOpts { +pub struct WorkerOpts { #[serde(default = "default_url")] pub url: String, #[serde(default = "default_internal_ch_size")] @@ -24,7 +25,7 @@ fn default_internal_ch_size() -> usize { DEFAULT_CHANNEL_SIZE } -impl Default for KrakenWorkerBuilderOpts { +impl Default for WorkerOpts { fn default() -> Self { Self { url: default_url(), diff --git a/bothan-kraken/src/worker/types.rs b/bothan-kraken/src/worker/types.rs deleted file mode 100644 index 1e216542..00000000 --- a/bothan-kraken/src/worker/types.rs +++ /dev/null @@ -1,5 +0,0 @@ -use tokio::time::Duration; - -pub const DEFAULT_CHANNEL_SIZE: usize = 1000; -pub const DEFAULT_TIMEOUT: Duration = Duration::from_secs(60); -pub const RECONNECT_BUFFER: Duration = Duration::from_secs(5); diff --git a/bothan-lib/Cargo.toml b/bothan-lib/Cargo.toml new file mode 100644 index 00000000..2fc4d160 --- /dev/null +++ b/bothan-lib/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "bothan-lib" +version = "0.0.1-alpha.4" +authors.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true + +[dependencies] +async-trait = { workspace = true } +bincode = { workspace = true } +derive_more = { workspace = true } +num-traits = { workspace = true } +serde = { workspace = true, features = ["rc"] } +serde_json = { workspace = true } +thiserror = { workspace = true } +rust_decimal = { workspace = true, features = ["maths", "serde-str"] } +tokio = { workspace = true } +tracing = { workspace = true } diff --git a/bothan-lib/src/lib.rs b/bothan-lib/src/lib.rs new file mode 100644 index 00000000..9cec3d1b --- /dev/null +++ b/bothan-lib/src/lib.rs @@ -0,0 +1,4 @@ +pub mod registry; +pub mod store; +pub mod types; +pub mod worker; diff --git a/bothan-core/src/registry.rs b/bothan-lib/src/registry.rs similarity index 87% rename from bothan-core/src/registry.rs rename to bothan-lib/src/registry.rs index c477b5c1..e720d06d 100644 --- a/bothan-core/src/registry.rs +++ b/bothan-lib/src/registry.rs @@ -8,7 +8,8 @@ use bincode::{Decode, Encode}; use serde::{Deserialize, Serialize}; use crate::registry::signal::Signal; -use crate::registry::validate::{validate_signal, ValidationError}; +pub use crate::registry::validate::ValidationError; +use crate::registry::validate::validate_signal; pub mod post_processor; pub mod processor; @@ -45,23 +46,26 @@ impl Registry { } impl Registry { + /// Returns the signal for a given signal id. pub fn get(&self, signal_id: &str) -> Option<&Signal> { self.inner.get(signal_id) } + /// Returns `true` if the registry contains the given signal id. pub fn contains(&self, signal_id: &str) -> bool { self.inner.contains_key(signal_id) } - - pub fn keys(&self) -> impl Iterator { + /// An iterator visiting all signal ids in the registry in arbitrary order. + pub fn signal_ids(&self) -> impl Iterator { self.inner.keys() } - + /// An iterator visiting all the signal ids and their signals in the registry in arbitrary + /// order. pub fn iter(&self) -> impl Iterator { self.inner.iter() } } -impl Default for Registry { +impl Default for Registry { fn default() -> Self { Registry { inner: HashMap::new(), @@ -90,22 +94,22 @@ impl Decode for Registry { pub(crate) mod tests { use crate::registry::{Invalid, Registry, ValidationError}; - pub(crate) fn valid_mock_registry() -> Registry { + pub fn valid_mock_registry() -> Registry { let json_string = "{\"CS:USDT-USD\":{\"sources\":[{\"source_id\":\"coingecko\",\"id\":\"tether\",\"routes\":[]}],\"processor\":{\"function\":\"median\",\"params\":{\"min_source_count\":1}},\"post_processors\":[]},\"CS:BTC-USD\":{\"sources\":[{\"source_id\":\"binance\",\"id\":\"btcusdt\",\"routes\":[{\"signal_id\":\"CS:USDT-USD\",\"operation\":\"*\"}]},{\"source_id\":\"coingecko\",\"id\":\"bitcoin\",\"routes\":[]}],\"processor\":{\"function\":\"median\",\"params\":{\"min_source_count\":1}},\"post_processors\":[]}}"; serde_json::from_str::(json_string).unwrap() } - pub(crate) fn invalid_dependency_mock_registry() -> Registry { + pub fn invalid_dependency_mock_registry() -> Registry { let json_string = "{\"CS:BTC-USD\":{\"sources\":[{\"source_id\":\"binance\",\"id\":\"btcusdt\",\"routes\":[{\"signal_id\":\"CS:USDT-USD\",\"operation\":\"*\"}]},{\"source_id\":\"coingecko\",\"id\":\"bitcoin\",\"routes\":[]}],\"processor\":{\"function\":\"median\",\"params\":{\"min_source_count\":1}},\"post_processors\":[]}}"; serde_json::from_str::(json_string).unwrap() } - pub(crate) fn complete_circular_dependency_mock_registry() -> Registry { + pub fn complete_circular_dependency_mock_registry() -> Registry { let json_string = "{\"CS:USDT-USD\":{\"sources\":[{\"source_id\":\"binance\",\"id\":\"usdtusdc\",\"routes\":[{\"signal_id\":\"CS:USDC-USD\",\"operation\":\"*\"}]}],\"processor\":{\"function\":\"median\",\"params\":{\"min_source_count\":1}},\"post_processors\":[]},\"CS:USDC-USD\":{\"sources\":[{\"source_id\":\"binance\",\"id\":\"usdcusdt\",\"routes\":[{\"signal_id\":\"CS:USDT-USD\",\"operation\":\"*\"}]}],\"processor\":{\"function\":\"median\",\"params\":{\"min_source_count\":1}},\"post_processors\":[]}}"; serde_json::from_str::(json_string).unwrap() } - pub(crate) fn circular_dependency_mock_registry() -> Registry { + pub fn circular_dependency_mock_registry() -> Registry { let json_string = "{\"CS:USDT-USD\":{\"sources\":[{\"source_id\":\"binance\",\"id\":\"usdtusdc\",\"routes\":[{\"signal_id\":\"CS:USDC-USD\",\"operation\":\"*\"}]}],\"processor\":{\"function\":\"median\",\"params\":{\"min_source_count\":1}},\"post_processors\":[]},\"CS:USDC-USD\":{\"sources\":[{\"source_id\":\"binance\",\"id\":\"usdcdai\",\"routes\":[{\"signal_id\":\"CS:DAI-USD\",\"operation\":\"*\"}]}],\"processor\":{\"function\":\"median\",\"params\":{\"min_source_count\":1}},\"post_processors\":[]},\"CS:DAI-USD\":{\"sources\":[{\"source_id\":\"binance\",\"id\":\"daiusdt\",\"routes\":[{\"signal_id\":\"CS:USDT-USD\",\"operation\":\"*\"}]}],\"processor\":{\"function\":\"median\",\"params\":{\"min_source_count\":1}},\"post_processors\":[]}}"; serde_json::from_str::(json_string).unwrap() } diff --git a/bothan-core/src/registry/post_processor.rs b/bothan-lib/src/registry/post_processor.rs similarity index 94% rename from bothan-core/src/registry/post_processor.rs rename to bothan-lib/src/registry/post_processor.rs index 2a35ce52..a1c9cf6a 100644 --- a/bothan-core/src/registry/post_processor.rs +++ b/bothan-lib/src/registry/post_processor.rs @@ -25,12 +25,14 @@ pub enum PostProcessor { } impl PostProcessor { + /// Returns the name of the post-processor. pub fn name(&self) -> &str { match self { PostProcessor::TickConvertor(_) => "tick_convertor", } } + /// Runs the post-processor on the given data. pub fn post_process(&self, data: Decimal) -> Result { match self { PostProcessor::TickConvertor(tick) => tick.process(data), diff --git a/bothan-core/src/registry/post_processor/tick.rs b/bothan-lib/src/registry/post_processor/tick.rs similarity index 99% rename from bothan-core/src/registry/post_processor/tick.rs rename to bothan-lib/src/registry/post_processor/tick.rs index 67a24382..233f0a33 100644 --- a/bothan-core/src/registry/post_processor/tick.rs +++ b/bothan-lib/src/registry/post_processor/tick.rs @@ -35,10 +35,9 @@ impl TickPostProcessor { #[cfg(test)] mod tests { - use crate::registry::post_processor::tick::TickPostProcessor; - use crate::registry::post_processor::PostProcessor; - use super::*; + use crate::registry::post_processor::PostProcessor; + use crate::registry::post_processor::tick::TickPostProcessor; #[test] fn test_process() { diff --git a/bothan-core/src/registry/processor.rs b/bothan-lib/src/registry/processor.rs similarity index 93% rename from bothan-core/src/registry/processor.rs rename to bothan-lib/src/registry/processor.rs index d5645d67..117d8aab 100644 --- a/bothan-core/src/registry/processor.rs +++ b/bothan-lib/src/registry/processor.rs @@ -27,6 +27,7 @@ pub enum Processor { } impl Processor { + /// Returns the name of the processor. pub fn name(&self) -> &str { match self { Processor::Median(_) => "median", @@ -34,6 +35,7 @@ impl Processor { } } + /// Runs the processor on the given data. pub fn process(&self, data: Vec<(String, Decimal)>) -> Result { match self { Processor::Median(median) => { diff --git a/bothan-core/src/registry/processor/median.rs b/bothan-lib/src/registry/processor/median.rs similarity index 91% rename from bothan-core/src/registry/processor/median.rs rename to bothan-lib/src/registry/processor/median.rs index 2e51416d..19a32ac4 100644 --- a/bothan-core/src/registry/processor/median.rs +++ b/bothan-lib/src/registry/processor/median.rs @@ -8,9 +8,9 @@ use serde::{Deserialize, Serialize}; use crate::registry::processor::ProcessError; -/// The `MedianProcessor` finds the median of a given data set. It also has a `min_source_count` which -/// is the minimum number of sources required to calculate the median. If the given data set has less -/// than `min_source_count` sources, it returns an error. +/// The `MedianProcessor` finds the median of a given data set. It also has a `min_source_count` +/// which is the minimum number of sources required to calculate the median. If the given data set +/// has less than `min_source_count` sources, it returns an error. #[derive(Clone, Debug, PartialEq, Serialize, Deserialize, Encode, Decode)] pub struct MedianProcessor { pub min_source_count: usize, diff --git a/bothan-core/src/registry/processor/weighted_median.rs b/bothan-lib/src/registry/processor/weighted_median.rs similarity index 100% rename from bothan-core/src/registry/processor/weighted_median.rs rename to bothan-lib/src/registry/processor/weighted_median.rs diff --git a/bothan-core/src/registry/signal.rs b/bothan-lib/src/registry/signal.rs similarity index 100% rename from bothan-core/src/registry/signal.rs rename to bothan-lib/src/registry/signal.rs diff --git a/bothan-core/src/registry/source.rs b/bothan-lib/src/registry/source.rs similarity index 100% rename from bothan-core/src/registry/source.rs rename to bothan-lib/src/registry/source.rs diff --git a/bothan-core/src/registry/validate.rs b/bothan-lib/src/registry/validate.rs similarity index 99% rename from bothan-core/src/registry/validate.rs rename to bothan-lib/src/registry/validate.rs index 53d7d75a..a542f6ea 100644 --- a/bothan-core/src/registry/validate.rs +++ b/bothan-lib/src/registry/validate.rs @@ -25,7 +25,7 @@ pub(crate) fn validate_signal( ) -> Result<(), ValidationError> { match visited.get(signal_id) { Some(VisitState::InProgress) => { - return Err(ValidationError::CycleDetected(signal_id.to_string())) + return Err(ValidationError::CycleDetected(signal_id.to_string())); } Some(VisitState::Complete) => return Ok(()), None => { diff --git a/bothan-lib/src/store.rs b/bothan-lib/src/store.rs new file mode 100644 index 00000000..0299e436 --- /dev/null +++ b/bothan-lib/src/store.rs @@ -0,0 +1,53 @@ +use std::collections::HashSet; +use std::error::Error as StdError; + +use async_trait::async_trait; +pub use registry::RegistryStore; +pub use worker::WorkerStore; + +use crate::registry::{Registry, Valid}; +use crate::types::AssetInfo; + +mod registry; +mod worker; + +/// The universal trait for all stores. All implementations must be thread-safe and atomic. +#[async_trait] +pub trait Store: Send + Sync + Clone { + type Error: StdError + Send + Sync + 'static; + + /// Set the registry in the store. + async fn set_registry( + &self, + registry: Registry, + ipfs_hash: String, + ) -> Result<(), Self::Error>; + /// Get the registry in the store. + async fn get_registry(&self) -> Registry; + /// Get the IPFS hash of the registry in the store. + async fn get_registry_ipfs_hash(&self) -> Result, Self::Error>; + /// Sets the query ids in the store under the given prefix. + async fn set_query_ids(&self, prefix: &str, ids: HashSet) -> Result<(), Self::Error>; + /// Gets the query ids in the store under the given prefix. + async fn get_query_ids(&self, prefix: &str) -> Result>, Self::Error>; + /// Checks if the store contains the query id under the given prefix. + async fn contains_query_id(&self, prefix: &str, id: &str) -> Result; + /// Gets the asset info in the store under the given prefix. + async fn get_asset_info( + &self, + prefix: &str, + id: &str, + ) -> Result, Self::Error>; + /// Inserts the asset info in the store under the given prefix. + async fn insert_asset_info( + &self, + prefix: &str, + asset_info: AssetInfo, + ) -> Result<(), Self::Error>; + /// Batch inserts the asset info in the store under the given prefix. + async fn insert_asset_infos( + &self, + prefix: &str, + asset_infos: Vec, + ) -> Result<(), Self::Error>; +} diff --git a/bothan-lib/src/store/registry.rs b/bothan-lib/src/store/registry.rs new file mode 100644 index 00000000..1f79674f --- /dev/null +++ b/bothan-lib/src/store/registry.rs @@ -0,0 +1,33 @@ +use crate::registry::{Registry, Valid}; +use crate::store::Store; + +#[derive(Clone)] +pub struct RegistryStore { + store: S, +} + +impl RegistryStore { + /// Creates a new RegistryStore from an existing store. + pub fn new(store: S) -> Self { + Self { store } + } + + /// Sets the current registry and its hash. + pub async fn set_registry( + &self, + registry: Registry, + hash: String, + ) -> Result<(), S::Error> { + self.store.set_registry(registry, hash).await + } + + /// Gets the current registry. + pub async fn get_registry(&self) -> Registry { + self.store.get_registry().await + } + + /// Gets the current registry hash. + pub async fn get_registry_ipfs_hash(&self) -> Result, S::Error> { + self.store.get_registry_ipfs_hash().await + } +} diff --git a/bothan-lib/src/store/worker.rs b/bothan-lib/src/store/worker.rs new file mode 100644 index 00000000..bfb6beea --- /dev/null +++ b/bothan-lib/src/store/worker.rs @@ -0,0 +1,134 @@ +use std::collections::HashSet; +use std::hash::RandomState; +use std::sync::Arc; + +use tokio::sync::Mutex; + +use crate::store::Store; +use crate::types::{AssetInfo, AssetState}; + +#[derive(Clone)] +pub struct WorkerStore { + store: S, + prefix: String, + query_ids_lock: Arc>, +} + +impl WorkerStore { + /// Creates a new WorkerStore with the specified store and unique prefix key. + pub fn new>(store: &S, prefix: T) -> Self { + Self { + store: store.clone(), + prefix: prefix.into(), + query_ids_lock: Arc::new(Mutex::new(())), + } + } + + /// Get the asset state for the specified query id. + pub async fn get_asset(&self, id: &str) -> Result { + if !self.store.contains_query_id(&self.prefix, id).await? { + return Ok(AssetState::Unsupported); + } + + match self.store.get_asset_info(&self.prefix, id).await? { + Some(asset) => Ok(AssetState::Available(asset)), + None => Ok(AssetState::Pending), + } + } + + /// Set the asset state for the specified query id. + pub async fn set_asset_info(&self, asset: AssetInfo) -> Result<(), S::Error> { + self.store.insert_asset_info(&self.prefix, asset).await + } + + /// Sets multiple asset states for the specified query ids. + pub async fn set_asset_infos(&self, assets: Vec) -> Result<(), S::Error> { + self.store.insert_asset_infos(&self.prefix, assets).await + } + + /// Gets multiple asset states for the specified query ids. + pub async fn get_query_ids(&self) -> Result, S::Error> { + let query_ids = self + .store + .get_query_ids(&self.prefix) + .await? + .unwrap_or_default(); + Ok(query_ids) + } + + /// Calculates the [Difference] between the current query ids and a new set of query ids. + pub async fn compute_query_id_difference( + &self, + ids: HashSet, + ) -> Result { + let query_ids = self.get_query_ids().await?; + let current_ids: HashSet = HashSet::from_iter(query_ids.into_iter()); + + let added = ids + .difference(¤t_ids) + .cloned() + .collect::>(); + let removed = current_ids + .difference(&ids) + .cloned() + .collect::>(); + + Ok(Difference { added, removed }) + } + + /// Adds the specified query ids to the current set of query ids. + pub async fn add_query_ids(&self, ids: Vec) -> Result<(), S::Error> { + if ids.is_empty() { + return Ok(()); + } + + // Guard will be dropped at the end of this + let _ = self.query_ids_lock.lock().await; + let mut query_ids = self + .store + .get_query_ids(&self.prefix) + .await? + .unwrap_or_default(); + query_ids.extend(ids.into_iter()); + self.set_query_ids(query_ids).await + } + + /// Removes the specified query ids from the current set of query ids. + pub async fn remove_query_ids(&self, ids: &[String]) -> Result<(), S::Error> { + if ids.is_empty() { + return Ok(()); + } + + let _ = self.query_ids_lock.lock().await; + let mut query_ids = self + .store + .get_query_ids(&self.prefix) + .await? + .unwrap_or_default(); + + let curr_len = query_ids.len(); + for id in ids { + query_ids.remove(id); + } + + // Value is not overwritten if no changes are made to query id set + if curr_len == query_ids.len() { + Ok(()) + } else { + self.set_query_ids(query_ids).await + } + } + + /// Completely overwrite the current query ids with the new set of query ids. + pub async fn set_query_ids(&self, ids: HashSet) -> Result<(), S::Error> { + self.store.set_query_ids(&self.prefix, ids).await + } +} + +/// Contains the query ids that would be added and removed relative to the old set. +pub struct Difference { + /// The query ids that would be added relative to the old set + pub added: Vec, + /// The query ids that would be removed relative to the old set + pub removed: Vec, +} diff --git a/bothan-lib/src/types.rs b/bothan-lib/src/types.rs new file mode 100644 index 00000000..7ea00ab5 --- /dev/null +++ b/bothan-lib/src/types.rs @@ -0,0 +1,5 @@ +pub use asset_info::AssetInfo; +pub use asset_state::AssetState; + +mod asset_info; +mod asset_state; diff --git a/bothan-core/src/types.rs b/bothan-lib/src/types/asset_info.rs similarity index 100% rename from bothan-core/src/types.rs rename to bothan-lib/src/types/asset_info.rs diff --git a/bothan-lib/src/types/asset_state.rs b/bothan-lib/src/types/asset_state.rs new file mode 100644 index 00000000..99dc6c59 --- /dev/null +++ b/bothan-lib/src/types/asset_state.rs @@ -0,0 +1,10 @@ +use serde::{Deserialize, Serialize}; + +use crate::types::asset_info::AssetInfo; + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub enum AssetState { + Unsupported, + Pending, + Available(AssetInfo), +} diff --git a/bothan-lib/src/worker.rs b/bothan-lib/src/worker.rs new file mode 100644 index 00000000..7f2f97b0 --- /dev/null +++ b/bothan-lib/src/worker.rs @@ -0,0 +1,24 @@ +use std::collections::HashSet; + +use error::AssetWorkerError; + +use crate::store::Store; +use crate::types::AssetState; + +pub mod error; +pub mod rest; + +/// The universal trait for all workers that provide asset info. +#[async_trait::async_trait] +pub trait AssetWorker: Send + Sync + Sized { + type Opts; + + /// The name of the worker. + fn name(&self) -> &'static str; + /// Build the worker with the given options. + async fn build(opts: Self::Opts, store: &S) -> Result; + /// Get the asset info for the given id. + async fn get_asset(&self, id: &str) -> Result; + /// Set the query ids for the worker. + async fn set_query_ids(&self, ids: HashSet) -> Result<(), AssetWorkerError>; +} diff --git a/bothan-lib/src/worker/error.rs b/bothan-lib/src/worker/error.rs new file mode 100644 index 00000000..a9268af2 --- /dev/null +++ b/bothan-lib/src/worker/error.rs @@ -0,0 +1,57 @@ +use std::error::Error as StdError; +use std::fmt; + +/// A custom error type that wraps another error with an optional message. +pub struct AssetWorkerError { + pub msg: String, + pub source: Option>, +} + +impl AssetWorkerError { + /// Create a new `AssetWorkerError` with a message. + pub fn new(msg: impl Into) -> Self { + Self { + msg: msg.into(), + source: None, + } + } + + /// Create a new `AssetWorkerError` with a message and a source error. + pub fn with_source(msg: impl Into, source: E) -> Self + where + E: StdError + Send + Sync + 'static, + { + Self { + msg: msg.into(), + source: Some(Box::new(source)), + } + } +} + +impl fmt::Display for AssetWorkerError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.msg)?; + if let Some(source) = &self.source { + write!(f, ": {}", source)?; + } + Ok(()) + } +} + +impl fmt::Debug for AssetWorkerError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(self, f) + } +} + +impl From for AssetWorkerError +where + E: StdError + Send + Sync + 'static, +{ + fn from(err: E) -> Self { + Self { + msg: format!("An error occurred: {}", err), + source: Some(Box::new(err)), + } + } +} diff --git a/bothan-lib/src/worker/rest.rs b/bothan-lib/src/worker/rest.rs new file mode 100644 index 00000000..de7186b3 --- /dev/null +++ b/bothan-lib/src/worker/rest.rs @@ -0,0 +1,64 @@ +use std::fmt::Display; +use std::sync::Weak; +use std::time::Duration; + +use tokio::time::{interval, timeout}; +use tracing::{debug, error}; + +use crate::store::{Store, WorkerStore}; +use crate::types::AssetInfo; + +#[async_trait::async_trait] +pub trait AssetInfoProvider: Send + Sync { + type Error: Display; + + async fn get_asset_info(&self, ids: &[String]) -> Result, Self::Error>; +} + +/// Starts polling asset information from a provider at the specified update interval. This function +/// will not return until the asset_info_provider is dropped. +/// Any errors that occur during the polling process will be logged. +pub async fn start_polling( + update_interval: Duration, + asset_info_provider: Weak>, + store: WorkerStore, +) { + let mut interval = interval(update_interval); + while let Some(provider) = asset_info_provider.upgrade() { + interval.tick().await; + + let ids = match store.get_query_ids().await { + Ok(ids) => ids.into_iter().collect::>(), + Err(e) => { + error!("failed to get query ids with error: {}", e); + continue; + } + }; + + if let Err(e) = store.get_query_ids().await { + error!("failed to get query ids with error: {}", e); + continue; + } + + if ids.is_empty() { + debug!("no ids to update, skipping update"); + continue; + } + + match timeout(interval.period(), provider.get_asset_info(&ids)).await { + Ok(Ok(asset_info)) => { + if let Err(e) = store.set_asset_infos(asset_info).await { + error!("failed to update asset info with error: {e}"); + } else { + debug!("asset info updated successfully"); + } + } + Ok(Err(e)) => { + error!("failed to update asset info with error: {e}"); + } + Err(_) => { + error!("updating interval exceeded timeout"); + } + } + } +} diff --git a/bothan-okx/Cargo.toml b/bothan-okx/Cargo.toml index b2734630..2883639b 100644 --- a/bothan-okx/Cargo.toml +++ b/bothan-okx/Cargo.toml @@ -1,13 +1,13 @@ [package] name = "bothan-okx" -version = "0.0.1-beta.1" +version = "0.0.1-beta.2" edition.workspace = true license.workspace = true repository.workspace = true [dependencies] async-trait = { workspace = true } -bothan-core = { workspace = true } +bothan-lib = { workspace = true } chrono = { workspace = true } futures-util = { workspace = true } rust_decimal = { workspace = true } @@ -17,7 +17,6 @@ thiserror = { workspace = true } tokio = { workspace = true } tokio-tungstenite = { workspace = true } tracing = { workspace = true } -tracing-subscriber = { workspace = true } [dev-dependencies] ws-mock = { git = "https://github.com/bandprotocol/ws-mock.git", branch = "master" } diff --git a/bothan-okx/examples/okx_basic.rs b/bothan-okx/examples/okx_basic.rs deleted file mode 100644 index e131c88f..00000000 --- a/bothan-okx/examples/okx_basic.rs +++ /dev/null @@ -1,39 +0,0 @@ -use std::time::Duration; - -use tokio::time::sleep; -use tracing_subscriber::fmt::init; - -use bothan_core::registry::Registry; -use bothan_core::store::SharedStore; -use bothan_core::worker::{AssetWorker, AssetWorkerBuilder}; -use bothan_okx::{OkxWorkerBuilder, OkxWorkerBuilderOpts}; - -#[tokio::main] -async fn main() { - init(); - let path = std::env::current_dir().unwrap(); - let registry = Registry::default().validate().unwrap(); - let store = SharedStore::new(registry, path.as_path()).await.unwrap(); - - let worker_store = store.create_worker_store(OkxWorkerBuilder::worker_name()); - let opts = OkxWorkerBuilderOpts::default(); - - let worker = OkxWorkerBuilder::new(worker_store, opts) - .build() - .await - .unwrap(); - - worker - .set_query_ids(vec!["BTC-USDT".to_string(), "ETH-USDT".to_string()]) - .await - .unwrap(); - - sleep(Duration::from_secs(2)).await; - - loop { - let btc_data = worker.get_asset("BTC-USDT").await; - let eth_data = worker.get_asset("ETH-USDT").await; - println!("{:?}, {:?}", btc_data, eth_data); - sleep(Duration::from_secs(5)).await; - } -} diff --git a/bothan-okx/src/api.rs b/bothan-okx/src/api.rs index 9a16018d..c320003f 100644 --- a/bothan-okx/src/api.rs +++ b/bothan-okx/src/api.rs @@ -1,5 +1,5 @@ pub use error::{ConnectionError, MessageError, SendError}; -pub use websocket::{OkxWebSocketConnection, OkxWebSocketConnector}; +pub use websocket::{WebSocketConnection, WebsocketConnector}; pub mod error; pub mod types; diff --git a/bothan-okx/src/api/types.rs b/bothan-okx/src/api/types.rs index d5f33d8a..89e85cf1 100644 --- a/bothan-okx/src/api/types.rs +++ b/bothan-okx/src/api/types.rs @@ -1,7 +1,6 @@ -use serde::{Deserialize, Serialize}; - pub use channel::{ChannelArgument, ChannelResponse, PushData, TickerData}; pub use message::WebSocketMessageResponse; +use serde::{Deserialize, Serialize}; pub mod channel; pub mod message; diff --git a/bothan-okx/src/api/websocket.rs b/bothan-okx/src/api/websocket.rs index 7988f0df..9c7be3f6 100644 --- a/bothan-okx/src/api/websocket.rs +++ b/bothan-okx/src/api/websocket.rs @@ -1,29 +1,29 @@ use futures_util::stream::{SplitSink, SplitStream}; use futures_util::{SinkExt, StreamExt}; use tokio::net::TcpStream; +use tokio_tungstenite::tungstenite::Message; use tokio_tungstenite::tungstenite::error::Error as TungsteniteError; use tokio_tungstenite::tungstenite::http::StatusCode; -use tokio_tungstenite::tungstenite::Message; -use tokio_tungstenite::{connect_async, MaybeTlsStream, WebSocketStream}; +use tokio_tungstenite::{MaybeTlsStream, WebSocketStream, connect_async}; use tracing::warn; use crate::api::error::{ConnectionError, MessageError, SendError}; -use crate::api::types::message::{InstrumentType, Op, PriceRequestArgument, WebSocketMessage}; use crate::api::types::OkxResponse; +use crate::api::types::message::{InstrumentType, Op, PriceRequestArgument, WebSocketMessage}; /// A connector for establishing a WebSocket connection to the OKX API. -pub struct OkxWebSocketConnector { +pub struct WebsocketConnector { url: String, } -impl OkxWebSocketConnector { +impl WebsocketConnector { /// Creates a new instance of `OkxWebSocketConnector`. pub fn new(url: impl Into) -> Self { Self { url: url.into() } } /// Connects to the OKX WebSocket API. - pub async fn connect(&self) -> Result { + pub async fn connect(&self) -> Result { let (wss, resp) = connect_async(self.url.clone()).await?; let status = resp.status(); @@ -34,17 +34,17 @@ impl OkxWebSocketConnector { )); } - Ok(OkxWebSocketConnection::new(wss)) + Ok(WebSocketConnection::new(wss)) } } /// Represents an active WebSocket connection to the OKX API. -pub struct OkxWebSocketConnection { +pub struct WebSocketConnection { sender: SplitSink>, Message>, receiver: SplitStream>>, } -impl OkxWebSocketConnection { +impl WebSocketConnection { /// Creates a new `OkxWebSocketConnection` instance. pub fn new(web_socket_stream: WebSocketStream>) -> Self { let (sender, receiver) = web_socket_stream.split(); @@ -125,9 +125,8 @@ pub(crate) mod test { use tokio::sync::mpsc; use ws_mock::ws_mock_server::{WsMock, WsMockServer}; - use crate::api::types::{ChannelArgument, ChannelResponse, OkxResponse, PushData, TickerData}; - use super::*; + use crate::api::types::{ChannelArgument, ChannelResponse, OkxResponse, PushData, TickerData}; pub(crate) async fn setup_mock_server() -> WsMockServer { WsMockServer::start().await @@ -137,7 +136,7 @@ pub(crate) mod test { async fn test_recv_ticker() { // Set up the mock server and the WebSocket connector. let server = setup_mock_server().await; - let connector = OkxWebSocketConnector::new(server.uri().await); + let connector = WebsocketConnector::new(server.uri().await); let (mpsc_send, mpsc_recv) = mpsc::channel::(32); // Create a mock ticker data. @@ -187,7 +186,7 @@ pub(crate) mod test { async fn test_recv_close() { // Set up the mock server and the WebSocket connector. let server = setup_mock_server().await; - let connector = OkxWebSocketConnector::new(server.uri().await); + let connector = WebsocketConnector::new(server.uri().await); let (mpsc_send, mpsc_recv) = mpsc::channel::(32); // Mount the mock WebSocket server and send a close message. diff --git a/bothan-okx/src/lib.rs b/bothan-okx/src/lib.rs index 1e4ffaa6..00b3e216 100644 --- a/bothan-okx/src/lib.rs +++ b/bothan-okx/src/lib.rs @@ -1,8 +1,6 @@ -pub use api::websocket::{OkxWebSocketConnection, OkxWebSocketConnector}; -pub use worker::builder::OkxWorkerBuilder; -pub use worker::error::BuildError; -pub use worker::opts::OkxWorkerBuilderOpts; -pub use worker::OkxWorker; +pub use api::websocket::{WebSocketConnection, WebsocketConnector}; +pub use worker::Worker; +pub use worker::opts::WorkerOpts; pub mod api; pub mod worker; diff --git a/bothan-okx/src/worker.rs b/bothan-okx/src/worker.rs index 43cce7b7..bb9b49fc 100644 --- a/bothan-okx/src/worker.rs +++ b/bothan-okx/src/worker.rs @@ -1,68 +1,88 @@ -use tokio::sync::mpsc::Sender; +use std::collections::HashSet; +use std::sync::Arc; -use bothan_core::store::error::Error as StoreError; -use bothan_core::store::WorkerStore; -use bothan_core::worker::{AssetState, AssetWorker, SetQueryIDError}; +use bothan_lib::store::{Store, WorkerStore}; +use bothan_lib::types::AssetState; +use bothan_lib::worker::AssetWorker; +use bothan_lib::worker::error::AssetWorkerError; +use tokio::sync::mpsc::{Sender, channel}; -use crate::api::websocket::OkxWebSocketConnector; +use crate::WorkerOpts; +use crate::api::websocket::WebsocketConnector; +use crate::worker::asset_worker::start_asset_worker; mod asset_worker; -pub mod builder; -pub(crate) mod error; pub mod opts; -mod types; -/// A worker that fetches and stores the asset information from Okx's API. -pub struct OkxWorker { - connector: OkxWebSocketConnector, - store: WorkerStore, - subscribe_tx: Sender>, - unsubscribe_tx: Sender>, +const WORKER_NAME: &str = "okx"; + +pub struct Worker { + inner: Arc>, } -impl OkxWorker { - /// Create a new worker with the specified connector, store and channels. - pub fn new( - connector: OkxWebSocketConnector, - store: WorkerStore, - subscribe_tx: Sender>, - unsubscribe_tx: Sender>, - ) -> Self { - Self { - connector, - store, - subscribe_tx, - unsubscribe_tx, +#[async_trait::async_trait] +impl AssetWorker for Worker { + type Opts = WorkerOpts; + + fn name(&self) -> &'static str { + WORKER_NAME + } + + async fn build(opts: Self::Opts, store: &S) -> Result { + let url = opts.url; + let ch_size = opts.internal_ch_size; + + let connector = WebsocketConnector::new(url); + let connection = connector.connect().await?; + + let (sub_tx, sub_rx) = channel(ch_size); + let (unsub_tx, unsub_rx) = channel(ch_size); + + let worker_store = WorkerStore::new(store, WORKER_NAME); + let to_sub = worker_store + .get_query_ids() + .await? + .into_iter() + .collect::>(); + + if !to_sub.is_empty() { + sub_tx.send(to_sub).await?; } + + let inner = Arc::new(InnerWorker { + connector, + store: worker_store, + subscribe_tx: sub_tx, + unsubscribe_tx: unsub_tx, + }); + + tokio::spawn(start_asset_worker( + Arc::downgrade(&inner), + connection, + sub_rx, + unsub_rx, + )); + + Ok(Worker { inner }) } -} -#[async_trait::async_trait] -impl AssetWorker for OkxWorker { - /// Fetches the AssetStatus for the given cryptocurrency id. - async fn get_asset(&self, id: &str) -> Result { - self.store.get_asset(&id).await + async fn get_asset(&self, id: &str) -> Result { + Ok(self.inner.store.get_asset(id).await?) } - /// Sets the specified cryptocurrency IDs to the query. If the ids are already in the query set, - /// it will not be resubscribed. - async fn set_query_ids(&self, ids: Vec) -> Result<(), SetQueryIDError> { - let (to_sub, to_unsub) = self - .store - .set_query_ids(ids) - .await - .map_err(|e| SetQueryIDError::new(e.to_string()))?; - - self.subscribe_tx - .send(to_sub) - .await - .map_err(|e| SetQueryIDError::new(e.to_string()))?; - - self.unsubscribe_tx - .send(to_unsub) - .await - .map_err(|e| SetQueryIDError::new(e.to_string()))?; + async fn set_query_ids(&self, ids: HashSet) -> Result<(), AssetWorkerError> { + let diff = self.inner.store.compute_query_id_difference(ids).await?; + + self.inner.subscribe_tx.send(diff.added).await?; + self.inner.unsubscribe_tx.send(diff.removed).await?; Ok(()) } } + +pub struct InnerWorker { + connector: WebsocketConnector, + store: WorkerStore, + subscribe_tx: Sender>, + unsubscribe_tx: Sender>, +} diff --git a/bothan-okx/src/worker/asset_worker.rs b/bothan-okx/src/worker/asset_worker.rs index 4844fd93..b7d9f960 100644 --- a/bothan-okx/src/worker/asset_worker.rs +++ b/bothan-okx/src/worker/asset_worker.rs @@ -1,24 +1,25 @@ use std::sync::Weak; +use std::time::Duration; +use bothan_lib::store::{Store, WorkerStore}; +use bothan_lib::types::AssetInfo; use rust_decimal::Decimal; use tokio::select; use tokio::sync::mpsc::Receiver; use tokio::time::{sleep, timeout}; use tracing::{debug, error, info, warn}; -use bothan_core::store::WorkerStore; -use bothan_core::types::AssetInfo; - use crate::api::error::{MessageError, SendError}; use crate::api::types::{ChannelResponse, OkxResponse, TickerData}; -use crate::api::{OkxWebSocketConnection, OkxWebSocketConnector}; -use crate::worker::error::WorkerError; -use crate::worker::types::{DEFAULT_TIMEOUT, RECONNECT_BUFFER}; -use crate::worker::OkxWorker; - -pub(crate) async fn start_asset_worker( - worker: Weak, - mut connection: OkxWebSocketConnection, +use crate::api::{WebSocketConnection, WebsocketConnector}; +use crate::worker::InnerWorker; + +const DEFAULT_TIMEOUT: Duration = Duration::from_secs(60); +const RECONNECT_BUFFER: Duration = Duration::from_secs(5); + +pub(crate) async fn start_asset_worker( + worker: Weak>, + mut connection: WebSocketConnection, mut subscribe_rx: Receiver>, mut unsubscribe_rx: Receiver>, ) { @@ -49,10 +50,7 @@ pub(crate) async fn start_asset_worker( debug!("asset worker has been dropped, stopping asset worker"); } -async fn subscribe( - ids: &[String], - connection: &mut OkxWebSocketConnection, -) -> Result<(), SendError> { +async fn subscribe(ids: &[String], connection: &mut WebSocketConnection) -> Result<(), SendError> { if !ids.is_empty() { let ids_vec = ids.iter().map(|s| s.as_str()).collect::>(); connection.subscribe_ticker(&ids_vec).await? @@ -61,7 +59,7 @@ async fn subscribe( Ok(()) } -async fn handle_subscribe_recv(ids: Vec, connection: &mut OkxWebSocketConnection) { +async fn handle_subscribe_recv(ids: Vec, connection: &mut WebSocketConnection) { if let Err(e) = subscribe(&ids, connection).await { error!("failed to subscribe to ids {:?}: {}", ids, e); } else { @@ -71,7 +69,7 @@ async fn handle_subscribe_recv(ids: Vec, connection: &mut OkxWebSocketCo async fn unsubscribe( ids: &[String], - connection: &mut OkxWebSocketConnection, + connection: &mut WebSocketConnection, ) -> Result<(), SendError> { if !ids.is_empty() { connection @@ -82,7 +80,7 @@ async fn unsubscribe( Ok(()) } -async fn handle_unsubscribe_recv(ids: Vec, connection: &mut OkxWebSocketConnection) { +async fn handle_unsubscribe_recv(ids: Vec, connection: &mut WebSocketConnection) { if let Err(e) = unsubscribe(&ids, connection).await { error!("failed to unsubscribe to ids {:?}: {}", ids, e); } else { @@ -90,10 +88,10 @@ async fn handle_unsubscribe_recv(ids: Vec, connection: &mut OkxWebSocket } } -async fn handle_reconnect( - connector: &OkxWebSocketConnector, - connection: &mut OkxWebSocketConnection, - query_ids: &WorkerStore, +async fn handle_reconnect( + connector: &WebsocketConnector, + connection: &mut WebSocketConnection, + query_ids: &WorkerStore, ) { let mut retry_count: usize = 1; loop { @@ -128,31 +126,30 @@ async fn handle_reconnect( } } -fn parse_ticker(ticker: TickerData) -> Result { +async fn store_ticker(store: &WorkerStore, ticker: TickerData, timestamp: i64) { let id = ticker.inst_id.clone(); - let price_value = Decimal::from_str_exact(&ticker.last)?; - let timestamp = chrono::Utc::now().timestamp(); - Ok(AssetInfo::new(id, price_value, timestamp)) -} - -async fn store_ticker(store: &WorkerStore, ticker: TickerData) -> Result<(), WorkerError> { - let id = ticker.inst_id.clone(); - store.set_asset(id.clone(), parse_ticker(ticker)?).await?; - debug!("stored data for id {}", id); - Ok(()) + match Decimal::from_str_exact(&ticker.last) { + Ok(price) => { + let asset_info = AssetInfo::new(id.clone(), price, timestamp); + if let Err(e) = store.set_asset_info(asset_info).await { + error!("failed to store data for id {}: {}", id, e); + } else { + debug!("stored data for id {}", id); + } + } + Err(e) => { + error!("failed to parse price for id {}: {}", id, e); + } + } } -/// Processes the response from the Okx API. -async fn process_response(resp: OkxResponse, store: &WorkerStore) { +async fn process_response(resp: OkxResponse, store: &WorkerStore) { match resp { OkxResponse::ChannelResponse(resp) => match resp { ChannelResponse::Ticker(push_data) => { - let tickers = push_data.data; - for ticker in tickers { - match store_ticker(store, ticker).await { - Ok(_) => debug!("saved data"), - Err(e) => error!("failed to save data: {}", e), - } + let timestamp = chrono::Utc::now().timestamp(); + for ticker in push_data.data { + store_ticker(store, ticker, timestamp).await } } }, @@ -162,11 +159,11 @@ async fn process_response(resp: OkxResponse, store: &WorkerStore) { } } -async fn handle_connection_recv( +async fn handle_connection_recv( recv_result: Result, - connector: &OkxWebSocketConnector, - connection: &mut OkxWebSocketConnection, - store: &WorkerStore, + connector: &WebsocketConnector, + connection: &mut WebSocketConnection, + store: &WorkerStore, ) { match recv_result { Ok(resp) => { @@ -183,61 +180,3 @@ async fn handle_connection_recv( } } } - -#[cfg(test)] -mod test { - use super::*; - - #[test] - fn test_parse_market() { - let ticker = TickerData { - inst_type: "SPOT".to_string(), - inst_id: "BTC-USDT".to_string(), - last: "42000.99".to_string(), - last_sz: "5000".to_string(), - ask_px: "10001".to_string(), - ask_sz: "5000".to_string(), - bid_px: "9999".to_string(), - bid_sz: "5000".to_string(), - open_24h: "10000".to_string(), - high_24h: "10000".to_string(), - low_24h: "10000".to_string(), - vol_ccy_24h: "10000".to_string(), - vol_24h: "10000".to_string(), - sod_utc0: "10000".to_string(), - sod_utc8: "10000".to_string(), - ts: "10000".to_string(), - }; - let result = parse_ticker(ticker); - let expected = AssetInfo::new( - "BTC-USDT".to_string(), - Decimal::from_str_exact("42000.99").unwrap(), - 0, - ); - assert_eq!(result.as_ref().unwrap().id, expected.id); - assert_eq!(result.unwrap().price, expected.price); - } - - #[test] - fn test_parse_market_with_failure() { - let ticker = TickerData { - inst_type: "SPOT".to_string(), - inst_id: "BTC-USDT".to_string(), - last: f64::INFINITY.to_string(), - last_sz: "5000".to_string(), - ask_px: "10001".to_string(), - ask_sz: "5000".to_string(), - bid_px: "9999".to_string(), - bid_sz: "5000".to_string(), - open_24h: "10000".to_string(), - high_24h: "10000".to_string(), - low_24h: "10000".to_string(), - vol_ccy_24h: "10000".to_string(), - vol_24h: "10000".to_string(), - sod_utc0: "10000".to_string(), - sod_utc8: "10000".to_string(), - ts: "10000".to_string(), - }; - assert!(parse_ticker(ticker).is_err()); - } -} diff --git a/bothan-okx/src/worker/builder.rs b/bothan-okx/src/worker/builder.rs deleted file mode 100644 index 95284552..00000000 --- a/bothan-okx/src/worker/builder.rs +++ /dev/null @@ -1,99 +0,0 @@ -use std::sync::Arc; - -use tokio::sync::mpsc::channel; - -use crate::api::OkxWebSocketConnector; -use crate::worker::asset_worker::start_asset_worker; -use crate::worker::error::BuildError; -use crate::worker::opts::OkxWorkerBuilderOpts; -use crate::worker::OkxWorker; -use bothan_core::store::WorkerStore; -use bothan_core::worker::AssetWorkerBuilder; - -/// Builds a `OkxWorker` with custom options. -/// Methods can be chained to set the configuration values and the -/// service is constructed by calling the [`build`](OkxWorkerBuilder::build) method. -pub struct OkxWorkerBuilder { - store: WorkerStore, - opts: OkxWorkerBuilderOpts, -} - -impl OkxWorkerBuilder { - /// Returns a new `OkxWorkerBuilder` with the given options. - pub fn new(store: WorkerStore, opts: OkxWorkerBuilderOpts) -> Self { - Self { store, opts } - } - - /// Set the URL for the `OkxWorker`. - /// The default URL is `DEFAULT_URL`. - pub fn with_url>(mut self, url: T) -> Self { - self.opts.url = url.into(); - self - } - - /// Set the internal channel size for the `OkxWorker`. - /// The default size is `DEFAULT_CHANNEL_SIZE`. - pub fn with_internal_ch_size(mut self, size: usize) -> Self { - self.opts.internal_ch_size = size; - self - } - - /// Sets the store for the `OkxWorker`. - /// If not set, the store is created and owned by the worker. - pub fn with_store(mut self, store: WorkerStore) -> Self { - self.store = store; - self - } -} - -#[async_trait::async_trait] -impl<'a> AssetWorkerBuilder<'a> for OkxWorkerBuilder { - type Opts = OkxWorkerBuilderOpts; - type Worker = OkxWorker; - type Error = BuildError; - - /// Returns a new `OkxWorkerBuilder` with the given options. - fn new(store: WorkerStore, opts: Self::Opts) -> Self { - Self { store, opts } - } - - /// Returns the name of the worker. - fn worker_name() -> &'static str { - "okx" - } - - /// Creates the configured `OkxWorker`. - async fn build(self) -> Result, BuildError> { - let url = self.opts.url; - let ch_size = self.opts.internal_ch_size; - - let connector = OkxWebSocketConnector::new(url); - let connection = connector.connect().await?; - - let (sub_tx, sub_rx) = channel(ch_size); - let (unsub_tx, unsub_rx) = channel(ch_size); - - let to_sub = self - .store - .get_query_ids() - .await? - .into_iter() - .collect::>(); - - if !to_sub.is_empty() { - // Unwrap here as the channel is guaranteed to be open - sub_tx.send(to_sub).await.unwrap(); - } - - let worker = Arc::new(OkxWorker::new(connector, self.store, sub_tx, unsub_tx)); - - tokio::spawn(start_asset_worker( - Arc::downgrade(&worker), - connection, - sub_rx, - unsub_rx, - )); - - Ok(worker) - } -} diff --git a/bothan-okx/src/worker/error.rs b/bothan-okx/src/worker/error.rs deleted file mode 100644 index ff6cdc33..00000000 --- a/bothan-okx/src/worker/error.rs +++ /dev/null @@ -1,21 +0,0 @@ -use crate::api; -use bothan_core::store; -use thiserror::Error; - -#[derive(Error, Debug)] -pub(crate) enum WorkerError { - #[error("value is not a valid decimal: {0}")] - InvalidDecimal(#[from] rust_decimal::Error), - - #[error("failed to set data to the store: {0}")] - SetFailed(#[from] store::error::Error), -} - -#[derive(Debug, Error)] -pub enum BuildError { - #[error("failed to connect: {0}")] - FailedToConnect(#[from] api::ConnectionError), - - #[error("store error: {0}")] - StoreError(#[from] store::error::Error), -} diff --git a/bothan-okx/src/worker/opts.rs b/bothan-okx/src/worker/opts.rs index 0142a763..5441cc96 100644 --- a/bothan-okx/src/worker/opts.rs +++ b/bothan-okx/src/worker/opts.rs @@ -1,15 +1,11 @@ use serde::{Deserialize, Serialize}; use crate::api::types::DEFAULT_URL; -use crate::worker::types::DEFAULT_CHANNEL_SIZE; -/// Options for configuring the `OkxWorkerBuilder`. -/// -/// `OkxWorkerBuilderOpts` provides a way to specify custom settings for creating a `OkxWorker`. -/// This struct allows users to set optional parameters such as the WebSocket URL and the internal channel size, -/// which will be used during the construction of the `OkxWorker`. +pub const DEFAULT_CHANNEL_SIZE: usize = 1000; + #[derive(Clone, Debug, Serialize, Deserialize)] -pub struct OkxWorkerBuilderOpts { +pub struct WorkerOpts { #[serde(default = "default_url")] pub url: String, #[serde(default = "default_internal_ch_size")] @@ -24,7 +20,7 @@ fn default_internal_ch_size() -> usize { DEFAULT_CHANNEL_SIZE } -impl Default for OkxWorkerBuilderOpts { +impl Default for WorkerOpts { fn default() -> Self { Self { url: default_url(), diff --git a/bothan-okx/src/worker/types.rs b/bothan-okx/src/worker/types.rs deleted file mode 100644 index 1e216542..00000000 --- a/bothan-okx/src/worker/types.rs +++ /dev/null @@ -1,5 +0,0 @@ -use tokio::time::Duration; - -pub const DEFAULT_CHANNEL_SIZE: usize = 1000; -pub const DEFAULT_TIMEOUT: Duration = Duration::from_secs(60); -pub const RECONNECT_BUFFER: Duration = Duration::from_secs(5); diff --git a/clippy.toml b/clippy.toml index 5e90250c..4972822f 100644 --- a/clippy.toml +++ b/clippy.toml @@ -1 +1 @@ -msrv = "1.81.0" +msrv = "1.85.0" diff --git a/rustfmt.toml b/rustfmt.toml index 9abcaba7..47164116 100644 --- a/rustfmt.toml +++ b/rustfmt.toml @@ -1,2 +1,6 @@ +unstable_features = true + max_width = 100 -reorder_imports = true +group_imports = "StdExternalCrate" +imports_granularity = "Module" +imports_layout = "Mixed"