From ad941f4c06f39184d77401fdcde647ff32d1b942 Mon Sep 17 00:00:00 2001 From: n-dusan Date: Mon, 15 Jan 2024 14:07:38 +0100 Subject: [PATCH 01/15] feat: add initial database support *use sqlx* Intended for serving historical documents and loading data from Postgres or SQlite *add generic connection over database* - Introduce a `db` module responsible for managing database - Add `connect` function that's generic over SQLite and Postgres *init database and apply migrations* Defaults to local SQLite database if `DATABASE_URL` is missing. After connecting, run any/all migrations that are necessary. *generate initial empty migration* To manage migrations, it's first necessary to install `sqlx-cli`. - `cargo install sqlx-cli`. Then, `sqlx migrate add ` to add a new migration. *introduce build.rs build script* Adding new migrations won't trigger cargo recompilation, so sqlx strongly suggests we trigger a recompilation on new migrations. *add top level db module* *connect to db on app start* After adding another field to the `AppState` struct the integration tests started failing. However, for the archive tests we don't really care about having database connection. To resolve, introduce a trait `GlobalState` that `init_app` implements and initializes the archive. This way we're flexibile in our testing framework and can use different structs to initialize the global app state. *create db.sqlite3 if not exists* *add postgres directory* We are not sure whether we'll support Postgres, but add directory --- Cargo.lock | 762 +++++++++++++++++- Cargo.toml | 3 +- build.rs | 5 + migrations/postgres/.keep | 0 .../sqlite/20240115152953_initial_db.sql | 1 + src/db/init.rs | 26 + src/db/mod.rs | 41 + src/lib.rs | 1 + src/server/publish.rs | 52 +- src/stelae/stele.rs | 1 - src/stelae/types/repositories.rs | 2 +- tests/common/mod.rs | 18 +- 12 files changed, 863 insertions(+), 49 deletions(-) create mode 100644 build.rs create mode 100644 migrations/postgres/.keep create mode 100644 migrations/sqlite/20240115152953_initial_db.sql create mode 100644 src/db/init.rs create mode 100644 src/db/mod.rs diff --git a/Cargo.lock b/Cargo.lock index 0b19dea..8b4e890 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -29,7 +29,7 @@ dependencies = [ "actix-rt", "actix-service", "actix-utils", - "ahash", + "ahash 0.7.6", "base64", "bitflags 1.3.2", "brotli", @@ -63,7 +63,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "465a6172cf69b960917811022d8f29bc0b7fa1398bc4f78b3c466673db1213b6" dependencies = [ "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -143,7 +143,7 @@ dependencies = [ "actix-service", "actix-utils", "actix-web-codegen", - "ahash", + "ahash 0.7.6", "bytes", "bytestring", "cfg-if", @@ -178,7 +178,7 @@ dependencies = [ "actix-router", "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -198,6 +198,18 @@ dependencies = [ "version_check", ] +[[package]] +name = "ahash" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77c3a9648d43b9cd48db467b3f87fdd6e146bcc88ab0180006cef2179fe11d01" +dependencies = [ + "cfg-if", + "once_cell", + "version_check", + "zerocopy", +] + [[package]] name = "aho-corasick" version = "0.7.19" @@ -222,12 +234,27 @@ dependencies = [ "alloc-no-stdlib", ] +[[package]] +name = "allocator-api2" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0942ffc6dcaadf03badf6e6a2d0228460359d5e34b57ccdc720b7382dfbd5ec5" + [[package]] name = "anyhow" version = "1.0.66" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "216261ddc8289130e551ddcd5ce8a064710c0d064a4d2895c67151c92b5443f6" +[[package]] +name = "atoi" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7c57d12312ff59c811c0643f4d80830505833c9ffaebd193d819392b265be8e" +dependencies = [ + "num-traits", +] + [[package]] name = "atty" version = "0.2.14" @@ -311,6 +338,12 @@ version = "3.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "572f695136211188308f16ad2ca5c851a712c464060ae6974944458eb83880ba" +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + [[package]] name = "bytes" version = "1.2.1" @@ -383,7 +416,7 @@ dependencies = [ "proc-macro-error", "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -412,6 +445,22 @@ dependencies = [ "version_check", ] +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" + [[package]] name = "cpufeatures" version = "0.2.5" @@ -421,6 +470,21 @@ dependencies = [ "libc", ] +[[package]] +name = "crc" +version = "3.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86ec7a15cbe22e59248fc7eadb1907dab5ba09372595da4d73dd805ed4417dfe" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" + [[package]] name = "crc32fast" version = "1.3.2" @@ -441,7 +505,7 @@ dependencies = [ "clap 2.34.0", "criterion-plot", "csv", - "itertools", + "itertools 0.10.5", "lazy_static", "num-traits", "oorandom", @@ -463,7 +527,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2673cc8207403546f45f5fd319a974b1e6983ad1a3ee7e6041650013be041876" dependencies = [ "cast", - "itertools", + "itertools 0.10.5", ] [[package]] @@ -501,14 +565,20 @@ dependencies = [ ] [[package]] -name = "crossbeam-utils" -version = "0.8.14" +name = "crossbeam-queue" +version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fb766fa798726286dbbb842f174001dab8abc7b627a1dd86e0b7222a95d929f" +checksum = "df0346b5d5e76ac2fe4e327c5fd1118d6be7c51dfb18f9b7922923f287471e35" dependencies = [ - "cfg-if", + "crossbeam-utils", ] +[[package]] +name = "crossbeam-utils" +version = "0.8.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "248e3bacc7dc6baa3b21e405ee045c3047101a49145e7e9eca583ab4c2ca5345" + [[package]] name = "crypto-common" version = "0.1.6" @@ -551,7 +621,7 @@ dependencies = [ "proc-macro2", "quote", "rustc_version", - "syn", + "syn 1.0.109", ] [[package]] @@ -562,8 +632,35 @@ checksum = "8168378f4e5023e7218c89c891c0fd8ecdb5e5e4f18cb78f38cf245dd021e76f" dependencies = [ "block-buffer", "crypto-common", + "subtle", +] + +[[package]] +name = "dirs" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca3aa72a6f96ea37bbc5aa912f6788242832f75369bdfdadcb0e38423f100059" +dependencies = [ + "dirs-sys", ] +[[package]] +name = "dirs-sys" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b1d1d91c932ef41c0f2663aa8b0ca0342d444d842c06914aa0a7e352d0bada6" +dependencies = [ + "libc", + "redox_users", + "winapi", +] + +[[package]] +name = "dotenvy" +version = "0.15.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" + [[package]] name = "either" version = "1.8.0" @@ -616,12 +713,24 @@ dependencies = [ "libc", ] +[[package]] +name = "event-listener" +version = "2.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" + [[package]] name = "fastrand" version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5" +[[package]] +name = "finl_unicode" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fcfdc7a0362c9f4444381a9e697c79d435fe65b52a37466fc2c1184cee9edc6" + [[package]] name = "flate2" version = "1.0.24" @@ -632,12 +741,39 @@ dependencies = [ "miniz_oxide", ] +[[package]] +name = "flume" +version = "0.10.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1657b4441c3403d9f7b3409e47575237dac27b1b5726df654a6ecbf92f0f7577" +dependencies = [ + "futures-core", + "futures-sink", + "pin-project", + "spin", +] + [[package]] name = "fnv" version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + [[package]] name = "form_urlencoded" version = "1.1.0" @@ -647,17 +783,49 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "futures-channel" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" +dependencies = [ + "futures-core", + "futures-sink", +] + [[package]] name = "futures-core" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" + +[[package]] +name = "futures-executor" version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04909a7a7e4633ae6c4a9ab280aeb86da1236243a77b694a49eacd659a4bd3ac" +checksum = "7acc85df6714c176ab5edf386123fafe217be88c0840ec11f199441134a074e2" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-intrusive" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a604f7a68fbf8103337523b1fadc8ade7361ee3f112f7c680ad179651616aed5" +dependencies = [ + "futures-core", + "lock_api", + "parking_lot 0.11.2", +] [[package]] name = "futures-sink" -version = "0.3.25" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39c15cf1a4aa79df40f1bb462fb39676d0ad9e366c2a33b590d7c66f4f81fcf9" +checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5" [[package]] name = "futures-task" @@ -672,9 +840,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "197676987abd2f9cadff84926f410af1c183608d36641465df73ae8211dc65d6" dependencies = [ "futures-core", + "futures-sink", "futures-task", "pin-project-lite", "pin-utils", + "slab", ] [[package]] @@ -749,12 +919,28 @@ name = "hashbrown" version = "0.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f93e7192158dbcda357bdec5fb5788eebf8bbac027f3f33e719d29135ae84156" +dependencies = [ + "ahash 0.8.7", + "allocator-api2", +] + +[[package]] +name = "hashlink" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8094feaf31ff591f651a2664fb9cfd92bba7a60ce3197265e9482ebe753c8f7" +dependencies = [ + "hashbrown 0.14.2", +] [[package]] name = "heck" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2540771e65fc8cb83cd6e8a237f70c319bd5c29f78ed1084ba5d50eeac86f7f9" +dependencies = [ + "unicode-segmentation", +] [[package]] name = "hermit-abi" @@ -774,6 +960,30 @@ dependencies = [ "libc", ] +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "hkdf" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b5f8eb2ad728638ea2c7d47a21db23b7b58a72ed6a38256b8a1849f15fbbdf7" +dependencies = [ + "hmac", +] + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + [[package]] name = "http" version = "0.2.8" @@ -827,6 +1037,15 @@ dependencies = [ "hashbrown 0.14.2", ] +[[package]] +name = "instant" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" +dependencies = [ + "cfg-if", +] + [[package]] name = "io-lifetimes" version = "1.0.2" @@ -858,6 +1077,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25db6b064527c5d482d0423354fcd07a89a2dfe07b67892e62411946db7f07b0" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "0.4.8" @@ -920,6 +1148,28 @@ dependencies = [ "pkg-config", ] +[[package]] +name = "libredox" +version = "0.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85c833ca1e66078851dba29046874e38f08b2c883700aa29a03ddd3b23814ee8" +dependencies = [ + "bitflags 2.4.1", + "libc", + "redox_syscall 0.4.1", +] + +[[package]] +name = "libsqlite3-sys" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "898745e570c7d0453cc1fbc4a701eb6c662ed54e8fec8b7d14be137ebeeb9d14" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + [[package]] name = "libssh2-sys" version = "0.3.0" @@ -995,6 +1245,15 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "md-5" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6365506850d44bff6e2fbcb5176cf63650e48bd45ef2fe2665ae1570e0f4b9ca" +dependencies = [ + "digest", +] + [[package]] name = "memchr" version = "2.5.0" @@ -1026,6 +1285,12 @@ dependencies = [ "unicase", ] +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + [[package]] name = "miniz_oxide" version = "0.5.4" @@ -1047,6 +1312,34 @@ dependencies = [ "windows-sys 0.42.0", ] +[[package]] +name = "native-tls" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07226173c32f2926027b63cce4bcd8076c3552846cbe7925f3aaffeac0a3b92e" +dependencies = [ + "lazy_static", + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + [[package]] name = "nu-ansi-term" version = "0.46.0" @@ -1078,9 +1371,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.16.0" +version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86f0b0d4bf799edbc74508c1e8bf170ff5f41238e5f8225603ca7caaae2b7860" +checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" [[package]] name = "oorandom" @@ -1088,6 +1381,32 @@ version = "11.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575" +[[package]] +name = "openssl" +version = "0.10.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12fc0523e3bd51a692c8850d075d74dc062ccf251c0110668cbd921917118a13" +dependencies = [ + "bitflags 1.3.2", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.48", +] + [[package]] name = "openssl-probe" version = "0.1.5" @@ -1119,6 +1438,17 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" +[[package]] +name = "parking_lot" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" +dependencies = [ + "instant", + "lock_api", + "parking_lot_core 0.8.6", +] + [[package]] name = "parking_lot" version = "0.12.1" @@ -1126,7 +1456,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" dependencies = [ "lock_api", - "parking_lot_core", + "parking_lot_core 0.9.4", +] + +[[package]] +name = "parking_lot_core" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc" +dependencies = [ + "cfg-if", + "instant", + "libc", + "redox_syscall 0.2.16", + "smallvec", + "winapi", ] [[package]] @@ -1171,7 +1515,7 @@ checksum = "069bdb1e05adc7a8990dce9cc75370895fbe4e3d58b9b73bf1aee56359344a55" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -1235,7 +1579,7 @@ dependencies = [ "proc-macro-error-attr", "proc-macro2", "quote", - "syn", + "syn 1.0.109", "version_check", ] @@ -1252,18 +1596,18 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.47" +version = "1.0.76" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ea3d908b0e36316caf9e9e2c4625cdde190a7e6f440d794667ed17a1855e725" +checksum = "95fc56cda0b5c3325f5fbbd7ff9fda9e02bb00bb3dac51252d2f1bfa1cb8cc8c" dependencies = [ "unicode-ident", ] [[package]] name = "quote" -version = "1.0.21" +version = "1.0.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbe448f377a7d6961e30f5955f9b8d106c3f5e449d493ee1b125c1d43c2b5179" +checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef" dependencies = [ "proc-macro2", ] @@ -1339,6 +1683,17 @@ dependencies = [ "bitflags 1.3.2", ] +[[package]] +name = "redox_users" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a18479200779601e498ada4e8c1e1f50e3ee19deb0259c25825a98b5603b2cb4" +dependencies = [ + "getrandom", + "libredox", + "thiserror", +] + [[package]] name = "regex" version = "1.7.0" @@ -1413,12 +1768,44 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "schannel" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbc91545643bcf3a0bbb6569265615222618bdf33ce4ffbbd13c4bbd4c093534" +dependencies = [ + "windows-sys 0.52.0", +] + [[package]] name = "scopeguard" version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" +[[package]] +name = "security-framework" +version = "2.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05b64fb303737d99b81884b2c63433e9ae28abebe5eb5045dcdd175dc2ecf4de" +dependencies = [ + "bitflags 1.3.2", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e932934257d3b408ed8f30db49d85ea163bfe74961f017f405b025af298f0c7a" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "semver" version = "1.0.14" @@ -1427,9 +1814,12 @@ checksum = "e25dfac463d778e353db5be2449d1cce89bd6fd23c9f1ea21310ce6e5a1b29c4" [[package]] name = "serde" -version = "1.0.147" +version = "1.0.152" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d193d69bae983fc11a79df82342761dfbf28a99fc8d203dca4c3c1b590948965" +checksum = "bb7d1f0d3021d347a83e556fc4683dea2ea09d87bccdf88ff5c12545d89d5efb" +dependencies = [ + "serde_derive", +] [[package]] name = "serde_cbor" @@ -1449,7 +1839,7 @@ checksum = "af487d118eecd09402d70a5d72551860e788df87b464af30e5ea6a38c75c541e" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -1495,6 +1885,17 @@ dependencies = [ "digest", ] +[[package]] +name = "sha2" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82e6b795fe2e3b1e845bafcb27aa35405c4d47cdfc92af5fc8d3002f76cebdc0" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "sharded-slab" version = "0.1.4" @@ -1538,6 +1939,120 @@ dependencies = [ "winapi", ] +[[package]] +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +dependencies = [ + "lock_api", +] + +[[package]] +name = "sqlformat" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce81b7bd7c4493975347ef60d8c7e8b742d4694f4c49f93e0a12ea263938176c" +dependencies = [ + "itertools 0.12.0", + "nom", + "unicode_categories", +] + +[[package]] +name = "sqlx" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8de3b03a925878ed54a954f621e64bf55a3c1bd29652d0d1a17830405350188" +dependencies = [ + "sqlx-core", + "sqlx-macros", +] + +[[package]] +name = "sqlx-core" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa8241483a83a3f33aa5fff7e7d9def398ff9990b2752b6c6112b83c6d246029" +dependencies = [ + "ahash 0.7.6", + "atoi", + "base64", + "bitflags 1.3.2", + "byteorder", + "bytes", + "crc", + "crossbeam-queue", + "dirs", + "dotenvy", + "either", + "event-listener", + "flume", + "futures-channel", + "futures-core", + "futures-executor", + "futures-intrusive", + "futures-util", + "hashlink", + "hex", + "hkdf", + "hmac", + "indexmap 1.9.2", + "itoa 1.0.4", + "libc", + "libsqlite3-sys", + "log", + "md-5", + "memchr", + "once_cell", + "paste", + "percent-encoding", + "rand", + "serde", + "serde_json", + "sha1", + "sha2", + "smallvec", + "sqlformat", + "sqlx-rt", + "stringprep", + "thiserror", + "tokio-stream", + "url", + "whoami", +] + +[[package]] +name = "sqlx-macros" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9966e64ae989e7e575b19d7265cb79d7fc3cbbdf179835cb0d716f294c2049c9" +dependencies = [ + "dotenvy", + "either", + "heck", + "once_cell", + "proc-macro2", + "quote", + "sha2", + "sqlx-core", + "sqlx-rt", + "syn 1.0.109", + "url", +] + +[[package]] +name = "sqlx-rt" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "804d3f245f894e61b1e6263c84b23ca675d96753b5abfd5cc8597d86806e8024" +dependencies = [ + "native-tls", + "once_cell", + "tokio", + "tokio-native-tls", +] + [[package]] name = "stelae" version = "0.2.1" @@ -1557,6 +2072,7 @@ dependencies = [ "serde", "serde_derive", "serde_json", + "sqlx", "tempfile", "toml", "toml_edit", @@ -1565,17 +2081,45 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "stringprep" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb41d74e231a107a1b4ee36bd1214b11285b77768d2e3824aedafa988fd36ee6" +dependencies = [ + "finl_unicode", + "unicode-bidi", + "unicode-normalization", +] + [[package]] name = "strsim" version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" +[[package]] +name = "subtle" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "syn" -version = "1.0.107" +version = "2.0.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f4064b5b16e03ae50984a5a8ed5d4f8803e6bc1fd170a3cda91a1be4b18e3f5" +checksum = "0f3531638e407dfc0814761abb7c00a5b54992b849452a0646b7f65c9f770f3f" dependencies = [ "proc-macro2", "quote", @@ -1613,6 +2157,26 @@ dependencies = [ "unicode-width", ] +[[package]] +name = "thiserror" +version = "1.0.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d54378c645627613241d077a3a79db965db602882668f9136ac42af9ecb730ad" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa0faa943b50f3db30a20aa7e265dbc66076993efed8463e8de414e5d06d3471" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.48", +] + [[package]] name = "thread_local" version = "1.1.4" @@ -1685,13 +2249,35 @@ dependencies = [ "libc", "memchr", "mio", - "parking_lot", + "num_cpus", + "parking_lot 0.12.1", "pin-project-lite", "signal-hook-registry", "socket2", "winapi", ] +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", +] + +[[package]] +name = "tokio-stream" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "397c988d37662c7dda6d2208364a706264bf3d6138b11d436cbac0ad38832842" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + [[package]] name = "tokio-util" version = "0.7.4" @@ -1773,7 +2359,7 @@ checksum = "4017f8f45139870ca7e672686113917c71c7a6e02d4924eda67186083c03081a" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -1847,12 +2433,24 @@ dependencies = [ "tinyvec", ] +[[package]] +name = "unicode-segmentation" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1dd624098567895118886609431a7c3b8f516e41d30e0643f03d94592a147e36" + [[package]] name = "unicode-width" version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" +[[package]] +name = "unicode_categories" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" + [[package]] name = "url" version = "2.3.1" @@ -1929,7 +2527,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn", + "syn 1.0.109", "wasm-bindgen-shared", ] @@ -1951,7 +2549,7 @@ checksum = "07bc0c051dc5f23e307b13285f9d75df86bfdf816c5721e573dec1f9b8aa193c" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.109", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -1972,6 +2570,16 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "whoami" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22fc3756b8a9133049b26c7f61ab35416c130e8c09b660f5b3958b446f52cc50" +dependencies = [ + "wasm-bindgen", + "web-sys", +] + [[package]] name = "winapi" version = "0.3.9" @@ -2024,7 +2632,16 @@ version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" dependencies = [ - "windows-targets", + "windows-targets 0.48.5", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.0", ] [[package]] @@ -2042,6 +2659,21 @@ dependencies = [ "windows_x86_64_msvc 0.48.5", ] +[[package]] +name = "windows-targets" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a18201040b24831fbb9e4eb208f8892e1f50a37feb53cc7ff887feb8f50e7cd" +dependencies = [ + "windows_aarch64_gnullvm 0.52.0", + "windows_aarch64_msvc 0.52.0", + "windows_i686_gnu 0.52.0", + "windows_i686_msvc 0.52.0", + "windows_x86_64_gnu 0.52.0", + "windows_x86_64_gnullvm 0.52.0", + "windows_x86_64_msvc 0.52.0", +] + [[package]] name = "windows_aarch64_gnullvm" version = "0.42.0" @@ -2054,6 +2686,12 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7764e35d4db8a7921e09562a0304bf2f93e0a51bfccee0bd0bb0b666b015ea" + [[package]] name = "windows_aarch64_msvc" version = "0.42.0" @@ -2066,6 +2704,12 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbaa0368d4f1d2aaefc55b6fcfee13f41544ddf36801e793edbbfd7d7df075ef" + [[package]] name = "windows_i686_gnu" version = "0.42.0" @@ -2078,6 +2722,12 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" +[[package]] +name = "windows_i686_gnu" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a28637cb1fa3560a16915793afb20081aba2c92ee8af57b4d5f28e4b3e7df313" + [[package]] name = "windows_i686_msvc" version = "0.42.0" @@ -2090,6 +2740,12 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" +[[package]] +name = "windows_i686_msvc" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffe5e8e31046ce6230cc7215707b816e339ff4d4d67c65dffa206fd0f7aa7b9a" + [[package]] name = "windows_x86_64_gnu" version = "0.42.0" @@ -2102,6 +2758,12 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d6fa32db2bc4a2f5abeacf2b69f7992cd09dca97498da74a151a3132c26befd" + [[package]] name = "windows_x86_64_gnullvm" version = "0.42.0" @@ -2114,6 +2776,12 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a657e1e9d3f514745a572a6846d3c7aa7dbe1658c056ed9c3344c4109a6949e" + [[package]] name = "windows_x86_64_msvc" version = "0.42.0" @@ -2126,6 +2794,12 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04" + [[package]] name = "winnow" version = "0.5.19" @@ -2135,6 +2809,26 @@ dependencies = [ "memchr", ] +[[package]] +name = "zerocopy" +version = "0.7.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74d4d3961e53fa4c9a25a8637fc2bfaf2595b3d3ae34875568a5cf64787716be" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.7.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.48", +] + [[package]] name = "zstd" version = "0.11.2+zstd.1.5.2" diff --git a/Cargo.toml b/Cargo.toml index bf9586b..c457a75 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "stelae" description = "A collection of tools in Rust and Python for preserving, authenticating, and accessing laws in perpetuity." -version = "0.2.1" +version = "0.3.0-alpha.3" edition = "2021" readme = "README.md" license = "AGPL-3.0" @@ -31,6 +31,7 @@ derive_more = "0.99.17" toml = "0.8.8" toml_edit = "0.21.0" serde_derive = "1.0.152" +sqlx = { version = "0.6", features = ["runtime-actix-native-tls", "postgres", "sqlite"] } [dev-dependencies] criterion = "0.3" diff --git a/build.rs b/build.rs new file mode 100644 index 0000000..d506869 --- /dev/null +++ b/build.rs @@ -0,0 +1,5 @@ +// generated by `sqlx migrate build-script` +fn main() { + // trigger recompilation when a new migration is added + println!("cargo:rerun-if-changed=migrations"); +} diff --git a/migrations/postgres/.keep b/migrations/postgres/.keep new file mode 100644 index 0000000..e69de29 diff --git a/migrations/sqlite/20240115152953_initial_db.sql b/migrations/sqlite/20240115152953_initial_db.sql new file mode 100644 index 0000000..8ddc1d3 --- /dev/null +++ b/migrations/sqlite/20240115152953_initial_db.sql @@ -0,0 +1 @@ +-- Add migration script here diff --git a/src/db/init.rs b/src/db/init.rs new file mode 100644 index 0000000..c213f57 --- /dev/null +++ b/src/db/init.rs @@ -0,0 +1,26 @@ +use crate::db::{Database, DatabaseConnection}; +use std::env; +use std::path::{Path, PathBuf}; +/// Connects to a database and applies migrations. +/// We use `SQLite` by default, but we can override this by setting the `DATABASE_URL` environment variable. +/// +/// # Errors +/// Errors if connection to database fails. +/// Connections can fail if the database is not running, or if the database URL is invalid. +pub async fn connect(archive_path: &Path) -> anyhow::Result { + let db_url = env::var("DATABASE_URL").unwrap_or_else(|_| { + let sqlite_db_path = &archive_path.join(PathBuf::from(".stelae/db.sqlite3")); + format!("sqlite://{}?mode=rwc", sqlite_db_path.to_string_lossy()) + }); + let connection = Database::connect(&db_url).await?; + tracing::info!("Connected to database"); + match connection { + DatabaseConnection::Sqlite(ref pool) => { + sqlx::migrate!("./migrations/sqlite").run(pool).await?; + } + DatabaseConnection::Postgres(ref pool) => { + sqlx::migrate!("./migrations/postgres").run(pool).await?; + } + } + Ok(connection) +} diff --git a/src/db/mod.rs b/src/db/mod.rs new file mode 100644 index 0000000..2b77e7f --- /dev/null +++ b/src/db/mod.rs @@ -0,0 +1,41 @@ +//! Database related module. +#![allow(clippy::unreachable)] +use tracing::instrument; + +/// Database initialization. +pub mod init; + +/// Generic Database +pub struct Database; + +/// Database connection. +#[derive(Debug, Clone)] +pub enum DatabaseConnection { + /// SQLite connection. + Sqlite(sqlx::SqlitePool), + /// Postgres connection. + Postgres(sqlx::PgPool), +} + +impl Database { + /// Connects to a database. + /// + /// # Errors + /// Errors if connection to database fails. + #[instrument(level = "trace")] + pub async fn connect(db_url: &str) -> anyhow::Result { + let connection = match db_url { + url if url.starts_with("sqlite://") => { + let pool = sqlx::SqlitePool::connect(url).await?; + DatabaseConnection::Sqlite(pool) + } + url if url.starts_with("postgres://") => { + let pool = sqlx::PgPool::connect(url).await?; + DatabaseConnection::Postgres(pool) + } + _ => anyhow::bail!("Unsupported database URL: {}", db_url), + }; + + Ok(connection) + } +} diff --git a/src/lib.rs b/src/lib.rs index e8f912b..8e645bf 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -85,6 +85,7 @@ clippy::semicolon_outside_block, )] +pub mod db; pub mod server; pub mod stelae; pub mod utils; diff --git a/src/server/publish.rs b/src/server/publish.rs index 20f5382..3711bbc 100644 --- a/src/server/publish.rs +++ b/src/server/publish.rs @@ -1,6 +1,7 @@ //! Serve documents in a Stelae archive. #![allow(clippy::exit)] #![allow(clippy::unused_async)] +use crate::db; use crate::stelae::archive::Archive; use crate::stelae::types::repositories::{Repositories, Repository}; use crate::utils::archive::get_name_parts; @@ -37,10 +38,30 @@ fn clean_path(path: &str) -> String { } /// Global, read-only state +pub trait GlobalState { + /// Fully initialized Stelae archive + fn archive(&self) -> &Archive; + /// Database connection + fn db(&self) -> &db::DatabaseConnection; +} + +/// Application state #[derive(Debug, Clone)] pub struct AppState { /// Fully initialized Stelae archive pub archive: Archive, + /// Database connection + pub db: db::DatabaseConnection, +} + +impl GlobalState for AppState { + fn archive(&self) -> &Archive { + &self.archive + } + + fn db(&self) -> &db::DatabaseConnection { + &self.db + } } /// Git repository to serve @@ -161,13 +182,24 @@ pub async fn serve_archive( let message = "Running Publish Server on a Stelae archive at"; tracing::info!("{message} '{raw_archive_path}' on http://{bind}:{port}.",); + let db = match db::init::connect(&archive_path).await { + Ok(db) => db, + Err(err) => { + tracing::error!( + "error: could not connect to database. Confirm that DATABASE_URL env var is set correctly." + ); + tracing::error!("Error: {:?}", err); + std::process::exit(1); + } + }; + let archive = Archive::parse(archive_path, &PathBuf::from(raw_archive_path), individual) .unwrap_or_else(|err| { tracing::error!("Unable to parse archive at '{raw_archive_path}'."); tracing::error!("Error: {:?}", err); std::process::exit(1); }); - let state = AppState { archive }; + let state = AppState { archive, db }; HttpServer::new(move || { init_app(&state).unwrap_or_else(|err| { @@ -187,8 +219,8 @@ pub async fn serve_archive( /// * `state` - The application state /// # Errors /// Will error if unable to initialize the application -pub fn init_app( - state: &AppState, +pub fn init_app( + state: &T, ) -> anyhow::Result< App< impl ServiceFactory< @@ -200,7 +232,7 @@ pub fn init_app( >, >, > { - let config = state.archive.get_config()?; + let config = state.archive().get_config()?; let stelae_guard = config .headers .and_then(|headers| headers.current_documents_guard); @@ -208,7 +240,7 @@ pub fn init_app( stelae_guard.map_or_else( || { tracing::info!("Initializing app"); - let root = state.archive.get_root()?; + let root = state.archive().get_root()?; let shared_state = init_shared_app_state(root)?; Ok(App::new().service( web::scope("") @@ -233,7 +265,7 @@ pub fn init_app( HEADER_NAME.get_or_init(|| guard); HEADER_VALUES.get_or_init(|| { state - .archive + .archive() .stelae .keys() .map(ToString::to_string) @@ -244,7 +276,7 @@ pub fn init_app( if let (Some(guard_name), Some(guard_values)) = (HEADER_NAME.get(), HEADER_VALUES.get()) { for guard_value in guard_values { - let stele = state.archive.stelae.get(guard_value); + let stele = state.archive().stelae.get(guard_value); if let Some(guarded_stele) = stele { let shared_state = init_shared_app_state(guarded_stele)?; let mut stelae_scope = web::scope(""); @@ -302,8 +334,8 @@ fn init_repo_state(repo: &Repository, stele: &Stele) -> anyhow::Result anyhow::Result<()> { - for stele in state.archive.stelae.values() { +fn register_routes(cfg: &mut web::ServiceConfig, state: &T) -> anyhow::Result<()> { + for stele in state.archive().stelae.values() { if let Some(repositories) = stele.repositories.as_ref() { if stele.is_root() { continue; @@ -311,7 +343,7 @@ fn register_routes(cfg: &mut web::ServiceConfig, state: &AppState) -> anyhow::Re register_dependent_routes(cfg, stele, repositories)?; } } - let root = state.archive.get_root()?; + let root = state.archive().get_root()?; register_root_routes(cfg, root)?; Ok(()) } diff --git a/src/stelae/stele.rs b/src/stelae/stele.rs index fb68ee2..b131fba 100644 --- a/src/stelae/stele.rs +++ b/src/stelae/stele.rs @@ -21,7 +21,6 @@ pub struct Stele { /// Stele's repositories (as specified in repositories.json). pub repositories: Option, /// Indicates whether or not the Stele is the root Stele. - /// TODO: this does not seem correct pub root: bool, /// Stele's authentication repo. pub auth_repo: Repo, diff --git a/src/stelae/types/repositories.rs b/src/stelae/types/repositories.rs index c2aafd8..1683f67 100644 --- a/src/stelae/types/repositories.rs +++ b/src/stelae/types/repositories.rs @@ -5,7 +5,7 @@ use serde::{ de::{self, MapAccess, Visitor}, Deserialize, Deserializer, }; -use serde_derive::{Deserialize, Serialize}; +use serde_derive::Serialize; use serde_json::Value; /// Repositories object diff --git a/tests/common/mod.rs b/tests/common/mod.rs index e96e238..3cfa90e 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -9,12 +9,13 @@ use actix_web::{ use anyhow::Result; use std::path::{Path, PathBuf}; use std::sync::Once; +use stelae::{db, server::publish::GlobalState}; use tempfile::Builder; static INIT: Once = Once::new(); use actix_http::body::MessageBody; -use stelae::server::publish::{init_app, AppState}; +use stelae::server::publish::init_app; use stelae::stelae::archive::Archive; pub const BASIC_MODULE_NAME: &str = "basic"; @@ -30,11 +31,24 @@ pub fn blob_to_string(blob: Vec) -> String { // to manually inspect state of test environment at present, // we use anyhow::bail!() which aborts the entire test suite. +pub struct TestAppState { + archive: Archive, +} + +impl GlobalState for TestAppState { + fn archive(&self) -> &Archive { + &self.archive + } + fn db(&self) -> &db::DatabaseConnection { + unimplemented!() + } +} + pub async fn initialize_app( archive_path: &Path, ) -> impl Service, Error = Error> { let archive = Archive::parse(archive_path.to_path_buf(), archive_path, false).unwrap(); - let state = AppState { archive }; + let state = TestAppState { archive }; let app = init_app(&state).unwrap(); test::init_service(app).await } From 9a32331d49dd5c72f7126f93d304f2350ceed429 Mon Sep 17 00:00:00 2001 From: n-dusan Date: Mon, 26 Feb 2024 16:48:39 +0100 Subject: [PATCH 02/15] feat: initial work on inserting changes into db - Add `sophia` dependency for working with the rdf - Add `history` module - Use opaque `Any` type for database This abstracts away the concrete database pool that's used by underlying sqlx. Trade-off is that we need the database-url to differentiate between databases --- Cargo.lock | 253 +++++++++++++++++++++++++++---- Cargo.toml | 10 +- src/db/init.rs | 18 ++- src/db/mod.rs | 69 +++++++-- src/db/queries.rs | 98 ++++++++++++ src/history/changes.rs | 191 +++++++++++++++++++++++ src/history/mod.rs | 3 + src/lib.rs | 1 + src/server/publish.rs | 6 +- src/stelae/types/repositories.rs | 8 + src/utils/cli.rs | 12 ++ 11 files changed, 616 insertions(+), 53 deletions(-) create mode 100644 src/db/queries.rs create mode 100644 src/history/changes.rs create mode 100644 src/history/mod.rs diff --git a/Cargo.lock b/Cargo.lock index 8b4e890..065e899 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -246,6 +246,17 @@ version = "1.0.66" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "216261ddc8289130e551ddcd5ce8a064710c0d064a4d2895c67151c92b5443f6" +[[package]] +name = "async-trait" +version = "0.1.77" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c980ee35e870bd1a4d2c8294d4c04d0499e67bca1e4b5cefcc693c2fa00caea9" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.48", +] + [[package]] name = "atoi" version = "1.0.0" @@ -532,36 +543,30 @@ dependencies = [ [[package]] name = "crossbeam-channel" -version = "0.5.6" +version = "0.5.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2dd04ddaf88237dc3b8d8f9a3c1004b506b54b3313403944054d23c0870c521" +checksum = "ab3db02a9c5b5121e1e42fbdb1aeb65f5e02624cc58c43f2884c6ccac0b82f95" dependencies = [ - "cfg-if", "crossbeam-utils", ] [[package]] name = "crossbeam-deque" -version = "0.8.2" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "715e8152b692bba2d374b53d4875445368fdf21a94751410af607a5ac677d1fc" +checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d" dependencies = [ - "cfg-if", "crossbeam-epoch", "crossbeam-utils", ] [[package]] name = "crossbeam-epoch" -version = "0.9.13" +version = "0.9.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01a9af1f4c2ef74bb8aa1f7e19706bc72d03598c8a570bb5de72243c7a9d9d5a" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" dependencies = [ - "autocfg", - "cfg-if", "crossbeam-utils", - "memoffset", - "scopeguard", ] [[package]] @@ -626,9 +631,9 @@ dependencies = [ [[package]] name = "digest" -version = "0.10.6" +version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8168378f4e5023e7218c89c891c0fd8ecdb5e5e4f18cb78f38cf245dd021e76f" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer", "crypto-common", @@ -1260,15 +1265,6 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" -[[package]] -name = "memoffset" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5de893c32cde5f383baa4c04c5d6dbdd735cfd4a794b0debdb2bb1b421da5ff4" -dependencies = [ - "autocfg", -] - [[package]] name = "mime" version = "0.3.17" @@ -1312,6 +1308,12 @@ dependencies = [ "windows-sys 0.42.0", ] +[[package]] +name = "mownstr" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bc45ce96192b5d8b20cffb10ccd85cc431c283a7d171a0d843ac0bd7d444598" + [[package]] name = "native-tls" version = "0.2.11" @@ -1438,6 +1440,18 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" +[[package]] +name = "oxilangtag" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d91edf4fbb970279443471345a4e8c491bf05bb283b3e6c88e4e606fd8c181b" + +[[package]] +name = "oxiri" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb175ec8981211357b7b379869c2f8d555881c55ea62311428ec0de46d89bd5c" + [[package]] name = "parking_lot" version = "0.11.2" @@ -1603,6 +1617,15 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "quick-xml" +version = "0.28.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ce5e73202a820a31f8a0ee32ada5e21029c81fd9e3ebf668a40832e4219d9d1" +dependencies = [ + "memchr", +] + [[package]] name = "quote" version = "1.0.35" @@ -1717,6 +1740,41 @@ version = "0.6.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "456c603be3e8d448b072f410900c09faf164fbce2d480456f50eea6e25f9c848" +[[package]] +name = "resiter" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbc95d56eb1865f69288945759cc0879d60ee68168dce676730275804ad2b276" + +[[package]] +name = "rio_api" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1924fa1f0e6d851f9b73b3c569e607c368a0d92995d99d563ad7bf1414696603" + +[[package]] +name = "rio_turtle" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cec59971eafd99b9c7e3544bfcabafea81a7072ac51c9f46985ca0bd7ba6016" +dependencies = [ + "oxilangtag", + "oxiri", + "rio_api", +] + +[[package]] +name = "rio_xml" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2edda57b877119dc326c612ba822e3ca1ee22bfc86781a4e9dc0884756b58c3" +dependencies = [ + "oxilangtag", + "oxiri", + "quick-xml", + "rio_api", +] + [[package]] name = "rustc_version" version = "0.4.0" @@ -1887,9 +1945,9 @@ dependencies = [ [[package]] name = "sha2" -version = "0.10.6" +version = "0.10.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82e6b795fe2e3b1e845bafcb27aa35405c4d47cdfc92af5fc8d3002f76cebdc0" +checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" dependencies = [ "cfg-if", "cpufeatures", @@ -1939,6 +1997,146 @@ dependencies = [ "winapi", ] +[[package]] +name = "sophia" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d9d3e79754eeda3fc7e3610afcc492613fa0a5581d286d7545094e3e7ce1608" +dependencies = [ + "sophia_api", + "sophia_c14n", + "sophia_inmem", + "sophia_iri", + "sophia_isomorphism", + "sophia_resource", + "sophia_rio", + "sophia_term", + "sophia_turtle", + "sophia_xml", +] + +[[package]] +name = "sophia_api" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e41b1197b9dbd2e5e2a7d8dc62fd6bab001724576463831920b13567bde2a4c" +dependencies = [ + "lazy_static", + "mownstr", + "regex", + "resiter", + "serde", + "sophia_iri", + "thiserror", +] + +[[package]] +name = "sophia_c14n" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04e4ebf65104879fc9f3f1f54224b42ad1f9e40b6a5dc26c5a17bf43846d6a1d" +dependencies = [ + "sha2", + "sophia_api", + "sophia_iri", + "thiserror", +] + +[[package]] +name = "sophia_inmem" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba3f836b898bbd5d5a73a977995e1d5dab8e2cf96a017890954864ece18b1e8c" +dependencies = [ + "sophia_api", + "thiserror", +] + +[[package]] +name = "sophia_iri" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb62d2fcd10fc3a44b646b1893ad478df0cb771f7ceb0331a5f3cee25f37ba7e" +dependencies = [ + "lazy_static", + "oxiri", + "regex", + "serde", + "thiserror", +] + +[[package]] +name = "sophia_isomorphism" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2b3876c71756d3dd94590c51090a036298a983f6b1e5c316f9eca514b2f6a5e" +dependencies = [ + "sophia_api", + "sophia_iri", +] + +[[package]] +name = "sophia_resource" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "807253d3a4144e1f3eacf6976849dab04cbd493ff568414df07c33fd10886ce2" +dependencies = [ + "sophia_api", + "sophia_iri", + "sophia_turtle", + "sophia_xml", + "thiserror", +] + +[[package]] +name = "sophia_rio" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93e107086ca2b3e329dbe0f85f9ce504b1fbb478c85341338942b9dff613d4d8" +dependencies = [ + "rio_api", + "sophia_api", + "sophia_iri", +] + +[[package]] +name = "sophia_term" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca3a77f8523038dc1204a59dd461f27322cbe14bea3da873d41b6cc40c3e63e9" +dependencies = [ + "lazy_static", + "sophia_api", +] + +[[package]] +name = "sophia_turtle" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a7b7ce7aeb34f55867599544622420b2a6b6488b16811e06fd3755fc9cae4d0" +dependencies = [ + "lazy_static", + "oxiri", + "regex", + "rio_turtle", + "sophia_api", + "sophia_iri", + "sophia_rio", +] + +[[package]] +name = "sophia_xml" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9df98b3ae2636fb806dcc8ac76eee27d567a3167e7c548a8a16aa23e43367518" +dependencies = [ + "oxiri", + "rio_xml", + "sophia_api", + "sophia_iri", + "sophia_rio", +] + [[package]] name = "spin" version = "0.9.8" @@ -2055,12 +2253,13 @@ dependencies = [ [[package]] name = "stelae" -version = "0.2.1" +version = "0.3.0-alpha.3" dependencies = [ "actix-http", "actix-service", "actix-web", "anyhow", + "async-trait", "clap 4.0.27", "criterion", "derive_more", @@ -2072,6 +2271,7 @@ dependencies = [ "serde", "serde_derive", "serde_json", + "sophia", "sqlx", "tempfile", "toml", @@ -2079,6 +2279,7 @@ dependencies = [ "tracing", "tracing-actix-web", "tracing-subscriber", + "walkdir", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index c457a75..859355d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,6 +15,7 @@ rust-version = "1.70" actix-web = "4" actix-service = "2.0" actix-http = "3.2" +async-trait = "0.1.77" mime = "0.3.17" mime_guess = "2.0.4" anyhow = "1.0" @@ -31,7 +32,14 @@ derive_more = "0.99.17" toml = "0.8.8" toml_edit = "0.21.0" serde_derive = "1.0.152" -sqlx = { version = "0.6", features = ["runtime-actix-native-tls", "postgres", "sqlite"] } +sqlx = { version = "0.6", features = [ + "runtime-actix-native-tls", + "any", + "postgres", + "sqlite", +] } +sophia = { version = "0.8.0", features = ["xml"] } +walkdir = "2" [dev-dependencies] criterion = "0.3" diff --git a/src/db/init.rs b/src/db/init.rs index c213f57..b75b475 100644 --- a/src/db/init.rs +++ b/src/db/init.rs @@ -1,4 +1,4 @@ -use crate::db::{Database, DatabaseConnection}; +use crate::db::{DatabaseConnection, DatabaseKind, Db}; use std::env; use std::path::{Path, PathBuf}; /// Connects to a database and applies migrations. @@ -12,14 +12,18 @@ pub async fn connect(archive_path: &Path) -> anyhow::Result let sqlite_db_path = &archive_path.join(PathBuf::from(".stelae/db.sqlite3")); format!("sqlite://{}?mode=rwc", sqlite_db_path.to_string_lossy()) }); - let connection = Database::connect(&db_url).await?; + let connection = DatabaseConnection::connect(&db_url).await?; tracing::info!("Connected to database"); - match connection { - DatabaseConnection::Sqlite(ref pool) => { - sqlx::migrate!("./migrations/sqlite").run(pool).await?; + match connection.kind { + DatabaseKind::Sqlite => { + sqlx::migrate!("./migrations/sqlite") + .run(&connection.pool) + .await?; } - DatabaseConnection::Postgres(ref pool) => { - sqlx::migrate!("./migrations/postgres").run(pool).await?; + DatabaseKind::Postgres => { + sqlx::migrate!("./migrations/postgres") + .run(&connection.pool) + .await?; } } Ok(connection) diff --git a/src/db/mod.rs b/src/db/mod.rs index 2b77e7f..dc1f41f 100644 --- a/src/db/mod.rs +++ b/src/db/mod.rs @@ -1,38 +1,75 @@ //! Database related module. #![allow(clippy::unreachable)] +use async_trait::async_trait; +use std::str::FromStr; + +use sqlx::any::{AnyPool, AnyPoolOptions}; +use sqlx::ConnectOptions; use tracing::instrument; /// Database initialization. pub mod init; +/// Queries for the database. +pub mod queries; +#[async_trait] /// Generic Database -pub struct Database; +pub trait Db { + /// Connects to a database. + /// + /// # Errors + /// Errors if connection to database fails. + async fn connect(url: &str) -> anyhow::Result; + + // async fn execute_statement(statement: &str, conn: &DatabaseConnection) -> anyhow::Result<()>; + + // async fn begin(&self) -> anyhow::Result<()>; + + // async fn close(&self) -> anyhow::Result<()>; +} + +/// Type of database connection. +#[derive(Debug, Clone)] +pub enum DatabaseKind { + /// Sqlite database. + Sqlite, + /// Postgres database. + Postgres, +} /// Database connection. #[derive(Debug, Clone)] -pub enum DatabaseConnection { - /// SQLite connection. - Sqlite(sqlx::SqlitePool), - /// Postgres connection. - Postgres(sqlx::PgPool), +pub struct DatabaseConnection { + /// Database connection pool. + pub pool: AnyPool, + /// Type of database connection. + pub kind: DatabaseKind, } -impl Database { +#[async_trait] +impl Db for DatabaseConnection { /// Connects to a database. /// /// # Errors /// Errors if connection to database fails. #[instrument(level = "trace")] - pub async fn connect(db_url: &str) -> anyhow::Result { + async fn connect(db_url: &str) -> anyhow::Result { + let options = sqlx::any::AnyConnectOptions::from_str(db_url)? + .disable_statement_logging() + .clone(); + let pool = AnyPoolOptions::new() + .max_connections(50) + .connect_with(options) + .await?; let connection = match db_url { - url if url.starts_with("sqlite://") => { - let pool = sqlx::SqlitePool::connect(url).await?; - DatabaseConnection::Sqlite(pool) - } - url if url.starts_with("postgres://") => { - let pool = sqlx::PgPool::connect(url).await?; - DatabaseConnection::Postgres(pool) - } + url if url.starts_with("sqlite://") => Self { + pool, + kind: DatabaseKind::Sqlite, + }, + url if url.starts_with("postgres://") => Self { + pool, + kind: DatabaseKind::Postgres, + }, _ => anyhow::bail!("Unsupported database URL: {}", db_url), }; diff --git a/src/db/queries.rs b/src/db/queries.rs new file mode 100644 index 0000000..46221b9 --- /dev/null +++ b/src/db/queries.rs @@ -0,0 +1,98 @@ +//! Central place for database queries in Stelae +use sqlx::Row; + +use crate::db::DatabaseConnection; + +use super::DatabaseKind; + +/// Inserts a new document into the database. +/// +/// # Errors +/// Errors if the document cannot be inserted into the database. +pub async fn insert_new_document(conn: &DatabaseConnection, doc_id: &str) -> anyhow::Result<()> { + let statement: &'static str = r#" + INSERT OR IGNORE INTO document ( doc_id ) + VALUES ( $1 ) + "#; + match &conn.kind { + &DatabaseKind::Sqlite => { + let mut connection = conn.pool.acquire().await?; + sqlx::query(statement) + .bind(doc_id) + .execute(&mut *connection) + .await?; + } + DatabaseKind::Postgres => { + let mut connection = conn.pool.acquire().await?; + sqlx::query(statement) + .bind(doc_id) + .execute(&mut *connection) + .await?; + } + }; + Ok(()) +} + +/// Inserts a new stele into the database. +/// +/// # Errors +/// Errors if the stele cannot be inserted into the database. +pub async fn insert_new_stele(conn: &DatabaseConnection, stele_id: &str) -> anyhow::Result<()> { + let statement: &'static str = r#" + INSERT OR IGNORE INTO stele ( name ) + VALUES ( $1 ) + "#; + match &conn.kind { + DatabaseKind::Sqlite => { + let mut connection = conn.pool.acquire().await?; + sqlx::query(statement) + .bind(stele_id) + .execute(&mut *connection) + .await?; + } + DatabaseKind::Postgres => { + let mut connection = conn.pool.acquire().await?; + sqlx::query(statement) + .bind(stele_id) + .execute(&mut *connection) + .await?; + } + }; + Ok(()) +} + +/// Find a stele by name. +/// +/// # Errors +/// Errors if the stele cannot be found in the database. +pub async fn find_stele_by_name( + conn: &DatabaseConnection, + name: &str, +) -> anyhow::Result> { + let statement: &'static str = r#" + SELECT id + FROM stele + WHERE name = $1 + "#; + let row: Option = match &conn.kind { + &DatabaseKind::Sqlite => { + let mut connection = conn.pool.acquire().await?; + let row = sqlx::query(statement) + .bind(name) + .fetch_one(&mut *connection) + .await + .ok(); + row.map(|r| r.get(0)) + } + DatabaseKind::Postgres => { + let mut connection = conn.pool.acquire().await?; + let row = sqlx::query(statement) + .bind(name) + .fetch_one(&mut *connection) + .await + .ok(); + row.map(|r| r.get(0)) + } + }; + Ok(row) +} diff --git a/src/history/changes.rs b/src/history/changes.rs new file mode 100644 index 0000000..c85b112 --- /dev/null +++ b/src/history/changes.rs @@ -0,0 +1,191 @@ +//! Module for inserting changes into the database +use crate::db::queries::{find_stele_by_name, insert_new_document, insert_new_stele}; +use crate::history::rdf_namespaces::{OLL_DOCUMENT_VERSION, OLL_DOC_ID}; +use crate::{ + db::{self, DatabaseConnection}, + stelae::{archive::Archive, types::repositories::Repository}, +}; +use sophia::api::{ns::NsTerm, prelude::*, term::SimpleTerm, MownStr}; +use sophia::xml::parser; +use sophia::{api::ns::Namespace, inmem::graph::FastGraph}; +use std::{ + collections::HashMap, + path::{Path, PathBuf}, +}; +use walkdir::WalkDir; + +/// Inserts changes from the archive into the database +/// +/// # Errors +/// Errors if the changes cannot be inserted into the archive +#[actix_web::main] +pub async fn insert( + raw_archive_path: &str, + archive_path: PathBuf, + stele: Option, +) -> std::io::Result<()> { + let conn = match db::init::connect(&archive_path).await { + Ok(conn) => conn, + Err(err) => { + tracing::error!( + "error: could not connect to database. Confirm that DATABASE_URL env var is set correctly." + ); + tracing::error!("Error: {:?}", err); + std::process::exit(1); + } + }; + if let Some(stele) = stele { + insert_changes_single_stele()?; + } else { + insert_changes_archive(&conn, raw_archive_path, &archive_path) + .await + .unwrap_or_else(|err| { + tracing::error!("Failed to insert changes into archive"); + tracing::error!("{:?}", err); + }); + } + Ok(()) +} + +fn insert_changes_single_stele() -> std::io::Result<()> { + Ok(()) +} + +/// Insert changes from the archive into the database +async fn insert_changes_archive( + conn: &DatabaseConnection, + raw_archive_path: &str, + archive_path: &Path, +) -> anyhow::Result<()> { + let archive = Archive::parse( + archive_path.to_path_buf(), + &PathBuf::from(raw_archive_path), + false, + )?; + + for (name, mut stele) in archive.stelae { + if let Some(repositories) = stele.get_repositories()? { + let Some(rdf_repo) = repositories.get_rdf_repository() else { + continue; + }; + let rdf_repo_path = archive_path.to_path_buf().join(&rdf_repo.name); + if !rdf_repo_path.exists() { + anyhow::bail!( + "RDF repository should exist on disk but not found: {}", + rdf_repo_path.display() + ); + } + insert_changes_from_rdf_repository(conn, rdf_repo_path, &name, rdf_repo).await?; + } + } + Ok(()) +} + +/// Insert changes from the RDF repository into the database +async fn insert_changes_from_rdf_repository( + conn: &DatabaseConnection, + rdf_repo_path: PathBuf, + name: &str, + rdf_repo: &Repository, +) -> anyhow::Result<()> { + tracing::info!("Inserting changes from RDF repository: {}", name); + tracing::info!("RDF repository path: {}", rdf_repo_path.display()); + + // let response = reqwest::get(NAMESPACE_URL).await?.text().await?; + let mut graph = FastGraph::new(); + + for entry in WalkDir::new(&rdf_repo_path) { + match entry { + Ok(entry) if is_rdf(&entry) => { + tracing::debug!("Parsing file: {:?}", entry.path()); + let file = std::fs::File::open(entry.path())?; + let reader = std::io::BufReader::new(file); + parser::parse_bufread(reader).add_to_graph(&mut graph)?; + } + Ok(entry) => { + tracing::debug!("Skipping non-RDF file: {:?}", entry.path()); + continue; + } + Err(err) => { + tracing::error!("Error reading file: {:?}", err); + } + } + } + let namespace_url = "https://open.law/us/ngo/oll/_ontology/v0.1/ontology.owl#"; + let oll: Namespace<&str> = Namespace::new(namespace_url).unwrap(); + + let oll_document_version: NsTerm = oll.get("DocumentVersion").unwrap(); + let oll_doc_id = oll.get("docId").unwrap(); + + let documents = graph.triples_matching(Any, Any, [oll_document_version]); + let mut doc_to_versions: HashMap> = HashMap::new(); + for triple in documents { + let triple = triple.unwrap(); + let document = triple.s(); + let mut doc_id_triples = graph.triples_matching([document], [oll_doc_id], Any); + if let Some(doc_id_triple) = doc_id_triples.next() { + let object = doc_id_triple.unwrap().o(); + let document_iri = document.iri().unwrap().to_string(); + if let SimpleTerm::LiteralDatatype(doc_id, _) = object { + doc_to_versions + .entry(doc_id.to_string()) + .or_insert_with(Vec::new) + .push(document_iri); + } + } + } + for versions in doc_to_versions.values() { + // Find the version with the maximum docId + let doc_version = versions + .iter() + .max_by_key(|&v| { + let mut doc_id_triples = graph.triples_matching([v.as_str()], [oll_doc_id], Any); + doc_id_triples + .next() + .map_or_else(String::new, |doc_id_triple| { + let object = doc_id_triple.unwrap().o(); + if let SimpleTerm::LiteralDatatype(doc_id, _) = object { + doc_id.to_string() + } else { + String::new() + } + }) + }) + .unwrap(); + // Get the docId for this version + // dbg!(&doc_version); + let doc_version_iri_ref = IriRef::new_unchecked(MownStr::from_str(doc_version.as_str())); + let mut doc_id_triples = + graph.triples_matching([SimpleTerm::Iri(doc_version_iri_ref)], [oll_doc_id], Any); + if let Some(doc_id_triple) = doc_id_triples.next() { + let object = doc_id_triple.unwrap().o(); + if let SimpleTerm::LiteralDatatype(doc_id, _) = object { + insert_new_document(conn, doc_id).await?; + } + } + } + + load_delta_from_publications(&mut graph, conn, rdf_repo_path.join("_publication"), name) + .await?; + Ok(()) +} + +/// Check if the entry is an RDF file +fn is_rdf(entry: &walkdir::DirEntry) -> bool { + entry.path().extension() == Some("rdf".as_ref()) +} + +/// Load deltas from the publications +async fn load_delta_from_publications( + graph: &mut FastGraph, + conn: &DatabaseConnection, + publication_path: PathBuf, + name: &str, +) -> anyhow::Result<()> { + insert_new_stele(conn, name).await?; + let id = find_stele_by_name(conn, name).await?; + tracing::info!("Inserting changes from publications for stele: {}", name); + dbg!(&id); + dbg!(&publication_path); + Ok(()) +} diff --git a/src/history/mod.rs b/src/history/mod.rs new file mode 100644 index 0000000..f61e8c3 --- /dev/null +++ b/src/history/mod.rs @@ -0,0 +1,3 @@ +//! The history module contains tools for interacting with the history of the Stele. +// The historical versions are loaded into the Stele and can be queried. +pub mod changes; \ No newline at end of file diff --git a/src/lib.rs b/src/lib.rs index 8e645bf..5df6641 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -86,6 +86,7 @@ )] pub mod db; +pub mod history; pub mod server; pub mod stelae; pub mod utils; diff --git a/src/server/publish.rs b/src/server/publish.rs index 3711bbc..bca013e 100644 --- a/src/server/publish.rs +++ b/src/server/publish.rs @@ -309,15 +309,15 @@ pub fn init_app( /// # Errors /// Will error if unable to initialize the data repository fn init_repo_state(repo: &Repository, stele: &Stele) -> anyhow::Result { - let name = &repo.name; let custom = &repo.custom; + let (org, name) = get_name_parts(&repo.name)?; let mut repo_path = stele.archive_path.to_string_lossy().into_owned(); - repo_path = format!("{repo_path}/{name}"); + repo_path = format!("{repo_path}/{org}/{name}"); Ok(RepoState { repo: Repo { archive_path: stele.archive_path.to_string_lossy().to_string(), path: PathBuf::from(&repo_path), - org: stele.auth_repo.org.clone(), + org: org.clone(), name: name.clone(), repo: GitRepository::open(&repo_path)?, }, diff --git a/src/stelae/types/repositories.rs b/src/stelae/types/repositories.rs index 1683f67..2cf008b 100644 --- a/src/stelae/types/repositories.rs +++ b/src/stelae/types/repositories.rs @@ -115,6 +115,14 @@ impl Repositories { }); result } + + /// Get the RDF repository from repositories. + #[must_use] + pub fn get_rdf_repository(&self) -> Option<&Repository> { + self.repositories + .values() + .find(|repository| repository.custom.repository_type.as_deref() == Some("rdf")) + } } #[allow(clippy::missing_trait_methods)] diff --git a/src/utils/cli.rs b/src/utils/cli.rs index fb3ae3d..eb41c8c 100644 --- a/src/utils/cli.rs +++ b/src/utils/cli.rs @@ -6,6 +6,7 @@ use crate::server::git::serve_git; use crate::server::publish::serve_archive; use crate::utils::archive::find_archive_path; +use crate::history::changes; use clap::Parser; use std::path::Path; use tracing; @@ -42,6 +43,13 @@ enum Subcommands { /// Serve an individual stele instead of the Stele specified in config.toml. individual: bool, }, + /// Insert historical information about the Steles in the archive. + /// Populates the database with change objects loaded in from RDF repository + /// By default inserts historical information for the root Stele (and all referenced stele) in the archive + InsertHistory { + /// Optionally insert historical information for this Stele only. + stele: Option, + }, } /// @@ -74,5 +82,9 @@ pub fn run() -> std::io::Result<()> { Subcommands::Serve { port, individual } => { serve_archive(&cli.archive_path, archive_path, port, individual) } + Subcommands::InsertHistory { stele } => { + tracing::info!("Inserting history into archive"); + changes::insert(&cli.archive_path, archive_path, stele) + } } } From 220a98626bea60ff552b37fe42a6e5b63733d884 Mon Sep 17 00:00:00 2001 From: n-dusan Date: Tue, 19 Mar 2024 20:52:30 +0100 Subject: [PATCH 03/15] feat: add more modules for inserting history The purpose of the submodule is to store any/all functions that work with rdf. Currently the underlying implementation relies on `sophia` to parse the graph and retrieve triples. Add `namespaces.rs` module that stores rdf namespaces. Currently stores the `oll` and `dcterms` ontology. I expected the `dcterms` to be supported out-of-the-box, but `sophia` supports only the most frequently used namespaces. *Add sqlx chrono feature* Will be necessary to work with dates in the database. *Add statements module for inserts and queries in db* The idea being that this module stores queries/insert statements for interacting with both databases. *Add chrono as dependency* Necessary because chrono exports serde Serialize and Deserialize traits. We then use these traits to map sqlx queries to structs. *Add models mod for sqlx structs* Created a centralized place for all database related structs. This approach is convenient because the sqlx queries will then serialize/deserialize to these structs using serde. *Rename from upsert to create* Much easier to reason about what the function is supposed to do. *Wrap insert changes into an atomic transaction* --- Cargo.lock | 134 +++++++++++++++++++++++++ Cargo.toml | 2 + src/db/mod.rs | 6 +- src/db/models/document.rs | 0 src/db/models/mod.rs | 8 ++ src/db/models/publication.rs | 21 ++++ src/db/models/stele.rs | 11 +++ src/db/queries.rs | 98 ------------------- src/db/statements/inserts.rs | 129 +++++++++++++++++++++++++ src/db/statements/mod.rs | 4 + src/db/statements/queries.rs | 119 +++++++++++++++++++++++ src/history/changes.rs | 177 +++++++++++++++++++++++++--------- src/history/mod.rs | 6 +- src/history/rdf/mod.rs | 4 + src/history/rdf/namespaces.rs | 24 +++++ 15 files changed, 594 insertions(+), 149 deletions(-) create mode 100644 src/db/models/document.rs create mode 100644 src/db/models/mod.rs create mode 100644 src/db/models/publication.rs create mode 100644 src/db/models/stele.rs delete mode 100644 src/db/queries.rs create mode 100644 src/db/statements/inserts.rs create mode 100644 src/db/statements/mod.rs create mode 100644 src/db/statements/queries.rs create mode 100644 src/history/rdf/mod.rs create mode 100644 src/history/rdf/namespaces.rs diff --git a/Cargo.lock b/Cargo.lock index 065e899..49b9641 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -240,6 +240,21 @@ version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0942ffc6dcaadf03badf6e6a2d0228460359d5e34b57ccdc720b7382dfbd5ec5" +[[package]] +name = "android-tzdata" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + [[package]] name = "anyhow" version = "1.0.66" @@ -391,6 +406,21 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "chrono" +version = "0.4.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f2c685bad3eb3d45a01354cedb7d5faa66194d1d58ba6e267a8de788f79db38" +dependencies = [ + "android-tzdata", + "iana-time-zone", + "js-sys", + "num-traits", + "serde", + "wasm-bindgen", + "windows-targets 0.48.5", +] + [[package]] name = "clap" version = "2.34.0" @@ -439,6 +469,16 @@ dependencies = [ "os_str_bytes", ] +[[package]] +name = "codespan-reporting" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3538270d33cc669650c4b093848450d380def10c331d38c768e34cac80576e6e" +dependencies = [ + "termcolor", + "unicode-width", +] + [[package]] name = "convert_case" version = "0.4.0" @@ -616,6 +656,50 @@ dependencies = [ "memchr", ] +[[package]] +name = "cxx" +version = "1.0.97" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e88abab2f5abbe4c56e8f1fb431b784d710b709888f35755a160e62e33fe38e8" +dependencies = [ + "cc", + "cxxbridge-flags", + "cxxbridge-macro", + "link-cplusplus", +] + +[[package]] +name = "cxx-build" +version = "1.0.97" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c0c11acd0e63bae27dcd2afced407063312771212b7a823b4fd72d633be30fb" +dependencies = [ + "cc", + "codespan-reporting", + "once_cell", + "proc-macro2", + "quote", + "scratch", + "syn 2.0.48", +] + +[[package]] +name = "cxxbridge-flags" +version = "1.0.97" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d3816ed957c008ccd4728485511e3d9aaf7db419aa321e3d2c5a2f3411e36c8" + +[[package]] +name = "cxxbridge-macro" +version = "1.0.97" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a26acccf6f445af85ea056362561a24ef56cdc15fcc685f03aec50b9c702cb6d" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.48", +] + [[package]] name = "derive_more" version = "0.99.17" @@ -1012,6 +1096,30 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" +[[package]] +name = "iana-time-zone" +version = "0.1.59" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6a67363e2aa4443928ce15e57ebae94fd8949958fd1223c4cfc0cd473ad7539" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0703ae284fc167426161c2e3f1da3ea71d94b21bedbcc9494e92b28e334e3dca" +dependencies = [ + "cxx", + "cxx-build", +] + [[package]] name = "idna" version = "0.3.0" @@ -1201,6 +1309,15 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "link-cplusplus" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d240c6f7e1ba3a28b0249f774e6a9dd0175054b52dfbb61b16eb8505c3785c9" +dependencies = [ + "cc", +] + [[package]] name = "linux-raw-sys" version = "0.1.3" @@ -1841,6 +1958,12 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" +[[package]] +name = "scratch" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3cf7c11c38cb994f3d40e8a8cde3bbd1f72a435e4c49e85d6553d8312306152" + [[package]] name = "security-framework" version = "2.9.2" @@ -2179,6 +2302,7 @@ dependencies = [ "bitflags 1.3.2", "byteorder", "bytes", + "chrono", "crc", "crossbeam-queue", "dirs", @@ -2260,6 +2384,7 @@ dependencies = [ "actix-web", "anyhow", "async-trait", + "chrono", "clap 4.0.27", "criterion", "derive_more", @@ -2812,6 +2937,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows-core" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" +dependencies = [ + "windows-targets 0.52.0", +] + [[package]] name = "windows-sys" version = "0.42.0" diff --git a/Cargo.toml b/Cargo.toml index 859355d..56607ec 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -32,7 +32,9 @@ derive_more = "0.99.17" toml = "0.8.8" toml_edit = "0.21.0" serde_derive = "1.0.152" +chrono = { version = "0.4.*", features = ["serde"] } sqlx = { version = "0.6", features = [ + "chrono", "runtime-actix-native-tls", "any", "postgres", diff --git a/src/db/mod.rs b/src/db/mod.rs index dc1f41f..e6163e9 100644 --- a/src/db/mod.rs +++ b/src/db/mod.rs @@ -9,8 +9,10 @@ use tracing::instrument; /// Database initialization. pub mod init; -/// Queries for the database. -pub mod queries; +/// Statements for the database. +pub mod statements; +/// Models for the database. +pub mod models; #[async_trait] /// Generic Database diff --git a/src/db/models/document.rs b/src/db/models/document.rs new file mode 100644 index 0000000..e69de29 diff --git a/src/db/models/mod.rs b/src/db/models/mod.rs new file mode 100644 index 0000000..6c897fd --- /dev/null +++ b/src/db/models/mod.rs @@ -0,0 +1,8 @@ +//! This module contains all the sqlx structs for the database tables. + +/// sqlx structs for document table. +pub mod document; +/// sqlx structs for publication table. +pub mod publication; +/// sqlx structs for stele table. +pub mod stele; diff --git a/src/db/models/publication.rs b/src/db/models/publication.rs new file mode 100644 index 0000000..67fe8ae --- /dev/null +++ b/src/db/models/publication.rs @@ -0,0 +1,21 @@ +use serde::{Deserialize, Serialize}; +use sqlx::types::chrono; + +#[derive(sqlx::FromRow, Deserialize, Serialize)] +/// Model for a Stele. +pub struct Publication { + /// Database id. + pub id: i32, + /// Name of the publication in %YYYY-%MM-%DD format + /// with optionally incrementing version numbers + /// when two publications exist on same date. + pub name: String, + /// Date of the publication. + pub date: chrono::NaiveDate, + /// FK reference to Stele id. + pub stele_id: i32, + /// Whether the publication has been revoked. + /// A publication is revoked if another publication exists + /// on the same date with a higher version number. + pub revoked: bool, +} diff --git a/src/db/models/stele.rs b/src/db/models/stele.rs new file mode 100644 index 0000000..a62f858 --- /dev/null +++ b/src/db/models/stele.rs @@ -0,0 +1,11 @@ +use serde::{Deserialize, Serialize}; + +#[derive(sqlx::FromRow, Deserialize, Serialize)] +/// Model for a Stele. +pub struct Stele { + /// Database id. + pub id: i32, + /// Stele identifier in / format. + /// Example: `org-name/repo-name-law`. + pub name: String, +} diff --git a/src/db/queries.rs b/src/db/queries.rs deleted file mode 100644 index 46221b9..0000000 --- a/src/db/queries.rs +++ /dev/null @@ -1,98 +0,0 @@ -//! Central place for database queries in Stelae -use sqlx::Row; - -use crate::db::DatabaseConnection; - -use super::DatabaseKind; - -/// Inserts a new document into the database. -/// -/// # Errors -/// Errors if the document cannot be inserted into the database. -pub async fn insert_new_document(conn: &DatabaseConnection, doc_id: &str) -> anyhow::Result<()> { - let statement: &'static str = r#" - INSERT OR IGNORE INTO document ( doc_id ) - VALUES ( $1 ) - "#; - match &conn.kind { - &DatabaseKind::Sqlite => { - let mut connection = conn.pool.acquire().await?; - sqlx::query(statement) - .bind(doc_id) - .execute(&mut *connection) - .await?; - } - DatabaseKind::Postgres => { - let mut connection = conn.pool.acquire().await?; - sqlx::query(statement) - .bind(doc_id) - .execute(&mut *connection) - .await?; - } - }; - Ok(()) -} - -/// Inserts a new stele into the database. -/// -/// # Errors -/// Errors if the stele cannot be inserted into the database. -pub async fn insert_new_stele(conn: &DatabaseConnection, stele_id: &str) -> anyhow::Result<()> { - let statement: &'static str = r#" - INSERT OR IGNORE INTO stele ( name ) - VALUES ( $1 ) - "#; - match &conn.kind { - DatabaseKind::Sqlite => { - let mut connection = conn.pool.acquire().await?; - sqlx::query(statement) - .bind(stele_id) - .execute(&mut *connection) - .await?; - } - DatabaseKind::Postgres => { - let mut connection = conn.pool.acquire().await?; - sqlx::query(statement) - .bind(stele_id) - .execute(&mut *connection) - .await?; - } - }; - Ok(()) -} - -/// Find a stele by name. -/// -/// # Errors -/// Errors if the stele cannot be found in the database. -pub async fn find_stele_by_name( - conn: &DatabaseConnection, - name: &str, -) -> anyhow::Result> { - let statement: &'static str = r#" - SELECT id - FROM stele - WHERE name = $1 - "#; - let row: Option = match &conn.kind { - &DatabaseKind::Sqlite => { - let mut connection = conn.pool.acquire().await?; - let row = sqlx::query(statement) - .bind(name) - .fetch_one(&mut *connection) - .await - .ok(); - row.map(|r| r.get(0)) - } - DatabaseKind::Postgres => { - let mut connection = conn.pool.acquire().await?; - let row = sqlx::query(statement) - .bind(name) - .fetch_one(&mut *connection) - .await - .ok(); - row.map(|r| r.get(0)) - } - }; - Ok(row) -} diff --git a/src/db/statements/inserts.rs b/src/db/statements/inserts.rs new file mode 100644 index 0000000..159d2ed --- /dev/null +++ b/src/db/statements/inserts.rs @@ -0,0 +1,129 @@ +//! Central place for database queries in Stelae +use sqlx::types::chrono::NaiveDate; + +use crate::db::DatabaseConnection; +use crate::db::DatabaseKind; + +/// Upsert a new document into the database. +/// +/// # Errors +/// Errors if the document cannot be inserted into the database. +pub async fn create_document(conn: &DatabaseConnection, doc_id: &str) -> anyhow::Result<()> { + let statement: &'static str = r#" + INSERT OR IGNORE INTO document ( doc_id ) + VALUES ( $1 ) + "#; + match conn.kind { + DatabaseKind::Sqlite => { + let mut connection = conn.pool.acquire().await?; + sqlx::query(statement) + .bind(doc_id) + .execute(&mut *connection) + .await?; + } + DatabaseKind::Postgres => { + let mut connection = conn.pool.acquire().await?; + sqlx::query(statement) + .bind(doc_id) + .execute(&mut *connection) + .await?; + } + }; + Ok(()) +} + +/// Upsert a new publication into the database. +/// # Errors +/// Errors if the publication cannot be inserted into the database. +pub async fn create_publication( + conn: &DatabaseConnection, + name: &str, + date: &NaiveDate, + stele_id: i32, +) -> anyhow::Result<()> { + let statement: &'static str = r#" + INSERT OR IGNORE INTO publication ( name, date, stele_id, revoked ) + VALUES ( $1, $2, $3, FALSE ) + "#; + match conn.kind { + DatabaseKind::Sqlite => { + let mut connection = conn.pool.acquire().await?; + sqlx::query(statement) + .bind(name) + .bind(date) + .bind(stele_id) + .execute(&mut *connection) + .await?; + } + DatabaseKind::Postgres => { + let mut connection = conn.pool.acquire().await?; + sqlx::query(statement) + .bind(name) + .bind(date) + .bind(stele_id) + .execute(&mut *connection) + .await?; + } + }; + Ok(()) +} + +/// Upsert a new stele into the database. +/// +/// # Errors +/// Errors if the stele cannot be inserted into the database. +pub async fn create_stele(conn: &DatabaseConnection, stele_id: &str) -> anyhow::Result<()> { + let statement: &'static str = r#" + INSERT OR IGNORE INTO stele ( name ) + VALUES ( $1 ) + "#; + match conn.kind { + DatabaseKind::Sqlite => { + let mut connection = conn.pool.acquire().await?; + sqlx::query(statement) + .bind(stele_id) + .execute(&mut *connection) + .await?; + } + DatabaseKind::Postgres => { + let mut connection = conn.pool.acquire().await?; + sqlx::query(statement) + .bind(stele_id) + .execute(&mut *connection) + .await?; + } + }; + Ok(()) +} + +/// Upsert a new version into the database. +/// +/// # Errors +/// Errors if the version cannot be inserted into the database. +pub async fn create_version(conn: &DatabaseConnection, codified_date: &str) -> anyhow::Result<()> { + let statement: &'static str = r#" + INSERT OR IGNORE INTO version ( codified_date ) + VALUES ( $1 ) + "#; + match conn.kind { + DatabaseKind::Sqlite => { + let mut connection = conn.pool.acquire().await?; + sqlx::query(statement) + .bind(codified_date) + .execute(&mut *connection) + .await?; + } + DatabaseKind::Postgres => { + let mut connection = conn.pool.acquire().await?; + sqlx::query(statement) + .bind(codified_date) + .execute(&mut *connection) + .await?; + } + }; + Ok(()) +} + +pub async fn create_publication_version() { + todo!() +} diff --git a/src/db/statements/mod.rs b/src/db/statements/mod.rs new file mode 100644 index 0000000..3116fd5 --- /dev/null +++ b/src/db/statements/mod.rs @@ -0,0 +1,4 @@ +// Contains all the statements for the database +pub mod inserts; +// Contains all the queries for the database +pub mod queries; diff --git a/src/db/statements/queries.rs b/src/db/statements/queries.rs new file mode 100644 index 0000000..b64b714 --- /dev/null +++ b/src/db/statements/queries.rs @@ -0,0 +1,119 @@ +//! Central place for database queries +use sqlx::types::chrono::NaiveDate; + +use crate::db::models::publication::Publication; +use crate::db::models::stele::Stele; +use crate::db::DatabaseConnection; + +use crate::db::DatabaseKind; + +/// Find a stele by `name`. +/// +/// # Errors +/// Errors if can't establish a connection to the database. +pub async fn find_stele_by_name( + conn: &DatabaseConnection, + name: &str, +) -> anyhow::Result> { + let statement: &'static str = r#" + SELECT * + FROM stele + WHERE name = $1 + "#; + let row = match conn.kind { + DatabaseKind::Sqlite => { + let mut connection = conn.pool.acquire().await?; + sqlx::query_as::<_, Stele>(statement) + .bind(name) + .fetch_one(&mut *connection) + .await + .ok() + } + DatabaseKind::Postgres => { + let mut connection = conn.pool.acquire().await?; + sqlx::query_as::<_, Stele>(statement) + .bind(name) + .fetch_one(&mut *connection) + .await + .ok() + } + }; + Ok(row) +} + +/// Find the last inserted publication by `stele_id`. +/// This function is then used to incrementally insert new change objects +/// +/// # Errors +/// Errors if can't establish a connection to the database. +pub async fn find_last_inserted_publication( + conn: &DatabaseConnection, + stele_id: i32, +) -> anyhow::Result> { + let statement: &'static str = r#" + SELECT * + FROM publication + WHERE revoked = 0 AND stele_id = $1 + ORDER BY date DESC + LIMIT 1 + "#; + let row = match conn.kind { + DatabaseKind::Sqlite => { + let mut connection = conn.pool.acquire().await?; + sqlx::query_as::<_, Publication>(statement) + .bind(stele_id) + .fetch_one(&mut *connection) + .await + .ok() + } + DatabaseKind::Postgres => { + let mut connection = conn.pool.acquire().await?; + sqlx::query_as::<_, Publication>(statement) + .bind(stele_id) + .fetch_one(&mut *connection) + .await + .ok() + } + }; + Ok(row) +} + +/// Find a publication by `name` and `date` and `stele_id`. +/// +/// # Errors +/// Errors if can't establish a connection to the database. +pub async fn find_publication_by_name_and_date_and_stele_id( + conn: &DatabaseConnection, + name: &str, + date: &NaiveDate, + stele_id: i32, +) -> anyhow::Result> { + let statement: &'static str = r#" + SELECT * + FROM publication + WHERE name = $1 AND date = $2 AND stele_id = $3 + "#; + let row = match conn.kind { + DatabaseKind::Sqlite => { + let mut connection = conn.pool.acquire().await?; + sqlx::query_as::<_, Publication>(statement) + .bind(name) + .bind(date) + .bind(stele_id) + .fetch_one(&mut *connection) + .await + .ok() + } + DatabaseKind::Postgres => { + let mut connection = conn.pool.acquire().await?; + sqlx::query_as::<_, Publication>(statement) + .bind(name) + .bind(date) + .bind(stele_id) + .fetch_one(&mut *connection) + .await + .ok() + } + }; + Ok(row) +} diff --git a/src/history/changes.rs b/src/history/changes.rs index c85b112..e3bf77b 100644 --- a/src/history/changes.rs +++ b/src/history/changes.rs @@ -1,18 +1,25 @@ //! Module for inserting changes into the database -use crate::db::queries::{find_stele_by_name, insert_new_document, insert_new_stele}; -use crate::history::rdf_namespaces::{OLL_DOCUMENT_VERSION, OLL_DOC_ID}; +#![allow(clippy::shadow_reuse)] +use crate::db::models::publication::Publication; +use crate::db::statements::queries::{find_last_inserted_publication, find_publication_by_name_and_date_and_stele_id, find_stele_by_name}; +use crate::db::statements::inserts::{create_document, create_publication, create_stele}; +use crate::utils::archive::get_name_parts; +use crate::utils::git::Repo; use crate::{ db::{self, DatabaseConnection}, - stelae::{archive::Archive, types::repositories::Repository}, + stelae::archive::Archive, }; -use sophia::api::{ns::NsTerm, prelude::*, term::SimpleTerm, MownStr}; +use anyhow::Context; +use sophia::api::{prelude::*, term::SimpleTerm, MownStr}; use sophia::xml::parser; -use sophia::{api::ns::Namespace, inmem::graph::FastGraph}; +use sophia::{api::ns::rdfs, inmem::graph::FastGraph}; use std::{ collections::HashMap, path::{Path, PathBuf}, }; +use sqlx::types::chrono::NaiveDate; use walkdir::WalkDir; +use crate::history::rdf::namespaces::{oll, dcterms}; /// Inserts changes from the archive into the database /// @@ -48,7 +55,7 @@ pub async fn insert( } fn insert_changes_single_stele() -> std::io::Result<()> { - Ok(()) + todo!() } /// Insert changes from the archive into the database @@ -65,17 +72,19 @@ async fn insert_changes_archive( for (name, mut stele) in archive.stelae { if let Some(repositories) = stele.get_repositories()? { - let Some(rdf_repo) = repositories.get_rdf_repository() else { + let Some(rdf_data) = repositories.get_rdf_repository() else { continue; }; - let rdf_repo_path = archive_path.to_path_buf().join(&rdf_repo.name); + let rdf_repo_path = archive_path.to_path_buf().join(&rdf_data.name); if !rdf_repo_path.exists() { anyhow::bail!( "RDF repository should exist on disk but not found: {}", rdf_repo_path.display() ); } - insert_changes_from_rdf_repository(conn, rdf_repo_path, &name, rdf_repo).await?; + let (rdf_org, rdf_name) = get_name_parts(&rdf_data.name)?; + let rdf_repo = Repo::new(archive_path, &rdf_org, &rdf_name)?; + insert_changes_from_rdf_repository(conn, rdf_repo, &name).await?; } } Ok(()) @@ -84,54 +93,128 @@ async fn insert_changes_archive( /// Insert changes from the RDF repository into the database async fn insert_changes_from_rdf_repository( conn: &DatabaseConnection, - rdf_repo_path: PathBuf, - name: &str, - rdf_repo: &Repository, + rdf_repo: Repo, + stele_id: &str, ) -> anyhow::Result<()> { - tracing::info!("Inserting changes from RDF repository: {}", name); - tracing::info!("RDF repository path: {}", rdf_repo_path.display()); - - // let response = reqwest::get(NAMESPACE_URL).await?.text().await?; - let mut graph = FastGraph::new(); - - for entry in WalkDir::new(&rdf_repo_path) { - match entry { - Ok(entry) if is_rdf(&entry) => { - tracing::debug!("Parsing file: {:?}", entry.path()); - let file = std::fs::File::open(entry.path())?; - let reader = std::io::BufReader::new(file); - parser::parse_bufread(reader).add_to_graph(&mut graph)?; + tracing::info!("Inserting changes from RDF repository: {}", stele_id); + tracing::info!("RDF repository path: {}", rdf_repo.path.display()); + let run_documents = false; + if run_documents { + let mut graph = FastGraph::new(); + let head_commit = rdf_repo.repo.head()?.peel_to_commit()?; + let tree = head_commit.tree()?; + tree.walk(git2::TreeWalkMode::PreOrder, |_, entry| { + let path_name = entry.name().unwrap(); + if path_name.contains(".rdf") { + let blob = rdf_repo.repo.find_blob(entry.id()).unwrap(); + let data = blob.content(); + let reader = std::io::BufReader::new(data); + parser::parse_bufread(reader) + .add_to_graph(&mut graph) + .unwrap(); } - Ok(entry) => { - tracing::debug!("Skipping non-RDF file: {:?}", entry.path()); - continue; + git2::TreeWalkResult::Ok + })?; + // for entry in WalkDir::new(&rdf_repo.path) { + // match entry { + // Ok(entry) if is_rdf(&entry) => { + // tracing::debug!("Parsing file: {:?}", entry.path()); + // let file = std::fs::File::open(entry.path())?; + // let reader = std::io::BufReader::new(file); + // parser::parse_bufread(reader).add_to_graph(&mut graph)?; + // } + // Ok(entry) => { + // tracing::debug!("Skipping non-RDF file: {:?}", entry.path()); + // continue; + // } + // Err(err) => { + // tracing::error!("Error reading file: {:?}", err); + // } + // } + // } + let documents = graph.triples_matching(Any, Any, [oll::DocumentVersion]); + let mut doc_to_versions: HashMap> = HashMap::new(); + for triple in documents { + let triple = triple.unwrap(); + let document = triple.s(); + let mut doc_id_triples = graph.triples_matching([document], [oll::docId], Any); + if let Some(doc_id_triple) = doc_id_triples.next() { + let object = doc_id_triple.unwrap().o(); + let document_iri = document.iri().unwrap().to_string(); + if let SimpleTerm::LiteralDatatype(doc_id, _) = object { + doc_to_versions + .entry(doc_id.to_string()) + .or_insert_with(Vec::new) + .push(document_iri); + } } - Err(err) => { - tracing::error!("Error reading file: {:?}", err); + } + for versions in doc_to_versions.values() { + // Find the version with the maximum docId + let doc_version = versions + .iter() + .max_by_key(|&v| { + let mut doc_id_triples = graph.triples_matching([v.as_str()], [oll::docId], Any); + doc_id_triples + .next() + .map_or_else(String::new, |doc_id_triple| { + let object = doc_id_triple.unwrap().o(); + if let SimpleTerm::LiteralDatatype(doc_id, _) = object { + doc_id.to_string() + } else { + String::new() + } + }) + }) + .unwrap(); + // Get the docId for this version + // dbg!(&doc_version); + let doc_version_iri_ref = IriRef::new_unchecked(MownStr::from_str(doc_version.as_str())); + let mut doc_id_triples = + graph.triples_matching([SimpleTerm::Iri(doc_version_iri_ref)], [oll::docId], Any); + if let Some(doc_id_triple) = doc_id_triples.next() { + let object = doc_id_triple.unwrap().o(); + if let SimpleTerm::LiteralDatatype(doc_id, _) = object { + create_document(conn, doc_id).await?; + } } } } - let namespace_url = "https://open.law/us/ngo/oll/_ontology/v0.1/ontology.owl#"; - let oll: Namespace<&str> = Namespace::new(namespace_url).unwrap(); + let tx = conn.pool.begin().await?; + match load_delta_from_publications(conn, &rdf_repo, stele_id).await { + Ok(_) => { + tx.commit().await?; + Ok(()) + } + Err(err) => { + tx.rollback().await?; + Err(err) + } + } +} let oll_document_version: NsTerm = oll.get("DocumentVersion").unwrap(); let oll_doc_id = oll.get("docId").unwrap(); - let documents = graph.triples_matching(Any, Any, [oll_document_version]); - let mut doc_to_versions: HashMap> = HashMap::new(); - for triple in documents { - let triple = triple.unwrap(); - let document = triple.s(); - let mut doc_id_triples = graph.triples_matching([document], [oll_doc_id], Any); - if let Some(doc_id_triple) = doc_id_triples.next() { - let object = doc_id_triple.unwrap().o(); - let document_iri = document.iri().unwrap().to_string(); - if let SimpleTerm::LiteralDatatype(doc_id, _) = object { - doc_to_versions - .entry(doc_id.to_string()) - .or_insert_with(Vec::new) - .push(document_iri); - } +/// Load deltas from the publications +async fn load_delta_from_publications( + conn: &DatabaseConnection, + rdf_repo: &Repo, + stele_name: &str, +) -> anyhow::Result<()> { + create_stele(conn, stele_name).await?; + let stele = find_stele_by_name(conn, stele_name).await?.unwrap(); + match find_last_inserted_publication(conn, stele.id).await? { + Some(publication) => { + tracing::info!("Inserting changes from last inserted publication"); + load_delta_from_publications_from_last_inserted_publication().await?; + }, + None => { + tracing::info!( + "Inserting changes from beginning for stele: {}", + stele_name + ); + load_delta_from_publications_from_beginning(conn, rdf_repo, stele.id).await?; } } for versions in doc_to_versions.values() { diff --git a/src/history/mod.rs b/src/history/mod.rs index f61e8c3..28e2a9d 100644 --- a/src/history/mod.rs +++ b/src/history/mod.rs @@ -1,3 +1,5 @@ //! The history module contains tools for interacting with the history of the Stele. -// The historical versions are loaded into the Stele and can be queried. -pub mod changes; \ No newline at end of file +// The changes module contains logic for inserting change objects into the database. +pub mod changes; +// The rdf module contains helper functions that work with loading, parsing and querying the RDF graph using `sophia`. +pub mod rdf; diff --git a/src/history/rdf/mod.rs b/src/history/rdf/mod.rs new file mode 100644 index 0000000..907ed27 --- /dev/null +++ b/src/history/rdf/mod.rs @@ -0,0 +1,4 @@ +//! The rdf module contains helper functions that work with loading, parsing and querying the RDF graph using `sophia`. + +/// RDF namespaces for the Stele ontology. +pub mod namespaces; diff --git a/src/history/rdf/namespaces.rs b/src/history/rdf/namespaces.rs new file mode 100644 index 0000000..979061e --- /dev/null +++ b/src/history/rdf/namespaces.rs @@ -0,0 +1,24 @@ +/// This module contains the RDF namespaces used by Stelae. +use sophia::api::namespace; + +/// Open Law Library ontology. +pub mod oll { + use super::namespace; + namespace! { + "https://open.law/us/ngo/oll/_ontology/v0.1/ontology.owl#", + CollectionVersion, + DocumentVersion, + docId, + codifiedDate + } +} + +/// Dublin Core Terms ontology. +pub mod dcterms { + use super::namespace; + + namespace! { + "http://purl.org/dc/terms/", + available + } +} From dc2ef1db6827912e0019125e56fee5410a4d8252 Mon Sep 17 00:00:00 2001 From: n-dusan Date: Thu, 21 Mar 2024 19:10:49 +0100 Subject: [PATCH 04/15] feat: further work on inserting changes into db The idea being we iterate over all publications using libgit2, and then for each publication we parse .rdf xml files into an rdf graph. After the graph is loaded, we iterate over document and collection versions from the graph and insert each into db. *Add more sqlx structs for database* *Wrap sophia graph with our StelaeGraph* Goal is to move similar code for working with rdf graph into a separate wrapper struct. This simplifies working with graphs as it hides away the rdf graph results complexity. *Use composite keys instead of autoincrement synthetic keys* *Split functionalities for working with rdf graph on* Since the graph queries will vary on the type of information that we need, the easiest is to create utility functions that return (s,p,o) depending on the queries. --- .../sqlite/20240115152953_initial_db.sql | 1 - src/db/models/document.rs | 8 + src/db/models/document_change.rs | 23 ++ src/db/models/mod.rs | 4 + src/db/models/publication.rs | 13 +- src/db/models/publication_version.rs | 14 + src/db/models/stele.rs | 2 - src/db/statements/inserts.rs | 166 +++++++--- src/db/statements/queries.rs | 73 ++++- src/history/changes.rs | 293 ++++++++++++++---- src/history/rdf/graph.rs | 130 ++++++++ src/history/rdf/mod.rs | 3 + src/history/rdf/namespaces.rs | 9 +- 13 files changed, 615 insertions(+), 124 deletions(-) delete mode 100644 migrations/sqlite/20240115152953_initial_db.sql create mode 100644 src/db/models/document_change.rs create mode 100644 src/db/models/publication_version.rs create mode 100644 src/history/rdf/graph.rs diff --git a/migrations/sqlite/20240115152953_initial_db.sql b/migrations/sqlite/20240115152953_initial_db.sql deleted file mode 100644 index 8ddc1d3..0000000 --- a/migrations/sqlite/20240115152953_initial_db.sql +++ /dev/null @@ -1 +0,0 @@ --- Add migration script here diff --git a/src/db/models/document.rs b/src/db/models/document.rs index e69de29..a427623 100644 --- a/src/db/models/document.rs +++ b/src/db/models/document.rs @@ -0,0 +1,8 @@ +use serde::{Deserialize, Serialize}; + +#[derive(sqlx::FromRow, Deserialize, Serialize)] +/// Model for documents. +pub struct Document { + /// Unique document identifier. + pub doc_id: String +} diff --git a/src/db/models/document_change.rs b/src/db/models/document_change.rs new file mode 100644 index 0000000..f38439e --- /dev/null +++ b/src/db/models/document_change.rs @@ -0,0 +1,23 @@ +use serde::{Deserialize, Serialize}; + +#[derive(sqlx::FromRow, Deserialize, Serialize)] +/// Model for document change events. +pub struct DocumentChange { + /// Materialized path to the document + pub doc_mpath: String, + /// Change status of the document. + /// Currently could be 'Element added', 'Element effective', 'Element changed' or 'Element removed'. + pub status: String, + /// Url to the document that was changed. + pub url: String, + /// Optional reason for the change event. + pub change_reason: Option, + /// Foreign key reference to the publication name. + pub publication: String, + /// Foreign key reference to codified date in a publication in %Y-%m-%d format + pub version: String, + /// Foreign key reference to stele identifier in / format. + pub stele: String, + /// Foreign key reference to document id. + pub doc_id: String, +} diff --git a/src/db/models/mod.rs b/src/db/models/mod.rs index 6c897fd..07e5de4 100644 --- a/src/db/models/mod.rs +++ b/src/db/models/mod.rs @@ -2,7 +2,11 @@ /// sqlx structs for document table. pub mod document; +/// sqlx structs for document_change table. +pub mod document_change; /// sqlx structs for publication table. pub mod publication; +/// sqlx structs for publication_version table +pub mod publication_version; /// sqlx structs for stele table. pub mod stele; diff --git a/src/db/models/publication.rs b/src/db/models/publication.rs index 67fe8ae..3b99bba 100644 --- a/src/db/models/publication.rs +++ b/src/db/models/publication.rs @@ -4,18 +4,23 @@ use sqlx::types::chrono; #[derive(sqlx::FromRow, Deserialize, Serialize)] /// Model for a Stele. pub struct Publication { - /// Database id. - pub id: i32, /// Name of the publication in %YYYY-%MM-%DD format /// with optionally incrementing version numbers /// when two publications exist on same date. pub name: String, /// Date of the publication. pub date: chrono::NaiveDate, - /// FK reference to Stele id. - pub stele_id: i32, + /// Foreign key reference to stele by name. + pub stele: String, /// Whether the publication has been revoked. /// A publication is revoked if another publication exists /// on the same date with a higher version number. pub revoked: bool, + /// If a publication is derived from another publication, + /// represents the last publication name that was valid before this publication. + pub last_valid_publication_name: Option, + /// If a publication is derived from another publication, + /// represents the last publication version (codified date) from the previous publication + /// that the current publication is derived from. + pub last_valid_version: Option } diff --git a/src/db/models/publication_version.rs b/src/db/models/publication_version.rs new file mode 100644 index 0000000..2a3dd7d --- /dev/null +++ b/src/db/models/publication_version.rs @@ -0,0 +1,14 @@ +use serde::{Deserialize, Serialize}; + +#[derive(sqlx::FromRow, Deserialize, Serialize)] +/// Model for a Stele. +pub struct PublicationVersion { + /// Database id. + pub id: i32, + /// Date in a publication in %Y-%m-%d format + pub version: String, + /// Foreign key reference to publication id. + pub publication_id: i32, + /// Reason for building the publication. + pub build_reason: Option, +} diff --git a/src/db/models/stele.rs b/src/db/models/stele.rs index a62f858..61a5093 100644 --- a/src/db/models/stele.rs +++ b/src/db/models/stele.rs @@ -3,8 +3,6 @@ use serde::{Deserialize, Serialize}; #[derive(sqlx::FromRow, Deserialize, Serialize)] /// Model for a Stele. pub struct Stele { - /// Database id. - pub id: i32, /// Stele identifier in / format. /// Example: `org-name/repo-name-law`. pub name: String, diff --git a/src/db/statements/inserts.rs b/src/db/statements/inserts.rs index 159d2ed..795ab72 100644 --- a/src/db/statements/inserts.rs +++ b/src/db/statements/inserts.rs @@ -8,28 +8,38 @@ use crate::db::DatabaseKind; /// /// # Errors /// Errors if the document cannot be inserted into the database. -pub async fn create_document(conn: &DatabaseConnection, doc_id: &str) -> anyhow::Result<()> { - let statement: &'static str = r#" - INSERT OR IGNORE INTO document ( doc_id ) - VALUES ( $1 ) - "#; - match conn.kind { +pub async fn create_document( + conn: &DatabaseConnection, + doc_id: &str, +) -> anyhow::Result> { + let id = match conn.kind { DatabaseKind::Sqlite => { + let statement: &'static str = r#" + INSERT OR IGNORE INTO document ( doc_id ) + VALUES ( $1 ) + "#; let mut connection = conn.pool.acquire().await?; sqlx::query(statement) .bind(doc_id) .execute(&mut *connection) - .await?; + .await? + .last_insert_id() } DatabaseKind::Postgres => { + let statement = r#" + INSERT INTO document ( doc_id ) + VALUES ( $1 ) + ON CONFLICT ( doc_id ) DO NOTHING; + "#; let mut connection = conn.pool.acquire().await?; sqlx::query(statement) .bind(doc_id) .execute(&mut *connection) - .await?; + .await? + .last_insert_id() } }; - Ok(()) + Ok(id) } /// Upsert a new publication into the database. @@ -39,91 +49,161 @@ pub async fn create_publication( conn: &DatabaseConnection, name: &str, date: &NaiveDate, - stele_id: i32, -) -> anyhow::Result<()> { - let statement: &'static str = r#" - INSERT OR IGNORE INTO publication ( name, date, stele_id, revoked ) - VALUES ( $1, $2, $3, FALSE ) - "#; - match conn.kind { + stele: &str, + last_valid_publication_name: Option, + last_valid_version: Option, +) -> anyhow::Result> { + let id = match conn.kind { DatabaseKind::Sqlite => { + let statement: &'static str = r#" + INSERT OR IGNORE INTO publication ( name, date, stele, revoked, last_valid_publication_name, last_valid_version ) + VALUES ( $1, $2, $3, FALSE, $4, $5 ) + "#; let mut connection = conn.pool.acquire().await?; sqlx::query(statement) .bind(name) .bind(date) - .bind(stele_id) + .bind(stele) + .bind(last_valid_publication_name) + .bind(last_valid_version) .execute(&mut *connection) - .await?; + .await? + .last_insert_id() } DatabaseKind::Postgres => { + let statement: &'static str = r#" + INSERT INTO publication ( name, date, stele, revoked, last_valid_publication_name, last_valid_version ) + VALUES ( $1, $2, $3, FALSE, $4, $5 ) + ON CONFLICT ( name, stele ) DO NOTHING; + "#; let mut connection = conn.pool.acquire().await?; sqlx::query(statement) .bind(name) .bind(date) - .bind(stele_id) + .bind(stele) + .bind(last_valid_publication_name) + .bind(last_valid_version) .execute(&mut *connection) - .await?; + .await? + .last_insert_id() } }; - Ok(()) + Ok(id) } /// Upsert a new stele into the database. /// /// # Errors /// Errors if the stele cannot be inserted into the database. -pub async fn create_stele(conn: &DatabaseConnection, stele_id: &str) -> anyhow::Result<()> { - let statement: &'static str = r#" - INSERT OR IGNORE INTO stele ( name ) - VALUES ( $1 ) - "#; - match conn.kind { +pub async fn create_stele(conn: &DatabaseConnection, stele: &str) -> anyhow::Result> { + let id = match conn.kind { DatabaseKind::Sqlite => { + let statement: &'static str = r#" + INSERT OR IGNORE INTO stele ( name ) + VALUES ( $1 ) + "#; let mut connection = conn.pool.acquire().await?; sqlx::query(statement) - .bind(stele_id) + .bind(stele) .execute(&mut *connection) - .await?; + .await? + .last_insert_id() } DatabaseKind::Postgres => { + let statement: &'static str = r#" + INSERT INTO stele ( name ) + VALUES ( $1 ) + ON CONFLICT ( name ) DO NOTHING; + "#; let mut connection = conn.pool.acquire().await?; sqlx::query(statement) - .bind(stele_id) + .bind(stele) .execute(&mut *connection) - .await?; + .await? + .last_insert_id() } }; - Ok(()) + Ok(id) } /// Upsert a new version into the database. /// /// # Errors /// Errors if the version cannot be inserted into the database. -pub async fn create_version(conn: &DatabaseConnection, codified_date: &str) -> anyhow::Result<()> { - let statement: &'static str = r#" - INSERT OR IGNORE INTO version ( codified_date ) - VALUES ( $1 ) - "#; - match conn.kind { +pub async fn create_version( + conn: &DatabaseConnection, + codified_date: &str, +) -> anyhow::Result> { + let id = match conn.kind { DatabaseKind::Sqlite => { + let statement: &'static str = r#" + INSERT OR IGNORE INTO version ( codified_date ) + VALUES ( $1 ) + "#; let mut connection = conn.pool.acquire().await?; sqlx::query(statement) .bind(codified_date) .execute(&mut *connection) - .await?; + .await? + .last_insert_id() } DatabaseKind::Postgres => { + let statement: &'static str = r#" + INSERT INTO version ( codified_date ) + VALUES ( $1 ) + ON CONFLICT ( codified_date ) DO NOTHING; + "#; let mut connection = conn.pool.acquire().await?; sqlx::query(statement) .bind(codified_date) .execute(&mut *connection) - .await?; + .await? + .last_insert_id() } }; - Ok(()) + Ok(id) } -pub async fn create_publication_version() { - todo!() +/// Upsert a new publication version into the database. +/// +/// # Errors +/// Errors if the publication version cannot be inserted into the database. +pub async fn create_publication_version( + conn: &DatabaseConnection, + publication: &str, + codified_date: &str, + stele: &str, +) -> anyhow::Result> { + let id = match conn.kind { + DatabaseKind::Sqlite => { + let statement = r#" + INSERT OR IGNORE INTO publication_version ( publication, version, stele ) + VALUES ( $1, $2, $3 ) + "#; + let mut connection = conn.pool.acquire().await?; + sqlx::query(statement) + .bind(publication) + .bind(codified_date) + .bind(stele) + .execute(&mut *connection) + .await? + .last_insert_id() + } + DatabaseKind::Postgres => { + let statement = r#" + INSERT INTO publication_version ( publication, version, stele ) + VALUES ( $1, $2, $3 ) + ON CONFLICT ( publication, version, stele ) DO NOTHING; + "#; + let mut connection = conn.pool.acquire().await?; + sqlx::query(statement) + .bind(publication) + .bind(codified_date) + .bind(stele) + .execute(&mut *connection) + .await? + .last_insert_id() + } + }; + Ok(id) } diff --git a/src/db/statements/queries.rs b/src/db/statements/queries.rs index b64b714..7251fd7 100644 --- a/src/db/statements/queries.rs +++ b/src/db/statements/queries.rs @@ -2,6 +2,7 @@ use sqlx::types::chrono::NaiveDate; use crate::db::models::publication::Publication; +use crate::db::models::publication_version::PublicationVersion; use crate::db::models::stele::Stele; use crate::db::DatabaseConnection; @@ -48,28 +49,35 @@ pub async fn find_stele_by_name( /// Errors if can't establish a connection to the database. pub async fn find_last_inserted_publication( conn: &DatabaseConnection, - stele_id: i32, + stele: &str, ) -> anyhow::Result> { - let statement: &'static str = r#" - SELECT * - FROM publication - WHERE revoked = 0 AND stele_id = $1 - ORDER BY date DESC - LIMIT 1 - "#; let row = match conn.kind { DatabaseKind::Sqlite => { + let statement: &'static str = r#" + SELECT * + FROM publication + WHERE revoked = 0 AND stele = $1 + ORDER BY date DESC + LIMIT 1 + "#; let mut connection = conn.pool.acquire().await?; sqlx::query_as::<_, Publication>(statement) - .bind(stele_id) + .bind(stele) .fetch_one(&mut *connection) .await .ok() } DatabaseKind::Postgres => { + let statement: &'static str = r#" + SELECT * + FROM publication + WHERE revoked = FALSE AND stele = $1 + ORDER BY date DESC + LIMIT 1 + "#; let mut connection = conn.pool.acquire().await?; sqlx::query_as::<_, Publication>(statement) - .bind(stele_id) + .bind(stele) .fetch_one(&mut *connection) .await .ok() @@ -86,12 +94,12 @@ pub async fn find_publication_by_name_and_date_and_stele_id( conn: &DatabaseConnection, name: &str, date: &NaiveDate, - stele_id: i32, + stele: &str, ) -> anyhow::Result> { let statement: &'static str = r#" SELECT * FROM publication - WHERE name = $1 AND date = $2 AND stele_id = $3 + WHERE name = $1 AND date = $2 AND stele = $3 "#; let row = match conn.kind { DatabaseKind::Sqlite => { @@ -99,7 +107,7 @@ pub async fn find_publication_by_name_and_date_and_stele_id( sqlx::query_as::<_, Publication>(statement) .bind(name) .bind(date) - .bind(stele_id) + .bind(stele) .fetch_one(&mut *connection) .await .ok() @@ -109,7 +117,44 @@ pub async fn find_publication_by_name_and_date_and_stele_id( sqlx::query_as::<_, Publication>(statement) .bind(name) .bind(date) - .bind(stele_id) + .bind(stele) + .fetch_one(&mut *connection) + .await + .ok() + } + }; + Ok(row) +} + +/// Find a publication version by `publication_id` and `version`. +/// +/// # Errors +/// Errors if can't establish a connection to the database. +pub async fn find_publication_version_by_publication_id_and_version( + conn: &DatabaseConnection, + publication_id: i32, + codified_date: &str, +) -> anyhow::Result> { + let statement: &'static str = r#" + SELECT * + FROM publication_version + WHERE publication_id = $1 AND version = $2 + "#; + let row = match conn.kind { + DatabaseKind::Sqlite => { + let mut connection = conn.pool.acquire().await?; + sqlx::query_as::<_, PublicationVersion>(statement) + .bind(publication_id) + .bind(codified_date) + .fetch_one(&mut *connection) + .await + .ok() + } + DatabaseKind::Postgres => { + let mut connection = conn.pool.acquire().await?; + sqlx::query_as::<_, PublicationVersion>(statement) + .bind(publication_id) + .bind(codified_date) .fetch_one(&mut *connection) .await .ok() diff --git a/src/history/changes.rs b/src/history/changes.rs index e3bf77b..2366028 100644 --- a/src/history/changes.rs +++ b/src/history/changes.rs @@ -1,8 +1,16 @@ //! Module for inserting changes into the database #![allow(clippy::shadow_reuse)] +use crate::db::models::document_change::DocumentChange; use crate::db::models::publication::Publication; -use crate::db::statements::queries::{find_last_inserted_publication, find_publication_by_name_and_date_and_stele_id, find_stele_by_name}; -use crate::db::statements::inserts::{create_document, create_publication, create_stele}; +use crate::db::statements::inserts::{ + create_document, create_publication, create_publication_version, create_stele, create_version, +}; +use crate::db::statements::queries::{ + find_last_inserted_publication, find_publication_by_name_and_date_and_stele_id, + find_publication_version_by_publication_id_and_version, find_stele_by_name, +}; +use crate::history::rdf::graph::StelaeGraph; +use crate::history::rdf::namespaces::{dcterms, oll}; use crate::utils::archive::get_name_parts; use crate::utils::git::Repo; use crate::{ @@ -10,16 +18,16 @@ use crate::{ stelae::archive::Archive, }; use anyhow::Context; -use sophia::api::{prelude::*, term::SimpleTerm, MownStr}; +use sophia::api::MownStr; +use sophia::api::{prelude::*, term::SimpleTerm}; use sophia::xml::parser; use sophia::{api::ns::rdfs, inmem::graph::FastGraph}; +use sqlx::types::chrono::NaiveDate; use std::{ collections::HashMap, path::{Path, PathBuf}, }; -use sqlx::types::chrono::NaiveDate; use walkdir::WalkDir; -use crate::history::rdf::namespaces::{oll, dcterms}; /// Inserts changes from the archive into the database /// @@ -200,75 +208,242 @@ async fn insert_changes_from_rdf_repository( async fn load_delta_from_publications( conn: &DatabaseConnection, rdf_repo: &Repo, - stele_name: &str, + stele: &str, ) -> anyhow::Result<()> { - create_stele(conn, stele_name).await?; - let stele = find_stele_by_name(conn, stele_name).await?.unwrap(); - match find_last_inserted_publication(conn, stele.id).await? { + create_stele(conn, stele).await?; + match find_last_inserted_publication(conn, stele).await? { Some(publication) => { tracing::info!("Inserting changes from last inserted publication"); load_delta_from_publications_from_last_inserted_publication().await?; - }, - None => { - tracing::info!( - "Inserting changes from beginning for stele: {}", - stele_name - ); - load_delta_from_publications_from_beginning(conn, rdf_repo, stele.id).await?; } - } - for versions in doc_to_versions.values() { - // Find the version with the maximum docId - let doc_version = versions - .iter() - .max_by_key(|&v| { - let mut doc_id_triples = graph.triples_matching([v.as_str()], [oll_doc_id], Any); - doc_id_triples - .next() - .map_or_else(String::new, |doc_id_triple| { - let object = doc_id_triple.unwrap().o(); - if let SimpleTerm::LiteralDatatype(doc_id, _) = object { - doc_id.to_string() - } else { - String::new() - } - }) - }) - .unwrap(); - // Get the docId for this version - // dbg!(&doc_version); - let doc_version_iri_ref = IriRef::new_unchecked(MownStr::from_str(doc_version.as_str())); - let mut doc_id_triples = - graph.triples_matching([SimpleTerm::Iri(doc_version_iri_ref)], [oll_doc_id], Any); - if let Some(doc_id_triple) = doc_id_triples.next() { - let object = doc_id_triple.unwrap().o(); - if let SimpleTerm::LiteralDatatype(doc_id, _) = object { - insert_new_document(conn, doc_id).await?; - } + None => { + tracing::info!("Inserting changes from beginning for stele: {}", stele); + load_delta_from_publications_from_beginning(conn, rdf_repo, stele).await?; } } + Ok(()) +} - load_delta_from_publications(&mut graph, conn, rdf_repo_path.join("_publication"), name) +/// Iterate and load delta from all publications in the `_publication` directory +/// +/// # Errors +/// Errors if the delta cannot be loaded from the publications +async fn load_delta_from_publications_from_beginning( + conn: &DatabaseConnection, + rdf_repo: &Repo, + stele: &str, +) -> anyhow::Result<()> { + let head_commit = rdf_repo.repo.head()?.peel_to_commit()?; + let tree = head_commit.tree()?; + let publications_dir_entry = tree.get_path(&PathBuf::from("_publication"))?; + let publications_subtree = rdf_repo.repo.find_tree(publications_dir_entry.id())?; + for publication_entry in publications_subtree.iter() { + let name = publication_entry.name().unwrap(); + let mut pub_graph = StelaeGraph::new(); + let object = publication_entry.to_object(&rdf_repo.repo)?; + let Some(publication_tree) = object.as_tree() else { + anyhow::bail!("Expected a tree but got something else"); + }; + let index_rdf = publication_tree.get_path(&PathBuf::from("index.rdf"))?; + let blob = rdf_repo.repo.find_blob(index_rdf.id())?; + let data = blob.content(); + let reader = std::io::BufReader::new(data); + parser::parse_bufread(reader).add_to_graph(&mut pub_graph.g)?; + let pub_label = pub_graph.literal_from_triple_matching(None, Some(rdfs::label), None)?; + let pub_name = pub_label + .strip_prefix("Publication ") + .context("Could not strip prefix")? + .to_string(); + tracing::info!("Publication: {pub_name}"); + let pub_date = + pub_graph.literal_from_triple_matching(None, Some(dcterms::available), None)?; + publication_tree.walk(git2::TreeWalkMode::PreOrder, |_, entry| { + let path_name = entry.name().unwrap(); + if path_name.contains(".rdf") { + let current_blob = rdf_repo.repo.find_blob(entry.id()).unwrap(); + let current_content = current_blob.content(); + parser::parse_bufread(std::io::BufReader::new(current_content)) + .add_to_graph(&mut pub_graph.g) + .unwrap(); + } + git2::TreeWalkResult::Ok + })?; + let pub_date = NaiveDate::parse_from_str(pub_date.as_str(), "%Y-%m-%d")?; + let (last_valid_pub_name, last_valid_codified_date) = + referenced_publication_information(&pub_graph); + create_publication( + conn, + &pub_name, + &pub_date, + stele, + last_valid_pub_name, + last_valid_codified_date, + ) .await?; + let publication = + find_publication_by_name_and_date_and_stele_id(conn, &pub_name, &pub_date, stele) + .await? + .unwrap(); + + load_delta_for_publication(conn, publication, &pub_graph, None).await?; + } Ok(()) } -/// Check if the entry is an RDF file -fn is_rdf(entry: &walkdir::DirEntry) -> bool { - entry.path().extension() == Some("rdf".as_ref()) +async fn load_delta_from_publications_from_last_inserted_publication() -> anyhow::Result<()> { + todo!() } -/// Load deltas from the publications -async fn load_delta_from_publications( - graph: &mut FastGraph, +/// +async fn load_delta_for_publication( + conn: &DatabaseConnection, + publication: Publication, + pub_graph: &StelaeGraph, + last_inserted_date: Option, +) -> anyhow::Result<()> { + let pub_document_versions = get_document_publication_versions(&pub_graph); + let pub_collection_versions = get_collection_publication_versions(&pub_graph); + + insert_document_changes( + conn, + &last_inserted_date, + pub_document_versions, + pub_graph, + &publication, + ) + .await?; + + // insert_library_changes(conn, &last_inserted_date, pub_collection_versions, pub_graph, &publication).await?; + // insert_shared_publication_versions_for_publication(con, pub.id, pub.last_valid_publication_name, pub.last_valid_version, pub.stele_id) + + // revoke_same_date_publications(conn, publication, stele_id).await?; + Ok(()) +} + +async fn insert_document_changes( conn: &DatabaseConnection, - publication_path: PathBuf, - name: &str, + last_inserted_date: &Option, + pub_document_versions: Vec<&SimpleTerm<'_>>, + pub_graph: &StelaeGraph, + publication: &Publication, ) -> anyhow::Result<()> { - insert_new_stele(conn, name).await?; - let id = find_stele_by_name(conn, name).await?; - tracing::info!("Inserting changes from publications for stele: {}", name); - dbg!(&id); - dbg!(&publication_path); + let mut document_changes_bulk: Vec = Vec::new(); + for version in pub_document_versions { + let codified_date = + pub_graph.literal_from_triple_matching(Some(version), Some(oll::codifiedDate), None)?; + if let Some(last_inserted_date) = last_inserted_date { + let codified_date = NaiveDate::parse_from_str(codified_date.as_str(), "%Y-%m-%d")?; + let last_inserted_date = + NaiveDate::parse_from_str(last_inserted_date.as_str(), "%Y-%m-%d")?; + if codified_date <= last_inserted_date { + // Date already inserted + continue; + } + } + create_version(conn, &codified_date).await?; + create_publication_version(conn, &publication.name, &codified_date, &publication.stele) + .await?; + let doc_id = + pub_graph.literal_from_triple_matching(Some(version), Some(oll::docId), None)?; + create_document(conn, &doc_id).await?; + + let changes_uri = + pub_graph.iri_from_triple_matching(Some(version), Some(oll::hasChanges), None)?; + let changes = pub_graph.subjects_from_triples_matching_subject(changes_uri); + for change in changes { + let doc_mpath = pub_graph.literal_from_triple_matching( + Some(&change), + Some(oll::documentMaterializedPath), + None, + )?; + let url = + pub_graph.literal_from_triple_matching(Some(&change), Some(oll::url), None)?; + let reason = pub_graph + .literal_from_triple_matching(Some(&change), Some(oll::reason), None) + .ok(); + let statuses = pub_graph.all_literals_from_triple_matching( + Some(&change), + Some(oll::status), + None, + )?; + for status in statuses { + document_changes_bulk.push(DocumentChange { + doc_mpath: doc_mpath.to_string(), + status: status.to_string(), + url: url.to_string(), + change_reason: reason.clone(), + publication: publication.name.clone(), + version: codified_date.to_string(), + stele: publication.stele.clone(), + doc_id: doc_id.to_string(), + }); + } + } + insert_document_changes_bulk(conn, &document_changes_bulk).await?; + // let publication_version = find_publication_version_by_publication_id_and_version( + // conn, + // publication.id, + // &codified_date, + // ) + // .await? + // .context("Could not find publication version")?; + } Ok(()) } + +async fn insert_library_changes( + conn: &DatabaseConnection, + last_inserted_date: &Option, + pub_collection_versions: Vec<&SimpleTerm<'_>>, + pub_graph: &FastGraph, + publication: &Publication, +) -> anyhow::Result<()> { + todo!() +} + +/// Get the last valid publication name and codified date from the graph +fn referenced_publication_information(pub_graph: &StelaeGraph) -> (Option, Option) { + let last_valid_pub = pub_graph + .literal_from_triple_matching(None, Some(oll::lastValidPublication), None) + .ok() + .and_then(|p: String| { + p.strip_prefix("Publication ") + .and_then(|s| Some(s.to_string())) + }); + let last_valid_version = pub_graph + .literal_from_triple_matching(None, Some(oll::lastValidCodifiedDate), None) + .ok(); + return (last_valid_pub, last_valid_version); +} + +async fn revoke_same_date_publications( + conn: &DatabaseConnection, + publication: Publication, + stele_id: i32, +) -> anyhow::Result<()> { + todo!() +} + +/// Get the document publication version IRIs from the graph +fn get_document_publication_versions(graph: &StelaeGraph) -> Vec<&SimpleTerm> { + let triples = graph.g.triples_matching(Any, Any, [oll::DocumentVersion]); + triples + .filter_map(|t| { + let t = t.ok()?; + let subject = t.s(); + Some(subject) + }) + .collect() +} + +/// Get the collection publication version IRIs from the graph +fn get_collection_publication_versions(graph: &StelaeGraph) -> Vec<&SimpleTerm> { + let triples = graph.g.triples_matching(Any, Any, [oll::CollectionVersion]); + triples + .filter_map(|t| { + let t = t.ok()?; + let subject = t.s(); + Some(subject) + }) + .collect() +} diff --git a/src/history/rdf/graph.rs b/src/history/rdf/graph.rs new file mode 100644 index 0000000..05d6d39 --- /dev/null +++ b/src/history/rdf/graph.rs @@ -0,0 +1,130 @@ +/// The helper methods for working with RDF in Stelae. +use anyhow::Context; +use sophia::api::graph::Graph; +use sophia::api::ns::rdf::li; +use sophia::api::ns::NsTerm; +use sophia::api::{prelude::*, term::SimpleTerm}; +use sophia::inmem::graph::FastGraph; +use sophia::inmem::index::TermIndexFullError; + +/// Stelae representation of an RDF graph. +pub struct StelaeGraph { + /// The underlying graph. + pub g: FastGraph, +} + +impl StelaeGraph { + /// Create a new graph. + pub fn new() -> Self { + Self { + g: FastGraph::new(), + } + } + /// Extract a literal from a triple matching. + /// + /// # Errors + /// Errors if the triple matching the object is not found. + /// Errors if the object is not an RDF literal. + pub fn literal_from_triple_matching( + &self, + subject: Option<&SimpleTerm>, + predicate: Option, + object: Option, + ) -> anyhow::Result { + let triple = self.get_next_triples_matching(subject, predicate, object)?; + let literal = match triple.o() { + SimpleTerm::LiteralLanguage(literal, _) => literal, + SimpleTerm::LiteralDatatype(literal, _) => literal, + _ => { + anyhow::bail!("Expected literal language, got - {:?}", triple.o()); + } + }; + Ok(literal.to_string()) + } + + /// Extract all literals from a triple matching. + /// + /// # Errors + /// Errors if the triple matching the object is not found. + /// Errors if the object is not an RDF literal. + pub fn all_literals_from_triple_matching( + &self, + subject: Option<&SimpleTerm>, + predicate: Option, + object: Option, + ) -> anyhow::Result> { + Ok(self + .literal_from_triple_matching(subject, predicate, object) + .into_iter() + .map(|t| t) + .collect()) + } + + /// Extract an IRI from a triple matching. + /// + /// # Errors + /// Errors if the triple matching the object is not found. + /// Errors if the object is not an RDF IRI. + pub fn iri_from_triple_matching<'graph>( + &'graph self, + subject: Option<&'graph SimpleTerm>, + predicate: Option>, + object: Option>, + ) -> anyhow::Result { + let triple = self.get_next_triples_matching(subject, predicate, object)?; + let iri = match triple.o() { + SimpleTerm::Iri(literal) => literal, + _ => { + anyhow::bail!("Expected literal language, got - {:?}", triple.o()); + } + }; + Ok(SimpleTerm::Iri(iri.clone())) + } + + /// Extract subjects from a triple matching a subject. + pub fn subjects_from_triples_matching_subject(&self, subject: SimpleTerm) -> Vec { + self.g + .triples_matching([subject], Any, Any) + .into_iter() + .filter_map(|t| { + let t = t.ok()?; + Some(t.s().clone()) + }) + .collect() + } + + fn get_next_triples_matching<'graph>( + &'graph self, + subject: Option<&'graph SimpleTerm>, + predicate: Option>, + object: Option>, + ) -> anyhow::Result<[&'graph SimpleTerm<'_>; 3]> { + let triple = match (subject, predicate, object) { + (Some(s), None, None) => { + self.g.triples_matching([s], Any, Any).next().context("Did not find a triple matching provided subject in the graph") + }, + (None, Some(p), None) => { + self.g.triples_matching(Any, [p], Any).next().context("Did not find a triple matching provided predicate in the graph") + }, + (None, None, Some(o)) => { + self.g.triples_matching(Any, Any, [o]).next().context("Did not find a triple matching provided object in the graph") + }, + (Some(s), Some(p), None) => { + self.g.triples_matching([s], [p], Any).next().context("Did not find a triple matching provided subject and predicate in the graph") + }, + (Some(s), None, Some(o)) => { + self.g.triples_matching([s], Any, [o]).next().context("Did not find a triple matching provided subject and object in the graph") + }, + (None, Some(p), Some(o)) => { + self.g.triples_matching(Any, [p], [o]).next().context("Did not find a triple matching provided predicate and object in the graph") + }, + (Some(s), Some(p), Some(o)) => { + self.g.triples_matching([s], [p], [o]).next().context("Did not find a triple matching provided subject, predicate and object in the graph") + }, + (None, None, None) => { + anyhow::bail!("No subject, predicate or object provided") + } + }?; + Ok(triple?) + } +} diff --git a/src/history/rdf/mod.rs b/src/history/rdf/mod.rs index 907ed27..c09bd7f 100644 --- a/src/history/rdf/mod.rs +++ b/src/history/rdf/mod.rs @@ -2,3 +2,6 @@ /// RDF namespaces for the Stele ontology. pub mod namespaces; + +/// The graph module contains the `Graph` struct which is used to interact with the RDF graph. +pub mod graph; diff --git a/src/history/rdf/namespaces.rs b/src/history/rdf/namespaces.rs index 979061e..58fc2ed 100644 --- a/src/history/rdf/namespaces.rs +++ b/src/history/rdf/namespaces.rs @@ -9,7 +9,14 @@ pub mod oll { CollectionVersion, DocumentVersion, docId, - codifiedDate + codifiedDate, + lastValidPublication, + lastValidCodifiedDate, + hasChanges, + documentMaterializedPath, + url, + reason, + status } } From bffdd23bf6dc9e93348a4d8184f0de41030755e1 Mon Sep 17 00:00:00 2001 From: n-dusan Date: Mon, 15 Apr 2024 21:23:13 +0200 Subject: [PATCH 05/15] feat: implement the rest of insert changes. This commit has the following significant changes: *Bulk insert multiple document and collection change objects into db* Idea being we create a query-builder and insert multiple values into an insert statement. On Windows, sqlx throws an error when inserting 4000+ rows at once. To resolve, chunk the rows to 2000 per bulk insert. A dd `BATCH_SIZE` constant, indicating how many rows will get inserted at once. Set to 2000 per insert. *Upgrade rustc compiler version* *Add utility methods for working with the `sophia` graph* RDF elements can be stored in a container, e.g. To resolve, introduce a `Bag` struct with `items` helper method. The idea being we iterate over all elements inside a container. This logic is similar to rdflib implementation of `items` in Python. To better understand this iteration logic, please see `rdflib/container.py` in rdflib Python. *Add missing sqlx structs to map objects to database rows* *Revoke same date publications* Set a boolean revoked to True for publications that exist on same date. Revoked publications aren't accounted for when querying for versions for a publication. *Support for publication partial update* Re-use the existing insert function, as most of the code is the same. The only difference is that we skip parsing/loading publications which are already inserted in the db. For publication directories which aren't in the DB, we continue inserting changes. --- .github/workflows/ci.yml | 2 +- Cargo.lock | 2086 ++++++++++------- Cargo.toml | 9 +- .../sqlite/20240115152953_initial_db.down.sql | 20 + .../sqlite/20240115152953_initial_db.up.sql | 121 + src/db/init.rs | 2 +- src/db/mod.rs | 20 +- src/db/models/changed_library_document.rs | 21 + src/db/models/document.rs | 2 +- src/db/models/library.rs | 8 + src/db/models/library_change.rs | 19 + src/db/models/mod.rs | 18 +- src/db/models/publication.rs | 24 +- .../publication_has_publication_versions.rs | 15 + src/db/models/publication_version.rs | 23 +- src/db/statements/inserts.rs | 245 +- src/db/statements/queries.rs | 244 +- src/history/changes.rs | 431 ++-- src/history/rdf/graph.rs | 205 +- src/history/rdf/namespaces.rs | 3 +- src/lib.rs | 5 + src/server/git.rs | 21 +- src/server/publish.rs | 46 +- src/server/tracing.rs | 2 +- src/stelae/archive.rs | 20 +- src/stelae/types/dependencies.rs | 10 + src/stelae/types/repositories.rs | 29 +- src/utils/cli.rs | 18 +- src/utils/git.rs | 4 +- src/utils/paths.rs | 4 +- 30 files changed, 2367 insertions(+), 1310 deletions(-) create mode 100644 migrations/sqlite/20240115152953_initial_db.down.sql create mode 100644 migrations/sqlite/20240115152953_initial_db.up.sql create mode 100644 src/db/models/changed_library_document.rs create mode 100644 src/db/models/library.rs create mode 100644 src/db/models/library_change.rs create mode 100644 src/db/models/publication_has_publication_versions.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d747371..5f96294 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,7 +39,7 @@ jobs: uses: actions/checkout@v4 - name: Install stable toolchain - uses: dtolnay/rust-toolchain@1.70 # Pin based on current `rust-version` in Cargo.toml. IMPORTANT: Upgrade version when `rust-version` changes. + uses: dtolnay/rust-toolchain@1.77 # Pin based on current `rust-version` in Cargo.toml. IMPORTANT: Upgrade version when `rust-version` changes. with: components: rustfmt, clippy diff --git a/Cargo.lock b/Cargo.lock index 49b9641..62f8bba 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4,34 +4,34 @@ version = 3 [[package]] name = "actix-codec" -version = "0.5.0" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57a7559404a7f3573127aab53c08ce37a6c6a315c374a31070f3c91cd1b4a7fe" +checksum = "5f7b0a21988c1bf877cf4759ef5ddaac04c1c9fe808c9142ecb78ba97d97a28a" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.5.0", "bytes", "futures-core", "futures-sink", - "log", "memchr", "pin-project-lite", "tokio", "tokio-util", + "tracing", ] [[package]] name = "actix-http" -version = "3.2.2" +version = "3.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c83abf9903e1f0ad9973cc4f7b9767fd5a03a583f51a5b7a339e07987cd2724" +checksum = "d223b13fd481fc0d1f83bb12659ae774d9e3601814c68a0bc539731698cca743" dependencies = [ "actix-codec", "actix-rt", "actix-service", "actix-utils", - "ahash 0.7.6", + "ahash", "base64", - "bitflags 1.3.2", + "bitflags 2.5.0", "brotli", "bytes", "bytestring", @@ -43,7 +43,7 @@ dependencies = [ "http", "httparse", "httpdate", - "itoa 1.0.4", + "itoa", "language-tags", "local-channel", "mime", @@ -52,25 +52,27 @@ dependencies = [ "rand", "sha1", "smallvec", + "tokio", + "tokio-util", "tracing", "zstd", ] [[package]] name = "actix-macros" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "465a6172cf69b960917811022d8f29bc0b7fa1398bc4f78b3c466673db1213b6" +checksum = "e01ed3140b2f8d422c68afa1ed2e85d996ea619c988ac834d255db32138655cb" dependencies = [ "quote", - "syn 1.0.109", + "syn 2.0.59", ] [[package]] name = "actix-router" -version = "0.5.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d66ff4d247d2b160861fa2866457e85706833527840e4133f8f49aa423a38799" +checksum = "d22475596539443685426b6bdadb926ad0ecaefdfc5fb05e5e3441f15463c511" dependencies = [ "bytestring", "http", @@ -81,9 +83,9 @@ dependencies = [ [[package]] name = "actix-rt" -version = "2.7.0" +version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ea16c295198e958ef31930a6ef37d0fb64e9ca3b6116e6b93a8bdae96ee1000" +checksum = "28f32d40287d3f402ae0028a9d54bef51af15c8769492826a69d28f81893151d" dependencies = [ "futures-core", "tokio", @@ -91,9 +93,9 @@ dependencies = [ [[package]] name = "actix-server" -version = "2.1.1" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0da34f8e659ea1b077bb4637948b815cd3768ad5a188fdcd74ff4d84240cd824" +checksum = "3eb13e7eef0423ea6eab0e59f6c72e7cb46d33691ad56a726b3cd07ddec2c2d4" dependencies = [ "actix-rt", "actix-service", @@ -101,8 +103,7 @@ dependencies = [ "futures-core", "futures-util", "mio", - "num_cpus", - "socket2", + "socket2 0.5.6", "tokio", "tracing", ] @@ -130,9 +131,9 @@ dependencies = [ [[package]] name = "actix-web" -version = "4.2.1" +version = "4.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d48f7b6534e06c7bfc72ee91db7917d4af6afe23e7d223b51e68fffbb21e96b9" +checksum = "43a6556ddebb638c2358714d853257ed226ece6023ef9364f23f0c70737ea984" dependencies = [ "actix-codec", "actix-http", @@ -143,7 +144,7 @@ dependencies = [ "actix-service", "actix-utils", "actix-web-codegen", - "ahash 0.7.6", + "ahash", "bytes", "bytestring", "cfg-if", @@ -152,8 +153,7 @@ dependencies = [ "encoding_rs", "futures-core", "futures-util", - "http", - "itoa 1.0.4", + "itoa", "language-tags", "log", "mime", @@ -164,47 +164,46 @@ dependencies = [ "serde_json", "serde_urlencoded", "smallvec", - "socket2", + "socket2 0.5.6", "time", "url", ] [[package]] name = "actix-web-codegen" -version = "4.1.0" +version = "4.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fa9362663c8643d67b2d5eafba49e4cb2c8a053a29ed00a0bea121f17c76b13" +checksum = "eb1f50ebbb30eca122b188319a4398b3f7bb4a8cdf50ecfb73bfc6a3c3ce54f5" dependencies = [ "actix-router", "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.59", ] [[package]] -name = "adler" -version = "1.0.2" +name = "addr2line" +version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb" +dependencies = [ + "gimli", +] [[package]] -name = "ahash" -version = "0.7.6" +name = "adler" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fcb51a0695d8f838b1ee009b3fbf66bda078cd64590202a864a8f3e8c4315c47" -dependencies = [ - "getrandom", - "once_cell", - "version_check", -] +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" [[package]] name = "ahash" -version = "0.8.7" +version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77c3a9648d43b9cd48db467b3f87fdd6e146bcc88ab0180006cef2179fe11d01" +checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" dependencies = [ "cfg-if", + "getrandom", "once_cell", "version_check", "zerocopy", @@ -212,9 +211,9 @@ dependencies = [ [[package]] name = "aho-corasick" -version = "0.7.19" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4f55bd91a0978cbfd91c457a164bab8b4001c833b7f323132c0a4e1922dd44e" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" dependencies = [ "memchr", ] @@ -236,9 +235,9 @@ dependencies = [ [[package]] name = "allocator-api2" -version = "0.2.16" +version = "0.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0942ffc6dcaadf03badf6e6a2d0228460359d5e34b57ccdc720b7382dfbd5ec5" +checksum = "5c6cb57a04249c6480766f7f7cef5467412af1490f8d1e243141daddada3264f" [[package]] name = "android-tzdata" @@ -255,32 +254,229 @@ dependencies = [ "libc", ] +[[package]] +name = "anstream" +version = "0.6.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d96bd03f33fe50a863e394ee9718a706f988b9079b20c3784fb726e7678b62fb" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8901269c6307e8d93993578286ac0edf7f195079ffff5ebdeea6a59ffb7e36bc" + +[[package]] +name = "anstyle-parse" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c75ac65da39e5fe5ab759307499ddad880d724eed2f6ce5b5e8a26f4f387928c" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e28923312444cdd728e4738b3f9c9cac739500909bb3d3c94b43551b16517648" +dependencies = [ + "windows-sys 0.52.0", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cd54b81ec8d6180e24654d0b371ad22fc3dd083b6ff8ba325b72e00c87660a7" +dependencies = [ + "anstyle", + "windows-sys 0.52.0", +] + [[package]] name = "anyhow" -version = "1.0.66" +version = "1.0.82" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f538837af36e6f6a9be0faa67f9a314f8119e4e4b5867c6ab40ed60360142519" + +[[package]] +name = "async-channel" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81953c529336010edd6d8e358f886d9581267795c61b19475b71314bffa46d35" +dependencies = [ + "concurrent-queue", + "event-listener 2.5.3", + "futures-core", +] + +[[package]] +name = "async-channel" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "136d4d23bcc79e27423727b36823d86233aad06dfea531837b038394d11e9928" +dependencies = [ + "concurrent-queue", + "event-listener 5.3.0", + "event-listener-strategy 0.5.1", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-executor" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b10202063978b3351199d68f8b22c4e47e4b1b822f8d43fd862d5ea8c006b29a" +dependencies = [ + "async-task", + "concurrent-queue", + "fastrand 2.0.2", + "futures-lite 2.3.0", + "slab", +] + +[[package]] +name = "async-global-executor" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "216261ddc8289130e551ddcd5ce8a064710c0d064a4d2895c67151c92b5443f6" +checksum = "05b1b633a2115cd122d73b955eadd9916c18c8f510ec9cd1686404c60ad1c29c" +dependencies = [ + "async-channel 2.2.1", + "async-executor", + "async-io 2.3.2", + "async-lock 3.3.0", + "blocking", + "futures-lite 2.3.0", + "once_cell", +] + +[[package]] +name = "async-io" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fc5b45d93ef0529756f812ca52e44c221b35341892d3dcc34132ac02f3dd2af" +dependencies = [ + "async-lock 2.8.0", + "autocfg", + "cfg-if", + "concurrent-queue", + "futures-lite 1.13.0", + "log", + "parking", + "polling 2.8.0", + "rustix 0.37.27", + "slab", + "socket2 0.4.10", + "waker-fn", +] + +[[package]] +name = "async-io" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcccb0f599cfa2f8ace422d3555572f47424da5648a4382a9dd0310ff8210884" +dependencies = [ + "async-lock 3.3.0", + "cfg-if", + "concurrent-queue", + "futures-io", + "futures-lite 2.3.0", + "parking", + "polling 3.6.0", + "rustix 0.38.32", + "slab", + "tracing", + "windows-sys 0.52.0", +] + +[[package]] +name = "async-lock" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "287272293e9d8c41773cec55e365490fe034813a2f172f502d6ddcf75b2f582b" +dependencies = [ + "event-listener 2.5.3", +] + +[[package]] +name = "async-lock" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d034b430882f8381900d3fe6f0aaa3ad94f2cb4ac519b429692a1bc2dda4ae7b" +dependencies = [ + "event-listener 4.0.3", + "event-listener-strategy 0.4.0", + "pin-project-lite", +] + +[[package]] +name = "async-std" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62565bb4402e926b29953c785397c6dc0391b7b446e45008b0049eb43cec6f5d" +dependencies = [ + "async-channel 1.9.0", + "async-global-executor", + "async-io 1.13.0", + "async-lock 2.8.0", + "crossbeam-utils", + "futures-channel", + "futures-core", + "futures-io", + "futures-lite 1.13.0", + "gloo-timers", + "kv-log-macro", + "log", + "memchr", + "once_cell", + "pin-project-lite", + "pin-utils", + "slab", + "wasm-bindgen-futures", +] + +[[package]] +name = "async-task" +version = "4.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbb36e985947064623dbd357f727af08ffd077f93d696782f3c56365fa2e2799" [[package]] name = "async-trait" -version = "0.1.77" +version = "0.1.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c980ee35e870bd1a4d2c8294d4c04d0499e67bca1e4b5cefcc693c2fa00caea9" +checksum = "c6fa2087f2753a7da8cc1c0dbfcf89579dd57458e36769de5ac750b4671737ca" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.59", ] [[package]] name = "atoi" -version = "1.0.0" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7c57d12312ff59c811c0643f4d80830505833c9ffaebd193d819392b265be8e" +checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" dependencies = [ "num-traits", ] +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + [[package]] name = "atty" version = "0.2.14" @@ -294,15 +490,36 @@ dependencies = [ [[package]] name = "autocfg" -version = "1.1.0" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1fdabc7756949593fe60f30ec81974b613357de856987752631dea1e3394c80" + +[[package]] +name = "backtrace" +version = "0.3.71" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +checksum = "26b05800d2e817c8b3b4b54abd461726265fa9789ae34330622f2db9ee696f9d" +dependencies = [ + "addr2line", + "cc", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", +] [[package]] name = "base64" -version = "0.13.1" +version = "0.21.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" +checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" + +[[package]] +name = "base64ct" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" [[package]] name = "bitflags" @@ -312,57 +529,64 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.4.1" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07" +checksum = "cf4b9d6a944f767f8e5e0db018570623c85f3d925ac718db4e06d0187adb21c1" +dependencies = [ + "serde", +] [[package]] name = "block-buffer" -version = "0.10.3" +version = "0.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69cce20737498f97b993470a6e536b8523f0af7892a4f928cceb1ac5e52ebe7e" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" dependencies = [ "generic-array", ] [[package]] -name = "brotli" -version = "3.3.4" +name = "blocking" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1a0b1dbcc8ae29329621f8d4f0d835787c1c38bb1401979b49d13b0b305ff68" +checksum = "6a37913e8dc4ddcc604f0c6d3bf2887c995153af3611de9e23c352b44c1b9118" dependencies = [ - "alloc-no-stdlib", - "alloc-stdlib", - "brotli-decompressor", + "async-channel 2.2.1", + "async-lock 3.3.0", + "async-task", + "fastrand 2.0.2", + "futures-io", + "futures-lite 2.3.0", + "piper", + "tracing", ] [[package]] -name = "brotli-decompressor" -version = "2.3.2" +name = "brotli" +version = "3.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59ad2d4653bf5ca36ae797b1f4bb4dbddb60ce49ca4aed8a2ce4829f60425b80" +checksum = "d640d25bc63c50fb1f0b545ffd80207d2e10a4c965530809b40ba3386825c391" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", + "brotli-decompressor", ] [[package]] -name = "bstr" -version = "0.2.17" +name = "brotli-decompressor" +version = "2.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba3569f383e8f1598449f1a423e72e99569137b47740b1da11ef19af3d5c3223" +checksum = "4e2e4afe60d7dd600fdd3de8d0f08c2b7ec039712e3b6137ff98b7004e82de4f" dependencies = [ - "lazy_static", - "memchr", - "regex-automata", - "serde", + "alloc-no-stdlib", + "alloc-stdlib", ] [[package]] name = "bumpalo" -version = "3.11.1" +version = "3.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "572f695136211188308f16ad2ca5c851a712c464060ae6974944458eb83880ba" +checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" [[package]] name = "byteorder" @@ -372,15 +596,15 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.2.1" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec8a7b6a70fde80372154c65702f00a0f56f3e1c36abbc6c440484be248856db" +checksum = "514de17de45fdb8dc022b1a7975556c53c86f9f0aa5f534b98977b171857c2c9" [[package]] name = "bytestring" -version = "1.2.0" +version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7f83e57d9154148e355404702e2694463241880b939570d7c97c014da7a69a1" +checksum = "74d80203ea6b29df88012294f62733de21cfeab47f17b41af3a38bc30a03ee72" dependencies = [ "bytes", ] @@ -393,11 +617,12 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cc" -version = "1.0.76" +version = "1.0.94" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76a284da2e6fe2092f2353e51713435363112dfd60030e22add80be333fb928f" +checksum = "17f6e324229dc011159fcc089755d1e2e216a90d43a7dea6853ca740b84f35e7" dependencies = [ "jobserver", + "libc", ] [[package]] @@ -408,9 +633,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "chrono" -version = "0.4.31" +version = "0.4.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f2c685bad3eb3d45a01354cedb7d5faa66194d1d58ba6e267a8de788f79db38" +checksum = "a21f936df1771bf62b77f047b726c4625ff2e8aa607c01ec06e5a05bd8463401" dependencies = [ "android-tzdata", "iana-time-zone", @@ -418,7 +643,7 @@ dependencies = [ "num-traits", "serde", "wasm-bindgen", - "windows-targets 0.48.5", + "windows-targets 0.52.5", ] [[package]] @@ -434,51 +659,65 @@ dependencies = [ [[package]] name = "clap" -version = "4.0.27" +version = "4.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0acbd8d28a0a60d7108d7ae850af6ba34cf2d1257fc646980e5f97ce14275966" +checksum = "90bc066a67923782aa8515dbaea16946c5bcc5addbd668bb80af688e53e548a0" dependencies = [ - "bitflags 1.3.2", + "clap_builder", "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae129e2e766ae0ec03484e609954119f123cc1fe650337e155d03b022f24f7b4" +dependencies = [ + "anstream", + "anstyle", "clap_lex", - "is-terminal", - "once_cell", "strsim", - "termcolor", ] [[package]] name = "clap_derive" -version = "4.0.21" +version = "4.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0177313f9f02afc995627906bbd8967e2be069f5261954222dac78290c2b9014" +checksum = "528131438037fd55894f62d6e9f068b8f45ac57ffa77517819645d10aed04f64" dependencies = [ - "heck", - "proc-macro-error", + "heck 0.5.0", "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.59", ] [[package]] name = "clap_lex" -version = "0.3.0" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d4198f73e42b4936b35b5bb248d81d2b595ecb170da0bac7655c54eedfa8da8" -dependencies = [ - "os_str_bytes", -] +checksum = "98cc8fbded0c607b7ba9dd60cd98df59af97e84d24e49c8557331cfc26d301ce" [[package]] -name = "codespan-reporting" -version = "0.11.1" +name = "colorchoice" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3538270d33cc669650c4b093848450d380def10c331d38c768e34cac80576e6e" +checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" + +[[package]] +name = "concurrent-queue" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d16048cd947b08fa32c24458a22f5dc5e835264f689f4f5653210c69fd107363" dependencies = [ - "termcolor", - "unicode-width", + "crossbeam-utils", ] +[[package]] +name = "const-oid" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" + [[package]] name = "convert_case" version = "0.4.0" @@ -487,25 +726,15 @@ checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e" [[package]] name = "cookie" -version = "0.16.1" +version = "0.16.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "344adc371239ef32293cb1c4fe519592fcf21206c79c02854320afcdf3ab4917" +checksum = "e859cd57d0710d9e06c381b550c06e76992472a8c6d527aecd2fc673dcc231fb" dependencies = [ "percent-encoding", "time", "version_check", ] -[[package]] -name = "core-foundation" -version = "0.9.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" -dependencies = [ - "core-foundation-sys", - "libc", -] - [[package]] name = "core-foundation-sys" version = "0.8.6" @@ -514,18 +743,18 @@ checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" [[package]] name = "cpufeatures" -version = "0.2.5" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28d997bd5e24a5928dd43e46dc529867e207907fe0b239c3477d924f7f2ca320" +checksum = "53fe5e26ff1b7aef8bca9c6080520cfb8d9333c7568e1829cef191a9723e5504" dependencies = [ "libc", ] [[package]] name = "crc" -version = "3.0.1" +version = "3.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86ec7a15cbe22e59248fc7eadb1907dab5ba09372595da4d73dd805ed4417dfe" +checksum = "69e6e4d7b33a94f0991c26729976b10ebde1d34c3ee82408fb536164fa10d636" dependencies = [ "crc-catalog", ] @@ -538,9 +767,9 @@ checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" [[package]] name = "crc32fast" -version = "1.3.2" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" +checksum = "b3855a8a784b474f333699ef2bbca9db2c4a1f6d9088a90a2d25b1eb53111eaa" dependencies = [ "cfg-if", ] @@ -581,15 +810,6 @@ dependencies = [ "itertools 0.10.5", ] -[[package]] -name = "crossbeam-channel" -version = "0.5.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab3db02a9c5b5121e1e42fbdb1aeb65f5e02624cc58c43f2884c6ccac0b82f95" -dependencies = [ - "crossbeam-utils", -] - [[package]] name = "crossbeam-deque" version = "0.8.5" @@ -636,68 +856,43 @@ dependencies = [ [[package]] name = "csv" -version = "1.1.6" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22813a6dc45b335f9bade10bf7271dc477e81113e89eb251a0bc2a8a81c536e1" +checksum = "ac574ff4d437a7b5ad237ef331c17ccca63c46479e5b5453eb8e10bb99a759fe" dependencies = [ - "bstr", "csv-core", - "itoa 0.4.8", + "itoa", "ryu", "serde", ] [[package]] name = "csv-core" -version = "0.1.10" +version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b2466559f260f48ad25fe6317b3c8dac77b5bdb5763ac7d9d6103530663bc90" +checksum = "5efa2b3d7902f4b634a20cae3c9c4e6209dc4779feb6863329607560143efa70" dependencies = [ "memchr", ] [[package]] -name = "cxx" -version = "1.0.97" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e88abab2f5abbe4c56e8f1fb431b784d710b709888f35755a160e62e33fe38e8" -dependencies = [ - "cc", - "cxxbridge-flags", - "cxxbridge-macro", - "link-cplusplus", -] - -[[package]] -name = "cxx-build" -version = "1.0.97" +name = "der" +version = "0.7.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c0c11acd0e63bae27dcd2afced407063312771212b7a823b4fd72d633be30fb" +checksum = "f55bf8e7b65898637379c1b74eb1551107c8294ed26d855ceb9fd1a09cfc9bc0" dependencies = [ - "cc", - "codespan-reporting", - "once_cell", - "proc-macro2", - "quote", - "scratch", - "syn 2.0.48", + "const-oid", + "pem-rfc7468", + "zeroize", ] [[package]] -name = "cxxbridge-flags" -version = "1.0.97" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d3816ed957c008ccd4728485511e3d9aaf7db419aa321e3d2c5a2f3411e36c8" - -[[package]] -name = "cxxbridge-macro" -version = "1.0.97" +name = "deranged" +version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a26acccf6f445af85ea056362561a24ef56cdc15fcc685f03aec50b9c702cb6d" +checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4" dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.48", + "powerfmt", ] [[package]] @@ -720,30 +915,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer", + "const-oid", "crypto-common", "subtle", ] -[[package]] -name = "dirs" -version = "4.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca3aa72a6f96ea37bbc5aa912f6788242832f75369bdfdadcb0e38423f100059" -dependencies = [ - "dirs-sys", -] - -[[package]] -name = "dirs-sys" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b1d1d91c932ef41c0f2663aa8b0ca0342d444d842c06914aa0a7e352d0bada6" -dependencies = [ - "libc", - "redox_users", - "winapi", -] - [[package]] name = "dotenvy" version = "0.15.7" @@ -752,15 +928,18 @@ checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" [[package]] name = "either" -version = "1.8.0" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90e5c1c8368803113bf0c9584fc495a58b86dc8a29edbf8fe877d21d9507e797" +checksum = "a47c1c47d2f5964e29c61246e81db715514cd532db6b5116a25ea3c03d6780a2" +dependencies = [ + "serde", +] [[package]] name = "encoding_rs" -version = "0.8.31" +version = "0.8.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9852635589dc9f9ea1b6fe9f05b50ef208c85c834a562f0c6abb1c475736ec2b" +checksum = "b45de904aa0b010bce2ab45264d0631681847fa7b6f2eaa7dab7619943bc4f59" dependencies = [ "cfg-if", ] @@ -773,46 +952,87 @@ checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" [[package]] name = "errno" -version = "0.2.8" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f639046355ee4f37944e44f60642c6f3a7efa3cf6b78c78a0d989a8ce6c396a1" +checksum = "a258e46cdc063eb8519c00b9fc845fc47bcfca4130e2f08e88665ceda8474245" dependencies = [ - "errno-dragonfly", "libc", - "winapi", + "windows-sys 0.52.0", ] [[package]] -name = "errno" -version = "0.3.7" +name = "etcetera" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f258a7194e7f7c2a7837a8913aeab7fd8c383457034fa20ce4dd3dcb813e8eb8" +checksum = "136d1b5283a1ab77bd9257427ffd09d8667ced0570b6f938942bc7568ed5b943" dependencies = [ - "libc", + "cfg-if", + "home", "windows-sys 0.48.0", ] [[package]] -name = "errno-dragonfly" -version = "0.1.2" +name = "event-listener" +version = "2.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf" +checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" + +[[package]] +name = "event-listener" +version = "4.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67b215c49b2b248c855fb73579eb1f4f26c38ffdc12973e20e07b91d78d5646e" dependencies = [ - "cc", - "libc", + "concurrent-queue", + "parking", + "pin-project-lite", ] [[package]] name = "event-listener" -version = "2.5.3" +version = "5.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" +checksum = "6d9944b8ca13534cdfb2800775f8dd4902ff3fc75a50101466decadfdf322a24" +dependencies = [ + "concurrent-queue", + "parking", + "pin-project-lite", +] + +[[package]] +name = "event-listener-strategy" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "958e4d70b6d5e81971bebec42271ec641e7ff4e170a6fa605f2b8a8b65cb97d3" +dependencies = [ + "event-listener 4.0.3", + "pin-project-lite", +] + +[[package]] +name = "event-listener-strategy" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "332f51cb23d20b0de8458b86580878211da09bcd4503cb579c225b3d124cabb3" +dependencies = [ + "event-listener 5.3.0", + "pin-project-lite", +] [[package]] name = "fastrand" -version = "2.0.1" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5" +checksum = "e51093e27b0797c359783294ca4f0a911c270184cb10f85783b118614a1501be" +dependencies = [ + "instant", +] + +[[package]] +name = "fastrand" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "658bd65b1cf4c852a3cc96f18a8ce7b5640f6b703f905c7d74532294c2a63984" [[package]] name = "finl_unicode" @@ -822,9 +1042,9 @@ checksum = "8fcfdc7a0362c9f4444381a9e697c79d435fe65b52a37466fc2c1184cee9edc6" [[package]] name = "flate2" -version = "1.0.24" +version = "1.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f82b0f4c27ad9f8bfd1f3208d882da2b09c301bc1c828fd3a00d0216d2fbbff6" +checksum = "46303f565772937ffe1d394a4fac6f411c6013172fadde9dcdb1e147a086940e" dependencies = [ "crc32fast", "miniz_oxide", @@ -832,14 +1052,13 @@ dependencies = [ [[package]] name = "flume" -version = "0.10.14" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1657b4441c3403d9f7b3409e47575237dac27b1b5726df654a6ecbf92f0f7577" +checksum = "55ac459de2512911e4b674ce33cf20befaba382d05b62b008afc1c8b57cbf181" dependencies = [ "futures-core", "futures-sink", - "pin-project", - "spin", + "spin 0.9.8", ] [[package]] @@ -848,26 +1067,11 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" -[[package]] -name = "foreign-types" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" -dependencies = [ - "foreign-types-shared", -] - -[[package]] -name = "foreign-types-shared" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" - [[package]] name = "form_urlencoded" -version = "1.1.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9c384f161156f5260c24a097c56119f9be8c798586aecc13afbcbe7b7e26bf8" +checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" dependencies = [ "percent-encoding", ] @@ -890,9 +1094,9 @@ checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" [[package]] name = "futures-executor" -version = "0.3.25" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7acc85df6714c176ab5edf386123fafe217be88c0840ec11f199441134a074e2" +checksum = "a576fc72ae164fca6b9db127eaa9a9dda0d61316034f33a0a0d4eda41f02b01d" dependencies = [ "futures-core", "futures-task", @@ -901,13 +1105,47 @@ dependencies = [ [[package]] name = "futures-intrusive" -version = "0.4.2" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a604f7a68fbf8103337523b1fadc8ade7361ee3f112f7c680ad179651616aed5" +checksum = "1d930c203dd0b6ff06e0201a4a2fe9149b43c684fd4420555b26d21b1a02956f" dependencies = [ "futures-core", "lock_api", - "parking_lot 0.11.2", + "parking_lot", +] + +[[package]] +name = "futures-io" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" + +[[package]] +name = "futures-lite" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49a9d51ce47660b1e808d3c990b4709f2f415d928835a17dfd16991515c46bce" +dependencies = [ + "fastrand 1.9.0", + "futures-core", + "futures-io", + "memchr", + "parking", + "pin-project-lite", + "waker-fn", +] + +[[package]] +name = "futures-lite" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52527eb5074e35e9339c6b4e8d12600c7128b68fb25dcb9fa9dec18f7c25f3a5" +dependencies = [ + "fastrand 2.0.2", + "futures-core", + "futures-io", + "parking", + "pin-project-lite", ] [[package]] @@ -918,19 +1156,21 @@ checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5" [[package]] name = "futures-task" -version = "0.3.25" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ffb393ac5d9a6eaa9d3fdf37ae2776656b706e200c8e16b1bdb227f5198e6ea" +checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" [[package]] name = "futures-util" -version = "0.3.25" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "197676987abd2f9cadff84926f410af1c183608d36641465df73ae8211dc65d6" +checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" dependencies = [ "futures-core", + "futures-io", "futures-sink", "futures-task", + "memchr", "pin-project-lite", "pin-utils", "slab", @@ -938,9 +1178,9 @@ dependencies = [ [[package]] name = "generic-array" -version = "0.14.6" +version = "0.14.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bff49e947297f3312447abdca79f45f4738097cc82b06e72054d2223f601f1b9" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" dependencies = [ "typenum", "version_check", @@ -948,15 +1188,21 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.8" +version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c05aeb6a22b8f62540c194aac980f2115af067bfe15a0734d7277a768d396b31" +checksum = "94b22e06ecb0110981051723910cbf0b5f5e09a2062dd7663334ee79a9d1286c" dependencies = [ "cfg-if", "libc", "wasi", ] +[[package]] +name = "gimli" +version = "0.28.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" + [[package]] name = "git2" version = "0.17.2" @@ -972,11 +1218,23 @@ dependencies = [ "url", ] +[[package]] +name = "gloo-timers" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b995a66bb87bebce9a0f4a95aed01daca4872c050bfcb21653361c03bc35e5c" +dependencies = [ + "futures-channel", + "futures-core", + "js-sys", + "wasm-bindgen", +] + [[package]] name = "h2" -version = "0.3.15" +version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f9f29bc9dda355256b2916cf526ab02ce0aeaaaf2bad60d65ef3f12f11dd0f4" +checksum = "81fe527a889e1532da5c525686d96d4c2e74cdd345badf8dfef9f6b39dd5f5e8" dependencies = [ "bytes", "fnv", @@ -984,7 +1242,7 @@ dependencies = [ "futures-sink", "futures-util", "http", - "indexmap 1.9.2", + "indexmap", "slab", "tokio", "tokio-util", @@ -993,23 +1251,17 @@ dependencies = [ [[package]] name = "half" -version = "1.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eabb4a44450da02c90444cf74558da904edde8fb4e9035a9a6a4e15445af0bd7" - -[[package]] -name = "hashbrown" -version = "0.12.3" +version = "1.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" +checksum = "1b43ede17f21864e81be2fa654110bf1e793774238d86ef8555c37e6519c0403" [[package]] name = "hashbrown" -version = "0.14.2" +version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f93e7192158dbcda357bdec5fb5788eebf8bbac027f3f33e719d29135ae84156" +checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" dependencies = [ - "ahash 0.8.7", + "ahash", "allocator-api2", ] @@ -1019,18 +1271,24 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8094feaf31ff591f651a2664fb9cfd92bba7a60ce3197265e9482ebe753c8f7" dependencies = [ - "hashbrown 0.14.2", + "hashbrown", ] [[package]] name = "heck" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2540771e65fc8cb83cd6e8a237f70c319bd5c29f78ed1084ba5d50eeac86f7f9" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" dependencies = [ "unicode-segmentation", ] +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + [[package]] name = "hermit-abi" version = "0.1.19" @@ -1042,12 +1300,9 @@ dependencies = [ [[package]] name = "hermit-abi" -version = "0.2.6" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee512640fe35acbfb4bb779db6f0d80704c2cacfa2e39b601ef3e3f47d1ae4c7" -dependencies = [ - "libc", -] +checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" [[package]] name = "hex" @@ -1073,15 +1328,24 @@ dependencies = [ "digest", ] +[[package]] +name = "home" +version = "0.5.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3d1354bf6b7235cb4a0576c2619fd4ed18183f689b12b006a0ee7329eeff9a5" +dependencies = [ + "windows-sys 0.52.0", +] + [[package]] name = "http" -version = "0.2.8" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75f43d41e26995c17e71ee126451dd3941010b0514a81a9d11f3b341debc2399" +checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" dependencies = [ "bytes", "fnv", - "itoa 1.0.4", + "itoa", ] [[package]] @@ -1092,15 +1356,15 @@ checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" [[package]] name = "httpdate" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" [[package]] name = "iana-time-zone" -version = "0.1.59" +version = "0.1.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6a67363e2aa4443928ce15e57ebae94fd8949958fd1223c4cfc0cd473ad7539" +checksum = "e7ffbb5a1b541ea2561f8c41c087286cc091e21e556a4f09a8f6cbf17b69b141" dependencies = [ "android_system_properties", "core-foundation-sys", @@ -1112,19 +1376,18 @@ dependencies = [ [[package]] name = "iana-time-zone-haiku" -version = "0.1.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0703ae284fc167426161c2e3f1da3ea71d94b21bedbcc9494e92b28e334e3dca" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" dependencies = [ - "cxx", - "cxx-build", + "cc", ] [[package]] name = "idna" -version = "0.3.0" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e14ddfc70884202db2244c223200c204c2bda1bc6e0998d11b5e024d657209e6" +checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6" dependencies = [ "unicode-bidi", "unicode-normalization", @@ -1132,22 +1395,12 @@ dependencies = [ [[package]] name = "indexmap" -version = "1.9.2" +version = "2.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1885e79c1fc4b10f0e172c475f458b7f7b93061064d98c3293e98c5ba0c8b399" -dependencies = [ - "autocfg", - "hashbrown 0.12.3", -] - -[[package]] -name = "indexmap" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d530e1a18b1cb4c484e6e34556a0d948706958449fca0cab753d649f2bce3d1f" +checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26" dependencies = [ "equivalent", - "hashbrown 0.14.2", + "hashbrown", ] [[package]] @@ -1161,24 +1414,13 @@ dependencies = [ [[package]] name = "io-lifetimes" -version = "1.0.2" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e394faa0efb47f9f227f1cd89978f854542b318a6f64fa695489c9c993056656" +checksum = "eae7b9aee968036d54dce06cebaefd919e4472e753296daccd6d344e3e2df0c2" dependencies = [ + "hermit-abi 0.3.9", "libc", - "windows-sys 0.42.0", -] - -[[package]] -name = "is-terminal" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aae5bc6e2eb41c9def29a3e0f1306382807764b9b53112030eff57435667352d" -dependencies = [ - "hermit-abi 0.2.6", - "io-lifetimes", - "rustix 0.36.3", - "windows-sys 0.42.0", + "windows-sys 0.48.0", ] [[package]] @@ -1192,43 +1434,46 @@ dependencies = [ [[package]] name = "itertools" -version = "0.12.0" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25db6b064527c5d482d0423354fcd07a89a2dfe07b67892e62411946db7f07b0" +checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" dependencies = [ "either", ] [[package]] name = "itoa" -version = "0.4.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4" - -[[package]] -name = "itoa" -version = "1.0.4" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4217ad341ebadf8d8e724e264f13e593e0648f5b3e94b3896a5df283be015ecc" +checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" [[package]] name = "jobserver" -version = "0.1.25" +version = "0.1.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "068b1ee6743e4d11fb9c6a1e6064b3693a1b600e7f5f5988047d98b3dc9fb90b" +checksum = "685a7d121ee3f65ae4fddd72b25a04bb36b6af81bc0828f7d5434c0fe60fa3a2" dependencies = [ "libc", ] [[package]] name = "js-sys" -version = "0.3.60" +version = "0.3.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49409df3e3bf0856b916e2ceaca09ee28e6871cf7d9ce97a692cacfdb2a25a47" +checksum = "29c15563dc2726973df627357ce0c9ddddbea194836909d655df6a75d2cf296d" dependencies = [ "wasm-bindgen", ] +[[package]] +name = "kv-log-macro" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0de8b303297635ad57c9f5059fd9cee7a47f8e8daa09df0fcd07dd39fb22977f" +dependencies = [ + "log", +] + [[package]] name = "language-tags" version = "0.3.2" @@ -1240,12 +1485,15 @@ name = "lazy_static" version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" +dependencies = [ + "spin 0.5.2", +] [[package]] name = "libc" -version = "0.2.150" +version = "0.2.153" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89d92a4743f9a61002fae18374ed11e7973f530cb3a3255fb354818118b2203c" +checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" [[package]] name = "libgit2-sys" @@ -1262,21 +1510,16 @@ dependencies = [ ] [[package]] -name = "libredox" -version = "0.0.1" +name = "libm" +version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85c833ca1e66078851dba29046874e38f08b2c883700aa29a03ddd3b23814ee8" -dependencies = [ - "bitflags 2.4.1", - "libc", - "redox_syscall 0.4.1", -] +checksum = "4ec2a862134d2a7d32d7983ddcdd1c4923530833c9f2ea1a44fc5fa473989058" [[package]] name = "libsqlite3-sys" -version = "0.24.2" +version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "898745e570c7d0453cc1fbc4a701eb6c662ed54e8fec8b7d14be137ebeeb9d14" +checksum = "cf4e226dcd58b4be396f7bd3c20da8fdee2911400705297ba7d2d7cc2c30f716" dependencies = [ "cc", "pkg-config", @@ -1299,9 +1542,9 @@ dependencies = [ [[package]] name = "libz-sys" -version = "1.1.8" +version = "1.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9702761c3935f8cc2f101793272e202c72b99da8f4224a19ddcf1279a6450bbf" +checksum = "5e143b5e666b2695d28f6bca6497720813f699c9602dd7f5cac91008b8ada7f9" dependencies = [ "cc", "libc", @@ -1309,50 +1552,40 @@ dependencies = [ "vcpkg", ] -[[package]] -name = "link-cplusplus" -version = "1.0.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d240c6f7e1ba3a28b0249f774e6a9dd0175054b52dfbb61b16eb8505c3785c9" -dependencies = [ - "cc", -] - [[package]] name = "linux-raw-sys" -version = "0.1.3" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f9f08d8963a6c613f4b1a78f4f4a4dbfadf8e6545b2d72861731e4858b8b47f" +checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519" [[package]] name = "linux-raw-sys" -version = "0.4.11" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "969488b55f8ac402214f3f5fd243ebb7206cf82de60d3172994707a4bcc2b829" +checksum = "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c" [[package]] name = "local-channel" -version = "0.1.3" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f303ec0e94c6c54447f84f3b0ef7af769858a9c4ef56ef2a986d3dcd4c3fc9c" +checksum = "b6cbc85e69b8df4b8bb8b89ec634e7189099cea8927a276b7384ce5488e53ec8" dependencies = [ "futures-core", "futures-sink", - "futures-util", "local-waker", ] [[package]] name = "local-waker" -version = "0.1.3" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e34f76eb3611940e0e7d53a9aaa4e6a3151f69541a282fd0dad5571420c53ff1" +checksum = "4d873d7c67ce09b42110d801813efbc9364414e356be9935700d368351657487" [[package]] name = "lock_api" -version = "0.4.9" +version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "435011366fe56583b16cf956f9df0095b405b82d76425bc8981c0e22e60ec4df" +checksum = "3c168f8615b12bc01f9c17e2eb0cc07dcae1940121185446edc3744920e8ef45" dependencies = [ "autocfg", "scopeguard", @@ -1360,27 +1593,28 @@ dependencies = [ [[package]] name = "log" -version = "0.4.17" +version = "0.4.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" +checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c" dependencies = [ - "cfg-if", + "value-bag", ] [[package]] name = "md-5" -version = "0.10.5" +version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6365506850d44bff6e2fbcb5176cf63650e48bd45ef2fe2665ae1570e0f4b9ca" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" dependencies = [ + "cfg-if", "digest", ] [[package]] name = "memchr" -version = "2.5.0" +version = "2.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" +checksum = "6c8640c5d730cb13ebd907d8d04b52f55ac9a2eec55b440c8892f40d56c76c1d" [[package]] name = "mime" @@ -1406,23 +1640,23 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "miniz_oxide" -version = "0.5.4" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96590ba8f175222643a85693f33d26e9c8a015f599c216509b1a6894af675d34" +checksum = "9d811f3e15f28568be3407c8e7fdb6514c1cda3cb30683f15b6a1a1dc4ea14a7" dependencies = [ "adler", ] [[package]] name = "mio" -version = "0.8.5" +version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5d732bc30207a6423068df043e3d02e0735b155ad7ce1a6f76fe2baa5b158de" +checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c" dependencies = [ "libc", "log", "wasi", - "windows-sys 0.42.0", + "windows-sys 0.48.0", ] [[package]] @@ -1431,24 +1665,6 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8bc45ce96192b5d8b20cffb10ccd85cc431c283a7d171a0d843ac0bd7d444598" -[[package]] -name = "native-tls" -version = "0.2.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07226173c32f2926027b63cce4bcd8076c3552846cbe7925f3aaffeac0a3b92e" -dependencies = [ - "lazy_static", - "libc", - "log", - "openssl", - "openssl-probe", - "openssl-sys", - "schannel", - "security-framework", - "security-framework-sys", - "tempfile", -] - [[package]] name = "nom" version = "7.1.3" @@ -1470,62 +1686,79 @@ dependencies = [ ] [[package]] -name = "num-traits" -version = "0.2.15" +name = "num-bigint-dig" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" +checksum = "dc84195820f291c7697304f3cbdadd1cb7199c0efc917ff5eafd71225c136151" dependencies = [ - "autocfg", + "byteorder", + "lazy_static", + "libm", + "num-integer", + "num-iter", + "num-traits", + "rand", + "smallvec", + "zeroize", ] [[package]] -name = "num_cpus" -version = "1.14.0" +name = "num-conv" +version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6058e64324c71e02bc2b150e4f3bc8286db6c83092132ffa3f6b1eab0f9def5" -dependencies = [ - "hermit-abi 0.1.19", - "libc", -] +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" [[package]] -name = "once_cell" -version = "1.19.0" +name = "num-integer" +version = "0.1.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] [[package]] -name = "oorandom" -version = "11.1.3" +name = "num-iter" +version = "0.1.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575" +checksum = "d869c01cc0c455284163fd0092f1f93835385ccab5a98a0dcc497b2f8bf055a9" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] [[package]] -name = "openssl" -version = "0.10.42" +name = "num-traits" +version = "0.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12fc0523e3bd51a692c8850d075d74dc062ccf251c0110668cbd921917118a13" +checksum = "da0df0e5185db44f69b44f26786fe401b6c293d1907744beaa7fa62b2e5a517a" dependencies = [ - "bitflags 1.3.2", - "cfg-if", - "foreign-types", - "libc", - "once_cell", - "openssl-macros", - "openssl-sys", + "autocfg", + "libm", ] [[package]] -name = "openssl-macros" -version = "0.1.1" +name = "object" +version = "0.32.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +checksum = "a6a622008b6e321afc04970976f62ee297fdbaa6f95318ca343e3eebb9648441" dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.48", + "memchr", ] +[[package]] +name = "once_cell" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" + +[[package]] +name = "oorandom" +version = "11.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575" + [[package]] name = "openssl-probe" version = "0.1.5" @@ -1534,23 +1767,16 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-sys" -version = "0.9.77" +version = "0.9.102" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b03b84c3b2d099b81f0953422b4d4ad58761589d0229b5506356afca05a3670a" +checksum = "c597637d56fbc83893a35eb0dd04b2b8e7a50c91e64e9493e398b5df4fb45fa2" dependencies = [ - "autocfg", "cc", "libc", "pkg-config", "vcpkg", ] -[[package]] -name = "os_str_bytes" -version = "6.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b7820b9daea5457c9f21c69448905d723fbd21136ccf521748f23fd49e723ee" - [[package]] name = "overload" version = "0.1.1" @@ -1559,26 +1785,24 @@ checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" [[package]] name = "oxilangtag" -version = "0.1.3" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d91edf4fbb970279443471345a4e8c491bf05bb283b3e6c88e4e606fd8c181b" +checksum = "23f3f87617a86af77fa3691e6350483e7154c2ead9f1261b75130e21ca0f8acb" +dependencies = [ + "serde", +] [[package]] name = "oxiri" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb175ec8981211357b7b379869c2f8d555881c55ea62311428ec0de46d89bd5c" +checksum = "d05417ee46e2eb40dd9d590b4d67fc2408208b3a48a6b7f71d2bc1d7ce12a3e0" [[package]] -name = "parking_lot" -version = "0.11.2" +name = "parking" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" -dependencies = [ - "instant", - "lock_api", - "parking_lot_core 0.8.6", -] +checksum = "bb813b8af86854136c6922af0598d719255ecb2179515e6e7730d468f05c9cae" [[package]] name = "parking_lot" @@ -1587,73 +1811,68 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" dependencies = [ "lock_api", - "parking_lot_core 0.9.4", + "parking_lot_core", ] [[package]] name = "parking_lot_core" -version = "0.8.6" +version = "0.9.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc" +checksum = "4c42a9226546d68acdd9c0a280d17ce19bfe27a46bf68784e4066115788d008e" dependencies = [ "cfg-if", - "instant", "libc", - "redox_syscall 0.2.16", + "redox_syscall", "smallvec", - "winapi", + "windows-targets 0.48.5", ] [[package]] -name = "parking_lot_core" -version = "0.9.4" +name = "paste" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4dc9e0dc2adc1c69d09143aff38d3d30c5c3f0df0dad82e6d25547af174ebec0" -dependencies = [ - "cfg-if", - "libc", - "redox_syscall 0.2.16", - "smallvec", - "windows-sys 0.42.0", -] +checksum = "de3145af08024dea9fa9914f381a17b8fc6034dfb00f3a84013f7ff43f29ed4c" [[package]] -name = "paste" -version = "1.0.9" +name = "pem-rfc7468" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1de2e551fb905ac83f73f7aedf2f0cb4a0da7e35efa24a202a936269f1f18e1" +checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" +dependencies = [ + "base64ct", +] [[package]] name = "percent-encoding" -version = "2.2.0" +version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e" +checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" [[package]] name = "pin-project" -version = "1.0.12" +version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad29a609b6bcd67fee905812e544992d216af9d755757c05ed2d0e15a74c6ecc" +checksum = "b6bf43b791c5b9e34c3d182969b4abb522f9343702850a2e57f460d00d09b4b3" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.0.12" +version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "069bdb1e05adc7a8990dce9cc75370895fbe4e3d58b9b73bf1aee56359344a55" +checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.59", ] [[package]] name = "pin-project-lite" -version = "0.2.9" +version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" +checksum = "bda66fc9667c18cb2758a2ac84d1167245054bcf85d5d1aaa6923f45801bdd02" [[package]] name = "pin-utils" @@ -1661,17 +1880,49 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "piper" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "668d31b1c4eba19242f2088b2bf3316b82ca31082a8335764db4e083db7485d4" +dependencies = [ + "atomic-waker", + "fastrand 2.0.2", + "futures-io", +] + +[[package]] +name = "pkcs1" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" +dependencies = [ + "der", + "pkcs8", + "spki", +] + +[[package]] +name = "pkcs8" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" +dependencies = [ + "der", + "spki", +] + [[package]] name = "pkg-config" -version = "0.3.26" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160" +checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec" [[package]] name = "plotters" -version = "0.3.4" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2538b639e642295546c50fcd545198c9d64ee2a38620a628724a3b266d5fbf97" +checksum = "d2c224ba00d7cadd4d5c660deaf2098e5e80e07846537c51f9cfa4be50c1fd45" dependencies = [ "num-traits", "plotters-backend", @@ -1682,54 +1933,67 @@ dependencies = [ [[package]] name = "plotters-backend" -version = "0.3.4" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "193228616381fecdc1224c62e96946dfbc73ff4384fba576e052ff8c1bea8142" +checksum = "9e76628b4d3a7581389a35d5b6e2139607ad7c75b17aed325f210aa91f4a9609" [[package]] name = "plotters-svg" -version = "0.3.3" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9a81d2759aae1dae668f783c308bc5c8ebd191ff4184aaa1b37f65a6ae5a56f" +checksum = "38f6d39893cca0701371e3c27294f09797214b86f1fb951b89ade8ec04e2abab" dependencies = [ "plotters-backend", ] [[package]] -name = "ppv-lite86" -version = "0.2.17" +name = "polling" +version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" +checksum = "4b2d323e8ca7996b3e23126511a523f7e62924d93ecd5ae73b333815b0eb3dce" +dependencies = [ + "autocfg", + "bitflags 1.3.2", + "cfg-if", + "concurrent-queue", + "libc", + "log", + "pin-project-lite", + "windows-sys 0.48.0", +] [[package]] -name = "proc-macro-error" -version = "1.0.4" +name = "polling" +version = "3.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" +checksum = "e0c976a60b2d7e99d6f229e414670a9b85d13ac305cc6d1e9c134de58c5aaaf6" dependencies = [ - "proc-macro-error-attr", - "proc-macro2", - "quote", - "syn 1.0.109", - "version_check", + "cfg-if", + "concurrent-queue", + "hermit-abi 0.3.9", + "pin-project-lite", + "rustix 0.38.32", + "tracing", + "windows-sys 0.52.0", ] [[package]] -name = "proc-macro-error-attr" -version = "1.0.4" +name = "powerfmt" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" -dependencies = [ - "proc-macro2", - "quote", - "version_check", -] +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + +[[package]] +name = "ppv-lite86" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" [[package]] name = "proc-macro2" -version = "1.0.76" +version = "1.0.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95fc56cda0b5c3325f5fbbd7ff9fda9e02bb00bb3dac51252d2f1bfa1cb8cc8c" +checksum = "a56dea16b0a29e94408b9aa5e2940a4eedbd128a1ba20e8f7ae60fd3d465af0e" dependencies = [ "unicode-ident", ] @@ -1745,9 +2009,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.35" +version = "1.0.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef" +checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7" dependencies = [ "proc-macro2", ] @@ -1784,34 +2048,22 @@ dependencies = [ [[package]] name = "rayon" -version = "1.6.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e060280438193c554f654141c9ea9417886713b7acd75974c85b18a69a88e0b" +checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" dependencies = [ - "crossbeam-deque", "either", "rayon-core", ] [[package]] name = "rayon-core" -version = "1.10.1" +version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cac410af5d00ab6884528b4ab69d1e8e146e8d471201800fa1b4524126de6ad3" +checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" dependencies = [ - "crossbeam-channel", "crossbeam-deque", "crossbeam-utils", - "num_cpus", -] - -[[package]] -name = "redox_syscall" -version = "0.2.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" -dependencies = [ - "bitflags 1.3.2", ] [[package]] @@ -1823,39 +2075,34 @@ dependencies = [ "bitflags 1.3.2", ] -[[package]] -name = "redox_users" -version = "0.4.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a18479200779601e498ada4e8c1e1f50e3ee19deb0259c25825a98b5603b2cb4" -dependencies = [ - "getrandom", - "libredox", - "thiserror", -] - [[package]] name = "regex" -version = "1.7.0" +version = "1.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e076559ef8e241f2ae3479e36f97bd5741c0330689e217ad51ce2c76808b868a" +checksum = "c117dbdfde9c8308975b6a18d71f3f385c89461f7b3fb054288ecf2a2058ba4c" dependencies = [ "aho-corasick", "memchr", + "regex-automata", "regex-syntax", ] [[package]] name = "regex-automata" -version = "0.1.10" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" +checksum = "86b83b8b9847f9bf95ef68afb0b8e6cdb80f498442f5179a29fad448fcc1eaea" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] [[package]] name = "regex-syntax" -version = "0.6.28" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "456c603be3e8d448b072f410900c09faf164fbce2d480456f50eea6e25f9c848" +checksum = "adad44e29e4c806119491a7f06f03de4d1af22c3a680dd47f1e6e179439d1f56" [[package]] name = "resiter" @@ -1892,6 +2139,32 @@ dependencies = [ "rio_api", ] +[[package]] +name = "rsa" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d0e5124fcb30e76a7e79bfee683a2746db83784b86289f6251b54b7950a0dfc" +dependencies = [ + "const-oid", + "digest", + "num-bigint-dig", + "num-integer", + "num-traits", + "pkcs1", + "pkcs8", + "rand_core", + "signature", + "spki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustc-demangle" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" + [[package]] name = "rustc_version" version = "0.4.0" @@ -1903,36 +2176,36 @@ dependencies = [ [[package]] name = "rustix" -version = "0.36.3" +version = "0.37.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b1fbb4dfc4eb1d390c02df47760bb19a84bb80b301ecc947ab5406394d8223e" +checksum = "fea8ca367a3a01fe35e6943c400addf443c0f57670e6ec51196f71a4b8762dd2" dependencies = [ "bitflags 1.3.2", - "errno 0.2.8", + "errno", "io-lifetimes", "libc", - "linux-raw-sys 0.1.3", - "windows-sys 0.42.0", + "linux-raw-sys 0.3.8", + "windows-sys 0.48.0", ] [[package]] name = "rustix" -version = "0.38.25" +version = "0.38.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc99bc2d4f1fed22595588a013687477aedf3cdcfb26558c559edb67b4d9b22e" +checksum = "65e04861e65f21776e67888bfbea442b3642beaa0138fdb1dd7a84a52dffdb89" dependencies = [ - "bitflags 2.4.1", - "errno 0.3.7", + "bitflags 2.5.0", + "errno", "libc", - "linux-raw-sys 0.4.11", - "windows-sys 0.48.0", + "linux-raw-sys 0.4.13", + "windows-sys 0.52.0", ] [[package]] name = "ryu" -version = "1.0.11" +version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4501abdff3ae82a1c1b477a17252eb69cee9e66eb915c1abaa4f44d873df9f09" +checksum = "e86697c916019a8588c99b5fac3cead74ec0b4b819707a682fd4d23fa0ce1ba1" [[package]] name = "same-file" @@ -1943,61 +2216,23 @@ dependencies = [ "winapi-util", ] -[[package]] -name = "schannel" -version = "0.1.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbc91545643bcf3a0bbb6569265615222618bdf33ce4ffbbd13c4bbd4c093534" -dependencies = [ - "windows-sys 0.52.0", -] - [[package]] name = "scopeguard" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" - -[[package]] -name = "scratch" -version = "1.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3cf7c11c38cb994f3d40e8a8cde3bbd1f72a435e4c49e85d6553d8312306152" - -[[package]] -name = "security-framework" -version = "2.9.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05b64fb303737d99b81884b2c63433e9ae28abebe5eb5045dcdd175dc2ecf4de" -dependencies = [ - "bitflags 1.3.2", - "core-foundation", - "core-foundation-sys", - "libc", - "security-framework-sys", -] - -[[package]] -name = "security-framework-sys" -version = "2.9.1" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e932934257d3b408ed8f30db49d85ea163bfe74961f017f405b025af298f0c7a" -dependencies = [ - "core-foundation-sys", - "libc", -] +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] name = "semver" -version = "1.0.14" +version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e25dfac463d778e353db5be2449d1cce89bd6fd23c9f1ea21310ce6e5a1b29c4" +checksum = "92d43fe69e652f3df9bdc2b85b2854a0825b86e4fb76bc44d945137d053639ca" [[package]] name = "serde" -version = "1.0.152" +version = "1.0.197" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb7d1f0d3021d347a83e556fc4683dea2ea09d87bccdf88ff5c12545d89d5efb" +checksum = "3fb1c873e1b9b056a4dc4c0c198b24c3ffa059243875552b2bd0933b1aee4ce2" dependencies = [ "serde_derive", ] @@ -2014,31 +2249,31 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.152" +version = "1.0.197" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af487d118eecd09402d70a5d72551860e788df87b464af30e5ea6a38c75c541e" +checksum = "7eb0b34b42edc17f6b7cac84a52a1c5f0e1bb2227e997ca9011ea3dd34e8610b" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.59", ] [[package]] name = "serde_json" -version = "1.0.88" +version = "1.0.116" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e8b3801309262e8184d9687fb697586833e939767aea0dda89f5a8e650e8bd7" +checksum = "3e17db7126d17feb94eb3fad46bf1a96b034e8aacbc2e775fe81505f8b0b2813" dependencies = [ - "itoa 1.0.4", + "itoa", "ryu", "serde", ] [[package]] name = "serde_spanned" -version = "0.6.4" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12022b835073e5b11e90a14f86838ceb1c8fb0325b72416845c487ac0fa95e80" +checksum = "eb3622f419d1296904700073ea6cc23ad690adbd66f13ea683df73298736f0c1" dependencies = [ "serde", ] @@ -2050,16 +2285,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" dependencies = [ "form_urlencoded", - "itoa 1.0.4", + "itoa", "ryu", "serde", ] [[package]] name = "sha1" -version = "0.10.5" +version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f04293dc80c3993519f2d7f6f511707ee7094fe0c6d3406feb330cdb3540eba3" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" dependencies = [ "cfg-if", "cpufeatures", @@ -2079,47 +2314,67 @@ dependencies = [ [[package]] name = "sharded-slab" -version = "0.1.4" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "900fba806f70c630b0a382d0d825e17a0f19fcd059a2ade1ff237bcddf446b31" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" dependencies = [ "lazy_static", ] [[package]] name = "signal-hook-registry" -version = "1.4.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e51e73328dc4ac0c7ccbda3a494dfa03df1de2f46018127f60c693f2648455b0" +checksum = "d8229b473baa5980ac72ef434c4415e70c4b5e71b423043adb4ba059f89c99a1" dependencies = [ "libc", ] +[[package]] +name = "signature" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" +dependencies = [ + "digest", + "rand_core", +] + [[package]] name = "slab" -version = "0.4.7" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4614a76b2a8be0058caa9dbbaf66d988527d86d003c11a94fbd335d7661edcef" +checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" dependencies = [ "autocfg", ] [[package]] name = "smallvec" -version = "1.10.0" +version = "1.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" +checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" [[package]] name = "socket2" -version = "0.4.7" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02e2d2db9033d13a1567121ddd7a095ee144db4e1ca1b1bda3419bc0da294ebd" +checksum = "9f7916fc008ca5542385b89a3d3ce689953c143e9304a9bf8beec1de48994c0d" dependencies = [ "libc", "winapi", ] +[[package]] +name = "socket2" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05ffd9c0a93b7543e062e759284fcf5f5e3b098501104bfbdde4d404db792871" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + [[package]] name = "sophia" version = "0.8.0" @@ -2260,6 +2515,12 @@ dependencies = [ "sophia_rio", ] +[[package]] +name = "spin" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" + [[package]] name = "spin" version = "0.9.8" @@ -2269,110 +2530,223 @@ dependencies = [ "lock_api", ] +[[package]] +name = "spki" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" +dependencies = [ + "base64ct", + "der", +] + [[package]] name = "sqlformat" version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ce81b7bd7c4493975347ef60d8c7e8b742d4694f4c49f93e0a12ea263938176c" dependencies = [ - "itertools 0.12.0", + "itertools 0.12.1", "nom", "unicode_categories", ] [[package]] name = "sqlx" -version = "0.6.3" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8de3b03a925878ed54a954f621e64bf55a3c1bd29652d0d1a17830405350188" +checksum = "c9a2ccff1a000a5a59cd33da541d9f2fdcd9e6e8229cc200565942bff36d0aaa" dependencies = [ "sqlx-core", "sqlx-macros", + "sqlx-mysql", + "sqlx-postgres", + "sqlx-sqlite", ] [[package]] name = "sqlx-core" -version = "0.6.3" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa8241483a83a3f33aa5fff7e7d9def398ff9990b2752b6c6112b83c6d246029" +checksum = "24ba59a9342a3d9bab6c56c118be528b27c9b60e490080e9711a04dccac83ef6" dependencies = [ - "ahash 0.7.6", + "ahash", + "async-io 1.13.0", + "async-std", "atoi", - "base64", - "bitflags 1.3.2", "byteorder", "bytes", "chrono", "crc", "crossbeam-queue", - "dirs", - "dotenvy", "either", - "event-listener", - "flume", + "event-listener 2.5.3", "futures-channel", "futures-core", - "futures-executor", "futures-intrusive", + "futures-io", "futures-util", "hashlink", "hex", - "hkdf", - "hmac", - "indexmap 1.9.2", - "itoa 1.0.4", - "libc", - "libsqlite3-sys", + "indexmap", "log", - "md-5", "memchr", "once_cell", "paste", "percent-encoding", - "rand", "serde", "serde_json", - "sha1", "sha2", "smallvec", "sqlformat", - "sqlx-rt", - "stringprep", "thiserror", - "tokio-stream", + "tracing", "url", - "whoami", ] [[package]] name = "sqlx-macros" -version = "0.6.3" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ea40e2345eb2faa9e1e5e326db8c34711317d2b5e08d0d5741619048a803127" +dependencies = [ + "proc-macro2", + "quote", + "sqlx-core", + "sqlx-macros-core", + "syn 1.0.109", +] + +[[package]] +name = "sqlx-macros-core" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9966e64ae989e7e575b19d7265cb79d7fc3cbbdf179835cb0d716f294c2049c9" +checksum = "5833ef53aaa16d860e92123292f1f6a3d53c34ba8b1969f152ef1a7bb803f3c8" dependencies = [ + "async-std", "dotenvy", "either", - "heck", + "heck 0.4.1", + "hex", "once_cell", "proc-macro2", "quote", + "serde", + "serde_json", "sha2", "sqlx-core", - "sqlx-rt", + "sqlx-mysql", + "sqlx-postgres", + "sqlx-sqlite", "syn 1.0.109", + "tempfile", "url", ] [[package]] -name = "sqlx-rt" -version = "0.6.3" +name = "sqlx-mysql" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "804d3f245f894e61b1e6263c84b23ca675d96753b5abfd5cc8597d86806e8024" +checksum = "1ed31390216d20e538e447a7a9b959e06ed9fc51c37b514b46eb758016ecd418" dependencies = [ - "native-tls", + "atoi", + "base64", + "bitflags 2.5.0", + "byteorder", + "bytes", + "chrono", + "crc", + "digest", + "dotenvy", + "either", + "futures-channel", + "futures-core", + "futures-io", + "futures-util", + "generic-array", + "hex", + "hkdf", + "hmac", + "itoa", + "log", + "md-5", + "memchr", "once_cell", - "tokio", - "tokio-native-tls", + "percent-encoding", + "rand", + "rsa", + "serde", + "sha1", + "sha2", + "smallvec", + "sqlx-core", + "stringprep", + "thiserror", + "tracing", + "whoami", +] + +[[package]] +name = "sqlx-postgres" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c824eb80b894f926f89a0b9da0c7f435d27cdd35b8c655b114e58223918577e" +dependencies = [ + "atoi", + "base64", + "bitflags 2.5.0", + "byteorder", + "chrono", + "crc", + "dotenvy", + "etcetera", + "futures-channel", + "futures-core", + "futures-io", + "futures-util", + "hex", + "hkdf", + "hmac", + "home", + "itoa", + "log", + "md-5", + "memchr", + "once_cell", + "rand", + "serde", + "serde_json", + "sha2", + "smallvec", + "sqlx-core", + "stringprep", + "thiserror", + "tracing", + "whoami", +] + +[[package]] +name = "sqlx-sqlite" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b244ef0a8414da0bed4bb1910426e890b19e5e9bccc27ada6b797d05c55ae0aa" +dependencies = [ + "atoi", + "chrono", + "flume", + "futures-channel", + "futures-core", + "futures-executor", + "futures-intrusive", + "futures-util", + "libsqlite3-sys", + "log", + "percent-encoding", + "serde", + "sqlx-core", + "tracing", + "url", + "urlencoding", ] [[package]] @@ -2385,7 +2759,7 @@ dependencies = [ "anyhow", "async-trait", "chrono", - "clap 4.0.27", + "clap 4.5.4", "criterion", "derive_more", "git2", @@ -2404,7 +2778,6 @@ dependencies = [ "tracing", "tracing-actix-web", "tracing-subscriber", - "walkdir", ] [[package]] @@ -2420,15 +2793,15 @@ dependencies = [ [[package]] name = "strsim" -version = "0.10.0" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" [[package]] name = "subtle" -version = "2.4.1" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" +checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" [[package]] name = "syn" @@ -2443,9 +2816,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.48" +version = "2.0.59" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f3531638e407dfc0814761abb7c00a5b54992b849452a0646b7f65c9f770f3f" +checksum = "4a6531ffc7b071655e4ce2e04bd464c4830bb585a61cabb96cf808f05172615a" dependencies = [ "proc-macro2", "quote", @@ -2454,24 +2827,14 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.8.1" +version = "3.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ef1adac450ad7f4b3c28589471ade84f25f731a7a0fe30d71dfa9f60fd808e5" +checksum = "85b77fafb263dd9d05cbeac119526425676db3784113aa9295c88498cbf8bff1" dependencies = [ "cfg-if", - "fastrand", - "redox_syscall 0.4.1", - "rustix 0.38.25", - "windows-sys 0.48.0", -] - -[[package]] -name = "termcolor" -version = "1.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bab24d30b911b2376f3a13cc2cd443142f0c81dda04c118693e35b3835757755" -dependencies = [ - "winapi-util", + "fastrand 2.0.2", + "rustix 0.38.32", + "windows-sys 0.52.0", ] [[package]] @@ -2485,40 +2848,44 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.56" +version = "1.0.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d54378c645627613241d077a3a79db965db602882668f9136ac42af9ecb730ad" +checksum = "03468839009160513471e86a034bb2c5c0e4baae3b43f79ffc55c4a5427b3297" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.56" +version = "1.0.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa0faa943b50f3db30a20aa7e265dbc66076993efed8463e8de414e5d06d3471" +checksum = "c61f3ba182994efc43764a46c018c347bc492c79f024e705f46567b418f6d4f7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.59", ] [[package]] name = "thread_local" -version = "1.1.4" +version = "1.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5516c27b78311c50bf42c071425c560ac799b11c30b31f87e3081965fe5e0180" +checksum = "8b9ef9bad013ada3808854ceac7b46812a6465ba368859a37e2100283d2d719c" dependencies = [ + "cfg-if", "once_cell", ] [[package]] name = "time" -version = "0.3.17" +version = "0.3.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a561bf4617eebd33bca6434b988f39ed798e527f51a1e797d0ee4f61c0a38376" +checksum = "5dfd88e563464686c916c7e46e623e520ddc6d79fa6641390f2e3fa86e83e885" dependencies = [ - "itoa 1.0.4", + "deranged", + "itoa", + "num-conv", + "powerfmt", "serde", "time-core", "time-macros", @@ -2526,16 +2893,17 @@ dependencies = [ [[package]] name = "time-core" -version = "0.1.0" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e153e1f1acaef8acc537e68b44906d2db6436e2b35ac2c6b42640fff91f00fd" +checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" [[package]] name = "time-macros" -version = "0.2.6" +version = "0.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d967f99f534ca7e495c575c62638eebc2898a8c84c119b89e250477bc4ba16b2" +checksum = "3f252a68540fde3a3877aeea552b832b40ab9a69e318efd078774a01ddee1ccf" dependencies = [ + "num-conv", "time-core", ] @@ -2560,55 +2928,32 @@ dependencies = [ [[package]] name = "tinyvec_macros" -version = "0.1.0" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.22.0" +version = "1.37.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d76ce4a75fb488c605c54bf610f221cea8b0dafb53333c1a67e8ee199dcd2ae3" +checksum = "1adbebffeca75fcfd058afa480fb6c0b81e165a0323f9c9d39c9697e37c46787" dependencies = [ - "autocfg", + "backtrace", "bytes", "libc", - "memchr", "mio", - "num_cpus", - "parking_lot 0.12.1", + "parking_lot", "pin-project-lite", "signal-hook-registry", - "socket2", - "winapi", -] - -[[package]] -name = "tokio-native-tls" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" -dependencies = [ - "native-tls", - "tokio", -] - -[[package]] -name = "tokio-stream" -version = "0.1.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "397c988d37662c7dda6d2208364a706264bf3d6138b11d436cbac0ad38832842" -dependencies = [ - "futures-core", - "pin-project-lite", - "tokio", + "socket2 0.5.6", + "windows-sys 0.48.0", ] [[package]] name = "tokio-util" -version = "0.7.4" +version = "0.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bb2e075f03b3d66d8d8785356224ba688d2906a371015e225beeb65ca92c740" +checksum = "5419f34732d9eb6ee4c3578b7989078579b7f039cbbb9ca2c4da015749371e15" dependencies = [ "bytes", "futures-core", @@ -2620,9 +2965,9 @@ dependencies = [ [[package]] name = "toml" -version = "0.8.8" +version = "0.8.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1a195ec8c9da26928f773888e0742ca3ca1040c6cd859c919c9f59c1954ab35" +checksum = "e9dd1545e8208b4a5af1aa9bbd0b4cf7e9ea08fabc5d0a5c67fcaafa17433aa3" dependencies = [ "serde", "serde_spanned", @@ -2641,11 +2986,11 @@ dependencies = [ [[package]] name = "toml_edit" -version = "0.21.0" +version = "0.22.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d34d383cd00a163b4a5b85053df514d45bc330f6de7737edfe0a93311d1eaa03" +checksum = "8e40bb779c5187258fd7aad0eb68cb8706a0a81fa712fbea808ab43c4b8374c4" dependencies = [ - "indexmap 2.1.0", + "indexmap", "serde", "serde_spanned", "toml_datetime", @@ -2654,11 +2999,10 @@ dependencies = [ [[package]] name = "tracing" -version = "0.1.37" +version = "0.1.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" +checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" dependencies = [ - "cfg-if", "log", "pin-project-lite", "tracing-attributes", @@ -2679,20 +3023,20 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.23" +version = "0.1.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4017f8f45139870ca7e672686113917c71c7a6e02d4924eda67186083c03081a" +checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.59", ] [[package]] name = "tracing-core" -version = "0.1.30" +version = "0.1.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24eb03ba0eab1fd845050058ce5e616558e8f8d8fca633e6b163fe25c797213a" +checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" dependencies = [ "once_cell", "valuable", @@ -2700,20 +3044,20 @@ dependencies = [ [[package]] name = "tracing-log" -version = "0.1.3" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78ddad33d2d10b1ed7eb9d1f518a5674713876e97e5bb9b7345a7984fbb4f922" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" dependencies = [ - "lazy_static", "log", + "once_cell", "tracing-core", ] [[package]] name = "tracing-subscriber" -version = "0.3.16" +version = "0.3.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6176eae26dd70d0c919749377897b54a9276bd7061339665dd68777926b5a70" +checksum = "ad0f048c97dbd9faa9b7df56362b8ebcaa52adb06b498c050d2f4e32f90a7a8b" dependencies = [ "nu-ansi-term", "sharded-slab", @@ -2725,51 +3069,51 @@ dependencies = [ [[package]] name = "typenum" -version = "1.15.0" +version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcf81ac59edc17cc8697ff311e8f5ef2d99fcbd9817b34cec66f90b6c3dfd987" +checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" [[package]] name = "unicase" -version = "2.6.0" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50f37be617794602aabbeee0be4f259dc1778fabe05e2d67ee8f79326d5cb4f6" +checksum = "f7d2d4dafb69621809a81864c9c1b864479e1235c0dd4e199924b9742439ed89" dependencies = [ "version_check", ] [[package]] name = "unicode-bidi" -version = "0.3.8" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "099b7128301d285f79ddd55b9a83d5e6b9e97c92e0ea0daebee7263e932de992" +checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" [[package]] name = "unicode-ident" -version = "1.0.5" +version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ceab39d59e4c9499d4e5a8ee0e2735b891bb7308ac83dfb4e80cad195c9f6f3" +checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" [[package]] name = "unicode-normalization" -version = "0.1.22" +version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921" +checksum = "a56d1686db2308d901306f92a263857ef59ea39678a5458e7cb17f01415101f5" dependencies = [ "tinyvec", ] [[package]] name = "unicode-segmentation" -version = "1.10.1" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1dd624098567895118886609431a7c3b8f516e41d30e0643f03d94592a147e36" +checksum = "d4c87d22b6e3f4a18d4d40ef354e97c90fcb14dd91d7dc0aa9d8a1172ebf7202" [[package]] name = "unicode-width" -version = "0.1.10" +version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" +checksum = "e51733f11c9c4f72aa0c160008246859e340b00807569a0da0e7a1079b27ba85" [[package]] name = "unicode_categories" @@ -2779,20 +3123,32 @@ checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" [[package]] name = "url" -version = "2.3.1" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d68c799ae75762b8c3fe375feb6600ef5602c883c5d21eb51c09f22b83c4643" +checksum = "31e6302e3bb753d46e83516cae55ae196fc0c309407cf11ab35cc51a4c2a4633" dependencies = [ "form_urlencoded", "idna", "percent-encoding", ] +[[package]] +name = "urlencoding" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" + +[[package]] +name = "utf8parse" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" + [[package]] name = "uuid" -version = "1.2.2" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "422ee0de9031b5b948b97a8fc04e3aa35230001a722ddd27943e0be31564ce4c" +checksum = "a183cf7feeba97b4dd1c0d46788634f6221d87fa961b305bed08c851829efcc0" dependencies = [ "getrandom", ] @@ -2803,6 +3159,12 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" +[[package]] +name = "value-bag" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74797339c3b98616c009c7c3eb53a0ce41e85c8ec66bd3db96ed132d20cfdee8" + [[package]] name = "vcpkg" version = "0.2.15" @@ -2815,14 +3177,19 @@ version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" +[[package]] +name = "waker-fn" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3c4517f54858c779bbcbf228f4fca63d121bf85fbecb2dc578cdf4a39395690" + [[package]] name = "walkdir" -version = "2.3.2" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "808cf2735cd4b6866113f648b791c6adc5714537bc222d9347bb203386ffda56" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" dependencies = [ "same-file", - "winapi", "winapi-util", ] @@ -2832,11 +3199,17 @@ version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +[[package]] +name = "wasite" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" + [[package]] name = "wasm-bindgen" -version = "0.2.83" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eaf9f5aceeec8be17c128b2e93e031fb8a4d469bb9c4ae2d7dc1888b26887268" +checksum = "4be2531df63900aeb2bca0daaaddec08491ee64ceecbee5076636a3b026795a8" dependencies = [ "cfg-if", "wasm-bindgen-macro", @@ -2844,24 +3217,36 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.83" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c8ffb332579b0557b52d268b91feab8df3615f265d5270fec2a8c95b17c1142" +checksum = "614d787b966d3989fa7bb98a654e369c762374fd3213d212cfc0251257e747da" dependencies = [ "bumpalo", "log", "once_cell", "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.59", "wasm-bindgen-shared", ] +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76bc14366121efc8dbb487ab05bcc9d346b3b5ec0eaa76e46594cabbe51762c0" +dependencies = [ + "cfg-if", + "js-sys", + "wasm-bindgen", + "web-sys", +] + [[package]] name = "wasm-bindgen-macro" -version = "0.2.83" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "052be0f94026e6cbc75cdefc9bae13fd6052cdcaf532fa6c45e7ae33a1e6c810" +checksum = "a1f8823de937b71b9460c0c34e25f3da88250760bec0ebac694b49997550d726" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -2869,28 +3254,28 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.83" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07bc0c051dc5f23e307b13285f9d75df86bfdf816c5721e573dec1f9b8aa193c" +checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.59", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.83" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c38c045535d93ec4f0b4defec448e4291638ee608530863b1e2ba115d4fff7f" +checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96" [[package]] name = "web-sys" -version = "0.3.60" +version = "0.3.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bcda906d8be16e728fd5adc5b729afad4e444e106ab28cd1c7256e54fa61510f" +checksum = "77afa9a11836342370f4817622a2f0f418b134426d91a82dfb48f532d2ec13ef" dependencies = [ "js-sys", "wasm-bindgen", @@ -2898,12 +3283,12 @@ dependencies = [ [[package]] name = "whoami" -version = "1.4.1" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22fc3756b8a9133049b26c7f61ab35416c130e8c09b660f5b3958b446f52cc50" +checksum = "a44ab49fad634e88f55bf8f9bb3abd2f27d7204172a112c7c9987e01c1c94ea9" dependencies = [ - "wasm-bindgen", - "web-sys", + "redox_syscall", + "wasite", ] [[package]] @@ -2924,9 +3309,9 @@ checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" [[package]] name = "winapi-util" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" +checksum = "f29e6f9198ba0d26b4c9f07dbe6f9ed633e1f3d5b8b414090084349e46a52596" dependencies = [ "winapi", ] @@ -2943,22 +3328,7 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" dependencies = [ - "windows-targets 0.52.0", -] - -[[package]] -name = "windows-sys" -version = "0.42.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7" -dependencies = [ - "windows_aarch64_gnullvm 0.42.0", - "windows_aarch64_msvc 0.42.0", - "windows_i686_gnu 0.42.0", - "windows_i686_msvc 0.42.0", - "windows_x86_64_gnu 0.42.0", - "windows_x86_64_gnullvm 0.42.0", - "windows_x86_64_msvc 0.42.0", + "windows-targets 0.52.5", ] [[package]] @@ -2976,7 +3346,7 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets 0.52.0", + "windows-targets 0.52.5", ] [[package]] @@ -2996,25 +3366,20 @@ dependencies = [ [[package]] name = "windows-targets" -version = "0.52.0" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a18201040b24831fbb9e4eb208f8892e1f50a37feb53cc7ff887feb8f50e7cd" +checksum = "6f0713a46559409d202e70e28227288446bf7841d3211583a4b53e3f6d96e7eb" dependencies = [ - "windows_aarch64_gnullvm 0.52.0", - "windows_aarch64_msvc 0.52.0", - "windows_i686_gnu 0.52.0", - "windows_i686_msvc 0.52.0", - "windows_x86_64_gnu 0.52.0", - "windows_x86_64_gnullvm 0.52.0", - "windows_x86_64_msvc 0.52.0", + "windows_aarch64_gnullvm 0.52.5", + "windows_aarch64_msvc 0.52.5", + "windows_i686_gnu 0.52.5", + "windows_i686_gnullvm", + "windows_i686_msvc 0.52.5", + "windows_x86_64_gnu 0.52.5", + "windows_x86_64_gnullvm 0.52.5", + "windows_x86_64_msvc 0.52.5", ] -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.42.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41d2aa71f6f0cbe00ae5167d90ef3cfe66527d6f613ca78ac8024c3ccab9a19e" - [[package]] name = "windows_aarch64_gnullvm" version = "0.48.5" @@ -3023,15 +3388,9 @@ checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" [[package]] name = "windows_aarch64_gnullvm" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb7764e35d4db8a7921e09562a0304bf2f93e0a51bfccee0bd0bb0b666b015ea" - -[[package]] -name = "windows_aarch64_msvc" -version = "0.42.0" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd0f252f5a35cac83d6311b2e795981f5ee6e67eb1f9a7f64eb4500fbc4dcdb4" +checksum = "7088eed71e8b8dda258ecc8bac5fb1153c5cffaf2578fc8ff5d61e23578d3263" [[package]] name = "windows_aarch64_msvc" @@ -3041,15 +3400,9 @@ checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" [[package]] name = "windows_aarch64_msvc" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbaa0368d4f1d2aaefc55b6fcfee13f41544ddf36801e793edbbfd7d7df075ef" - -[[package]] -name = "windows_i686_gnu" -version = "0.42.0" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbeae19f6716841636c28d695375df17562ca208b2b7d0dc47635a50ae6c5de7" +checksum = "9985fd1504e250c615ca5f281c3f7a6da76213ebd5ccc9561496568a2752afb6" [[package]] name = "windows_i686_gnu" @@ -3059,15 +3412,15 @@ checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" [[package]] name = "windows_i686_gnu" -version = "0.52.0" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a28637cb1fa3560a16915793afb20081aba2c92ee8af57b4d5f28e4b3e7df313" +checksum = "88ba073cf16d5372720ec942a8ccbf61626074c6d4dd2e745299726ce8b89670" [[package]] -name = "windows_i686_msvc" -version = "0.42.0" +name = "windows_i686_gnullvm" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84c12f65daa39dd2babe6e442988fc329d6243fdce47d7d2d155b8d874862246" +checksum = "87f4261229030a858f36b459e748ae97545d6f1ec60e5e0d6a3d32e0dc232ee9" [[package]] name = "windows_i686_msvc" @@ -3077,15 +3430,9 @@ checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" [[package]] name = "windows_i686_msvc" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffe5e8e31046ce6230cc7215707b816e339ff4d4d67c65dffa206fd0f7aa7b9a" - -[[package]] -name = "windows_x86_64_gnu" -version = "0.42.0" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf7b1b21b5362cbc318f686150e5bcea75ecedc74dd157d874d754a2ca44b0ed" +checksum = "db3c2bf3d13d5b658be73463284eaf12830ac9a26a90c717b7f771dfe97487bf" [[package]] name = "windows_x86_64_gnu" @@ -3095,15 +3442,9 @@ checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" [[package]] name = "windows_x86_64_gnu" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d6fa32db2bc4a2f5abeacf2b69f7992cd09dca97498da74a151a3132c26befd" - -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.42.0" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09d525d2ba30eeb3297665bd434a54297e4170c7f1a44cad4ef58095b4cd2028" +checksum = "4e4246f76bdeff09eb48875a0fd3e2af6aada79d409d33011886d3e1581517d9" [[package]] name = "windows_x86_64_gnullvm" @@ -3113,15 +3454,9 @@ checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" [[package]] name = "windows_x86_64_gnullvm" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a657e1e9d3f514745a572a6846d3c7aa7dbe1658c056ed9c3344c4109a6949e" - -[[package]] -name = "windows_x86_64_msvc" -version = "0.42.0" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f40009d85759725a34da6d89a94e63d7bdc50a862acf0dbc7c8e488f1edcb6f5" +checksum = "852298e482cd67c356ddd9570386e2862b5673c85bd5f88df9ab6802b334c596" [[package]] name = "windows_x86_64_msvc" @@ -3131,15 +3466,15 @@ checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" [[package]] name = "windows_x86_64_msvc" -version = "0.52.0" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04" +checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0" [[package]] name = "winnow" -version = "0.5.19" +version = "0.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "829846f3e3db426d4cee4510841b71a8e58aa2a76b1132579487ae430ccd9c7b" +checksum = "f0c976aaaa0e1f90dbb21e9587cdaf1d9679a1cde8875c0d6bd83ab96a208352" dependencies = [ "memchr", ] @@ -3161,34 +3496,39 @@ checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.59", ] +[[package]] +name = "zeroize" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "525b4ec142c6b68a2d10f01f7bbf6755599ca3f81ea53b8431b7dd348f5fdb2d" + [[package]] name = "zstd" -version = "0.11.2+zstd.1.5.2" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20cc960326ece64f010d2d2107537f26dc589a6573a316bd5b1dba685fa5fde4" +checksum = "2d789b1514203a1120ad2429eae43a7bd32b90976a7bb8a05f7ec02fa88cc23a" dependencies = [ "zstd-safe", ] [[package]] name = "zstd-safe" -version = "5.0.2+zstd.1.5.2" +version = "7.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d2a5585e04f9eea4b2a3d1eca508c4dee9592a89ef6f450c11719da0726f4db" +checksum = "1cd99b45c6bc03a018c8b8a86025678c87e55526064e38f9df301989dce7ec0a" dependencies = [ - "libc", "zstd-sys", ] [[package]] name = "zstd-sys" -version = "2.0.1+zstd.1.5.2" +version = "2.0.10+zstd.1.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fd07cbbc53846d9145dbffdf6dd09a7a0aa52be46741825f5c97bdd4f73f12b" +checksum = "c253a4914af5bafc8fa8c86ee400827e83cf6ec01195ec1f1ed8441bf00d65aa" dependencies = [ "cc", - "libc", + "pkg-config", ] diff --git a/Cargo.toml b/Cargo.toml index 56607ec..1b4ffe3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,7 +8,7 @@ license = "AGPL-3.0" keywords = ["authentication", "laws", "preservation"] categories = ["authentication", "web-programming::http-server"] repository = "https://github.com/openlawlibrary/stelae" -rust-version = "1.70" +rust-version = "1.77" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] @@ -30,18 +30,17 @@ tracing-subscriber = "0.3.16" tracing-actix-web = "0.6.2" derive_more = "0.99.17" toml = "0.8.8" -toml_edit = "0.21.0" +toml_edit = "0.22" serde_derive = "1.0.152" chrono = { version = "0.4.*", features = ["serde"] } -sqlx = { version = "0.6", features = [ +sqlx = { version = "0.7", features = [ "chrono", - "runtime-actix-native-tls", + "runtime-async-std", "any", "postgres", "sqlite", ] } sophia = { version = "0.8.0", features = ["xml"] } -walkdir = "2" [dev-dependencies] criterion = "0.3" diff --git a/migrations/sqlite/20240115152953_initial_db.down.sql b/migrations/sqlite/20240115152953_initial_db.down.sql new file mode 100644 index 0000000..a6e8623 --- /dev/null +++ b/migrations/sqlite/20240115152953_initial_db.down.sql @@ -0,0 +1,20 @@ +-- Add down migration script here +PRAGMA foreign_keys = OFF; + +DROP INDEX IF EXISTS changed_library_document_library_mpath_idx; +DROP INDEX IF EXISTS library_change_library_mpath_idx; +DROP INDEX IF EXISTS document_change_doc_mpath_idx; + +DROP TABLE IF EXISTS changed_library_document; +DROP TABLE IF EXISTS library_change; +DROP TABLE IF EXISTS document_change; +DROP TABLE IF EXISTS publication_has_publication_versions; +DROP TABLE IF EXISTS publication_version; +DROP TABLE IF EXISTS publication; +DROP TABLE IF EXISTS version; +DROP TABLE IF EXISTS library_document; +DROP TABLE IF EXISTS library; +DROP TABLE IF EXISTS document; +DROP TABLE IF EXISTS stele; + +PRAGMA optimize; \ No newline at end of file diff --git a/migrations/sqlite/20240115152953_initial_db.up.sql b/migrations/sqlite/20240115152953_initial_db.up.sql new file mode 100644 index 0000000..77e5fad --- /dev/null +++ b/migrations/sqlite/20240115152953_initial_db.up.sql @@ -0,0 +1,121 @@ +-- Add migration script here +PRAGMA foreign_keys = ON; + +CREATE TABLE stele ( + name TEXT PRIMARY KEY +); +CREATE TABLE document ( + doc_id TEXT PRIMARY KEY +); +CREATE TABLE library ( + mpath TEXT PRIMARY KEY +); +CREATE TABLE library_document ( + collection_mpath TEXT, + doc_id TEXT, + start DATE, + end DATE, + CONSTRAINT fk_coll_mpath + FOREIGN KEY (collection_mpath) + REFERENCES library(mpath), + CONSTRAINT fk_doc_id + FOREIGN KEY (doc_id) + REFERENCES document(doc_id), + PRIMARY KEY (collection_mpath, doc_id) +); +CREATE TABLE publication ( + name TEXT, + date INTEGER, + stele TEXT, + revoked INTEGER, + last_valid_publication_name TEXT, + last_valid_version TEXT, + CONSTRAINT fk_last_valid_version + FOREIGN KEY (last_valid_version) + REFERENCES version(codified_date), + CONSTRAINT fk_last_valid_publication + FOREIGN KEY (last_valid_publication_name, stele) + REFERENCES publication(name, stele), + CONSTRAINT fk_stele + FOREIGN KEY (stele) + REFERENCES stele(name) + ON DELETE CASCADE, + PRIMARY KEY (name, stele) +); +CREATE TABLE publication_version ( + version TEXT, + publication TEXT, + stele TEXT, + build_reason TEXT, + CONSTRAINT fk_publication + FOREIGN KEY (publication, stele) + REFERENCES publication(name, stele) + ON DELETE CASCADE, + CONSTRAINT fk_version + FOREIGN KEY (version) + REFERENCES version(codified_date), + PRIMARY KEY (publication, version, stele) +); +CREATE TABLE publication_has_publication_versions ( + publication TEXT, + referenced_publication TEXT, + referenced_version TEXT, + stele TEXT, + CONSTRAINT fk_publication FOREIGN KEY (publication, stele) REFERENCES publication(name, stele) ON DELETE CASCADE, + CONSTRAINT fk_referenced_publication FOREIGN KEY (referenced_publication, referenced_version, stele) REFERENCES publication_version(publication, version, stele) ON DELETE CASCADE, + PRIMARY KEY (publication, referenced_publication, referenced_version, stele) +); +CREATE TABLE version( + codified_date TEXT PRIMARY KEY +); +CREATE TABLE document_change ( + doc_mpath TEXT, + status TEXT, + url TEXT, + change_reason TEXT, + publication TEXT, + version TEXT, + stele TEXT, + doc_id TEXT, + CONSTRAINT fk_doc_id + FOREIGN KEY (doc_id) + REFERENCES document(doc_id) + ON DELETE CASCADE, + CONSTRAINT fk_publication_version + FOREIGN KEY (publication, version, stele) + REFERENCES publication_version(publication, version, stele) + ON DELETE CASCADE, + PRIMARY KEY (doc_mpath, status, publication, version, stele) +); +CREATE INDEX document_change_doc_mpath_idx ON document_change(doc_mpath COLLATE NOCASE); +CREATE TABLE library_change ( + publication TEXT, + version TEXT, + stele TEXT, + status TEXT, + library_mpath TEXT, + url TEXT, + CONSTRAINT fk_publication_version + FOREIGN KEY (publication, version, stele) + REFERENCES publication_version(publication, version, stele) + ON DELETE CASCADE, + PRIMARY KEY (publication, version, stele, library_mpath, status) +); +CREATE TABLE changed_library_document ( + publication TEXT, + version TEXT, + stele TEXT, + doc_mpath TEXT, + status TEXT, + library_mpath TEXT, + url TEXT, + CONSTRAINT fk_document_change + FOREIGN KEY (publication, version, stele, doc_mpath, status) + REFERENCES document_change(publication, version, stele, doc_mpath, status) + ON DELETE CASCADE, + PRIMARY KEY (publication, version, stele, library_mpath, doc_mpath, status) +); +CREATE INDEX library_change_library_mpath_idx ON library_change(library_mpath COLLATE NOCASE); +CREATE INDEX changed_library_document_library_mpath_idx ON changed_library_document(library_mpath COLLATE NOCASE); + +PRAGMA optimize; \ No newline at end of file diff --git a/src/db/init.rs b/src/db/init.rs index b75b475..fd3f6d8 100644 --- a/src/db/init.rs +++ b/src/db/init.rs @@ -10,7 +10,7 @@ use std::path::{Path, PathBuf}; pub async fn connect(archive_path: &Path) -> anyhow::Result { let db_url = env::var("DATABASE_URL").unwrap_or_else(|_| { let sqlite_db_path = &archive_path.join(PathBuf::from(".stelae/db.sqlite3")); - format!("sqlite://{}?mode=rwc", sqlite_db_path.to_string_lossy()) + format!("sqlite:///{}?mode=rwc", sqlite_db_path.to_string_lossy()) }); let connection = DatabaseConnection::connect(&db_url).await?; tracing::info!("Connected to database"); diff --git a/src/db/mod.rs b/src/db/mod.rs index e6163e9..ee4d149 100644 --- a/src/db/mod.rs +++ b/src/db/mod.rs @@ -3,16 +3,17 @@ use async_trait::async_trait; use std::str::FromStr; -use sqlx::any::{AnyPool, AnyPoolOptions}; +use sqlx::any::{self, AnyPoolOptions}; +use sqlx::AnyPool; use sqlx::ConnectOptions; use tracing::instrument; /// Database initialization. pub mod init; -/// Statements for the database. -pub mod statements; /// Models for the database. pub mod models; +/// Statements for the database. +pub mod statements; #[async_trait] /// Generic Database @@ -22,12 +23,6 @@ pub trait Db { /// # Errors /// Errors if connection to database fails. async fn connect(url: &str) -> anyhow::Result; - - // async fn execute_statement(statement: &str, conn: &DatabaseConnection) -> anyhow::Result<()>; - - // async fn begin(&self) -> anyhow::Result<()>; - - // async fn close(&self) -> anyhow::Result<()>; } /// Type of database connection. @@ -56,15 +51,14 @@ impl Db for DatabaseConnection { /// Errors if connection to database fails. #[instrument(level = "trace")] async fn connect(db_url: &str) -> anyhow::Result { - let options = sqlx::any::AnyConnectOptions::from_str(db_url)? - .disable_statement_logging() - .clone(); + any::install_default_drivers(); + let options = any::AnyConnectOptions::from_str(db_url)?.disable_statement_logging(); let pool = AnyPoolOptions::new() .max_connections(50) .connect_with(options) .await?; let connection = match db_url { - url if url.starts_with("sqlite://") => Self { + url if url.starts_with("sqlite:///") => Self { pool, kind: DatabaseKind::Sqlite, }, diff --git a/src/db/models/changed_library_document.rs b/src/db/models/changed_library_document.rs new file mode 100644 index 0000000..7a734db --- /dev/null +++ b/src/db/models/changed_library_document.rs @@ -0,0 +1,21 @@ +use serde::{Deserialize, Serialize}; + +#[derive(sqlx::FromRow, Deserialize, Serialize)] +/// Model for library (collection) change events. +pub struct ChangedLibraryDocument { + /// Foreign key reference to publication name + pub publication: String, + /// Foreign key reference to codified date in a publication in %Y-%m-%d format + pub version: String, + /// Foreign key reference to stele identifier in / format. + pub stele: String, + /// Materialized path to the document + pub doc_mpath: String, + /// Change status of the document. + /// Currently could be 'Element added', 'Element effective', 'Element changed' or 'Element removed'. + pub status: String, + /// Materialized path to the library + pub library_mpath: String, + /// Url to the library that was changed. + pub url: String, +} diff --git a/src/db/models/document.rs b/src/db/models/document.rs index a427623..5499822 100644 --- a/src/db/models/document.rs +++ b/src/db/models/document.rs @@ -4,5 +4,5 @@ use serde::{Deserialize, Serialize}; /// Model for documents. pub struct Document { /// Unique document identifier. - pub doc_id: String + pub doc_id: String, } diff --git a/src/db/models/library.rs b/src/db/models/library.rs new file mode 100644 index 0000000..d6643bc --- /dev/null +++ b/src/db/models/library.rs @@ -0,0 +1,8 @@ +use serde::{Deserialize, Serialize}; + +#[derive(sqlx::FromRow, Deserialize, Serialize)] +/// Model for library (collection). +pub struct Library { + /// Materialized path to the library + pub mpath: String, +} diff --git a/src/db/models/library_change.rs b/src/db/models/library_change.rs new file mode 100644 index 0000000..bd11982 --- /dev/null +++ b/src/db/models/library_change.rs @@ -0,0 +1,19 @@ +use serde::{Deserialize, Serialize}; + +#[derive(sqlx::FromRow, Deserialize, Serialize, Debug)] +/// Model for library (collection) change events. +pub struct LibraryChange { + /// Foreign key reference to publication name + pub publication: String, + /// Foreign key reference to codified date in a publication in %Y-%m-%d format + pub version: String, + /// Foreign key reference to stele identifier in / format. + pub stele: String, + /// Change status of the document. + /// Currently could be 'Element added', 'Element effective', 'Element changed' or 'Element removed'. + pub status: String, + /// Url to the library that was changed. + pub url: String, + /// Materialized path to the library + pub library_mpath: String, +} diff --git a/src/db/models/mod.rs b/src/db/models/mod.rs index 07e5de4..dac6adc 100644 --- a/src/db/models/mod.rs +++ b/src/db/models/mod.rs @@ -1,12 +1,20 @@ //! This module contains all the sqlx structs for the database tables. -/// sqlx structs for document table. +/// sqlx structs for `changed_library_document` table. +pub mod changed_library_document; +/// sqlx structs for `document` table. pub mod document; -/// sqlx structs for document_change table. +/// sqlx structs for `document_change` table. pub mod document_change; -/// sqlx structs for publication table. +/// sqlx structs for `library` table. +pub mod library; +/// sqlx structs for `library_change` table. +pub mod library_change; +/// sqlx structs for `publication` table. pub mod publication; -/// sqlx structs for publication_version table +/// sqlx structs for `publication_has_publication_versions` table. +pub mod publication_has_publication_versions; +/// sqlx structs for `publication_version` table pub mod publication_version; -/// sqlx structs for stele table. +/// sqlx structs for `stele` table. pub mod stele; diff --git a/src/db/models/publication.rs b/src/db/models/publication.rs index 3b99bba..a69c803 100644 --- a/src/db/models/publication.rs +++ b/src/db/models/publication.rs @@ -1,7 +1,7 @@ use serde::{Deserialize, Serialize}; -use sqlx::types::chrono; +use sqlx::{any::AnyRow, FromRow, Row}; -#[derive(sqlx::FromRow, Deserialize, Serialize)] +#[derive(Deserialize, Serialize, Debug)] /// Model for a Stele. pub struct Publication { /// Name of the publication in %YYYY-%MM-%DD format @@ -9,18 +9,32 @@ pub struct Publication { /// when two publications exist on same date. pub name: String, /// Date of the publication. - pub date: chrono::NaiveDate, + pub date: String, /// Foreign key reference to stele by name. pub stele: String, /// Whether the publication has been revoked. /// A publication is revoked if another publication exists /// on the same date with a higher version number. - pub revoked: bool, + pub revoked: i64, /// If a publication is derived from another publication, /// represents the last publication name that was valid before this publication. pub last_valid_publication_name: Option, /// If a publication is derived from another publication, /// represents the last publication version (codified date) from the previous publication /// that the current publication is derived from. - pub last_valid_version: Option + pub last_valid_version: Option, +} + +impl FromRow<'_, AnyRow> for Publication { + #[allow(clippy::unwrap_in_result, clippy::unwrap_used)] + fn from_row(row: &AnyRow) -> Result { + Ok(Self { + name: row.try_get("name").unwrap(), + date: row.try_get("date").unwrap(), + stele: row.try_get("stele").unwrap(), + revoked: row.try_get("revoked").unwrap(), + last_valid_publication_name: row.try_get("last_valid_publication_name").ok(), + last_valid_version: row.try_get("last_valid_version").ok(), + }) + } } diff --git a/src/db/models/publication_has_publication_versions.rs b/src/db/models/publication_has_publication_versions.rs new file mode 100644 index 0000000..ee035bf --- /dev/null +++ b/src/db/models/publication_has_publication_versions.rs @@ -0,0 +1,15 @@ +use serde::{Deserialize, Serialize}; + +#[derive(sqlx::FromRow, Deserialize, Serialize, Clone, Debug)] +/// Model for publication which contain publication versions. +pub struct PublicationHasPublicationVersions { + /// Foreign key reference to publication name. + pub publication: String, + /// Publication can reference another publication. + /// Foreign key reference to the referenced publication name. + pub referenced_publication: String, + /// Date in a publication in %Y-%m-%d format + pub referenced_version: String, + /// Foreign key reference to stele. + pub stele: String, +} diff --git a/src/db/models/publication_version.rs b/src/db/models/publication_version.rs index 2a3dd7d..025ba01 100644 --- a/src/db/models/publication_version.rs +++ b/src/db/models/publication_version.rs @@ -1,14 +1,27 @@ use serde::{Deserialize, Serialize}; +use sqlx::{any::AnyRow, FromRow, Row}; -#[derive(sqlx::FromRow, Deserialize, Serialize)] +#[derive(Deserialize, Serialize, Hash, Eq, PartialEq, Clone)] /// Model for a Stele. pub struct PublicationVersion { - /// Database id. - pub id: i32, /// Date in a publication in %Y-%m-%d format pub version: String, - /// Foreign key reference to publication id. - pub publication_id: i32, + /// Foreign key reference to publication name. + pub publication: String, + /// Foreign key reference to stele. + pub stele: String, /// Reason for building the publication. pub build_reason: Option, } + +impl FromRow<'_, AnyRow> for PublicationVersion { + #[allow(clippy::unwrap_in_result, clippy::unwrap_used)] + fn from_row(row: &AnyRow) -> Result { + Ok(Self { + version: row.try_get("version").unwrap(), + publication: row.try_get("publication").unwrap(), + stele: row.try_get("stele").unwrap(), + build_reason: row.try_get("build_reason").ok(), + }) + } +} diff --git a/src/db/statements/inserts.rs b/src/db/statements/inserts.rs index 795ab72..1668c23 100644 --- a/src/db/statements/inserts.rs +++ b/src/db/statements/inserts.rs @@ -1,9 +1,18 @@ //! Central place for database queries in Stelae use sqlx::types::chrono::NaiveDate; +use sqlx::QueryBuilder; +use crate::db::models::changed_library_document::ChangedLibraryDocument; +use crate::db::models::document_change::DocumentChange; +use crate::db::models::library::Library; +use crate::db::models::library_change::LibraryChange; +use crate::db::models::publication_has_publication_versions::PublicationHasPublicationVersions; use crate::db::DatabaseConnection; use crate::db::DatabaseKind; +/// Size of the batch for bulk inserts. +const BATCH_SIZE: usize = 1000; + /// Upsert a new document into the database. /// /// # Errors @@ -14,10 +23,10 @@ pub async fn create_document( ) -> anyhow::Result> { let id = match conn.kind { DatabaseKind::Sqlite => { - let statement: &'static str = r#" + let statement = " INSERT OR IGNORE INTO document ( doc_id ) VALUES ( $1 ) - "#; + "; let mut connection = conn.pool.acquire().await?; sqlx::query(statement) .bind(doc_id) @@ -26,11 +35,11 @@ pub async fn create_document( .last_insert_id() } DatabaseKind::Postgres => { - let statement = r#" + let statement = " INSERT INTO document ( doc_id ) VALUES ( $1 ) ON CONFLICT ( doc_id ) DO NOTHING; - "#; + "; let mut connection = conn.pool.acquire().await?; sqlx::query(statement) .bind(doc_id) @@ -42,6 +51,168 @@ pub async fn create_document( Ok(id) } +/// Upsert a bulk of document changes into the database. +/// +/// # Errors +/// Errors if the document changes cannot be inserted into the database. +pub async fn insert_document_changes_bulk( + conn: &DatabaseConnection, + document_changes: Vec, +) -> anyhow::Result<()> { + match conn.kind { + DatabaseKind::Sqlite => { + let mut connection = conn.pool.acquire().await?; + let mut query_builder = QueryBuilder::new("INSERT OR IGNORE INTO document_change (doc_mpath, status, url, change_reason, publication, version, stele, doc_id) "); + for chunk in document_changes.chunks(BATCH_SIZE) { + query_builder.push_values(chunk, |mut bindings, dc| { + bindings + .push_bind(&dc.doc_mpath) + .push_bind(&dc.status) + .push_bind(&dc.url) + .push_bind(&dc.change_reason) + .push_bind(&dc.publication) + .push_bind(&dc.version) + .push_bind(&dc.stele) + .push_bind(&dc.doc_id); + }); + let query = query_builder.build(); + query.execute(&mut *connection).await?; + query_builder.reset(); + } + } + DatabaseKind::Postgres => { + anyhow::bail!("Not supported yet") + } + }; + + Ok(()) +} + +/// Upsert a bulk of libraries into the database. +/// +/// # Errors +/// Errors if the libraries cannot be inserted into the database. +pub async fn insert_library_bulk( + conn: &DatabaseConnection, + libraries: Vec, +) -> anyhow::Result<()> { + match conn.kind { + DatabaseKind::Sqlite => { + let mut connection = conn.pool.acquire().await?; + let mut query_builder = QueryBuilder::new("INSERT OR IGNORE INTO library (mpath) "); + for chunk in libraries.chunks(BATCH_SIZE) { + query_builder.push_values(chunk, |mut bindings, lb| { + bindings.push_bind(&lb.mpath); + }); + let query = query_builder.build(); + query.execute(&mut *connection).await?; + } + } + DatabaseKind::Postgres => { + anyhow::bail!("Not supported yet") + } + } + Ok(()) +} + +/// Upsert a bulk of library changes into the database. +/// +/// # Errors +/// Errors if the library changes cannot be inserted into the database. +pub async fn insert_library_changes_bulk( + conn: &DatabaseConnection, + library_changes: Vec, +) -> anyhow::Result<()> { + match conn.kind { + DatabaseKind::Sqlite => { + let mut connection = conn.pool.acquire().await?; + let mut query_builder = QueryBuilder::new("INSERT OR IGNORE INTO library_change (library_mpath, publication, version, stele, status, url) "); + for chunk in library_changes.chunks(BATCH_SIZE) { + query_builder.push_values(chunk, |mut bindings, lc| { + bindings + .push_bind(&lc.library_mpath) + .push_bind(&lc.publication) + .push_bind(&lc.version) + .push_bind(&lc.stele) + .push_bind(&lc.status) + .push_bind(&lc.url); + }); + let query = query_builder.build(); + query.execute(&mut *connection).await?; + } + } + DatabaseKind::Postgres => { + anyhow::bail!("Not supported yet") + } + } + Ok(()) +} + +/// Upsert a bulk of changed library documents into the database. +/// +/// # Errors +/// Errors if the changed library documents cannot be inserted into the database. +pub async fn insert_changed_library_document_bulk( + conn: &DatabaseConnection, + changed_library_document: Vec, +) -> anyhow::Result<()> { + match conn.kind { + DatabaseKind::Sqlite => { + let mut connection = conn.pool.acquire().await?; + let mut query_builder = QueryBuilder::new("INSERT OR IGNORE INTO changed_library_document (publication, version, stele, doc_mpath, status, library_mpath, url) "); + for chunk in changed_library_document.chunks(BATCH_SIZE) { + query_builder.push_values(chunk, |mut bindings, cl| { + bindings + .push_bind(&cl.publication) + .push_bind(&cl.version) + .push_bind(&cl.stele) + .push_bind(&cl.doc_mpath) + .push_bind(&cl.status) + .push_bind(&cl.library_mpath) + .push_bind(&cl.url); + }); + let query = query_builder.build(); + query.execute(&mut *connection).await?; + } + } + DatabaseKind::Postgres => { + anyhow::bail!("Not supported yet") + } + } + Ok(()) +} + +/// Upsert a bulk of `publication_has_publication_versions` into the database. +/// +/// # Errors +/// Errors if the `publication_has_publication_versions` cannot be inserted into the database. +pub async fn insert_publication_has_publication_versions_bulk( + conn: &DatabaseConnection, + publication_has_publication_versions: Vec, +) -> anyhow::Result<()> { + match conn.kind { + DatabaseKind::Sqlite => { + let mut connection = conn.pool.acquire().await?; + let mut query_builder = QueryBuilder::new("INSERT OR IGNORE INTO publication_has_publication_versions (publication, referenced_publication, referenced_version, stele) "); + for chunk in publication_has_publication_versions.chunks(BATCH_SIZE) { + query_builder.push_values(chunk, |mut bindings, pb| { + bindings + .push_bind(&pb.publication) + .push_bind(&pb.referenced_publication) + .push_bind(&pb.referenced_version) + .push_bind(&pb.stele); + }); + let query = query_builder.build(); + query.execute(&mut *connection).await?; + } + } + DatabaseKind::Postgres => { + anyhow::bail!("Not supported yet") + } + } + Ok(()) +} + /// Upsert a new publication into the database. /// # Errors /// Errors if the publication cannot be inserted into the database. @@ -55,14 +226,14 @@ pub async fn create_publication( ) -> anyhow::Result> { let id = match conn.kind { DatabaseKind::Sqlite => { - let statement: &'static str = r#" + let statement = " INSERT OR IGNORE INTO publication ( name, date, stele, revoked, last_valid_publication_name, last_valid_version ) VALUES ( $1, $2, $3, FALSE, $4, $5 ) - "#; + "; let mut connection = conn.pool.acquire().await?; sqlx::query(statement) .bind(name) - .bind(date) + .bind(date.to_string()) .bind(stele) .bind(last_valid_publication_name) .bind(last_valid_version) @@ -71,15 +242,15 @@ pub async fn create_publication( .last_insert_id() } DatabaseKind::Postgres => { - let statement: &'static str = r#" + let statement = " INSERT INTO publication ( name, date, stele, revoked, last_valid_publication_name, last_valid_version ) VALUES ( $1, $2, $3, FALSE, $4, $5 ) ON CONFLICT ( name, stele ) DO NOTHING; - "#; + "; let mut connection = conn.pool.acquire().await?; sqlx::query(statement) .bind(name) - .bind(date) + .bind(date.to_string()) .bind(stele) .bind(last_valid_publication_name) .bind(last_valid_version) @@ -98,10 +269,10 @@ pub async fn create_publication( pub async fn create_stele(conn: &DatabaseConnection, stele: &str) -> anyhow::Result> { let id = match conn.kind { DatabaseKind::Sqlite => { - let statement: &'static str = r#" + let statement = " INSERT OR IGNORE INTO stele ( name ) VALUES ( $1 ) - "#; + "; let mut connection = conn.pool.acquire().await?; sqlx::query(statement) .bind(stele) @@ -110,11 +281,11 @@ pub async fn create_stele(conn: &DatabaseConnection, stele: &str) -> anyhow::Res .last_insert_id() } DatabaseKind::Postgres => { - let statement: &'static str = r#" + let statement = " INSERT INTO stele ( name ) VALUES ( $1 ) ON CONFLICT ( name ) DO NOTHING; - "#; + "; let mut connection = conn.pool.acquire().await?; sqlx::query(statement) .bind(stele) @@ -136,10 +307,10 @@ pub async fn create_version( ) -> anyhow::Result> { let id = match conn.kind { DatabaseKind::Sqlite => { - let statement: &'static str = r#" + let statement = " INSERT OR IGNORE INTO version ( codified_date ) VALUES ( $1 ) - "#; + "; let mut connection = conn.pool.acquire().await?; sqlx::query(statement) .bind(codified_date) @@ -148,11 +319,11 @@ pub async fn create_version( .last_insert_id() } DatabaseKind::Postgres => { - let statement: &'static str = r#" + let statement = " INSERT INTO version ( codified_date ) VALUES ( $1 ) ON CONFLICT ( codified_date ) DO NOTHING; - "#; + "; let mut connection = conn.pool.acquire().await?; sqlx::query(statement) .bind(codified_date) @@ -176,10 +347,10 @@ pub async fn create_publication_version( ) -> anyhow::Result> { let id = match conn.kind { DatabaseKind::Sqlite => { - let statement = r#" + let statement = " INSERT OR IGNORE INTO publication_version ( publication, version, stele ) VALUES ( $1, $2, $3 ) - "#; + "; let mut connection = conn.pool.acquire().await?; sqlx::query(statement) .bind(publication) @@ -190,11 +361,11 @@ pub async fn create_publication_version( .last_insert_id() } DatabaseKind::Postgres => { - let statement = r#" + let statement = " INSERT INTO publication_version ( publication, version, stele ) VALUES ( $1, $2, $3 ) ON CONFLICT ( publication, version, stele ) DO NOTHING; - "#; + "; let mut connection = conn.pool.acquire().await?; sqlx::query(statement) .bind(publication) @@ -207,3 +378,33 @@ pub async fn create_publication_version( }; Ok(id) } + +/// Update a publication by name and stele to be revoked. +/// +/// # Errors +/// Errors if the publication cannot be updated. +pub async fn update_publication_by_name_and_stele_set_revoked_true( + conn: &DatabaseConnection, + name: &str, + stele: &str, +) -> anyhow::Result<()> { + let statement = " + UPDATE publication + SET revoked = TRUE + WHERE name = $1 AND stele = $2 + "; + match conn.kind { + DatabaseKind::Sqlite => { + let mut connection = conn.pool.acquire().await?; + sqlx::query(statement) + .bind(name) + .bind(stele) + .execute(&mut *connection) + .await?; + } + DatabaseKind::Postgres => { + unimplemented!() + } + } + Ok(()) +} diff --git a/src/db/statements/queries.rs b/src/db/statements/queries.rs index 7251fd7..4f4c9ae 100644 --- a/src/db/statements/queries.rs +++ b/src/db/statements/queries.rs @@ -1,10 +1,10 @@ //! Central place for database queries -use sqlx::types::chrono::NaiveDate; use crate::db::models::publication::Publication; use crate::db::models::publication_version::PublicationVersion; use crate::db::models::stele::Stele; use crate::db::DatabaseConnection; +use std::collections::HashSet; use crate::db::DatabaseKind; @@ -16,21 +16,13 @@ pub async fn find_stele_by_name( conn: &DatabaseConnection, name: &str, ) -> anyhow::Result> { - let statement: &'static str = r#" + let statement = " SELECT * FROM stele WHERE name = $1 - "#; + "; let row = match conn.kind { - DatabaseKind::Sqlite => { - let mut connection = conn.pool.acquire().await?; - sqlx::query_as::<_, Stele>(statement) - .bind(name) - .fetch_one(&mut *connection) - .await - .ok() - } - DatabaseKind::Postgres => { + DatabaseKind::Postgres | DatabaseKind::Sqlite => { let mut connection = conn.pool.acquire().await?; sqlx::query_as::<_, Stele>(statement) .bind(name) @@ -51,15 +43,15 @@ pub async fn find_last_inserted_publication( conn: &DatabaseConnection, stele: &str, ) -> anyhow::Result> { + let statement = " + SELECT * + FROM publication + WHERE revoked = 0 AND stele = $1 + ORDER BY date DESC + LIMIT 1 + "; let row = match conn.kind { DatabaseKind::Sqlite => { - let statement: &'static str = r#" - SELECT * - FROM publication - WHERE revoked = 0 AND stele = $1 - ORDER BY date DESC - LIMIT 1 - "#; let mut connection = conn.pool.acquire().await?; sqlx::query_as::<_, Publication>(statement) .bind(stele) @@ -68,19 +60,7 @@ pub async fn find_last_inserted_publication( .ok() } DatabaseKind::Postgres => { - let statement: &'static str = r#" - SELECT * - FROM publication - WHERE revoked = FALSE AND stele = $1 - ORDER BY date DESC - LIMIT 1 - "#; - let mut connection = conn.pool.acquire().await?; - sqlx::query_as::<_, Publication>(statement) - .bind(stele) - .fetch_one(&mut *connection) - .await - .ok() + unimplemented!() } }; Ok(row) @@ -90,74 +70,214 @@ pub async fn find_last_inserted_publication( /// /// # Errors /// Errors if can't establish a connection to the database. -pub async fn find_publication_by_name_and_date_and_stele_id( +pub async fn find_publication_by_name_and_stele( conn: &DatabaseConnection, name: &str, - date: &NaiveDate, stele: &str, -) -> anyhow::Result> { - let statement: &'static str = r#" +) -> anyhow::Result { + let statement = " SELECT * FROM publication - WHERE name = $1 AND date = $2 AND stele = $3 - "#; + WHERE name = $1 AND stele = $2 + "; let row = match conn.kind { DatabaseKind::Sqlite => { let mut connection = conn.pool.acquire().await?; sqlx::query_as::<_, Publication>(statement) .bind(name) - .bind(date) .bind(stele) .fetch_one(&mut *connection) - .await - .ok() + .await? } DatabaseKind::Postgres => { + unimplemented!() + } + }; + Ok(row) +} + +/// Find a publication version by `publication_id` and `version`. +/// +/// # Errors +/// Errors if can't establish a connection to the database. +pub async fn find_all_publication_versions_by_publication_name_and_stele( + conn: &DatabaseConnection, + publication: &str, + stele: &str, +) -> anyhow::Result> { + let statement = " + SELECT * + FROM publication_version + WHERE publication = $1 AND stele = $2 + "; + let rows = match conn.kind { + DatabaseKind::Sqlite => { + let mut connection = conn.pool.acquire().await?; + sqlx::query_as::<_, PublicationVersion>(statement) + .bind(publication) + .bind(stele) + .fetch_all(&mut *connection) + .await? + } + DatabaseKind::Postgres => { + unimplemented!() + } + }; + Ok(rows) +} + +/// Find all publication versions in `publications`. +async fn find_all_publication_versions_in_publication_has_publication_versions( + conn: &DatabaseConnection, + publications: Vec, + stele: &str, +) -> anyhow::Result> { + let parameters = publications + .iter() + .map(|_| "?") + .collect::>() + .join(", "); + let rows = match conn.kind { + DatabaseKind::Sqlite => { + let mut connection = conn.pool.acquire().await?; + + let statement = format!(" + SELECT DISTINCT pv.publication, pv.version + FROM publication_version pv + LEFT JOIN publication_has_publication_versions phpv ON pv.publication = phpv.referenced_publication AND pv.version = phpv.referenced_version + WHERE phpv.publication IN ({parameters} AND pv.stele = ?) + "); + + let mut query = sqlx::query_as::<_, PublicationVersion>(&statement); + for publication in publications { + query = query.bind(publication); + } + query = query.bind(stele); + + query.fetch_all(&mut *connection).await? + } + DatabaseKind::Postgres => { + unimplemented!() + } + }; + Ok(rows) +} + +/// Recursively find all publication versions starting from a given publication ID. + +/// This is necessary publication versions can be the same across publications. +/// To make versions query simpler, we walk the publication hierarchy starting from +/// `publication_name` looking for related publications. +/// The function returns all the `publication_version` IDs, even in simple cases where a publication +/// has no hierarchy. +/// +/// # Errors +/// Errors if can't establish a connection to the database. +pub async fn find_publication_versions_for_publication( + conn: &DatabaseConnection, + publication_name: String, + stele: String, +) -> anyhow::Result> { + let mut versions: HashSet = + find_all_publication_versions_by_publication_name_and_stele( + conn, + &publication_name, + &stele, + ) + .await? + .into_iter() + .collect(); + + let mut checked_publication_names = HashSet::new(); + checked_publication_names.insert(publication_name.clone()); + + let mut publication_names_to_check = HashSet::new(); + publication_names_to_check.insert(publication_name); + + while !publication_names_to_check.is_empty() { + let new_versions: HashSet = + find_all_publication_versions_in_publication_has_publication_versions( + conn, + publication_names_to_check.clone().into_iter().collect(), + &stele, + ) + .await? + .into_iter() + .collect(); + versions.extend(new_versions.clone()); + + checked_publication_names.extend(publication_names_to_check.clone()); + + publication_names_to_check = new_versions + .clone() + .into_iter() + .filter(|pv| !checked_publication_names.contains(&pv.publication.clone())) + .map(|pv| pv.publication) + .collect(); + } + Ok(versions.into_iter().collect()) +} + +/// Find all publication names by date and stele. +/// +/// # Errors +/// Errors if can't establish a connection to the database. +pub async fn find_all_publications_by_date_and_stele_order_by_name_desc( + conn: &DatabaseConnection, + date: String, + stele: String, +) -> anyhow::Result> { + let statement = " + SELECT * + FROM publication + WHERE date = $1 AND stele = $2 + ORDER BY name DESC + "; + let rows = match conn.kind { + DatabaseKind::Sqlite => { let mut connection = conn.pool.acquire().await?; sqlx::query_as::<_, Publication>(statement) - .bind(name) .bind(date) .bind(stele) - .fetch_one(&mut *connection) - .await - .ok() + .fetch_all(&mut *connection) + .await? + } + DatabaseKind::Postgres => { + unimplemented!(); } }; - Ok(row) + Ok(rows) } -/// Find a publication version by `publication_id` and `version`. +/// Find last inserted publication version in DB. +/// Used when partially inserted new changes to the database. /// /// # Errors /// Errors if can't establish a connection to the database. -pub async fn find_publication_version_by_publication_id_and_version( +pub async fn find_last_inserted_publication_version_by_publication_and_stele( conn: &DatabaseConnection, - publication_id: i32, - codified_date: &str, + publication: &str, + stele: &str, ) -> anyhow::Result> { - let statement: &'static str = r#" + let statement = " SELECT * FROM publication_version - WHERE publication_id = $1 AND version = $2 - "#; + WHERE publication = $1 AND stele = $2 + ORDER BY version DESC + LIMIT 1 + "; let row = match conn.kind { DatabaseKind::Sqlite => { let mut connection = conn.pool.acquire().await?; sqlx::query_as::<_, PublicationVersion>(statement) - .bind(publication_id) - .bind(codified_date) + .bind(publication) + .bind(stele) .fetch_one(&mut *connection) .await .ok() } DatabaseKind::Postgres => { - let mut connection = conn.pool.acquire().await?; - sqlx::query_as::<_, PublicationVersion>(statement) - .bind(publication_id) - .bind(codified_date) - .fetch_one(&mut *connection) - .await - .ok() + unimplemented!(); } }; Ok(row) diff --git a/src/history/changes.rs b/src/history/changes.rs index 2366028..27cc11a 100644 --- a/src/history/changes.rs +++ b/src/history/changes.rs @@ -1,13 +1,21 @@ //! Module for inserting changes into the database -#![allow(clippy::shadow_reuse)] +#![allow(clippy::exit, clippy::shadow_reuse, clippy::future_not_send)] +use crate::db::models::changed_library_document::ChangedLibraryDocument; use crate::db::models::document_change::DocumentChange; +use crate::db::models::library::Library; +use crate::db::models::library_change::LibraryChange; use crate::db::models::publication::Publication; +use crate::db::models::publication_has_publication_versions::PublicationHasPublicationVersions; use crate::db::statements::inserts::{ create_document, create_publication, create_publication_version, create_stele, create_version, + insert_changed_library_document_bulk, insert_document_changes_bulk, insert_library_bulk, + insert_library_changes_bulk, insert_publication_has_publication_versions_bulk, + update_publication_by_name_and_stele_set_revoked_true, }; use crate::db::statements::queries::{ - find_last_inserted_publication, find_publication_by_name_and_date_and_stele_id, - find_publication_version_by_publication_id_and_version, find_stele_by_name, + find_all_publications_by_date_and_stele_order_by_name_desc, find_last_inserted_publication, + find_last_inserted_publication_version_by_publication_and_stele, + find_publication_by_name_and_stele, find_publication_versions_for_publication, }; use crate::history::rdf::graph::StelaeGraph; use crate::history::rdf::namespaces::{dcterms, oll}; @@ -18,16 +26,20 @@ use crate::{ stelae::archive::Archive, }; use anyhow::Context; -use sophia::api::MownStr; +use git2::{TreeWalkMode, TreeWalkResult}; +use sophia::api::ns::rdfs; use sophia::api::{prelude::*, term::SimpleTerm}; use sophia::xml::parser; -use sophia::{api::ns::rdfs, inmem::graph::FastGraph}; use sqlx::types::chrono::NaiveDate; use std::{ - collections::HashMap, + borrow::ToOwned, + io::{self, BufReader}, path::{Path, PathBuf}, + process, + result::Result, }; -use walkdir::WalkDir; + +use super::rdf::graph::Bag; /// Inserts changes from the archive into the database /// @@ -38,7 +50,7 @@ pub async fn insert( raw_archive_path: &str, archive_path: PathBuf, stele: Option, -) -> std::io::Result<()> { +) -> io::Result<()> { let conn = match db::init::connect(&archive_path).await { Ok(conn) => conn, Err(err) => { @@ -46,10 +58,10 @@ pub async fn insert( "error: could not connect to database. Confirm that DATABASE_URL env var is set correctly." ); tracing::error!("Error: {:?}", err); - std::process::exit(1); + process::exit(1); } }; - if let Some(stele) = stele { + if let Some(_stele) = stele { insert_changes_single_stele()?; } else { insert_changes_archive(&conn, raw_archive_path, &archive_path) @@ -62,8 +74,9 @@ pub async fn insert( Ok(()) } -fn insert_changes_single_stele() -> std::io::Result<()> { - todo!() +/// Insert changes for a single stele instead of an entire archive +fn insert_changes_single_stele() -> io::Result<()> { + unimplemented!() } /// Insert changes from the archive into the database @@ -78,7 +91,7 @@ async fn insert_changes_archive( false, )?; - for (name, mut stele) in archive.stelae { + for (name, mut stele) in archive.get_stelae() { if let Some(repositories) = stele.get_repositories()? { let Some(rdf_data) = repositories.get_rdf_repository() else { continue; @@ -106,91 +119,9 @@ async fn insert_changes_from_rdf_repository( ) -> anyhow::Result<()> { tracing::info!("Inserting changes from RDF repository: {}", stele_id); tracing::info!("RDF repository path: {}", rdf_repo.path.display()); - let run_documents = false; - if run_documents { - let mut graph = FastGraph::new(); - let head_commit = rdf_repo.repo.head()?.peel_to_commit()?; - let tree = head_commit.tree()?; - tree.walk(git2::TreeWalkMode::PreOrder, |_, entry| { - let path_name = entry.name().unwrap(); - if path_name.contains(".rdf") { - let blob = rdf_repo.repo.find_blob(entry.id()).unwrap(); - let data = blob.content(); - let reader = std::io::BufReader::new(data); - parser::parse_bufread(reader) - .add_to_graph(&mut graph) - .unwrap(); - } - git2::TreeWalkResult::Ok - })?; - // for entry in WalkDir::new(&rdf_repo.path) { - // match entry { - // Ok(entry) if is_rdf(&entry) => { - // tracing::debug!("Parsing file: {:?}", entry.path()); - // let file = std::fs::File::open(entry.path())?; - // let reader = std::io::BufReader::new(file); - // parser::parse_bufread(reader).add_to_graph(&mut graph)?; - // } - // Ok(entry) => { - // tracing::debug!("Skipping non-RDF file: {:?}", entry.path()); - // continue; - // } - // Err(err) => { - // tracing::error!("Error reading file: {:?}", err); - // } - // } - // } - let documents = graph.triples_matching(Any, Any, [oll::DocumentVersion]); - let mut doc_to_versions: HashMap> = HashMap::new(); - for triple in documents { - let triple = triple.unwrap(); - let document = triple.s(); - let mut doc_id_triples = graph.triples_matching([document], [oll::docId], Any); - if let Some(doc_id_triple) = doc_id_triples.next() { - let object = doc_id_triple.unwrap().o(); - let document_iri = document.iri().unwrap().to_string(); - if let SimpleTerm::LiteralDatatype(doc_id, _) = object { - doc_to_versions - .entry(doc_id.to_string()) - .or_insert_with(Vec::new) - .push(document_iri); - } - } - } - for versions in doc_to_versions.values() { - // Find the version with the maximum docId - let doc_version = versions - .iter() - .max_by_key(|&v| { - let mut doc_id_triples = graph.triples_matching([v.as_str()], [oll::docId], Any); - doc_id_triples - .next() - .map_or_else(String::new, |doc_id_triple| { - let object = doc_id_triple.unwrap().o(); - if let SimpleTerm::LiteralDatatype(doc_id, _) = object { - doc_id.to_string() - } else { - String::new() - } - }) - }) - .unwrap(); - // Get the docId for this version - // dbg!(&doc_version); - let doc_version_iri_ref = IriRef::new_unchecked(MownStr::from_str(doc_version.as_str())); - let mut doc_id_triples = - graph.triples_matching([SimpleTerm::Iri(doc_version_iri_ref)], [oll::docId], Any); - if let Some(doc_id_triple) = doc_id_triples.next() { - let object = doc_id_triple.unwrap().o(); - if let SimpleTerm::LiteralDatatype(doc_id, _) = object { - create_document(conn, doc_id).await?; - } - } - } - } let tx = conn.pool.begin().await?; - match load_delta_from_publications(conn, &rdf_repo, stele_id).await { - Ok(_) => { + match load_delta_for_stele(conn, &rdf_repo, stele_id).await { + Ok(()) => { tx.commit().await?; Ok(()) } @@ -201,25 +132,19 @@ async fn insert_changes_from_rdf_repository( } } - let oll_document_version: NsTerm = oll.get("DocumentVersion").unwrap(); - let oll_doc_id = oll.get("docId").unwrap(); - /// Load deltas from the publications -async fn load_delta_from_publications( +async fn load_delta_for_stele( conn: &DatabaseConnection, rdf_repo: &Repo, stele: &str, ) -> anyhow::Result<()> { create_stele(conn, stele).await?; - match find_last_inserted_publication(conn, stele).await? { - Some(publication) => { - tracing::info!("Inserting changes from last inserted publication"); - load_delta_from_publications_from_last_inserted_publication().await?; - } - None => { - tracing::info!("Inserting changes from beginning for stele: {}", stele); - load_delta_from_publications_from_beginning(conn, rdf_repo, stele).await?; - } + if let Some(publication) = find_last_inserted_publication(conn, stele).await? { + tracing::info!("Inserting changes from last inserted publication"); + load_delta_from_publications(conn, rdf_repo, stele, Some(publication)).await?; + } else { + tracing::info!("Inserting changes from beginning for stele: {}", stele); + load_delta_from_publications(conn, rdf_repo, stele, None).await?; } Ok(()) } @@ -228,47 +153,68 @@ async fn load_delta_from_publications( /// /// # Errors /// Errors if the delta cannot be loaded from the publications -async fn load_delta_from_publications_from_beginning( +#[allow(clippy::unwrap_used)] +async fn load_delta_from_publications( conn: &DatabaseConnection, rdf_repo: &Repo, stele: &str, + last_inserted_publication: Option, ) -> anyhow::Result<()> { let head_commit = rdf_repo.repo.head()?.peel_to_commit()?; let tree = head_commit.tree()?; let publications_dir_entry = tree.get_path(&PathBuf::from("_publication"))?; let publications_subtree = rdf_repo.repo.find_tree(publications_dir_entry.id())?; - for publication_entry in publications_subtree.iter() { - let name = publication_entry.name().unwrap(); + let mut last_inserted_date: Option = None; + for publication_entry in &publications_subtree { let mut pub_graph = StelaeGraph::new(); let object = publication_entry.to_object(&rdf_repo.repo)?; - let Some(publication_tree) = object.as_tree() else { - anyhow::bail!("Expected a tree but got something else"); - }; + let publication_tree = object + .as_tree() + .context("Expected a tree but got something else")?; let index_rdf = publication_tree.get_path(&PathBuf::from("index.rdf"))?; let blob = rdf_repo.repo.find_blob(index_rdf.id())?; let data = blob.content(); - let reader = std::io::BufReader::new(data); + let reader = io::BufReader::new(data); parser::parse_bufread(reader).add_to_graph(&mut pub_graph.g)?; let pub_label = pub_graph.literal_from_triple_matching(None, Some(rdfs::label), None)?; let pub_name = pub_label .strip_prefix("Publication ") .context("Could not strip prefix")? - .to_string(); - tracing::info!("Publication: {pub_name}"); + .to_owned(); let pub_date = pub_graph.literal_from_triple_matching(None, Some(dcterms::available), None)?; - publication_tree.walk(git2::TreeWalkMode::PreOrder, |_, entry| { - let path_name = entry.name().unwrap(); + let pub_date = NaiveDate::parse_from_str(pub_date.as_str(), "%Y-%m-%d")?; + if let Some(last_inserted_pub) = last_inserted_publication.as_ref() { + let last_inserted_pub_date = + NaiveDate::parse_from_str(&last_inserted_pub.date, "%Y-%m-%d")?; + // continue from last inserted publication, since that publication can contain + // new changes (versions) that are not in db + if pub_date < last_inserted_pub_date { + // skip past publications since they are already in db + continue; + } + last_inserted_date = find_last_inserted_publication_version_by_publication_and_stele( + conn, &pub_name, stele, + ) + .await? + .map(|pv| { + NaiveDate::parse_from_str(&pv.version, "%Y-%m-%d").context("Could not parse date") + }) + .and_then(Result::ok); + } + tracing::info!("[{stele}] | Publication: {pub_name}"); + publication_tree.walk(TreeWalkMode::PreOrder, |_, entry| { + let path_name = entry.name().unwrap_or_default(); if path_name.contains(".rdf") { let current_blob = rdf_repo.repo.find_blob(entry.id()).unwrap(); let current_content = current_blob.content(); - parser::parse_bufread(std::io::BufReader::new(current_content)) + parser::parse_bufread(BufReader::new(current_content)) .add_to_graph(&mut pub_graph.g) .unwrap(); } - git2::TreeWalkResult::Ok + TreeWalkResult::Ok })?; - let pub_date = NaiveDate::parse_from_str(pub_date.as_str(), "%Y-%m-%d")?; + let (last_valid_pub_name, last_valid_codified_date) = referenced_publication_information(&pub_graph); create_publication( @@ -280,29 +226,26 @@ async fn load_delta_from_publications_from_beginning( last_valid_codified_date, ) .await?; - let publication = - find_publication_by_name_and_date_and_stele_id(conn, &pub_name, &pub_date, stele) - .await? - .unwrap(); - - load_delta_for_publication(conn, publication, &pub_graph, None).await?; + let publication = find_publication_by_name_and_stele(conn, &pub_name, stele).await?; + load_delta_for_publication(conn, publication, &pub_graph, last_inserted_date).await?; } Ok(()) } -async fn load_delta_from_publications_from_last_inserted_publication() -> anyhow::Result<()> { - todo!() -} - +/// Load all deltas for the publication given a stele /// +/// # Errors +/// Errors if database connection fails or if delta cannot be loaded for the publication async fn load_delta_for_publication( conn: &DatabaseConnection, publication: Publication, pub_graph: &StelaeGraph, - last_inserted_date: Option, + last_inserted_date: Option, ) -> anyhow::Result<()> { - let pub_document_versions = get_document_publication_versions(&pub_graph); - let pub_collection_versions = get_collection_publication_versions(&pub_graph); + let pub_document_versions = + pub_graph.all_iris_from_triple_matching(None, None, Some(oll::DocumentVersion))?; + let pub_collection_versions = + pub_graph.all_iris_from_triple_matching(None, None, Some(oll::CollectionVersion))?; insert_document_changes( conn, @@ -313,29 +256,35 @@ async fn load_delta_for_publication( ) .await?; - // insert_library_changes(conn, &last_inserted_date, pub_collection_versions, pub_graph, &publication).await?; - // insert_shared_publication_versions_for_publication(con, pub.id, pub.last_valid_publication_name, pub.last_valid_version, pub.stele_id) + insert_library_changes( + conn, + &last_inserted_date, + pub_collection_versions, + pub_graph, + &publication, + ) + .await?; + insert_shared_publication_versions_for_publication(conn, &publication).await?; - // revoke_same_date_publications(conn, publication, stele_id).await?; + revoke_same_date_publications(conn, publication).await?; Ok(()) } +/// Insert document changes into the database async fn insert_document_changes( conn: &DatabaseConnection, - last_inserted_date: &Option, + last_inserted_date: &Option, pub_document_versions: Vec<&SimpleTerm<'_>>, pub_graph: &StelaeGraph, publication: &Publication, ) -> anyhow::Result<()> { - let mut document_changes_bulk: Vec = Vec::new(); + let mut document_changes_bulk: Vec = vec![]; for version in pub_document_versions { let codified_date = pub_graph.literal_from_triple_matching(Some(version), Some(oll::codifiedDate), None)?; - if let Some(last_inserted_date) = last_inserted_date { + if let Some(last_inserted_date) = last_inserted_date.as_ref() { let codified_date = NaiveDate::parse_from_str(codified_date.as_str(), "%Y-%m-%d")?; - let last_inserted_date = - NaiveDate::parse_from_str(last_inserted_date.as_str(), "%Y-%m-%d")?; - if codified_date <= last_inserted_date { + if &codified_date <= last_inserted_date { // Date already inserted continue; } @@ -349,8 +298,8 @@ async fn insert_document_changes( let changes_uri = pub_graph.iri_from_triple_matching(Some(version), Some(oll::hasChanges), None)?; - let changes = pub_graph.subjects_from_triples_matching_subject(changes_uri); - for change in changes { + let changes = Bag::new(pub_graph, changes_uri); + for change in changes.items()? { let doc_mpath = pub_graph.literal_from_triple_matching( Some(&change), Some(oll::documentMaterializedPath), @@ -368,37 +317,137 @@ async fn insert_document_changes( )?; for status in statuses { document_changes_bulk.push(DocumentChange { - doc_mpath: doc_mpath.to_string(), - status: status.to_string(), - url: url.to_string(), + doc_mpath: doc_mpath.clone(), + status: status.clone(), + url: url.clone(), change_reason: reason.clone(), publication: publication.name.clone(), - version: codified_date.to_string(), + version: codified_date.clone(), stele: publication.stele.clone(), - doc_id: doc_id.to_string(), + doc_id: doc_id.clone(), }); } } - insert_document_changes_bulk(conn, &document_changes_bulk).await?; - // let publication_version = find_publication_version_by_publication_id_and_version( - // conn, - // publication.id, - // &codified_date, - // ) - // .await? - // .context("Could not find publication version")?; } + insert_document_changes_bulk(conn, document_changes_bulk).await?; Ok(()) } +/// Insert library changes into the database async fn insert_library_changes( conn: &DatabaseConnection, - last_inserted_date: &Option, + last_inserted_date: &Option, pub_collection_versions: Vec<&SimpleTerm<'_>>, - pub_graph: &FastGraph, + pub_graph: &StelaeGraph, publication: &Publication, ) -> anyhow::Result<()> { - todo!() + let mut library_changes_bulk: Vec = vec![]; + let mut changed_library_document_bulk: Vec = vec![]; + let mut library_bulk: Vec = vec![]; + for version in pub_collection_versions { + let codified_date = + pub_graph.literal_from_triple_matching(Some(version), Some(oll::codifiedDate), None)?; + if let Some(last_inserted_date) = last_inserted_date.as_ref() { + let codified_date = NaiveDate::parse_from_str(codified_date.as_str(), "%Y-%m-%d")?; + if &codified_date <= last_inserted_date { + // Date already inserted + continue; + } + } + let library_mpath = pub_graph.literal_from_triple_matching( + Some(version), + Some(oll::libraryMaterializedPath), + None, + )?; + let url = pub_graph.literal_from_triple_matching(Some(version), Some(oll::url), None)?; + let status = + pub_graph.literal_from_triple_matching(Some(version), Some(oll::status), None)?; + library_bulk.push(Library { + mpath: library_mpath.clone(), + }); + library_changes_bulk.push(LibraryChange { + library_mpath: library_mpath.clone(), + publication: publication.name.clone(), + version: codified_date.clone(), + stele: publication.stele.clone(), + status: status.clone(), + url: url.clone(), + }); + let changes_uri = + pub_graph.iri_from_triple_matching(Some(version), Some(oll::hasChanges), None)?; + let changes = Bag::new(pub_graph, changes_uri); + for change in changes.items()? { + let Ok(el_status) = + pub_graph.literal_from_triple_matching(Some(&change), Some(oll::status), None) + else { + continue; + }; + let Ok(doc_mpath) = pub_graph.literal_from_triple_matching( + Some(&change), + Some(oll::documentMaterializedPath), + None, + ) else { + continue; + }; + changed_library_document_bulk.push(ChangedLibraryDocument { + publication: publication.name.clone(), + version: codified_date.clone(), + stele: publication.stele.clone(), + doc_mpath: doc_mpath.clone(), + status: el_status.clone(), + library_mpath: library_mpath.clone(), + url: url.clone(), + }); + } + } + insert_library_bulk(conn, library_bulk).await?; + insert_library_changes_bulk(conn, library_changes_bulk).await?; + insert_changed_library_document_bulk(conn, changed_library_document_bulk).await?; + Ok(()) +} + +/// Insert shared publication versions for the publication +/// Support for lightweight publications. +/// Populate the many-to-many mapping between change objects and publications +async fn insert_shared_publication_versions_for_publication( + conn: &DatabaseConnection, + publication: &Publication, +) -> anyhow::Result<()> { + let mut publication_has_publication_versions_bulk: Vec = + vec![]; + let mut publication_versions = find_publication_versions_for_publication( + conn, + publication.name.clone(), + publication.stele.clone(), + ) + .await?; + if let (Some(last_valid_pub_name), Some(_)) = ( + publication.last_valid_publication_name.as_ref(), + publication.last_valid_version.as_ref(), + ) { + let publication_versions_last_valid = find_publication_versions_for_publication( + conn, + last_valid_pub_name.clone(), + publication.stele.clone(), + ) + .await?; + publication_versions.extend(publication_versions_last_valid); + } + publication_has_publication_versions_bulk.extend(publication_versions.iter().map(|pv| { + PublicationHasPublicationVersions { + publication: publication.name.clone(), + referenced_publication: pv.publication.clone(), + referenced_version: pv.version.clone(), + stele: publication.stele.clone(), + } + })); + insert_publication_has_publication_versions_bulk( + conn, + publication_has_publication_versions_bulk, + ) + .await?; + + Ok(()) } /// Get the last valid publication name and codified date from the graph @@ -406,44 +455,36 @@ fn referenced_publication_information(pub_graph: &StelaeGraph) -> (Option anyhow::Result<()> { - todo!() -} - -/// Get the document publication version IRIs from the graph -fn get_document_publication_versions(graph: &StelaeGraph) -> Vec<&SimpleTerm> { - let triples = graph.g.triples_matching(Any, Any, [oll::DocumentVersion]); - triples - .filter_map(|t| { - let t = t.ok()?; - let subject = t.s(); - Some(subject) - }) - .collect() -} - -/// Get the collection publication version IRIs from the graph -fn get_collection_publication_versions(graph: &StelaeGraph) -> Vec<&SimpleTerm> { - let triples = graph.g.triples_matching(Any, Any, [oll::CollectionVersion]); - triples - .filter_map(|t| { - let t = t.ok()?; - let subject = t.s(); - Some(subject) - }) - .collect() + let duplicate_publications = find_all_publications_by_date_and_stele_order_by_name_desc( + conn, + publication.date, + publication.stele, + ) + .await?; + if let Some(duplicate_publications_slice) = duplicate_publications.get(1..) { + for duplicate_pub in duplicate_publications_slice { + update_publication_by_name_and_stele_set_revoked_true( + conn, + &duplicate_pub.name, + &duplicate_pub.stele, + ) + .await?; + } + } + Ok(()) } diff --git a/src/history/rdf/graph.rs b/src/history/rdf/graph.rs index 05d6d39..a7c2ef2 100644 --- a/src/history/rdf/graph.rs +++ b/src/history/rdf/graph.rs @@ -1,20 +1,31 @@ +#![allow( + clippy::module_name_repetitions, + clippy::min_ident_chars, + clippy::pattern_type_mismatch +)] /// The helper methods for working with RDF in Stelae. use anyhow::Context; -use sophia::api::graph::Graph; -use sophia::api::ns::rdf::li; +use sophia::api::graph::{GTripleSource, Graph}; use sophia::api::ns::NsTerm; +use sophia::api::MownStr; use sophia::api::{prelude::*, term::SimpleTerm}; use sophia::inmem::graph::FastGraph; -use sophia::inmem::index::TermIndexFullError; - +use std::iter; /// Stelae representation of an RDF graph. pub struct StelaeGraph { /// The underlying graph. pub g: FastGraph, } +impl Default for StelaeGraph { + fn default() -> Self { + Self::new() + } +} + impl StelaeGraph { /// Create a new graph. + #[must_use] pub fn new() -> Self { Self { g: FastGraph::new(), @@ -32,14 +43,26 @@ impl StelaeGraph { object: Option, ) -> anyhow::Result { let triple = self.get_next_triples_matching(subject, predicate, object)?; - let literal = match triple.o() { - SimpleTerm::LiteralLanguage(literal, _) => literal, - SimpleTerm::LiteralDatatype(literal, _) => literal, - _ => { - anyhow::bail!("Expected literal language, got - {:?}", triple.o()); + let literal = self.term_to_literal(&triple)?; + Ok(literal) + } + + /// Convert a term to a literal. + /// + /// # Errors + /// Errors if the term is not an RDF literal. + pub fn term_to_literal(&self, term: &[&SimpleTerm<'_>; 3]) -> anyhow::Result { + match &term.o() { + SimpleTerm::LiteralDatatype(literal, _) | SimpleTerm::LiteralLanguage(literal, _) => { + Ok(literal.to_string()) } - }; - Ok(literal.to_string()) + SimpleTerm::Iri(_) + | SimpleTerm::BlankNode(_) + | SimpleTerm::Triple(_) + | SimpleTerm::Variable(_) => { + anyhow::bail!("Expected literal language, got - {:?}", term) + } + } } /// Extract all literals from a triple matching. @@ -53,11 +76,12 @@ impl StelaeGraph { predicate: Option, object: Option, ) -> anyhow::Result> { - Ok(self - .literal_from_triple_matching(subject, predicate, object) - .into_iter() - .map(|t| t) - .collect()) + let mut literals = Vec::new(); + let triples_iter = self.triples_matching_inner(subject, predicate, object); + for term in triples_iter { + literals.push(self.term_to_literal(&term?)?); + } + Ok(literals) } /// Extract an IRI from a triple matching. @@ -72,59 +96,122 @@ impl StelaeGraph { object: Option>, ) -> anyhow::Result { let triple = self.get_next_triples_matching(subject, predicate, object)?; - let iri = match triple.o() { - SimpleTerm::Iri(literal) => literal, - _ => { - anyhow::bail!("Expected literal language, got - {:?}", triple.o()); - } + let SimpleTerm::Iri(iri) = &triple.o() else { + anyhow::bail!("Expected literal language, got - {:?}", triple.o()); }; Ok(SimpleTerm::Iri(iri.clone())) } - /// Extract subjects from a triple matching a subject. - pub fn subjects_from_triples_matching_subject(&self, subject: SimpleTerm) -> Vec { - self.g - .triples_matching([subject], Any, Any) - .into_iter() - .filter_map(|t| { - let t = t.ok()?; - Some(t.s().clone()) - }) - .collect() - } - + /// Returns the next triple matching the given subject, predicate, and object. + /// + /// # Errors + /// Errors if the triple matching the object is not found. fn get_next_triples_matching<'graph>( &'graph self, subject: Option<&'graph SimpleTerm>, predicate: Option>, object: Option>, ) -> anyhow::Result<[&'graph SimpleTerm<'_>; 3]> { - let triple = match (subject, predicate, object) { - (Some(s), None, None) => { - self.g.triples_matching([s], Any, Any).next().context("Did not find a triple matching provided subject in the graph") - }, - (None, Some(p), None) => { - self.g.triples_matching(Any, [p], Any).next().context("Did not find a triple matching provided predicate in the graph") - }, - (None, None, Some(o)) => { - self.g.triples_matching(Any, Any, [o]).next().context("Did not find a triple matching provided object in the graph") - }, - (Some(s), Some(p), None) => { - self.g.triples_matching([s], [p], Any).next().context("Did not find a triple matching provided subject and predicate in the graph") - }, - (Some(s), None, Some(o)) => { - self.g.triples_matching([s], Any, [o]).next().context("Did not find a triple matching provided subject and object in the graph") - }, - (None, Some(p), Some(o)) => { - self.g.triples_matching(Any, [p], [o]).next().context("Did not find a triple matching provided predicate and object in the graph") - }, - (Some(s), Some(p), Some(o)) => { - self.g.triples_matching([s], [p], [o]).next().context("Did not find a triple matching provided subject, predicate and object in the graph") - }, - (None, None, None) => { - anyhow::bail!("No subject, predicate or object provided") - } - }?; + let triple = self + .triples_matching_inner(subject, predicate, object) + .next() + .context("Expected to find triple matching")?; Ok(triple?) } + + /// Utility method to convert from Option method arguments to a triple source. + fn triples_matching_inner<'graph>( + &'graph self, + subject: Option<&'graph SimpleTerm>, + predicate: Option>, + object: Option>, + ) -> GTripleSource<'graph, FastGraph> { + let triple = match (subject, predicate, object) { + (Some(s), None, None) => self.g.triples_matching([s], Any, Any), + (None, Some(p), None) => self.g.triples_matching(Any, [p], Any), + (None, None, Some(o)) => self.g.triples_matching(Any, Any, [o]), + (Some(s), Some(p), None) => self.g.triples_matching([s], [p], Any), + (Some(s), None, Some(o)) => self.g.triples_matching([s], Any, [o]), + (None, Some(p), Some(o)) => self.g.triples_matching(Any, [p], [o]), + (Some(s), Some(p), Some(o)) => self.g.triples_matching([s], [p], [o]), + (None, None, None) => Box::new(iter::empty()), + }; + triple + } + + /// Extract all IRIs from a triple matching. + /// + /// # Errors + /// Errors if the triple matching the object is not found. + /// Errors if the object is not an RDF IRI. + pub fn all_iris_from_triple_matching<'graph>( + &'graph self, + subject: Option<&'graph SimpleTerm>, + predicate: Option>, + object: Option>, + ) -> anyhow::Result> { + let triples_iter = self.triples_matching_inner(subject, predicate, object); + let iris = triples_iter + .into_iter() + .filter_map(|triple| { + let found_triple = triple.ok()?; + let subj = found_triple.s(); + Some(subj) + }) + .collect(); + Ok(iris) + } +} + +/// Unordered container of RDF items. +pub struct Bag<'graph> { + /// The container URI. + uri: SimpleTerm<'graph>, + /// The underlying graph. + graph: &'graph StelaeGraph, +} + +impl Bag<'_> { + /// Create a new Bag. + #[must_use] + pub const fn new<'graph>(graph: &'graph StelaeGraph, uri: SimpleTerm<'graph>) -> Bag<'graph> { + Bag { uri, graph } + } + + /// Extract items from the container. + /// + /// # Errors + /// Errors if the items are not found. + #[allow(clippy::separated_literal_suffix)] + pub fn items(&self) -> anyhow::Result> { + let container = &self.uri; + let mut i = 1_u32; + let mut l_ = vec![]; + loop { + let el_uri = format!("http://www.w3.org/1999/02/22-rdf-syntax-ns#_{i}"); + let elem_iri = SimpleTerm::Iri(IriRef::new_unchecked(MownStr::from_str(&el_uri))); + if self + .graph + .g + .triples_matching([container], Some(elem_iri.clone()), Any) + .next() + .is_some() + { + i += 1; + let item = self + .graph + .g + .triples_matching([container], Some(elem_iri), Any) + .next() + .context(format!("Expected to find item in {container:?}"))? + .context("Expected to find item in container")? + .o() + .clone(); + l_.push(item); + } else { + break; + } + } + Ok(l_) + } } diff --git a/src/history/rdf/namespaces.rs b/src/history/rdf/namespaces.rs index 58fc2ed..ba3f0d3 100644 --- a/src/history/rdf/namespaces.rs +++ b/src/history/rdf/namespaces.rs @@ -16,7 +16,8 @@ pub mod oll { documentMaterializedPath, url, reason, - status + status, + libraryMaterializedPath } } diff --git a/src/lib.rs b/src/lib.rs index 5df6641..6f55544 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -83,6 +83,11 @@ clippy::exhaustive_enums, clippy::question_mark_used, clippy::semicolon_outside_block, + // We tend to break up long functions into smaller ones, so this lint is not useful + clippy::single_call_fn, + clippy::arithmetic_side_effects, + // We'll allow unimplemented! in code, but disallow todo! + clippy::unimplemented, )] pub mod db; diff --git a/src/server/git.rs b/src/server/git.rs index 4f13a78..86b3c15 100644 --- a/src/server/git.rs +++ b/src/server/git.rs @@ -4,14 +4,19 @@ // Unused asyncs are the norm in Actix route definition files clippy::unused_async, clippy::unreachable, - clippy::let_with_type_underscore + clippy::let_with_type_underscore, + // Clippy wrongly detects the `infinite_loop` lint on functions with tracing::instrument! + clippy::infinite_loop )] use actix_web::{get, web, App, HttpResponse, HttpServer, Responder}; -use git2; +use git2::{self, ErrorCode}; use lazy_static::lazy_static; use regex::Regex; -use std::path::{Path, PathBuf}; +use std::{ + io, + path::{Path, PathBuf}, +}; use tracing_actix_web::TracingLogger; use super::errors::StelaeError; @@ -29,7 +34,7 @@ struct AppState { /// Remove leading and trailing `/`s from the `path` string. fn clean_path(path: &str) -> String { lazy_static! { - static ref RE: Regex = Regex::new(r"(?:^/*|/*$)").expect("Failed to compile regex!?!"); + static ref RE: Regex = Regex::new("(?:^/*|/*$)").expect("Failed to compile regex!?!"); } RE.replace_all(path, "").to_string() } @@ -92,7 +97,7 @@ fn blob_error_response(error: &anyhow::Error, namespace: &str, name: &str) -> Ht if let Some(git_error) = error.downcast_ref::() { return match git_error.code() { // TODO: check this is the right error - git2::ErrorCode::NotFound => { + ErrorCode::NotFound => { HttpResponse::NotFound().body(format!("repo {namespace}/{name} does not exist")) } _ => HttpResponse::InternalServerError().body("Unexpected Git error"), @@ -109,11 +114,7 @@ fn blob_error_response(error: &anyhow::Error, namespace: &str, name: &str) -> Ht /// Serve git repositories in the Stelae archive. #[actix_web::main] // or #[tokio::main] -pub async fn serve_git( - raw_archive_path: &str, - archive_path: PathBuf, - port: u16, -) -> std::io::Result<()> { +pub async fn serve_git(raw_archive_path: &str, archive_path: PathBuf, port: u16) -> io::Result<()> { let bind = "127.0.0.1"; let message = "Serving content from the Stelae archive at"; tracing::info!("{message} '{raw_archive_path}' on http://{bind}:{port}.",); diff --git a/src/server/publish.rs b/src/server/publish.rs index bca013e..8621641 100644 --- a/src/server/publish.rs +++ b/src/server/publish.rs @@ -1,6 +1,5 @@ //! Serve documents in a Stelae archive. -#![allow(clippy::exit)] -#![allow(clippy::unused_async)] +#![allow(clippy::exit, clippy::unused_async, clippy::infinite_loop)] use crate::db; use crate::stelae::archive::Archive; use crate::stelae::types::repositories::{Repositories, Repository}; @@ -13,7 +12,7 @@ use actix_web::{guard, web, App, Error, HttpRequest, HttpResponse, HttpServer, R use git2::Repository as GitRepository; use lazy_static::lazy_static; use regex::Regex; -use std::{fmt, path::PathBuf}; +use std::{fmt, io, path::PathBuf, process}; use tracing_actix_web::TracingLogger; use actix_http::body::MessageBody; @@ -32,7 +31,7 @@ const HEAD_COMMIT: &str = "HEAD"; /// Remove leading and trailing `/`s from the `path` string. fn clean_path(path: &str) -> String { lazy_static! { - static ref RE: Regex = Regex::new(r"(?:^/*|/*$)").expect("Failed to compile regex!?!"); + static ref RE: Regex = Regex::new("(?:^/*|/*$)").expect("Failed to compile regex!?!"); } RE.replace_all(path, "").to_string() } @@ -79,9 +78,9 @@ pub struct SharedState { } impl fmt::Debug for RepoState { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { write!( - f, + formatter, "Repo for {} in the archive at {}", self.repo.name, self.repo.path.display() @@ -90,16 +89,16 @@ impl fmt::Debug for RepoState { } impl fmt::Debug for SharedState { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { let fb = &self.fallback; - match *fb { - Some(ref fallback) => write!( - f, + match fb.as_ref() { + Some(fallback) => write!( + formatter, "Repo for {} in the archive at {}", fallback.repo.name, fallback.repo.path.display() ), - None => write!(f, "No fallback repo"), + None => write!(formatter, "No fallback repo"), } } } @@ -158,7 +157,7 @@ fn find_current_blob(repo: &Repo, shared: &SharedState, path: &str) -> anyhow::R match blob { Ok(content) => Ok(content), Err(error) => { - if let Some(ref fallback) = shared.fallback { + if let Some(fallback) = shared.fallback.as_ref() { let fallback_blob = fallback.repo.get_bytes_at_path(HEAD_COMMIT, path); return fallback_blob.map_or_else( |err| anyhow::bail!("No fallback blob found - {}", err.to_string()), @@ -177,7 +176,7 @@ pub async fn serve_archive( archive_path: PathBuf, port: u16, individual: bool, -) -> std::io::Result<()> { +) -> io::Result<()> { let bind = "127.0.0.1"; let message = "Running Publish Server on a Stelae archive at"; tracing::info!("{message} '{raw_archive_path}' on http://{bind}:{port}.",); @@ -189,7 +188,7 @@ pub async fn serve_archive( "error: could not connect to database. Confirm that DATABASE_URL env var is set correctly." ); tracing::error!("Error: {:?}", err); - std::process::exit(1); + process::exit(1); } }; @@ -197,7 +196,7 @@ pub async fn serve_archive( .unwrap_or_else(|err| { tracing::error!("Unable to parse archive at '{raw_archive_path}'."); tracing::error!("Error: {:?}", err); - std::process::exit(1); + process::exit(1); }); let state = AppState { archive, db }; @@ -205,7 +204,7 @@ pub async fn serve_archive( init_app(&state).unwrap_or_else(|err| { tracing::error!("Unable to initialize app."); tracing::error!("Error: {:?}", err); - std::process::exit(1); + process::exit(1); }) }) .bind((bind, port))? @@ -252,7 +251,7 @@ pub fn init_app( "Failed to initialize routes for root Stele: {}", root.get_qualified_name() ); - std::process::exit(1); + process::exit(1); }); }), )) @@ -291,7 +290,7 @@ pub fn init_app( "Failed to initialize routes for Stele: {}", guarded_stele.get_qualified_name() ); - std::process::exit(1); + process::exit(1); }); }), ); @@ -317,8 +316,8 @@ fn init_repo_state(repo: &Repository, stele: &Stele) -> anyhow::Result anyhow::Result(cfg: &mut web::ServiceConfig, state: &T) -> anyhow::Result<()> { for stele in state.archive().stelae.values() { if let Some(repositories) = stele.repositories.as_ref() { @@ -383,7 +383,7 @@ fn register_root_routes(cfg: &mut web::ServiceConfig, stele: &Stele) -> anyhow:: for repository in sorted_repositories { let custom = &repository.custom; let repo_state = init_repo_state(repository, stele)?; - for route in custom.routes.iter().flat_map(|r| r.iter()) { + for route in custom.routes.iter().flat_map(|routes| routes.iter()) { let actix_route = format!("/{{tail:{}}}", &route); root_scope = root_scope.service( web::resource(actix_route.as_str()) @@ -421,13 +421,13 @@ fn register_dependent_routes( repositories: &Repositories, ) -> anyhow::Result<()> { let sorted_repositories = repositories.get_sorted_repositories(); - for scope in repositories.scopes.iter().flat_map(|s| s.iter()) { + for scope in repositories.scopes.iter().flat_map(|scopes| scopes.iter()) { let scope_str = format!("/{{prefix:{}}}", &scope.as_str()); let mut actix_scope = web::scope(scope_str.as_str()); for repository in &sorted_repositories { let custom = &repository.custom; let repo_state = init_repo_state(repository, stele)?; - for route in custom.routes.iter().flat_map(|r| r.iter()) { + for route in custom.routes.iter().flat_map(|routes| routes.iter()) { if route.starts_with('_') { // Ignore routes in dependent Stele that start with underscore // These routes are handled by the root Stele. diff --git a/src/server/tracing.rs b/src/server/tracing.rs index f50c67c..38701a3 100644 --- a/src/server/tracing.rs +++ b/src/server/tracing.rs @@ -55,7 +55,7 @@ impl RootSpanBuilder for StelaeRootSpanBuilder { // error, as I assume that's considered a handled error. So maybe `outcome` is only ever // an error for an Actix-internal error? Either way, the root span and timings all work // normally for known and handled request errors. - outcome.as_ref().map_or((), |response| { + let () = outcome.as_ref().map_or((), |response| { if let Some(req_start) = response.request().extensions().get::() { let elapsed = req_start.0.elapsed(); let millis = elapsed.as_millis(); diff --git a/src/stelae/archive.rs b/src/stelae/archive.rs index 8756ff4..4867084 100644 --- a/src/stelae/archive.rs +++ b/src/stelae/archive.rs @@ -6,8 +6,9 @@ use crate::stelae::stele::Stele; use crate::utils::archive::{find_archive_path, get_name_parts}; use serde_derive::{Deserialize, Serialize}; use std::collections::HashMap; -use std::fs::{create_dir_all, read_to_string, write}; +use std::fs::{self, create_dir_all, read_to_string, write}; use std::path::{Path, PathBuf}; +use toml_edit::ser; /// The Archive struct is used for interacting with a Stelae Archive. #[derive(Debug, Clone)] @@ -36,7 +37,7 @@ impl Archive { let root = self .stelae .values() - .find(|s| s.is_root()) + .find(|stele| stele.is_root()) .ok_or_else(|| anyhow::anyhow!("No root Stele found in archive"))?; Ok(root) } @@ -70,6 +71,15 @@ impl Archive { Ok(()) } + /// Return sorted vector of all Stelae in the Archive. + #[must_use] + pub fn get_stelae(&self) -> Vec<(String, Stele)> { + let mut stelae = self.stelae.clone(); + let mut stelae_vec: Vec<(String, Stele)> = stelae.drain().collect(); + stelae_vec.sort_by(|first_stele, second_stele| first_stele.0.cmp(&second_stele.0)); + stelae_vec + } + /// Parse an Archive. /// # Errors /// Will raise error if unable to determine the current root stele or if unable to traverse the child steles. @@ -101,10 +111,10 @@ impl Archive { /// If unable to unwrap the parent directory of the current path. pub fn traverse_children(&mut self, current: &Stele) -> anyhow::Result<()> { if let Some(dependencies) = current.get_dependencies()? { - for (qualified_name, _) in dependencies.dependencies { + for qualified_name in dependencies.sorted_dependencies_names() { let parent_dir = self.path.clone(); let (org, name) = get_name_parts(&qualified_name)?; - if std::fs::metadata(parent_dir.join(&org).join(&name)).is_err() { + if fs::metadata(parent_dir.join(&org).join(&name)).is_err() { // Stele does not exist on the filesystem, continue to traverse other Steles continue; } @@ -182,7 +192,7 @@ pub fn init( shallow, headers, }; - let conf_str = toml_edit::ser::to_string_pretty(&conf)?; + let conf_str = ser::to_string_pretty(&conf)?; write(config_path, conf_str)?; let archive = Archive { path, diff --git a/src/stelae/types/dependencies.rs b/src/stelae/types/dependencies.rs index fbdae9b..9811471 100644 --- a/src/stelae/types/dependencies.rs +++ b/src/stelae/types/dependencies.rs @@ -18,3 +18,13 @@ pub struct Dependency { /// The default branch for a Stele. pub branch: String, } + +impl Dependencies { + /// Get the dependencies names for a given Stele. + #[must_use] + pub fn sorted_dependencies_names(&self) -> Vec { + let mut keys = self.dependencies.keys().cloned().collect::>(); + keys.sort(); + keys + } +} diff --git a/src/stelae/types/repositories.rs b/src/stelae/types/repositories.rs index 2cf008b..9872dc1 100644 --- a/src/stelae/types/repositories.rs +++ b/src/stelae/types/repositories.rs @@ -1,5 +1,5 @@ //! A Stele's data repositories. -use std::{collections::HashMap, fmt}; +use std::{collections::HashMap, fmt, string::String}; use serde::{ de::{self, MapAccess, Visitor}, @@ -99,17 +99,18 @@ impl Repositories { /// /// This is needed for serving current documents because Actix routes are matched in the order they are added. #[must_use] + #[allow(clippy::iter_over_hash_type)] pub fn get_sorted_repositories(&self) -> Vec<&Repository> { let mut result = Vec::new(); for repository in self.repositories.values() { result.push(repository); } result.sort_by(|repo1, repo2| { - let routes1 = repo1.custom.routes.as_ref().map_or(0, |r| { - r.iter().map(std::string::String::len).max().unwrap_or(0) + let routes1 = repo1.custom.routes.as_ref().map_or(0, |routes| { + routes.iter().map(String::len).max().unwrap_or(0) }); - let routes2 = repo2.custom.routes.as_ref().map_or(0, |r| { - r.iter().map(std::string::String::len).max().unwrap_or(0) + let routes2 = repo2.custom.routes.as_ref().map_or(0, |routes| { + routes.iter().map(String::len).max().unwrap_or(0) }); routes2.cmp(&routes1) }); @@ -184,20 +185,24 @@ impl<'de> Deserialize<'de> for Repositories { V: MapAccess<'de>, { let repositories_json: HashMap = map.next_value()?; + let mut keys = repositories_json.keys().clone().collect::>(); + keys.sort(); let mut repositories = HashMap::new(); - for (map_key, value) in repositories_json { - let custom_value = value + for key in keys { + let custom_value = repositories_json + .get(key) + .ok_or_else(|| de::Error::custom(format!("Missing {key} in JSON")))? .get("custom") - .ok_or_else(|| serde::de::Error::custom("Missing 'custom' field"))?; + .ok_or_else(|| de::Error::custom("Missing 'custom' field"))?; let custom: Custom = - serde_json::from_value(custom_value.clone()).map_err(|e| { - serde::de::Error::custom(format!("Failed to deserialize 'custom': {e}")) + serde_json::from_value(custom_value.clone()).map_err(|err| { + de::Error::custom(format!("Failed to deserialize 'custom': {err}")) })?; let repo = Repository { - name: map_key.clone(), + name: key.clone(), custom, }; - repositories.insert(map_key, repo); + repositories.insert(key.clone(), repo); } Ok(repositories) } diff --git a/src/utils/cli.rs b/src/utils/cli.rs index eb41c8c..93783b2 100644 --- a/src/utils/cli.rs +++ b/src/utils/cli.rs @@ -3,13 +3,17 @@ // Allow exits because in this file we ideally handle all errors with known exit codes #![allow(clippy::exit)] +use crate::history::changes; use crate::server::git::serve_git; use crate::server::publish::serve_archive; use crate::utils::archive::find_archive_path; -use crate::history::changes; use clap::Parser; +use std::env; +use std::io; use std::path::Path; +use std::process; use tracing; +use tracing_subscriber::fmt; /// Stelae is currently just a simple git server. /// run from the library directory or pass @@ -25,7 +29,7 @@ struct Cli { subcommands: Subcommands, } -/// +/// Subcommands for the Stelae CLI #[derive(Clone, clap::Subcommand)] enum Subcommands { /// Serve git repositories in the Stelae archive @@ -54,9 +58,9 @@ enum Subcommands { /// fn init_tracing() { - tracing_subscriber::fmt::init(); - if std::env::var("RUST_LOG").is_err() { - std::env::set_var("RUST_LOG", "info"); + fmt::init(); + if env::var("RUST_LOG").is_err() { + env::set_var("RUST_LOG", "info"); } } @@ -64,7 +68,7 @@ fn init_tracing() { /// /// # Errors /// TODO: This function should not return errors -pub fn run() -> std::io::Result<()> { +pub fn run() -> io::Result<()> { init_tracing(); tracing::debug!("Starting application"); let cli = Cli::parse(); @@ -74,7 +78,7 @@ pub fn run() -> std::io::Result<()> { "error: could not find `.stelae` folder in `{}` or any parent directory", &cli.archive_path ); - std::process::exit(1); + process::exit(1); }; match cli.subcommands { diff --git a/src/utils/git.rs b/src/utils/git.rs index 16d996c..fbbcd12 100644 --- a/src/utils/git.rs +++ b/src/utils/git.rs @@ -27,9 +27,9 @@ pub struct Repo { } impl fmt::Debug for Repo { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { write!( - f, + formatter, "Repo for {}/{} in the archive at {}", self.org, self.name, self.archive_path ) diff --git a/src/utils/paths.rs b/src/utils/paths.rs index 859acdf..9739d58 100644 --- a/src/utils/paths.rs +++ b/src/utils/paths.rs @@ -6,8 +6,8 @@ use std::path::{Path, PathBuf}; pub fn fix_unc_path(absolute_path: &Path) -> PathBuf { if cfg!(windows) { let absolute_path_str = absolute_path.display().to_string(); - if absolute_path_str.starts_with(r#"\\?"#) { - return PathBuf::from(absolute_path_str.replace(r#"\\?\"#, "")); + if absolute_path_str.starts_with(r"\\?") { + return PathBuf::from(absolute_path_str.replace(r"\\?\", "")); } } absolute_path.to_path_buf() From 5ea1eda45b8c4b60e8bcf1df7239e63a8aca0e52 Mon Sep 17 00:00:00 2001 From: n-dusan Date: Tue, 14 May 2024 11:36:51 +0200 Subject: [PATCH 06/15] refact: use `?` with `anyhow` for `FromRow` trait --- src/db/models/publication.rs | 11 +++++------ src/db/models/publication_version.rs | 9 ++++----- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/src/db/models/publication.rs b/src/db/models/publication.rs index a69c803..593178d 100644 --- a/src/db/models/publication.rs +++ b/src/db/models/publication.rs @@ -26,13 +26,12 @@ pub struct Publication { } impl FromRow<'_, AnyRow> for Publication { - #[allow(clippy::unwrap_in_result, clippy::unwrap_used)] - fn from_row(row: &AnyRow) -> Result { + fn from_row(row: &AnyRow) -> anyhow::Result { Ok(Self { - name: row.try_get("name").unwrap(), - date: row.try_get("date").unwrap(), - stele: row.try_get("stele").unwrap(), - revoked: row.try_get("revoked").unwrap(), + name: row.try_get("name")?, + date: row.try_get("date")?, + stele: row.try_get("stele")?, + revoked: row.try_get("revoked")?, last_valid_publication_name: row.try_get("last_valid_publication_name").ok(), last_valid_version: row.try_get("last_valid_version").ok(), }) diff --git a/src/db/models/publication_version.rs b/src/db/models/publication_version.rs index 025ba01..8893a23 100644 --- a/src/db/models/publication_version.rs +++ b/src/db/models/publication_version.rs @@ -15,12 +15,11 @@ pub struct PublicationVersion { } impl FromRow<'_, AnyRow> for PublicationVersion { - #[allow(clippy::unwrap_in_result, clippy::unwrap_used)] - fn from_row(row: &AnyRow) -> Result { + fn from_row(row: &AnyRow) -> anyhow::Result { Ok(Self { - version: row.try_get("version").unwrap(), - publication: row.try_get("publication").unwrap(), - stele: row.try_get("stele").unwrap(), + version: row.try_get("version")?, + publication: row.try_get("publication")?, + stele: row.try_get("stele")?, build_reason: row.try_get("build_reason").ok(), }) } From e8f7b812f2f6eae84105c3c74b5f6df033ecfe72 Mon Sep 17 00:00:00 2001 From: n-dusan Date: Tue, 14 May 2024 11:37:03 +0200 Subject: [PATCH 07/15] refact: use more descriptive names Makes the variables easier to read. --- src/history/changes.rs | 4 ++-- src/history/rdf/graph.rs | 44 +++++++++++++++++----------------------- 2 files changed, 21 insertions(+), 27 deletions(-) diff --git a/src/history/changes.rs b/src/history/changes.rs index 27cc11a..fc19831 100644 --- a/src/history/changes.rs +++ b/src/history/changes.rs @@ -175,7 +175,7 @@ async fn load_delta_from_publications( let blob = rdf_repo.repo.find_blob(index_rdf.id())?; let data = blob.content(); let reader = io::BufReader::new(data); - parser::parse_bufread(reader).add_to_graph(&mut pub_graph.g)?; + parser::parse_bufread(reader).add_to_graph(&mut pub_graph.fast_graph)?; let pub_label = pub_graph.literal_from_triple_matching(None, Some(rdfs::label), None)?; let pub_name = pub_label .strip_prefix("Publication ") @@ -209,7 +209,7 @@ async fn load_delta_from_publications( let current_blob = rdf_repo.repo.find_blob(entry.id()).unwrap(); let current_content = current_blob.content(); parser::parse_bufread(BufReader::new(current_content)) - .add_to_graph(&mut pub_graph.g) + .add_to_graph(&mut pub_graph.fast_graph) .unwrap(); } TreeWalkResult::Ok diff --git a/src/history/rdf/graph.rs b/src/history/rdf/graph.rs index a7c2ef2..ba838ed 100644 --- a/src/history/rdf/graph.rs +++ b/src/history/rdf/graph.rs @@ -13,8 +13,8 @@ use sophia::inmem::graph::FastGraph; use std::iter; /// Stelae representation of an RDF graph. pub struct StelaeGraph { - /// The underlying graph. - pub g: FastGraph, + /// The underlying `sophia` graph. + pub fast_graph: FastGraph, } impl Default for StelaeGraph { @@ -28,7 +28,7 @@ impl StelaeGraph { #[must_use] pub fn new() -> Self { Self { - g: FastGraph::new(), + fast_graph: FastGraph::new(), } } /// Extract a literal from a triple matching. @@ -127,13 +127,13 @@ impl StelaeGraph { object: Option>, ) -> GTripleSource<'graph, FastGraph> { let triple = match (subject, predicate, object) { - (Some(s), None, None) => self.g.triples_matching([s], Any, Any), - (None, Some(p), None) => self.g.triples_matching(Any, [p], Any), - (None, None, Some(o)) => self.g.triples_matching(Any, Any, [o]), - (Some(s), Some(p), None) => self.g.triples_matching([s], [p], Any), - (Some(s), None, Some(o)) => self.g.triples_matching([s], Any, [o]), - (None, Some(p), Some(o)) => self.g.triples_matching(Any, [p], [o]), - (Some(s), Some(p), Some(o)) => self.g.triples_matching([s], [p], [o]), + (Some(s), None, None) => self.fast_graph.triples_matching([s], Any, Any), + (None, Some(p), None) => self.fast_graph.triples_matching(Any, [p], Any), + (None, None, Some(o)) => self.fast_graph.triples_matching(Any, Any, [o]), + (Some(s), Some(p), None) => self.fast_graph.triples_matching([s], [p], Any), + (Some(s), None, Some(o)) => self.fast_graph.triples_matching([s], Any, [o]), + (None, Some(p), Some(o)) => self.fast_graph.triples_matching(Any, [p], [o]), + (Some(s), Some(p), Some(o)) => self.fast_graph.triples_matching([s], [p], [o]), (None, None, None) => Box::new(iter::empty()), }; triple @@ -186,32 +186,26 @@ impl Bag<'_> { pub fn items(&self) -> anyhow::Result> { let container = &self.uri; let mut i = 1_u32; - let mut l_ = vec![]; + let mut items = vec![]; loop { let el_uri = format!("http://www.w3.org/1999/02/22-rdf-syntax-ns#_{i}"); let elem_iri = SimpleTerm::Iri(IriRef::new_unchecked(MownStr::from_str(&el_uri))); - if self + let item_response = self .graph - .g - .triples_matching([container], Some(elem_iri.clone()), Any) - .next() - .is_some() - { + .fast_graph + .triples_matching([container], Some(elem_iri), Any) + .next(); + if let Some(found_item) = item_response { i += 1; - let item = self - .graph - .g - .triples_matching([container], Some(elem_iri), Any) - .next() + let item = found_item .context(format!("Expected to find item in {container:?}"))? - .context("Expected to find item in container")? .o() .clone(); - l_.push(item); + items.push(item); } else { break; } } - Ok(l_) + Ok(items) } } From a0f427bdea7dfde61f1939d02d85066bc34c7851 Mon Sep 17 00:00:00 2001 From: n-dusan Date: Wed, 24 Apr 2024 20:05:34 +0200 Subject: [PATCH 08/15] refact: reorganize routing - Introduce a `routing` module, intended as a central place to map all routing in stelae app. Introduced the `_api` scope with the versions endpoint - feat(api): initial versions endpoint - feat(app): call register_api routing function - feat(api): extract stele name from request --- src/server/history.rs | 48 +++++++++++++++++++++++++++++++++++++++++++ src/server/mod.rs | 4 +++- src/server/publish.rs | 4 +++- src/server/routes.rs | 26 +++++++++++++++++++++++ tests/common/mod.rs | 1 + 5 files changed, 81 insertions(+), 2 deletions(-) create mode 100644 src/server/history.rs create mode 100644 src/server/routes.rs diff --git a/src/server/history.rs b/src/server/history.rs new file mode 100644 index 0000000..a3c98ca --- /dev/null +++ b/src/server/history.rs @@ -0,0 +1,48 @@ +//! Handlers for serving historical documents. +#![allow(clippy::future_not_send)] +use actix_web::{web, HttpRequest, HttpResponse, Responder}; +use serde::Deserialize; + +use crate::stelae::archive::Archive; + +use super::publish::{AppState, GlobalState}; + +/// Request for the versions endpoint. +#[derive(Deserialize, Debug)] +pub struct VersionRequest { + /// Publication name. + pub publication: Option, + /// Date to compare. + pub date: Option, + /// Date to compare against. + pub compare_date: Option, + /// Path to document/collection. + pub path: Option, +} + +/// Handler for the versions endpoint. +pub async fn versions(req: HttpRequest, data: web::Data) -> impl Responder { + let stele = match get_stele_from_request(&req, data.archive()) { + Ok(stele) => stele, + Err(err) => return HttpResponse::BadRequest().body(format!("Error: {err}")), + }; + HttpResponse::Ok().body(format!("Hello world! - {stele}")) +} + +/// Extracts the stele from the request. +/// If the `X-Stelae` header is present, it will return the value of the header. +/// Otherwise, it will return the root stele. +fn get_stele_from_request(req: &HttpRequest, archive: &Archive) -> anyhow::Result { + let req_headers = req.headers(); + let stele = archive.get_root()?.get_qualified_name(); + + req_headers.get("X-Stelae").map_or_else( + || Ok(stele), + |value| { + value.to_str().map_or_else( + |_| anyhow::bail!("Invalid X-Stelae header value"), + |str| Ok(str.to_owned()), + ) + }, + ) +} diff --git a/src/server/mod.rs b/src/server/mod.rs index 2d9b564..bfa8e92 100644 --- a/src/server/mod.rs +++ b/src/server/mod.rs @@ -1,8 +1,10 @@ -//! Functionality for serving Stelae libraries. +//! Functionality for serving Stelae archive. //! //! Currently contains only a git microserver. pub mod errors; pub mod git; +pub mod history; pub mod publish; +pub mod routes; pub mod tracing; diff --git a/src/server/publish.rs b/src/server/publish.rs index 8621641..b638992 100644 --- a/src/server/publish.rs +++ b/src/server/publish.rs @@ -1,6 +1,7 @@ //! Serve documents in a Stelae archive. #![allow(clippy::exit, clippy::unused_async, clippy::infinite_loop)] use crate::db; +use crate::server::routes::register_api; use crate::stelae::archive::Archive; use crate::stelae::types::repositories::{Repositories, Repository}; use crate::utils::archive::get_name_parts; @@ -218,7 +219,7 @@ pub async fn serve_archive( /// * `state` - The application state /// # Errors /// Will error if unable to initialize the application -pub fn init_app( +pub fn init_app( state: &T, ) -> anyhow::Result< App< @@ -246,6 +247,7 @@ pub fn init_app( .app_data(web::Data::new(shared_state)) .wrap(TracingLogger::::new()) .configure(|cfg| { + register_api(cfg, state); register_routes(cfg, state).unwrap_or_else(|_| { tracing::error!( "Failed to initialize routes for root Stele: {}", diff --git a/src/server/routes.rs b/src/server/routes.rs new file mode 100644 index 0000000..7501722 --- /dev/null +++ b/src/server/routes.rs @@ -0,0 +1,26 @@ +//! A central place to register App routes. +use actix_web::web; + +use super::{history::versions, publish::GlobalState}; + +/// Central place to register _api/ routes. +pub fn register_api(cfg: &mut web::ServiceConfig, state: &T) { + cfg.service( + web::scope("/_api").service( + web::scope("/versions") + .service( + web::resource( + "/_publication/{publication}/_compare/{date}/{compare_date}/{path:.*}", + ) + .to(versions), + ) + .service(web::resource("/_publication/{publication}/_date/{date}").to(versions)) + .service(web::resource("/_publication/{publication}").to(versions)) + .service(web::resource("/_publication/{publication}/{path:.*}").to(versions)) + .service(web::resource("/_compare/{date}/{compare_date}/{path:.*}").to(versions)) + .service(web::resource("/_date/{date}/{path:.*}").to(versions)) + .service(web::resource("/{path:.*}").to(versions)), + ), + ) + .app_data(web::Data::new(state.clone())); +} diff --git a/tests/common/mod.rs b/tests/common/mod.rs index 3cfa90e..ddfb319 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -31,6 +31,7 @@ pub fn blob_to_string(blob: Vec) -> String { // to manually inspect state of test environment at present, // we use anyhow::bail!() which aborts the entire test suite. +#[derive(Debug, Clone)] pub struct TestAppState { archive: Archive, } From c0bee7efce9731f9333e1aa381592435bbd6d587 Mon Sep 17 00:00:00 2001 From: n-dusan Date: Fri, 26 Apr 2024 17:03:00 +0200 Subject: [PATCH 09/15] feat: introduce `_api/versions` endpoint This commit introduces a new endpoint in stelae to get all the meaningful dates on which a document or a collection changed. Below is a summary of significant changes: - Add a new `api` module as a centralized place to look at all the endpoints stelae has. - refact: rename publish to app - feat(db): add Manager traits for model The reasoning being it's easier visually figure out which database queries work with which data structure. - feat(db): add version struct - feat(db): implement document and collection version queries Idea being we query for all the versions in which document or a collection changed. - feat(api): get all versions for a publication Executes the database query for document or library versions. - feat(api): map request/response body to structs Idea being it's more legibile to reason about the structure of the api. Rely on serde to serialize/deserialize values from and into json. - Introduce the `requests` and `response` modules for mapping the endpoint response to historical data for presentation - feat(api): support for historical messages In addition to the versions, we also display HTML messages that: i) show on which historical publication the user is currently on, ii) through which dates the currently viewed version was valid from (start and end date). If comparing versions, we show how many changes happened between the compare and version date. --- src/db/models/document_change.rs | 16 + src/db/models/library_change.rs | 16 + src/db/models/mod.rs | 2 + src/db/models/publication.rs | 26 ++ src/db/models/version.rs | 14 + src/db/statements/queries.rs | 241 ++++++++++++- src/server/api/mod.rs | 2 + src/server/api/versions.rs | 579 ++++++++++++++++++++++++++++++ src/server/{publish.rs => app.rs} | 8 +- src/server/history.rs | 48 --- src/server/mod.rs | 4 +- src/server/routes.rs | 2 +- src/utils/cli.rs | 2 +- tests/common/mod.rs | 4 +- 14 files changed, 908 insertions(+), 56 deletions(-) create mode 100644 src/db/models/version.rs create mode 100644 src/server/api/mod.rs create mode 100644 src/server/api/versions.rs rename src/server/{publish.rs => app.rs} (98%) delete mode 100644 src/server/history.rs diff --git a/src/db/models/document_change.rs b/src/db/models/document_change.rs index f38439e..ce44ac2 100644 --- a/src/db/models/document_change.rs +++ b/src/db/models/document_change.rs @@ -1,5 +1,21 @@ +use async_trait::async_trait; use serde::{Deserialize, Serialize}; +use super::version::Version; + +/// Trait for managing document changes. +#[async_trait] +pub trait Manager { + /// Find one document materialized path by url. + async fn find_doc_mpath_by_url(&self, url: &str) -> anyhow::Result>; + /// All dates on which given document changed. + async fn find_all_document_versions_by_mpath_and_publication( + &self, + mpath: &str, + publication: &str, + ) -> anyhow::Result>; +} + #[derive(sqlx::FromRow, Deserialize, Serialize)] /// Model for document change events. pub struct DocumentChange { diff --git a/src/db/models/library_change.rs b/src/db/models/library_change.rs index bd11982..6b87d98 100644 --- a/src/db/models/library_change.rs +++ b/src/db/models/library_change.rs @@ -1,5 +1,21 @@ +use async_trait::async_trait; use serde::{Deserialize, Serialize}; +use super::version::Version; + +/// Trait for managing collection changes. +#[async_trait] +pub trait Manager { + /// Find one library materialized path by url. + async fn find_lib_mpath_by_url(&self, url: &str) -> anyhow::Result>; + /// All dates on which given documents within a collection changed. + async fn find_all_collection_versions_by_mpath_and_publication( + &self, + mpath: &str, + publication: &str, + ) -> anyhow::Result>; +} + #[derive(sqlx::FromRow, Deserialize, Serialize, Debug)] /// Model for library (collection) change events. pub struct LibraryChange { diff --git a/src/db/models/mod.rs b/src/db/models/mod.rs index dac6adc..de42bf7 100644 --- a/src/db/models/mod.rs +++ b/src/db/models/mod.rs @@ -18,3 +18,5 @@ pub mod publication_has_publication_versions; pub mod publication_version; /// sqlx structs for `stele` table. pub mod stele; +/// sqlx structs for `version` table. +pub mod version; diff --git a/src/db/models/publication.rs b/src/db/models/publication.rs index 593178d..c73df08 100644 --- a/src/db/models/publication.rs +++ b/src/db/models/publication.rs @@ -1,6 +1,17 @@ +use async_trait::async_trait; use serde::{Deserialize, Serialize}; use sqlx::{any::AnyRow, FromRow, Row}; +/// Trait for managing publications. +#[async_trait] +pub trait Manager { + /// Find all publications which are not revoked for a given stele. + async fn find_all_non_revoked_publications( + &self, + stele: &str, + ) -> anyhow::Result>; +} + #[derive(Deserialize, Serialize, Debug)] /// Model for a Stele. pub struct Publication { @@ -37,3 +48,18 @@ impl FromRow<'_, AnyRow> for Publication { }) } } + +impl Publication { + /// Create a new publication. + #[must_use] + pub const fn new(name: String, date: String, stele: String) -> Self { + Self { + name, + date, + stele, + revoked: 0, + last_valid_publication_name: None, + last_valid_version: None, + } + } +} diff --git a/src/db/models/version.rs b/src/db/models/version.rs new file mode 100644 index 0000000..c64b859 --- /dev/null +++ b/src/db/models/version.rs @@ -0,0 +1,14 @@ +use async_trait::async_trait; +use serde::{Deserialize, Serialize}; + +/// Trait for managing versions. +#[async_trait] +pub trait Manager {} + +#[derive(sqlx::FromRow, Deserialize, Serialize, Debug, Eq, PartialEq)] +/// Model for a version. +pub struct Version { + /// Significant codified date of any publication. + /// Used in the form %YYYY-%MM-%DD. + pub codified_date: String, +} diff --git a/src/db/statements/queries.rs b/src/db/statements/queries.rs index 4f4c9ae..c7e0342 100644 --- a/src/db/statements/queries.rs +++ b/src/db/statements/queries.rs @@ -1,8 +1,13 @@ //! Central place for database queries -use crate::db::models::publication::Publication; +use async_trait::async_trait; +use chrono::NaiveDate; + +use crate::db::models::publication::{self, Publication}; use crate::db::models::publication_version::PublicationVersion; use crate::db::models::stele::Stele; +use crate::db::models::version::Version; +use crate::db::models::{document_change, library_change}; use crate::db::DatabaseConnection; use std::collections::HashSet; @@ -282,3 +287,237 @@ pub async fn find_last_inserted_publication_version_by_publication_and_stele( }; Ok(row) } + +#[async_trait] +impl document_change::Manager for DatabaseConnection { + /// Find one document materialized path by url. + /// + /// # Errors + /// Errors if can't establish a connection to the database. + async fn find_doc_mpath_by_url(&self, url: &str) -> anyhow::Result> { + let statement = " + SELECT doc_mpath + FROM document_change + WHERE url = $1 + LIMIT 1 + "; + let row = match self.kind { + DatabaseKind::Postgres | DatabaseKind::Sqlite => { + let mut connection = self.pool.acquire().await?; + sqlx::query_as::<_, (String,)>(statement) + .bind(url) + .fetch_one(&mut *connection) + .await + .ok() + } + }; + Ok(row.map(|(doc_mpath,)| doc_mpath)) + } + + /// All dates on which given document changed. + /// + /// # Errors + /// Errors if can't establish a connection to the database. + async fn find_all_document_versions_by_mpath_and_publication( + &self, + mpath: &str, + publication: &str, + ) -> anyhow::Result> { + let mut statement = " + SELECT DISTINCT phpv.referenced_version as codified_date + FROM document_change dc + LEFT JOIN publication_has_publication_versions phpv + ON dc.publication = phpv.referenced_publication + AND dc.version = phpv.referenced_version + WHERE dc.doc_mpath LIKE $1 AND phpv.publication = $2 + "; + let mut rows = match self.kind { + DatabaseKind::Postgres | DatabaseKind::Sqlite => { + let mut connection = self.pool.acquire().await?; + sqlx::query_as::<_, Version>(statement) + .bind(format!("{mpath}%")) + .bind(publication) + .fetch_all(&mut *connection) + .await? + } + }; + statement = " + SELECT phpv.referenced_version as codified_date + FROM document_change dc + LEFT JOIN publication_has_publication_versions phpv + ON dc.publication = phpv.referenced_publication + AND dc.version = phpv.referenced_version + WHERE dc.doc_mpath = $1 + AND phpv.publication = $2 + AND dc.status = 'Element added' + LIMIT 1 + "; + let element_added = match self.kind { + DatabaseKind::Postgres | DatabaseKind::Sqlite => { + let mut connection = self.pool.acquire().await?; + sqlx::query_as::<_, Version>(statement) + .bind(mpath) + .bind(publication) + .fetch_one(&mut *connection) + .await + .ok() + } + }; + + if element_added.is_none() { + // When element doesn't have date added, it means we're looking + // at an old publication and this element doesn't yet exist in it + rows.sort_by(|v1, v2| v2.codified_date.cmp(&v1.codified_date)); + return Ok(rows); + } + + statement = " + SELECT phpv.referenced_version as codified_date + FROM document_change dc + LEFT JOIN publication_has_publication_versions phpv + ON dc.publication = phpv.referenced_publication + AND dc.version = phpv.referenced_version + WHERE dc.doc_mpath = $1 + AND phpv.publication = $2 + AND dc.status = 'Element effective' + LIMIT 1 + "; + let mut doc = mpath.split('|').next().unwrap_or("").to_owned(); + doc.push('|'); + + let document_effective = match self.kind { + DatabaseKind::Postgres | DatabaseKind::Sqlite => { + let mut connection = self.pool.acquire().await?; + sqlx::query_as::<_, Version>(statement) + .bind(doc) + .bind(publication) + .fetch_one(&mut *connection) + .await + .ok() + } + }; + + if let (Some(doc_effective), Some(el_added)) = (document_effective, element_added) { + if NaiveDate::parse_from_str(&doc_effective.codified_date, "%Y-%m-%d") + .unwrap_or_default() + > NaiveDate::parse_from_str(&el_added.codified_date, "%Y-%m-%d").unwrap_or_default() + { + rows.push(doc_effective); + } + } + rows.sort_by(|v1, v2| v2.codified_date.cmp(&v1.codified_date)); + Ok(rows) + } +} + +#[async_trait] +impl library_change::Manager for DatabaseConnection { + /// Find one library materialized path by url. + /// + /// # Errors + /// Errors if can't establish a connection to the database. + async fn find_lib_mpath_by_url(&self, url: &str) -> anyhow::Result> { + let statement = " + SELECT library_mpath + FROM library_change + WHERE url = $1 + LIMIT 1 + "; + let row = match self.kind { + DatabaseKind::Postgres | DatabaseKind::Sqlite => { + let mut connection = self.pool.acquire().await?; + sqlx::query_as::<_, (String,)>(statement) + .bind(url) + .fetch_one(&mut *connection) + .await + .ok() + } + }; + Ok(row.map(|(library_mpath,)| library_mpath)) + } + /// All dates on which documents from this collection changed. + /// + /// # Errors + /// Errors if can't establish a connection to the database. + async fn find_all_collection_versions_by_mpath_and_publication( + &self, + mpath: &str, + publication: &str, + ) -> anyhow::Result> { + let mut statement = " + SELECT DISTINCT phpv.referenced_version as codified_date + FROM changed_library_document cld + LEFT JOIN publication_has_publication_versions phpv + ON cld.publication = phpv.referenced_publication + AND cld.version = phpv.referenced_version + WHERE cld.library_mpath LIKE $1 AND phpv.publication = $2 + "; + let mut rows = match self.kind { + DatabaseKind::Postgres | DatabaseKind::Sqlite => { + let mut connection = self.pool.acquire().await?; + sqlx::query_as::<_, Version>(statement) + .bind(format!("{mpath}%")) + .bind(publication) + .fetch_all(&mut *connection) + .await? + } + }; + statement = " + SELECT DISTINCT phpv.referenced_version as codified_date + FROM library_change lc + LEFT JOIN publication_has_publication_versions phpv + ON lc.publication = phpv.referenced_publication + AND lc.version = phpv.referenced_version + WHERE lc.library_mpath LIKE $1 AND lc.status = 'Element added' AND phpv.publication = $2 + LIMIT 1 + "; + let element_added = match self.kind { + DatabaseKind::Postgres | DatabaseKind::Sqlite => { + let mut connection = self.pool.acquire().await?; + sqlx::query_as::<_, Version>(statement) + .bind(format!("{mpath}%")) + .bind(publication) + .fetch_one(&mut *connection) + .await + .ok() + } + }; + + if let Some(el_added) = element_added { + if !rows.contains(&el_added) { + rows.push(el_added); + } + } + rows.sort_by(|v1, v2| v2.codified_date.cmp(&v1.codified_date)); + Ok(rows) + } +} + +#[async_trait] +impl publication::Manager for DatabaseConnection { + /// Find all publications which are not revoked for a given stele. + /// + /// # Errors + /// Errors if can't establish a connection to the database. + async fn find_all_non_revoked_publications( + &self, + stele: &str, + ) -> anyhow::Result> { + let statement = " + SELECT * + FROM publication + WHERE revoked = 0 AND stele = $1 + ORDER BY name DESC + "; + let rows = match self.kind { + DatabaseKind::Postgres | DatabaseKind::Sqlite => { + let mut connection = self.pool.acquire().await?; + sqlx::query_as::<_, Publication>(statement) + .bind(stele) + .fetch_all(&mut *connection) + .await? + } + }; + Ok(rows) + } +} diff --git a/src/server/api/mod.rs b/src/server/api/mod.rs new file mode 100644 index 0000000..a752df8 --- /dev/null +++ b/src/server/api/mod.rs @@ -0,0 +1,2 @@ +//! This module contains the API endpoints for the server. +pub mod versions; diff --git a/src/server/api/versions.rs b/src/server/api/versions.rs new file mode 100644 index 0000000..08b4109 --- /dev/null +++ b/src/server/api/versions.rs @@ -0,0 +1,579 @@ +//! Handlers for serving historical documents. +#![allow(clippy::future_not_send)] +use actix_web::{web, HttpRequest, HttpResponse, Responder}; +use chrono::NaiveDate; +use std::convert::Into; +use std::time::Instant; + +use crate::{ + db::{ + models::{ + document_change, library_change, + publication::{self, Publication}, + }, + DatabaseConnection, + }, + server::app::{AppState, GlobalState}, + stelae::archive::Archive, +}; + +/// Name of the current publication. +pub const CURRENT_PUBLICATION_NAME: &str = "Current"; +/// Name of the current version. +pub const CURRENT_VERSION_NAME: &str = "Current"; +/// Date of the current version. +pub const CURRENT_VERSION_DATE: &str = "current"; + +/// Module that maps the HTTP web request body to structs. +mod request { + use serde::Deserialize; + /// Request for the versions endpoint. + #[derive(Deserialize, Debug)] + pub struct Version { + /// Publication name. + pub publication: Option, + /// Date to compare. + pub date: Option, + /// Date to compare against. + pub compare_date: Option, + /// Path to document/collection. + pub path: Option, + } +} + +/// Module that maps the HTTP web response to structs. +mod response { + use std::{cmp::Reverse, collections::BTreeMap}; + + use serde::Serialize; + + use crate::db::models; + + use super::format_date; + + /// Response for the versions endpoint. + #[derive(Serialize, Debug)] + #[serde(rename_all = "camelCase")] + pub struct Versions { + /// Currently selected publication. + /// Resolves to "Current" if the latest publication is selected. + pub active_publication: String, + /// Currently selected version. + /// Resolves to "current" if the latest version is selected. + pub active_version: String, + /// Currently selected version to compare against. + /// If compare_date is specified, this will be the date to compare against. + pub active_compare_to: Option, + /// Features for the versions endpoint. + pub features: Features, + /// URL path. + pub path: String, + /// List of all found publications in descending order. + pub publications: BTreeMap, Publication>, + /// Messages for the versions endpoint. + pub messages: HistoricalMessages, + } + /// Features for the versions endpoint. + #[derive(Serialize, Debug)] + #[serde(rename_all = "camelCase")] + pub struct Features { + /// Whether the compare feature is enabled. + pub compare: bool, + /// Whether the historical versions feature is enabled. + pub historical_versions: bool, + } + + /// Response for a publication. + #[derive(Serialize, Debug)] + #[serde(rename_all = "camelCase")] + pub struct Publication { + /// Whether the publication is currently active. + pub active: bool, + /// Date of the publication. + pub date: String, + /// Display name of the publication. + pub display: String, + /// Name of the publication. + pub name: String, + /// List of versions for the publication. + pub versions: Vec, + } + + /// Response for a version. + #[derive(Serialize, Debug, Clone)] + #[serde(rename_all = "camelCase")] + pub struct Version { + /// Codified date of the version. + pub date: String, + /// Display date of the version. + pub display: String, + /// Version number of the version. + #[serde(rename = "version")] + pub index: usize, + } + + impl From for Version { + fn from(value: models::version::Version) -> Self { + Self { + date: value.codified_date.clone(), + display: value.codified_date, + index: 0, + } + } + } + + impl Versions { + /// Build and returns an HTTP versions response converted into json. + pub fn build( + active_publication_name: &str, + active_version: String, + active_compare_to: Option, + url: &str, + publications: &[models::publication::Publication], + versions: &[Version], + messages: HistoricalMessages, + ) -> Self { + Self { + active_publication: active_publication_name.to_owned(), + active_version, + active_compare_to, + features: Features { + compare: true, + historical_versions: true, + }, + path: url.strip_prefix('/').unwrap_or_default().to_owned(), + publications: { + let mut sorted_publications = BTreeMap::new(); + for pb in publications { + let mut response_pub = Publication { + active: pb.name == active_publication_name, + date: pb.date.clone(), + display: format_date(&pb.date), + name: pb.name.clone(), + versions: vec![], + }; + if pb.name == active_publication_name { + response_pub.versions = versions.to_vec(); + } + sorted_publications.insert(Reverse(pb.name.clone()), response_pub); + } + sorted_publications + }, + messages, + } + } + } + + impl Version { + /// Create a new version. + pub const fn new(date: String, display: String, index: usize) -> Self { + Self { + date, + display, + index, + } + } + + /// Insert a new version if it is not present in the list of versions. + /// If the date is not in the list of versions, add it + /// This for compatibility purposes with the previous implementation of historical versions + pub fn insert_if_not_present(versions: &mut Vec, date: Option) { + if let Some(version_date) = date { + if versions.iter().all(|ver| ver.date != version_date) { + let version = Self::new(version_date.clone(), version_date, 0); + Self::insert_version_sorted(versions, version); + } + } + } + + /// Insert a new item into an already sorted collection. + /// The collection is sorted by date in descending order. + pub fn insert_version_sorted(collection: &mut Vec, item: Self) { + let mut idx = 0; + for i in collection.iter() { + if i.date < item.date { + break; + } + idx += 1; + } + collection.insert(idx, item); + } + + /// Utility function to find the index of a date in a list of versions. + pub fn find_index_or_closest(versions: &[Self], date: &str) -> usize { + versions + .iter() + .position(|ver| ver.date.as_str() == date) + .unwrap_or_else(|| { + let closest_date = versions + .iter() + .filter(|ver| ver.date.as_str() < date) + .max_by(|current, next| current.date.cmp(&next.date)) + .map_or_else(|| None, |ver| Some(ver.date.as_str())) + .unwrap_or("-1"); + versions + .iter() + .position(|ver| ver.date.as_str() == closest_date) + .unwrap_or(versions.len()) + }) + } + } + + /// Messages for the versions endpoint. + #[derive(Serialize, Debug)] + #[serde(rename_all = "camelCase")] + pub struct HistoricalMessages { + /// Message for an outdated publication. + pub publication: Option, + /// Message for an outdated version. + pub version: Option, + /// Message for a comparison between two versions. + pub comparison: Option, + } +} + +/// Handler for the versions endpoint. +pub async fn versions( + req: HttpRequest, + data: web::Data, + params: web::Path, +) -> impl Responder { + let first = Instant::now(); + let stele = match get_stele_from_request(&req, data.archive()) { + Ok(stele) => stele, + Err(err) => return HttpResponse::BadRequest().body(format!("Error: {err}")), + }; + let db = data.db(); + let mut publications = publication::Manager::find_all_non_revoked_publications(db, &stele) + .await + .unwrap_or_default(); + + let Some(current_publication) = publications.first() else { + return HttpResponse::NotFound().body("No publications found."); + }; + + let mut active_publication_name = params + .publication + .clone() + .unwrap_or_else(|| current_publication.name.clone()); + + let active_publication = publications + .iter() + .find(|pb| pb.name == active_publication_name); + + let mut url = String::from("/"); + url.push_str(params.path.clone().unwrap_or_default().as_str()); + + let before = Instant::now(); + let mut versions = if let Some(publication) = active_publication { + publication_versions(db, publication, url.clone()).await + } else { + vec![] + }; + println!("Versions query elapsed time: {:.2?}", before.elapsed()); + + // latest date in active publication + let current_date = versions + .first() + .map_or(String::new(), |ver| ver.date.clone()); + // active version is the version the user is looking at right now + let mut active_version = + NaiveDate::parse_from_str(params.date.as_deref().unwrap_or_default(), "%Y-%m-%d") + .map_or(current_date.clone(), |date| date.clone().to_string()); + let active_compare_to = params.compare_date.clone().map_or_else( + || { + NaiveDate::parse_from_str(¤t_date, "%Y-%m-%d") + .map(|date| date.to_string()) + .ok() + }, + |date_str| { + NaiveDate::parse_from_str(&date_str, "%Y-%m-%d") + .map(|date| date.to_string()) + .ok() + }, + ); + + if active_version == current_date { + active_version = CURRENT_VERSION_DATE.to_owned(); + } + + if active_publication_name == current_publication.name.clone() { + active_publication_name = CURRENT_PUBLICATION_NAME.to_owned(); + } + let messages = historical_messages( + &versions, + current_publication, + &active_publication_name, + ¶ms.date, + &active_compare_to, + ); + + response::Version::insert_if_not_present(&mut versions, params.date.clone()); + response::Version::insert_if_not_present(&mut versions, active_compare_to.clone()); + + let versions_size = versions.len(); + for (idx, version) in versions.iter_mut().enumerate() { + version.display = format_date(&version.date.clone()); + version.index = versions_size - idx; + } + if let Some(ver) = versions.first_mut() { + ver.display.push_str(" (last modified)"); + }; + + let current_version = response::Version::new( + CURRENT_VERSION_DATE.to_owned(), + CURRENT_VERSION_NAME.to_owned(), + versions.first().map_or(0, |ver| ver.index), + ); + + versions.insert(versions_size - current_version.index, current_version); + + // duplicate current publication with current label + publications.insert( + 0, + Publication::new( + CURRENT_PUBLICATION_NAME.to_owned(), + current_publication.date.clone(), + current_publication.stele.clone(), + ), + ); + + println!("Total elapsed time: {:.2?}", first.elapsed()); + HttpResponse::Ok().json(response::Versions::build( + &active_publication_name, + active_version, + active_compare_to, + &url, + &publications, + &versions, + messages, + )) +} + +/// Get all the versions of a publication. +async fn publication_versions( + db: &DatabaseConnection, + publication: &Publication, + url: String, +) -> Vec { + let mut versions = vec![]; + let before_url = Instant::now(); + let doc_mpath = document_change::Manager::find_doc_mpath_by_url(db, &url) + .await + .unwrap_or_default(); + println!("URL query elapsed time: {:.2?}", before_url.elapsed()); + if let Some(mpath) = doc_mpath { + let doc_versions = + document_change::Manager::find_all_document_versions_by_mpath_and_publication( + db, + &mpath, + &publication.name, + ) + .await + .unwrap_or_default(); + versions = doc_versions.into_iter().map(Into::into).collect(); + return versions; + } + + let lib_mpath = library_change::Manager::find_lib_mpath_by_url(db, &url) + .await + .unwrap_or_default(); + if let Some(mpath) = lib_mpath { + let coll_versions = + library_change::Manager::find_all_collection_versions_by_mpath_and_publication( + db, + &mpath, + &publication.name, + ) + .await + .unwrap_or_default(); + versions = coll_versions.into_iter().map(Into::into).collect(); + } + versions +} + +/// Extracts the stele from the request. +/// If the `X-Stelae` header is present, it will return the value of the header. +/// Otherwise, it will return the root stele. +fn get_stele_from_request(req: &HttpRequest, archive: &Archive) -> anyhow::Result { + let req_headers = req.headers(); + let stele = archive.get_root()?.get_qualified_name(); + + req_headers.get("X-Stelae").map_or_else( + || Ok(stele), + |value| { + value.to_str().map_or_else( + |_| anyhow::bail!("Invalid X-Stelae header value"), + |str| Ok(str.to_owned()), + ) + }, + ) +} + +/// Returns historical messages for the versions endpoint. +/// The historical messages currently include: +/// - A message for an outdated publication. +/// - A message for an outdated version. +/// - A message for a comparison between two versions. +fn historical_messages( + versions: &[response::Version], + current_publication: &Publication, + active_publication_name: &str, + version_date: &Option, + compare_to_date: &Option, +) -> response::HistoricalMessages { + let current_publication_name = current_publication.name.as_str(); + let current_version: &str = versions + .first() + .map(|lmv| lmv.date.as_str()) + .unwrap_or_default(); + + let publication = publication_message( + active_publication_name, + current_publication_name, + current_version, + ); + let version = version_date.as_ref().and_then(|found_version_date| { + version_message( + current_version, + found_version_date, + versions, + compare_to_date, + ) + }); + let comparison = compare_to_date.as_ref().and_then(|found_compare_to_date| { + version_date.as_ref().map(|found_version_date| { + comparison_message( + found_compare_to_date, + found_version_date, + current_version, + versions, + ) + }) + }); + response::HistoricalMessages { + publication, + version, + comparison, + } +} + +/// Returns a historical message for an outdated publication. +fn publication_message( + active_publication_name: &str, + current_publication_name: &str, + current_version: &str, +) -> Option { + if active_publication_name == current_publication_name { + return None; + } + Some(format!( + "You are viewing a historical publication that was last updated on {current_date} and is no longer being updated.", + current_date = format_date(current_version) + )) +} + +/// Returns a historical message for an outdated version. +/// Version is outdated if `version_date` is in the past. +fn version_message( + current_version: &str, + version_date: &str, + versions: &[response::Version], + compare_to_date: &Option, +) -> Option { + let is_current_version = { + let current_date = + NaiveDate::parse_from_str(current_version, "%Y-%m-%d").unwrap_or_default(); + let Ok(parsed_version_date) = NaiveDate::parse_from_str(version_date, "%Y-%m-%d") else { + return None; + }; + current_date <= parsed_version_date + }; + if compare_to_date.is_some() || is_current_version { + return None; + } + let version_date_idx = versions + .iter() + .position(|ver| ver.date.as_str() == version_date); + let (start_date, end_date) = version_date_idx.map_or_else( + || { + let end_date = versions + .iter() + .find(|ver| ver.date.as_str() > version_date) + .map(|ver| ver.date.as_str()); + let found_idx = versions + .iter() + .position(|ver| ver.date.as_str() == end_date.unwrap_or_default()) + .unwrap_or_default(); + let start_date = versions + .get(found_idx + 1) + .map_or_else(|| versions.last(), Some) + .map(|ver| ver.date.as_str()); + (start_date.unwrap_or_default(), end_date.unwrap_or_default()) + }, + |idx| { + let start_date = version_date; + let end_date = versions + .get(idx - 1) + .map_or_else(|| versions.first(), Some) + .map(|ver| ver.date.as_str()) + .unwrap_or_default(); + (start_date, end_date) + }, + ); + Some(format!("You are viewing this document as it appeared on {version_date}. This version was valid between {start_date} and {end_date}.", + version_date = format_date(version_date), start_date = format_date(start_date), end_date = format_date(end_date))) +} + +/// Returns a historical message for a comparison between two versions. +fn comparison_message( + compare_to_date: &str, + version_date: &str, + current_date: &str, + versions: &[response::Version], +) -> String { + let (compare_start_date, compare_end_date) = if version_date > compare_to_date { + (compare_to_date, version_date) + } else { + (version_date, compare_to_date) + }; + let start_idx = response::Version::find_index_or_closest(versions, compare_start_date); + let end_idx = response::Version::find_index_or_closest(versions, compare_start_date); + let num_of_changes = start_idx - end_idx; + let start_date = format_date(compare_start_date); + let end_date = if compare_end_date == current_date { + None + } else { + Some(format_date(compare_end_date)) + }; + get_messages_between(num_of_changes, &start_date, end_date) +} + +/// Returns a message for the number of changes between two dates. +fn get_messages_between( + num_of_changes: usize, + start_date: &str, + end_date: Option, +) -> String { + let changes = match num_of_changes { + 0 => "no updates".to_owned(), + 1 => "1 update".to_owned(), + _ => format!("{num_of_changes} updates"), + }; + + end_date.map_or_else( + || format!("There have been {changes} since {start_date}."), + |found_end_date| { + format!( + "There have been {changes} between {start_date} and {found_end_date}." + ) + }, + ) +} + +/// Format a date from %Y-%m-%d to %B %d, %Y. +fn format_date(date: &str) -> String { + NaiveDate::parse_from_str(date, "%Y-%m-%d").map_or(date.to_owned(), |found_date| { + found_date.format("%B %d, %Y").to_string() + }) +} diff --git a/src/server/publish.rs b/src/server/app.rs similarity index 98% rename from src/server/publish.rs rename to src/server/app.rs index b638992..061968c 100644 --- a/src/server/publish.rs +++ b/src/server/app.rs @@ -1,5 +1,10 @@ //! Serve documents in a Stelae archive. -#![allow(clippy::exit, clippy::unused_async, clippy::infinite_loop)] +#![allow( + clippy::exit, + clippy::unused_async, + clippy::infinite_loop, + clippy::module_name_repetitions +)] use crate::db; use crate::server::routes::register_api; use crate::stelae::archive::Archive; @@ -287,6 +292,7 @@ pub fn init_app( .app_data(web::Data::new(shared_state)) .wrap(TracingLogger::::new()) .configure(|cfg| { + register_api(cfg, state); register_root_routes(cfg, guarded_stele).unwrap_or_else(|_| { tracing::error!( "Failed to initialize routes for Stele: {}", diff --git a/src/server/history.rs b/src/server/history.rs deleted file mode 100644 index a3c98ca..0000000 --- a/src/server/history.rs +++ /dev/null @@ -1,48 +0,0 @@ -//! Handlers for serving historical documents. -#![allow(clippy::future_not_send)] -use actix_web::{web, HttpRequest, HttpResponse, Responder}; -use serde::Deserialize; - -use crate::stelae::archive::Archive; - -use super::publish::{AppState, GlobalState}; - -/// Request for the versions endpoint. -#[derive(Deserialize, Debug)] -pub struct VersionRequest { - /// Publication name. - pub publication: Option, - /// Date to compare. - pub date: Option, - /// Date to compare against. - pub compare_date: Option, - /// Path to document/collection. - pub path: Option, -} - -/// Handler for the versions endpoint. -pub async fn versions(req: HttpRequest, data: web::Data) -> impl Responder { - let stele = match get_stele_from_request(&req, data.archive()) { - Ok(stele) => stele, - Err(err) => return HttpResponse::BadRequest().body(format!("Error: {err}")), - }; - HttpResponse::Ok().body(format!("Hello world! - {stele}")) -} - -/// Extracts the stele from the request. -/// If the `X-Stelae` header is present, it will return the value of the header. -/// Otherwise, it will return the root stele. -fn get_stele_from_request(req: &HttpRequest, archive: &Archive) -> anyhow::Result { - let req_headers = req.headers(); - let stele = archive.get_root()?.get_qualified_name(); - - req_headers.get("X-Stelae").map_or_else( - || Ok(stele), - |value| { - value.to_str().map_or_else( - |_| anyhow::bail!("Invalid X-Stelae header value"), - |str| Ok(str.to_owned()), - ) - }, - ) -} diff --git a/src/server/mod.rs b/src/server/mod.rs index bfa8e92..0fccdab 100644 --- a/src/server/mod.rs +++ b/src/server/mod.rs @@ -2,9 +2,9 @@ //! //! Currently contains only a git microserver. +pub mod api; +pub mod app; pub mod errors; pub mod git; -pub mod history; -pub mod publish; pub mod routes; pub mod tracing; diff --git a/src/server/routes.rs b/src/server/routes.rs index 7501722..79f7dc5 100644 --- a/src/server/routes.rs +++ b/src/server/routes.rs @@ -1,7 +1,7 @@ //! A central place to register App routes. use actix_web::web; -use super::{history::versions, publish::GlobalState}; +use super::{api::versions::versions, app::GlobalState}; /// Central place to register _api/ routes. pub fn register_api(cfg: &mut web::ServiceConfig, state: &T) { diff --git a/src/utils/cli.rs b/src/utils/cli.rs index 93783b2..fec8e1f 100644 --- a/src/utils/cli.rs +++ b/src/utils/cli.rs @@ -4,8 +4,8 @@ #![allow(clippy::exit)] use crate::history::changes; +use crate::server::app::serve_archive; use crate::server::git::serve_git; -use crate::server::publish::serve_archive; use crate::utils::archive::find_archive_path; use clap::Parser; use std::env; diff --git a/tests/common/mod.rs b/tests/common/mod.rs index ddfb319..c86c538 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -9,13 +9,13 @@ use actix_web::{ use anyhow::Result; use std::path::{Path, PathBuf}; use std::sync::Once; -use stelae::{db, server::publish::GlobalState}; +use stelae::{db, server::app::GlobalState}; use tempfile::Builder; static INIT: Once = Once::new(); use actix_http::body::MessageBody; -use stelae::server::publish::init_app; +use stelae::server::app::init_app; use stelae::stelae::archive::Archive; pub const BASIC_MODULE_NAME: &str = "basic"; From 894f64c66bc5e011978f8abb8b4e3f827465e286 Mon Sep 17 00:00:00 2001 From: n-dusan Date: Thu, 9 May 2024 14:10:53 +0200 Subject: [PATCH 10/15] fix(api): correctly build the versions response - fix(api): assign Current value to variable after historical messages - refact: separate formatting logic into associated function `format_display_date` - lots of clippy lints --- src/db/statements/queries.rs | 8 ++-- src/server/api/versions.rs | 81 +++++++++++++++++++++--------------- 2 files changed, 53 insertions(+), 36 deletions(-) diff --git a/src/db/statements/queries.rs b/src/db/statements/queries.rs index c7e0342..9b30f81 100644 --- a/src/db/statements/queries.rs +++ b/src/db/statements/queries.rs @@ -398,9 +398,11 @@ impl document_change::Manager for DatabaseConnection { }; if let (Some(doc_effective), Some(el_added)) = (document_effective, element_added) { - if NaiveDate::parse_from_str(&doc_effective.codified_date, "%Y-%m-%d") - .unwrap_or_default() - > NaiveDate::parse_from_str(&el_added.codified_date, "%Y-%m-%d").unwrap_or_default() + if !rows.contains(&doc_effective) + && NaiveDate::parse_from_str(&doc_effective.codified_date, "%Y-%m-%d") + .unwrap_or_default() + > NaiveDate::parse_from_str(&el_added.codified_date, "%Y-%m-%d") + .unwrap_or_default() { rows.push(doc_effective); } diff --git a/src/server/api/versions.rs b/src/server/api/versions.rs index 08b4109..f636183 100644 --- a/src/server/api/versions.rs +++ b/src/server/api/versions.rs @@ -3,7 +3,6 @@ use actix_web::{web, HttpRequest, HttpResponse, Responder}; use chrono::NaiveDate; use std::convert::Into; -use std::time::Instant; use crate::{ db::{ @@ -50,6 +49,7 @@ mod response { use crate::db::models; use super::format_date; + use super::CURRENT_PUBLICATION_NAME; /// Response for the versions endpoint. #[derive(Serialize, Debug)] @@ -124,12 +124,14 @@ mod response { impl Versions { /// Build and returns an HTTP versions response converted into json. + #[allow(clippy::too_many_arguments)] pub fn build( active_publication_name: &str, active_version: String, active_compare_to: Option, url: &str, publications: &[models::publication::Publication], + current_publication_name: &str, versions: &[Version], messages: HistoricalMessages, ) -> Self { @@ -145,23 +147,45 @@ mod response { publications: { let mut sorted_publications = BTreeMap::new(); for pb in publications { - let mut response_pub = Publication { - active: pb.name == active_publication_name, - date: pb.date.clone(), - display: format_date(&pb.date), - name: pb.name.clone(), - versions: vec![], - }; - if pb.name == active_publication_name { - response_pub.versions = versions.to_vec(); - } - sorted_publications.insert(Reverse(pb.name.clone()), response_pub); + sorted_publications.insert( + Reverse(pb.name.clone()), + Publication { + active: pb.name == active_publication_name, + date: pb.date.clone(), + display: Self::format_display_date( + &pb.name, + current_publication_name, + ), + name: pb.name.clone(), + versions: { + if pb.name == active_publication_name { + versions.to_vec() + } else { + vec![] + } + }, + }, + ); } sorted_publications }, messages, } } + + /// Returns a formatted display date. + /// If the `date` is current, returns the date with `(current)` appended. + fn format_display_date(date: &str, current_date: &str) -> String { + if date == CURRENT_PUBLICATION_NAME { + CURRENT_PUBLICATION_NAME.to_owned() + } else { + let mut formatted_date = format_date(date); + if date == current_date { + formatted_date.push_str(" (current)"); + } + formatted_date + } + } } impl Version { @@ -238,7 +262,6 @@ pub async fn versions( data: web::Data, params: web::Path, ) -> impl Responder { - let first = Instant::now(); let stele = match get_stele_from_request(&req, data.archive()) { Ok(stele) => stele, Err(err) => return HttpResponse::BadRequest().body(format!("Error: {err}")), @@ -264,13 +287,11 @@ pub async fn versions( let mut url = String::from("/"); url.push_str(params.path.clone().unwrap_or_default().as_str()); - let before = Instant::now(); let mut versions = if let Some(publication) = active_publication { publication_versions(db, publication, url.clone()).await } else { vec![] }; - println!("Versions query elapsed time: {:.2?}", before.elapsed()); // latest date in active publication let current_date = versions @@ -280,26 +301,17 @@ pub async fn versions( let mut active_version = NaiveDate::parse_from_str(params.date.as_deref().unwrap_or_default(), "%Y-%m-%d") .map_or(current_date.clone(), |date| date.clone().to_string()); - let active_compare_to = params.compare_date.clone().map_or_else( - || { - NaiveDate::parse_from_str(¤t_date, "%Y-%m-%d") - .map(|date| date.to_string()) - .ok() - }, - |date_str| { - NaiveDate::parse_from_str(&date_str, "%Y-%m-%d") - .map(|date| date.to_string()) - .ok() - }, - ); + let active_compare_to = params.compare_date.clone().map(|date| { + NaiveDate::parse_from_str(&date, "%Y-%m-%d").map_or_else( + |_| current_date.clone(), + |active_date| active_date.to_string(), + ) + }); if active_version == current_date { active_version = CURRENT_VERSION_DATE.to_owned(); } - if active_publication_name == current_publication.name.clone() { - active_publication_name = CURRENT_PUBLICATION_NAME.to_owned(); - } let messages = historical_messages( &versions, current_publication, @@ -308,6 +320,10 @@ pub async fn versions( &active_compare_to, ); + if active_publication_name == current_publication.name.clone() { + active_publication_name = CURRENT_PUBLICATION_NAME.to_owned(); + } + response::Version::insert_if_not_present(&mut versions, params.date.clone()); response::Version::insert_if_not_present(&mut versions, active_compare_to.clone()); @@ -328,6 +344,7 @@ pub async fn versions( versions.insert(versions_size - current_version.index, current_version); + let current_publication_name = current_publication.name.clone(); // duplicate current publication with current label publications.insert( 0, @@ -338,13 +355,13 @@ pub async fn versions( ), ); - println!("Total elapsed time: {:.2?}", first.elapsed()); HttpResponse::Ok().json(response::Versions::build( &active_publication_name, active_version, active_compare_to, &url, &publications, + ¤t_publication_name, &versions, messages, )) @@ -357,11 +374,9 @@ async fn publication_versions( url: String, ) -> Vec { let mut versions = vec![]; - let before_url = Instant::now(); let doc_mpath = document_change::Manager::find_doc_mpath_by_url(db, &url) .await .unwrap_or_default(); - println!("URL query elapsed time: {:.2?}", before_url.elapsed()); if let Some(mpath) = doc_mpath { let doc_versions = document_change::Manager::find_all_document_versions_by_mpath_and_publication( From 676a9d826970d92699fb8ff55dd6111eda9e82d9 Mon Sep 17 00:00:00 2001 From: n-dusan Date: Tue, 14 May 2024 13:03:14 +0200 Subject: [PATCH 11/15] refact: re-organize versions endpoint For the API, move the `versions.rs` into `versions/mod.rs`. The two modules for request and response are now in `request/mod.rs` and `response/mod.rs`. Historical messages are in the `response/messages.rs` module --- src/server/api/versions.rs | 594 ------------------- src/server/api/versions/mod.rs | 208 +++++++ src/server/api/versions/request/mod.rs | 13 + src/server/api/versions/response/messages.rs | 193 ++++++ src/server/api/versions/response/mod.rs | 211 +++++++ tests/archive_testtools/config.rs | 2 +- 6 files changed, 626 insertions(+), 595 deletions(-) delete mode 100644 src/server/api/versions.rs create mode 100644 src/server/api/versions/mod.rs create mode 100644 src/server/api/versions/request/mod.rs create mode 100644 src/server/api/versions/response/messages.rs create mode 100644 src/server/api/versions/response/mod.rs diff --git a/src/server/api/versions.rs b/src/server/api/versions.rs deleted file mode 100644 index f636183..0000000 --- a/src/server/api/versions.rs +++ /dev/null @@ -1,594 +0,0 @@ -//! Handlers for serving historical documents. -#![allow(clippy::future_not_send)] -use actix_web::{web, HttpRequest, HttpResponse, Responder}; -use chrono::NaiveDate; -use std::convert::Into; - -use crate::{ - db::{ - models::{ - document_change, library_change, - publication::{self, Publication}, - }, - DatabaseConnection, - }, - server::app::{AppState, GlobalState}, - stelae::archive::Archive, -}; - -/// Name of the current publication. -pub const CURRENT_PUBLICATION_NAME: &str = "Current"; -/// Name of the current version. -pub const CURRENT_VERSION_NAME: &str = "Current"; -/// Date of the current version. -pub const CURRENT_VERSION_DATE: &str = "current"; - -/// Module that maps the HTTP web request body to structs. -mod request { - use serde::Deserialize; - /// Request for the versions endpoint. - #[derive(Deserialize, Debug)] - pub struct Version { - /// Publication name. - pub publication: Option, - /// Date to compare. - pub date: Option, - /// Date to compare against. - pub compare_date: Option, - /// Path to document/collection. - pub path: Option, - } -} - -/// Module that maps the HTTP web response to structs. -mod response { - use std::{cmp::Reverse, collections::BTreeMap}; - - use serde::Serialize; - - use crate::db::models; - - use super::format_date; - use super::CURRENT_PUBLICATION_NAME; - - /// Response for the versions endpoint. - #[derive(Serialize, Debug)] - #[serde(rename_all = "camelCase")] - pub struct Versions { - /// Currently selected publication. - /// Resolves to "Current" if the latest publication is selected. - pub active_publication: String, - /// Currently selected version. - /// Resolves to "current" if the latest version is selected. - pub active_version: String, - /// Currently selected version to compare against. - /// If compare_date is specified, this will be the date to compare against. - pub active_compare_to: Option, - /// Features for the versions endpoint. - pub features: Features, - /// URL path. - pub path: String, - /// List of all found publications in descending order. - pub publications: BTreeMap, Publication>, - /// Messages for the versions endpoint. - pub messages: HistoricalMessages, - } - /// Features for the versions endpoint. - #[derive(Serialize, Debug)] - #[serde(rename_all = "camelCase")] - pub struct Features { - /// Whether the compare feature is enabled. - pub compare: bool, - /// Whether the historical versions feature is enabled. - pub historical_versions: bool, - } - - /// Response for a publication. - #[derive(Serialize, Debug)] - #[serde(rename_all = "camelCase")] - pub struct Publication { - /// Whether the publication is currently active. - pub active: bool, - /// Date of the publication. - pub date: String, - /// Display name of the publication. - pub display: String, - /// Name of the publication. - pub name: String, - /// List of versions for the publication. - pub versions: Vec, - } - - /// Response for a version. - #[derive(Serialize, Debug, Clone)] - #[serde(rename_all = "camelCase")] - pub struct Version { - /// Codified date of the version. - pub date: String, - /// Display date of the version. - pub display: String, - /// Version number of the version. - #[serde(rename = "version")] - pub index: usize, - } - - impl From for Version { - fn from(value: models::version::Version) -> Self { - Self { - date: value.codified_date.clone(), - display: value.codified_date, - index: 0, - } - } - } - - impl Versions { - /// Build and returns an HTTP versions response converted into json. - #[allow(clippy::too_many_arguments)] - pub fn build( - active_publication_name: &str, - active_version: String, - active_compare_to: Option, - url: &str, - publications: &[models::publication::Publication], - current_publication_name: &str, - versions: &[Version], - messages: HistoricalMessages, - ) -> Self { - Self { - active_publication: active_publication_name.to_owned(), - active_version, - active_compare_to, - features: Features { - compare: true, - historical_versions: true, - }, - path: url.strip_prefix('/').unwrap_or_default().to_owned(), - publications: { - let mut sorted_publications = BTreeMap::new(); - for pb in publications { - sorted_publications.insert( - Reverse(pb.name.clone()), - Publication { - active: pb.name == active_publication_name, - date: pb.date.clone(), - display: Self::format_display_date( - &pb.name, - current_publication_name, - ), - name: pb.name.clone(), - versions: { - if pb.name == active_publication_name { - versions.to_vec() - } else { - vec![] - } - }, - }, - ); - } - sorted_publications - }, - messages, - } - } - - /// Returns a formatted display date. - /// If the `date` is current, returns the date with `(current)` appended. - fn format_display_date(date: &str, current_date: &str) -> String { - if date == CURRENT_PUBLICATION_NAME { - CURRENT_PUBLICATION_NAME.to_owned() - } else { - let mut formatted_date = format_date(date); - if date == current_date { - formatted_date.push_str(" (current)"); - } - formatted_date - } - } - } - - impl Version { - /// Create a new version. - pub const fn new(date: String, display: String, index: usize) -> Self { - Self { - date, - display, - index, - } - } - - /// Insert a new version if it is not present in the list of versions. - /// If the date is not in the list of versions, add it - /// This for compatibility purposes with the previous implementation of historical versions - pub fn insert_if_not_present(versions: &mut Vec, date: Option) { - if let Some(version_date) = date { - if versions.iter().all(|ver| ver.date != version_date) { - let version = Self::new(version_date.clone(), version_date, 0); - Self::insert_version_sorted(versions, version); - } - } - } - - /// Insert a new item into an already sorted collection. - /// The collection is sorted by date in descending order. - pub fn insert_version_sorted(collection: &mut Vec, item: Self) { - let mut idx = 0; - for i in collection.iter() { - if i.date < item.date { - break; - } - idx += 1; - } - collection.insert(idx, item); - } - - /// Utility function to find the index of a date in a list of versions. - pub fn find_index_or_closest(versions: &[Self], date: &str) -> usize { - versions - .iter() - .position(|ver| ver.date.as_str() == date) - .unwrap_or_else(|| { - let closest_date = versions - .iter() - .filter(|ver| ver.date.as_str() < date) - .max_by(|current, next| current.date.cmp(&next.date)) - .map_or_else(|| None, |ver| Some(ver.date.as_str())) - .unwrap_or("-1"); - versions - .iter() - .position(|ver| ver.date.as_str() == closest_date) - .unwrap_or(versions.len()) - }) - } - } - - /// Messages for the versions endpoint. - #[derive(Serialize, Debug)] - #[serde(rename_all = "camelCase")] - pub struct HistoricalMessages { - /// Message for an outdated publication. - pub publication: Option, - /// Message for an outdated version. - pub version: Option, - /// Message for a comparison between two versions. - pub comparison: Option, - } -} - -/// Handler for the versions endpoint. -pub async fn versions( - req: HttpRequest, - data: web::Data, - params: web::Path, -) -> impl Responder { - let stele = match get_stele_from_request(&req, data.archive()) { - Ok(stele) => stele, - Err(err) => return HttpResponse::BadRequest().body(format!("Error: {err}")), - }; - let db = data.db(); - let mut publications = publication::Manager::find_all_non_revoked_publications(db, &stele) - .await - .unwrap_or_default(); - - let Some(current_publication) = publications.first() else { - return HttpResponse::NotFound().body("No publications found."); - }; - - let mut active_publication_name = params - .publication - .clone() - .unwrap_or_else(|| current_publication.name.clone()); - - let active_publication = publications - .iter() - .find(|pb| pb.name == active_publication_name); - - let mut url = String::from("/"); - url.push_str(params.path.clone().unwrap_or_default().as_str()); - - let mut versions = if let Some(publication) = active_publication { - publication_versions(db, publication, url.clone()).await - } else { - vec![] - }; - - // latest date in active publication - let current_date = versions - .first() - .map_or(String::new(), |ver| ver.date.clone()); - // active version is the version the user is looking at right now - let mut active_version = - NaiveDate::parse_from_str(params.date.as_deref().unwrap_or_default(), "%Y-%m-%d") - .map_or(current_date.clone(), |date| date.clone().to_string()); - let active_compare_to = params.compare_date.clone().map(|date| { - NaiveDate::parse_from_str(&date, "%Y-%m-%d").map_or_else( - |_| current_date.clone(), - |active_date| active_date.to_string(), - ) - }); - - if active_version == current_date { - active_version = CURRENT_VERSION_DATE.to_owned(); - } - - let messages = historical_messages( - &versions, - current_publication, - &active_publication_name, - ¶ms.date, - &active_compare_to, - ); - - if active_publication_name == current_publication.name.clone() { - active_publication_name = CURRENT_PUBLICATION_NAME.to_owned(); - } - - response::Version::insert_if_not_present(&mut versions, params.date.clone()); - response::Version::insert_if_not_present(&mut versions, active_compare_to.clone()); - - let versions_size = versions.len(); - for (idx, version) in versions.iter_mut().enumerate() { - version.display = format_date(&version.date.clone()); - version.index = versions_size - idx; - } - if let Some(ver) = versions.first_mut() { - ver.display.push_str(" (last modified)"); - }; - - let current_version = response::Version::new( - CURRENT_VERSION_DATE.to_owned(), - CURRENT_VERSION_NAME.to_owned(), - versions.first().map_or(0, |ver| ver.index), - ); - - versions.insert(versions_size - current_version.index, current_version); - - let current_publication_name = current_publication.name.clone(); - // duplicate current publication with current label - publications.insert( - 0, - Publication::new( - CURRENT_PUBLICATION_NAME.to_owned(), - current_publication.date.clone(), - current_publication.stele.clone(), - ), - ); - - HttpResponse::Ok().json(response::Versions::build( - &active_publication_name, - active_version, - active_compare_to, - &url, - &publications, - ¤t_publication_name, - &versions, - messages, - )) -} - -/// Get all the versions of a publication. -async fn publication_versions( - db: &DatabaseConnection, - publication: &Publication, - url: String, -) -> Vec { - let mut versions = vec![]; - let doc_mpath = document_change::Manager::find_doc_mpath_by_url(db, &url) - .await - .unwrap_or_default(); - if let Some(mpath) = doc_mpath { - let doc_versions = - document_change::Manager::find_all_document_versions_by_mpath_and_publication( - db, - &mpath, - &publication.name, - ) - .await - .unwrap_or_default(); - versions = doc_versions.into_iter().map(Into::into).collect(); - return versions; - } - - let lib_mpath = library_change::Manager::find_lib_mpath_by_url(db, &url) - .await - .unwrap_or_default(); - if let Some(mpath) = lib_mpath { - let coll_versions = - library_change::Manager::find_all_collection_versions_by_mpath_and_publication( - db, - &mpath, - &publication.name, - ) - .await - .unwrap_or_default(); - versions = coll_versions.into_iter().map(Into::into).collect(); - } - versions -} - -/// Extracts the stele from the request. -/// If the `X-Stelae` header is present, it will return the value of the header. -/// Otherwise, it will return the root stele. -fn get_stele_from_request(req: &HttpRequest, archive: &Archive) -> anyhow::Result { - let req_headers = req.headers(); - let stele = archive.get_root()?.get_qualified_name(); - - req_headers.get("X-Stelae").map_or_else( - || Ok(stele), - |value| { - value.to_str().map_or_else( - |_| anyhow::bail!("Invalid X-Stelae header value"), - |str| Ok(str.to_owned()), - ) - }, - ) -} - -/// Returns historical messages for the versions endpoint. -/// The historical messages currently include: -/// - A message for an outdated publication. -/// - A message for an outdated version. -/// - A message for a comparison between two versions. -fn historical_messages( - versions: &[response::Version], - current_publication: &Publication, - active_publication_name: &str, - version_date: &Option, - compare_to_date: &Option, -) -> response::HistoricalMessages { - let current_publication_name = current_publication.name.as_str(); - let current_version: &str = versions - .first() - .map(|lmv| lmv.date.as_str()) - .unwrap_or_default(); - - let publication = publication_message( - active_publication_name, - current_publication_name, - current_version, - ); - let version = version_date.as_ref().and_then(|found_version_date| { - version_message( - current_version, - found_version_date, - versions, - compare_to_date, - ) - }); - let comparison = compare_to_date.as_ref().and_then(|found_compare_to_date| { - version_date.as_ref().map(|found_version_date| { - comparison_message( - found_compare_to_date, - found_version_date, - current_version, - versions, - ) - }) - }); - response::HistoricalMessages { - publication, - version, - comparison, - } -} - -/// Returns a historical message for an outdated publication. -fn publication_message( - active_publication_name: &str, - current_publication_name: &str, - current_version: &str, -) -> Option { - if active_publication_name == current_publication_name { - return None; - } - Some(format!( - "You are viewing a historical publication that was last updated on {current_date} and is no longer being updated.", - current_date = format_date(current_version) - )) -} - -/// Returns a historical message for an outdated version. -/// Version is outdated if `version_date` is in the past. -fn version_message( - current_version: &str, - version_date: &str, - versions: &[response::Version], - compare_to_date: &Option, -) -> Option { - let is_current_version = { - let current_date = - NaiveDate::parse_from_str(current_version, "%Y-%m-%d").unwrap_or_default(); - let Ok(parsed_version_date) = NaiveDate::parse_from_str(version_date, "%Y-%m-%d") else { - return None; - }; - current_date <= parsed_version_date - }; - if compare_to_date.is_some() || is_current_version { - return None; - } - let version_date_idx = versions - .iter() - .position(|ver| ver.date.as_str() == version_date); - let (start_date, end_date) = version_date_idx.map_or_else( - || { - let end_date = versions - .iter() - .find(|ver| ver.date.as_str() > version_date) - .map(|ver| ver.date.as_str()); - let found_idx = versions - .iter() - .position(|ver| ver.date.as_str() == end_date.unwrap_or_default()) - .unwrap_or_default(); - let start_date = versions - .get(found_idx + 1) - .map_or_else(|| versions.last(), Some) - .map(|ver| ver.date.as_str()); - (start_date.unwrap_or_default(), end_date.unwrap_or_default()) - }, - |idx| { - let start_date = version_date; - let end_date = versions - .get(idx - 1) - .map_or_else(|| versions.first(), Some) - .map(|ver| ver.date.as_str()) - .unwrap_or_default(); - (start_date, end_date) - }, - ); - Some(format!("You are viewing this document as it appeared on {version_date}. This version was valid between {start_date} and {end_date}.", - version_date = format_date(version_date), start_date = format_date(start_date), end_date = format_date(end_date))) -} - -/// Returns a historical message for a comparison between two versions. -fn comparison_message( - compare_to_date: &str, - version_date: &str, - current_date: &str, - versions: &[response::Version], -) -> String { - let (compare_start_date, compare_end_date) = if version_date > compare_to_date { - (compare_to_date, version_date) - } else { - (version_date, compare_to_date) - }; - let start_idx = response::Version::find_index_or_closest(versions, compare_start_date); - let end_idx = response::Version::find_index_or_closest(versions, compare_start_date); - let num_of_changes = start_idx - end_idx; - let start_date = format_date(compare_start_date); - let end_date = if compare_end_date == current_date { - None - } else { - Some(format_date(compare_end_date)) - }; - get_messages_between(num_of_changes, &start_date, end_date) -} - -/// Returns a message for the number of changes between two dates. -fn get_messages_between( - num_of_changes: usize, - start_date: &str, - end_date: Option, -) -> String { - let changes = match num_of_changes { - 0 => "no updates".to_owned(), - 1 => "1 update".to_owned(), - _ => format!("{num_of_changes} updates"), - }; - - end_date.map_or_else( - || format!("There have been {changes} since {start_date}."), - |found_end_date| { - format!( - "There have been {changes} between {start_date} and {found_end_date}." - ) - }, - ) -} - -/// Format a date from %Y-%m-%d to %B %d, %Y. -fn format_date(date: &str) -> String { - NaiveDate::parse_from_str(date, "%Y-%m-%d").map_or(date.to_owned(), |found_date| { - found_date.format("%B %d, %Y").to_string() - }) -} diff --git a/src/server/api/versions/mod.rs b/src/server/api/versions/mod.rs new file mode 100644 index 0000000..d5ada86 --- /dev/null +++ b/src/server/api/versions/mod.rs @@ -0,0 +1,208 @@ +//! Handlers for serving historical documents. +#![allow(clippy::future_not_send)] +use actix_web::{web, HttpRequest, HttpResponse, Responder}; +use chrono::NaiveDate; +use std::convert::Into; + +use crate::{ + db::{ + models::{ + document_change, library_change, + publication::{self, Publication}, + }, + DatabaseConnection, + }, + server::app::{AppState, GlobalState}, + stelae::archive::Archive, +}; + +use self::response::messages; + +/// Name of the current publication. +pub const CURRENT_PUBLICATION_NAME: &str = "Current"; +/// Name of the current version. +pub const CURRENT_VERSION_NAME: &str = "Current"; +/// Date of the current version. +pub const CURRENT_VERSION_DATE: &str = "current"; + +/// Module that maps the HTTP web request body to structs. +pub mod request; + +/// Module that maps the HTTP web response to structs. +pub mod response; + +/// Handler for the versions endpoint. +pub async fn versions( + req: HttpRequest, + data: web::Data, + params: web::Path, +) -> impl Responder { + let stele = match get_stele_from_request(&req, data.archive()) { + Ok(stele) => stele, + Err(err) => return HttpResponse::BadRequest().body(format!("Error: {err}")), + }; + let db = data.db(); + let mut publications = publication::Manager::find_all_non_revoked_publications(db, &stele) + .await + .unwrap_or_default(); + + let Some(current_publication) = publications.first() else { + return HttpResponse::NotFound().body("No publications found."); + }; + + let mut active_publication_name = params + .publication + .clone() + .unwrap_or_else(|| current_publication.name.clone()); + + let active_publication = publications + .iter() + .find(|pb| pb.name == active_publication_name); + + let mut url = String::from("/"); + url.push_str(params.path.clone().unwrap_or_default().as_str()); + + let mut versions = if let Some(publication) = active_publication { + publication_versions(db, publication, url.clone()).await + } else { + vec![] + }; + + // latest date in active publication + let current_date = versions + .first() + .map_or(String::new(), |ver| ver.date.clone()); + // active version is the version the user is looking at right now + let mut active_version = + NaiveDate::parse_from_str(params.date.as_deref().unwrap_or_default(), "%Y-%m-%d") + .map_or(current_date.clone(), |date| date.clone().to_string()); + let active_compare_to = params.compare_date.clone().map(|date| { + NaiveDate::parse_from_str(&date, "%Y-%m-%d").map_or_else( + |_| current_date.clone(), + |active_date| active_date.to_string(), + ) + }); + + if active_version == current_date { + active_version = CURRENT_VERSION_DATE.to_owned(); + } + + let messages = messages::historical( + &versions, + current_publication.name.as_str(), + &active_publication_name, + ¶ms.date, + &active_compare_to, + ); + + if active_publication_name == current_publication.name.clone() { + active_publication_name = CURRENT_PUBLICATION_NAME.to_owned(); + } + + response::Version::insert_if_not_present(&mut versions, params.date.clone()); + response::Version::insert_if_not_present(&mut versions, active_compare_to.clone()); + + let versions_size = versions.len(); + for (idx, version) in versions.iter_mut().enumerate() { + version.display = format_date(&version.date.clone()); + version.index = versions_size - idx; + } + if let Some(ver) = versions.first_mut() { + ver.display.push_str(" (last modified)"); + }; + + let current_version = response::Version::new( + CURRENT_VERSION_DATE.to_owned(), + CURRENT_VERSION_NAME.to_owned(), + versions.first().map_or(0, |ver| ver.index), + ); + + versions.insert(versions_size - current_version.index, current_version); + + let current_publication_name = current_publication.name.clone(); + // duplicate current publication with current label + publications.insert( + 0, + Publication::new( + CURRENT_PUBLICATION_NAME.to_owned(), + current_publication.date.clone(), + current_publication.stele.clone(), + ), + ); + + HttpResponse::Ok().json(response::Versions::build( + &active_publication_name, + active_version, + active_compare_to, + &url, + &publications, + ¤t_publication_name, + &versions, + messages, + )) +} + +/// Get all the versions of a publication. +async fn publication_versions( + db: &DatabaseConnection, + publication: &Publication, + url: String, +) -> Vec { + let mut versions = vec![]; + let doc_mpath = document_change::Manager::find_doc_mpath_by_url(db, &url) + .await + .unwrap_or_default(); + if let Some(mpath) = doc_mpath { + let doc_versions = + document_change::Manager::find_all_document_versions_by_mpath_and_publication( + db, + &mpath, + &publication.name, + ) + .await + .unwrap_or_default(); + versions = doc_versions.into_iter().map(Into::into).collect(); + return versions; + } + + let lib_mpath = library_change::Manager::find_lib_mpath_by_url(db, &url) + .await + .unwrap_or_default(); + if let Some(mpath) = lib_mpath { + let coll_versions = + library_change::Manager::find_all_collection_versions_by_mpath_and_publication( + db, + &mpath, + &publication.name, + ) + .await + .unwrap_or_default(); + versions = coll_versions.into_iter().map(Into::into).collect(); + } + versions +} + +/// Extracts the stele from the request. +/// If the `X-Stelae` header is present, it will return the value of the header. +/// Otherwise, it will return the root stele. +fn get_stele_from_request(req: &HttpRequest, archive: &Archive) -> anyhow::Result { + let req_headers = req.headers(); + let stele = archive.get_root()?.get_qualified_name(); + + req_headers.get("X-Stelae").map_or_else( + || Ok(stele), + |value| { + value.to_str().map_or_else( + |_| anyhow::bail!("Invalid X-Stelae header value"), + |str| Ok(str.to_owned()), + ) + }, + ) +} + +/// Format a date from %Y-%m-%d to %B %d, %Y. +fn format_date(date: &str) -> String { + NaiveDate::parse_from_str(date, "%Y-%m-%d").map_or(date.to_owned(), |found_date| { + found_date.format("%B %d, %Y").to_string() + }) +} diff --git a/src/server/api/versions/request/mod.rs b/src/server/api/versions/request/mod.rs new file mode 100644 index 0000000..f437b09 --- /dev/null +++ b/src/server/api/versions/request/mod.rs @@ -0,0 +1,13 @@ +use serde::Deserialize; +/// Request for the versions endpoint. +#[derive(Deserialize, Debug)] +pub struct Version { + /// Publication name. + pub publication: Option, + /// Date to compare. + pub date: Option, + /// Date to compare against. + pub compare_date: Option, + /// Path to document/collection. + pub path: Option, +} diff --git a/src/server/api/versions/response/messages.rs b/src/server/api/versions/response/messages.rs new file mode 100644 index 0000000..b5fca29 --- /dev/null +++ b/src/server/api/versions/response/messages.rs @@ -0,0 +1,193 @@ +use chrono::NaiveDate; +use serde::Serialize; + +use super::format_date; +use crate::server::api::versions::response::Version; + +/// Messages for the versions endpoint. +#[derive(Serialize, Debug, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub struct Historical { + /// Message for an outdated publication. + pub publication: Option, + /// Message for an outdated version. + pub version: Option, + /// Message for a comparison between two versions. + pub comparison: Option, +} + +/// Returns historical messages for the versions endpoint. +/// The historical messages currently include: +/// - A message for an outdated publication. +/// - A message for an outdated version. +/// - A message for a comparison between two versions. +#[must_use] +pub fn historical( + versions: &[Version], + current_publication_name: &str, + active_publication_name: &str, + version_date: &Option, + compare_to_date: &Option, +) -> Historical { + let current_version: &str = versions + .first() + .map(|lmv| lmv.date.as_str()) + .unwrap_or_default(); + + let publication = publication_message( + active_publication_name, + current_publication_name, + current_version, + ); + let version = version_date.as_ref().and_then(|found_version_date| { + version_message( + current_version, + found_version_date, + versions, + compare_to_date, + ) + }); + let comparison = compare_to_date.as_ref().and_then(|found_compare_to_date| { + version_date.as_ref().map(|found_version_date| { + comparison_message( + found_compare_to_date, + found_version_date, + current_version, + versions, + ) + }) + }); + Historical { + publication, + version, + comparison, + } +} + +/// Returns a historical message for an outdated publication. +fn publication_message( + active_publication_name: &str, + current_publication_name: &str, + current_version: &str, +) -> Option { + if active_publication_name == current_publication_name { + return None; + } + Some(publication_message_template(current_version)) +} + +/// Formats the response for an outdated publication. +fn publication_message_template(date: &str) -> String { + format!( + "You are viewing a historical publication that was last updated on {current_date} and is no longer being updated.", + current_date = format_date(date) + ) +} + +/// Returns a historical message for an outdated version. +/// Version is outdated if `version_date` is in the past. +fn version_message( + current_version: &str, + version_date: &str, + versions: &[Version], + compare_to_date: &Option, +) -> Option { + let is_current_version = { + let current_date = + NaiveDate::parse_from_str(current_version, "%Y-%m-%d").unwrap_or_default(); + let Ok(parsed_version_date) = NaiveDate::parse_from_str(version_date, "%Y-%m-%d") else { + return None; + }; + current_date <= parsed_version_date + }; + if compare_to_date.is_some() || is_current_version { + return None; + } + let version_date_idx = versions + .iter() + .position(|ver| ver.date.as_str() == version_date); + let (start_date, end_date) = version_date_idx.map_or_else( + || { + let end_date = versions + .iter() + .filter(|ver| ver.date.as_str() > version_date) + .map(|ver| ver.date.as_str()) + .min(); + let found_idx = versions + .iter() + .position(|ver| ver.date.as_str() == end_date.unwrap_or_default()) + .unwrap_or_default(); + let start_date = versions + .get(found_idx + 1) + .map_or_else(|| versions.last(), Some) + .map(|ver| ver.date.as_str()); + (start_date.unwrap_or_default(), end_date.unwrap_or_default()) + }, + |idx| { + let start_date = version_date; + let end_date = versions + .get(idx - 1) + .map_or_else(|| versions.first(), Some) + .map(|ver| ver.date.as_str()) + .unwrap_or_default(); + (start_date, end_date) + }, + ); + Some(version_message_template(version_date, start_date, end_date)) +} + +/// Formats the response for an outdated version. +fn version_message_template(version_date: &str, start_date: &str, end_date: &str) -> String { + format!( + "You are viewing this document as it appeared on {version_date}. This version was valid between {start_date} and {end_date}.", + version_date = format_date(version_date), + start_date = format_date(start_date), + end_date = format_date(end_date) + ) +} + +/// Returns a historical message for a comparison between two versions. +fn comparison_message( + compare_to_date: &str, + version_date: &str, + current_date: &str, + versions: &[Version], +) -> String { + let (compare_start_date, compare_end_date) = if version_date > compare_to_date { + (compare_to_date, version_date) + } else { + (version_date, compare_to_date) + }; + let start_idx = Version::find_index_or_closest(versions, compare_start_date); + let end_idx = Version::find_index_or_closest(versions, compare_end_date); + let num_of_changes = start_idx - end_idx; + let start_date = format_date(compare_start_date); + let end_date = if compare_end_date == current_date { + None + } else { + Some(format_date(compare_end_date)) + }; + messages_between_template(num_of_changes, &start_date, end_date) +} + +/// Formats and returns a message for the number of changes between two dates. +fn messages_between_template( + num_of_changes: usize, + start_date: &str, + end_date: Option, +) -> String { + let changes = match num_of_changes { + 0 => "no updates".to_owned(), + 1 => "1 update".to_owned(), + _ => format!("{num_of_changes} updates"), + }; + + end_date.map_or_else( + || format!("There have been {changes} since {start_date}."), + |found_end_date| { + format!( + "There have been {changes} between {start_date} and {found_end_date}." + ) + }, + ) +} diff --git a/src/server/api/versions/response/mod.rs b/src/server/api/versions/response/mod.rs new file mode 100644 index 0000000..debe6dc --- /dev/null +++ b/src/server/api/versions/response/mod.rs @@ -0,0 +1,211 @@ +use std::{cmp::Reverse, collections::BTreeMap}; + +use serde::Deserialize; +use serde::Serialize; + +use crate::db::models; + +use self::messages::Historical; + +use super::format_date; +use super::CURRENT_PUBLICATION_NAME; + +/// Historical messages for the versions endpoint. +pub mod messages; + +/// Response for the versions endpoint. +#[derive(Serialize, Debug)] +#[serde(rename_all = "camelCase")] +pub struct Versions { + /// Currently selected publication. + /// Resolves to "Current" if the latest publication is selected. + pub active_publication: String, + /// Currently selected version. + /// Resolves to "current" if the latest version is selected. + pub active_version: String, + /// Currently selected version to compare against. + /// If compare_date is specified, this will be the date to compare against. + pub active_compare_to: Option, + /// Features for the versions endpoint. + pub features: Features, + /// URL path. + pub path: String, + /// List of all found publications in descending order. + pub publications: BTreeMap, Publication>, + /// Messages for the versions endpoint. + pub messages: Historical, +} + +/// Features for the versions endpoint. +#[derive(Serialize, Debug)] +#[serde(rename_all = "camelCase")] +pub struct Features { + /// Whether the compare feature is enabled. + pub compare: bool, + /// Whether the historical versions feature is enabled. + pub historical_versions: bool, +} + +/// Response for a publication. +#[derive(Serialize, Deserialize, Debug)] +#[serde(rename_all = "camelCase")] +pub struct Publication { + /// Whether the publication is currently active. + pub active: bool, + /// Date of the publication. + pub date: String, + /// Display name of the publication. + pub display: String, + /// Name of the publication. + pub name: String, + /// List of versions for the publication. + pub versions: Vec, +} + +/// Response for a version. +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(rename_all = "camelCase")] +pub struct Version { + /// Codified date of the version. + pub date: String, + /// Display date of the version. + pub display: String, + /// Version number of the version. + #[serde(rename = "version")] + pub index: usize, +} + +impl From for Version { + fn from(value: models::version::Version) -> Self { + Self { + date: value.codified_date.clone(), + display: value.codified_date, + index: 0, + } + } +} + +impl Versions { + /// Build and returns an HTTP versions response converted into json. + #[allow(clippy::too_many_arguments)] + #[must_use] + pub fn build( + active_publication_name: &str, + active_version: String, + active_compare_to: Option, + url: &str, + publications: &[models::publication::Publication], + current_publication_name: &str, + versions: &[Version], + messages: Historical, + ) -> Self { + Self { + active_publication: active_publication_name.to_owned(), + active_version, + active_compare_to, + features: Features { + compare: true, + historical_versions: true, + }, + path: url.strip_prefix('/').unwrap_or_default().to_owned(), + publications: { + let mut sorted_publications = BTreeMap::new(); + for pb in publications { + sorted_publications.insert( + Reverse(pb.name.clone()), + Publication { + active: pb.name == active_publication_name, + date: pb.date.clone(), + display: Self::format_display_date( + &pb.name, + &pb.date, + current_publication_name, + ), + name: pb.name.clone(), + versions: { + if pb.name == active_publication_name { + versions.to_vec() + } else { + vec![] + } + }, + }, + ); + } + sorted_publications + }, + messages, + } + } + + /// Returns a formatted display date. + /// If the `date` is current, returns the date with `(current)` appended. + fn format_display_date(name: &str, date: &str, current_date: &str) -> String { + if name == CURRENT_PUBLICATION_NAME { + CURRENT_PUBLICATION_NAME.to_owned() + } else { + let mut formatted_date = format_date(date); + if date == current_date { + formatted_date.push_str(" (current)"); + } + formatted_date + } + } +} + +impl Version { + /// Create a new version. + #[must_use] + pub const fn new(date: String, display: String, index: usize) -> Self { + Self { + date, + display, + index, + } + } + + /// Insert a new version if it is not present in the list of versions. + /// If the date is not in the list of versions, add it + /// This for compatibility purposes with the previous implementation of historical versions + pub fn insert_if_not_present(versions: &mut Vec, date: Option) { + if let Some(version_date) = date { + if versions.iter().all(|ver| ver.date != version_date) { + let version = Self::new(version_date.clone(), version_date, 0); + Self::insert_version_sorted(versions, version); + } + } + } + + /// Insert a new item into an already sorted collection. + /// The collection is sorted by date in descending order. + pub fn insert_version_sorted(collection: &mut Vec, item: Self) { + let mut idx = 0; + for i in collection.iter() { + if i.date < item.date { + break; + } + idx += 1; + } + collection.insert(idx, item); + } + + /// Utility function to find the index of a date in a list of versions. + #[must_use] + pub fn find_index_or_closest(versions: &[Self], date: &str) -> usize { + versions + .iter() + .position(|ver| ver.date.as_str() == date) + .unwrap_or_else(|| { + let closest_date = versions + .iter() + .filter(|ver| ver.date.as_str() < date) + .max_by(|current, next| current.date.cmp(&next.date)) + .map_or_else(|| None, |ver| Some(ver.date.as_str())) + .unwrap_or("-1"); + versions + .iter() + .position(|ver| ver.date.as_str() == closest_date) + .unwrap_or(versions.len()) + }) + } +} diff --git a/tests/archive_testtools/config.rs b/tests/archive_testtools/config.rs index 4466137..90416ac 100644 --- a/tests/archive_testtools/config.rs +++ b/tests/archive_testtools/config.rs @@ -1,6 +1,6 @@ use anyhow::Result; use std::path::PathBuf; -pub use stelae::stelae::types::repositories::{Custom, Repositories, Repository}; +pub use stelae::stelae::types::repositories::{Custom, Repository}; pub enum ArchiveType { Basic(Jurisdiction), From 4400c15657cf693f022bec07d51871d4011fdf96 Mon Sep 17 00:00:00 2001 From: n-dusan Date: Tue, 14 May 2024 17:14:24 +0200 Subject: [PATCH 12/15] feat: add unit tests for historical messages --- src/server/api/versions/response/messages.rs | 276 +++++++++++++++++++ 1 file changed, 276 insertions(+) diff --git a/src/server/api/versions/response/messages.rs b/src/server/api/versions/response/messages.rs index b5fca29..9788c35 100644 --- a/src/server/api/versions/response/messages.rs +++ b/src/server/api/versions/response/messages.rs @@ -191,3 +191,279 @@ fn messages_between_template( }, ) } + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::*; + + use std::cmp::Reverse; + use std::collections::BTreeMap; + + use super::super::Publication; + + fn publication_to_versions() -> BTreeMap, Publication> { + let test_data = json!({ + "2023-12-30": { + "active": false, + "date": "2023-12-30", + "display": "2023-12-30", + "name": "2023-12-30", + "versions": [ + {"date": "2023-12-30", "display": "2023-12-30", "version": 0}, + {"date": "2023-12-11", "display": "2023-12-11", "version": 0}, + {"date": "2023-11-02", "display": "2023-11-02", "version": 0}, + {"date": "2023-10-22", "display": "2023-10-22", "version": 0}, + {"date": "2023-08-12", "display": "2023-08-12", "version": 0}, + {"date": "2023-08-10", "display": "2023-08-10", "version": 0}, + {"date": "2023-06-04", "display": "2023-06-04", "version": 0}, + {"date": "2023-01-01", "display": "2023-01-01", "version": 0} + ] + }, + "2023-10-22": { + "active": false, + "date": "2023-10-22", + "display": "2023-10-22", + "name": "2023-10-22", + "versions": [ + {"date": "2023-10-22", "display": "2023-10-22", "version": 0}, + {"date": "2023-08-12", "display": "2023-08-12", "version": 0}, + {"date": "2023-08-10", "display": "2023-08-10", "version": 0}, + {"date": "2023-06-04", "display": "2023-06-04", "version": 0}, + {"date": "2023-01-01", "display": "2023-01-01", "version": 0} + ] + } + }); + let map: BTreeMap, Publication> = + serde_json::from_value(test_data).unwrap(); + map + } + + fn current_publication_name() -> String { + "2023-12-30".to_string() + } + + #[test] + fn test_historical_when_current_publication_expect_no_historical_messages() { + let active_publication_name = "2023-12-30".to_string(); + let current_publication_name = current_publication_name(); + let publication_to_versions = publication_to_versions(); + let versions = &publication_to_versions + .get(&Reverse(active_publication_name.clone())) + .unwrap() + .versions; + let version_date: Option = None; + let compare_to_date: Option = None; + + let cut = historical; + + let actual = cut( + &versions, + ¤t_publication_name, + &active_publication_name, + &version_date, + &compare_to_date, + ); + let expected = Historical { + publication: None, + version: None, + comparison: None, + }; + + assert_eq!(actual, expected); + } + + #[test] + fn test_historical_when_outdated_publication_expect_publication_message_with_last_update() { + let test_cases = vec![ + None, + Some("2023-10-22".to_string()), + Some("2024-06-06".to_string()), + ]; + + for version_date in test_cases { + let active_publication_name = "2023-10-22".to_string(); + let current_publication_name = current_publication_name(); + let publication_to_versions = publication_to_versions(); + let versions = &publication_to_versions + .get(&Reverse(active_publication_name.clone())) + .unwrap() + .versions; + let compare_to_date: Option = None; + + let cut = historical; + + let actual = cut( + &versions, + ¤t_publication_name, + &active_publication_name, + &version_date, + &compare_to_date, + ); + let expected = Historical { + publication: Some(publication_message_template(&versions[0].date)), + version: None, + comparison: None, + }; + + assert_eq!(actual, expected); + } + } + + #[test] + fn test_historical_when_outdated_publication_and_outdated_date_expect_publication_and_version_message_with_last_update( + ) { + let test_cases = vec![ + ("2023-01-01", "2023-01-01", "2023-06-04"), // first date + ("2023-08-12", "2023-08-12", "2023-10-22"), // middle date + ("2023-02-02", "2023-01-01", "2023-06-04"), // non-existing date + ]; + + for (version_date, start_date, end_date) in test_cases { + let active_publication_name = "2023-10-22".to_string(); + let current_publication_name = current_publication_name(); + let publication_to_versions = publication_to_versions(); + let versions = &publication_to_versions + .get(&Reverse(active_publication_name.clone())) + .unwrap() + .versions; + let compare_to_date: Option = None; + + let cut = historical; + + let actual = cut( + &versions, + ¤t_publication_name, + &active_publication_name, + &Some(version_date.to_string()), + &compare_to_date, + ); + let expected = Historical { + publication: Some(publication_message_template(&versions[0].date)), + version: Some(version_message_template(version_date, start_date, end_date)), + comparison: None, + }; + + assert_eq!(actual, expected); + } + } + + #[test] + fn test_historical_when_comparing_with_latest_date_expect_historical_message_with_comparison_date( + ) { + let test_cases = vec![ + ("2023-10-22", "no updates"), + ("2023-08-12", "1 update"), + ("2023-08-10", "2 updates"), + ("2023-07-01", "3 updates"), // non-existing date + ("2023-01-01", "4 updates"), + ("2020-01-01", "5 updates"), // non-existing date before creation date + ]; + + for (version_date, changes) in test_cases { + let active_publication_name = "2023-10-22".to_string(); + let current_publication_name = current_publication_name(); + let publication_to_versions = publication_to_versions(); + let versions = &publication_to_versions + .get(&Reverse(active_publication_name.clone())) + .unwrap() + .versions; + let compare_to_date = Some("2023-10-22".to_string()); + let start_date = version_date; + + let cut = historical; + + let actual = cut( + &versions, + ¤t_publication_name, + &active_publication_name, + &Some(version_date.to_string()), + &compare_to_date, + ); + + let expected_comparison_message = messages_between_template( + match changes { + "no updates" => 0, + "1 update" => 1, + "2 updates" => 2, + "3 updates" => 3, + "4 updates" => 4, + "5 updates" => 5, + _ => 0, + }, + &format_date(start_date), + None, + ); + + let expected = Historical { + publication: Some(publication_message_template(&versions[0].date)), + version: None, + comparison: Some(expected_comparison_message), + }; + + assert_eq!(actual, expected); + } + } + + #[test] + fn test_historical_messages_when_comparing_with_non_latest_date_expect_historical_message() { + let test_cases = vec![ + ("2023-12-11", "2023-12-11", "no updates"), + ("2023-10-22", "2023-11-02", "1 update"), + ("2023-10-22", "2023-12-11", "2 updates"), + ("2023-07-01", "2023-12-11", "5 updates"), // non-existing start date + ("2023-06-04", "2023-09-11", "2 updates"), // non-existing end date + ("2023-07-01", "2023-09-11", "2 updates"), // non-existing start and end date + ("2020-01-01", "2023-06-04", "2 updates"), // non-existing start date before creation date + ("2020-01-01", "2020-06-04", "no updates"), // non-existing start and end date before creation date + ("2020-01-01", "2024-06-04", "8 updates"), // end date in the future + ("2023-07-01", "2024-06-04", "6 updates"), // non-existing start date and end date in the future + ]; + + for (version_date, compare_to_date, changes) in test_cases { + let active_publication_name = "2023-12-30".to_string(); + let current_publication_name = current_publication_name(); + let publication_to_versions = publication_to_versions(); + let versions = &publication_to_versions + .get(&Reverse(active_publication_name.clone())) + .unwrap() + .versions; + + let cut = historical; + + let actual = cut( + &versions, + ¤t_publication_name, + &active_publication_name, + &Some(version_date.to_string()), + &Some(compare_to_date.to_string()), + ); + + let expected_comparison_message = messages_between_template( + match changes { + "no updates" => 0, + "1 update" => 1, + "2 updates" => 2, + "3 updates" => 3, + "4 updates" => 4, + "5 updates" => 5, + "6 updates" => 6, + "7 updates" => 7, + "8 updates" => 8, + _ => 0, + }, + &format_date(version_date), + Some(format_date(compare_to_date)), + ); + + let expected = Historical { + publication: None, + version: None, + comparison: Some(expected_comparison_message), + }; + + assert_eq!(actual, expected); + } + } +} From cca5e72ad21e194dcfeb86538b7ba87c86974a73 Mon Sep 17 00:00:00 2001 From: n-dusan Date: Wed, 15 May 2024 19:55:21 +0200 Subject: [PATCH 13/15] refact: organize server endpoints into `server/api` mod These modules should make the code base more legibile. These changes include: - Moving `serve` current documents endpoint into `server/api/serve` module. This way we'll centralize all future endpoint logic in the `api` module - Centralize routing into `routes` mod. We'll expect to add new 'static' routing, while also keeping our dynamic logic in the same place. - Centralize global app state into `state` mod. - Small updates following the re-org of current codebase refact: rename structs in `state.rs` module to fix linting issues Address clippy lint warnings/pedantic errors related to generics in function arguments --- src/server/api/mod.rs | 3 + src/server/api/routes.rs | 275 +++++++++++++++++++++++ src/server/api/serve/mod.rs | 67 ++++++ src/server/api/state.rs | 138 ++++++++++++ src/server/api/versions/mod.rs | 3 +- src/server/app.rs | 387 +-------------------------------- src/server/mod.rs | 1 - src/server/routes.rs | 26 --- tests/common/mod.rs | 5 +- 9 files changed, 497 insertions(+), 408 deletions(-) create mode 100644 src/server/api/routes.rs create mode 100644 src/server/api/serve/mod.rs create mode 100644 src/server/api/state.rs delete mode 100644 src/server/routes.rs diff --git a/src/server/api/mod.rs b/src/server/api/mod.rs index a752df8..5349f87 100644 --- a/src/server/api/mod.rs +++ b/src/server/api/mod.rs @@ -1,2 +1,5 @@ //! This module contains the API endpoints for the server. +pub mod routes; +pub mod serve; +pub mod state; pub mod versions; diff --git a/src/server/api/routes.rs b/src/server/api/routes.rs new file mode 100644 index 0000000..5b685f8 --- /dev/null +++ b/src/server/api/routes.rs @@ -0,0 +1,275 @@ +//! A central place to register App routes. +#![allow(clippy::exit)] +use std::{process, sync::OnceLock}; + +use actix_service::ServiceFactory; +use actix_web::{ + body::MessageBody, + dev::{ServiceRequest, ServiceResponse}, + guard, web, App, Error, Scope, +}; +use tracing_actix_web::TracingLogger; + +use crate::server::{api::state, tracing::StelaeRootSpanBuilder}; +use crate::stelae::{stele::Stele, types::repositories::Repositories}; + +use super::{serve::serve, state::Global, versions::versions}; + +/// Name of the header to guard current documents +static HEADER_NAME: OnceLock = OnceLock::new(); +/// Values of the header to guard current documents +static HEADER_VALUES: OnceLock> = OnceLock::new(); + +/// Central place to register all the App routing. +/// +/// Registers all routes for the given Archive +/// Static routes should be registered first, followed by dynamic routes. +/// +/// # Errors +/// Errors if unable to register dynamic routes (e.g. if git repository cannot be opened) +pub fn register_app< + T: Global + Clone + 'static, + U: MessageBody, + V: ServiceFactory< + ServiceRequest, + Response = ServiceResponse, + Config = (), + InitError = (), + Error = Error, + >, +>( + mut app: App, + state: &T, +) -> anyhow::Result< + App< + impl ServiceFactory< + ServiceRequest, + Response = ServiceResponse, + Config = (), + InitError = (), + Error = Error, + >, + >, +> { + app = app + .service( + web::scope("/_api").service( + web::scope("/versions") + .service( + web::resource( + "/_publication/{publication}/_compare/{date}/{compare_date}/{path:.*}", + ) + .to(versions), + ) + .service(web::resource("/_publication/{publication}/_date/{date}").to(versions)) + .service(web::resource("/_publication/{publication}").to(versions)) + .service(web::resource("/_publication/{publication}/{path:.*}").to(versions)) + .service( + web::resource("/_compare/{date}/{compare_date}/{path:.*}").to(versions), + ) + .service(web::resource("/_date/{date}/{path:.*}").to(versions)) + .service(web::resource("/{path:.*}").to(versions)), + ), + ) + .app_data(web::Data::new(state.clone())); + + let initialized_app = register_dynamic_routes(app, state)?; + Ok(initialized_app) +} + +/// Initialize all dynamic routes for the given Archive. +/// +/// Dynamic routes are determined at runtime by looking at the stele's `dependencies.json` and `repositories.json` files +/// in the authentication (e.g. law) repository. +/// +/// # Errors +/// Errors if unable to register dynamic routes (e.g. if git repository cannot be opened) +fn register_dynamic_routes( + mut app: App< + impl ServiceFactory< + ServiceRequest, + Response = ServiceResponse, + Config = (), + InitError = (), + Error = Error, + >, + >, + state: &impl Global, +) -> anyhow::Result< + App< + impl ServiceFactory< + ServiceRequest, + Response = ServiceResponse, + Config = (), + InitError = (), + Error = Error, + >, + >, +> { + let config = state.archive().get_config()?; + let stelae_guard = config + .headers + .and_then(|headers| headers.current_documents_guard); + + if let Some(guard) = stelae_guard { + tracing::info!( + "Initializing guarded current documents with header: {}", + guard + ); + HEADER_NAME.get_or_init(|| guard); + HEADER_VALUES.get_or_init(|| { + state + .archive() + .stelae + .keys() + .map(ToString::to_string) + .collect() + }); + + if let (Some(guard_name), Some(guard_values)) = (HEADER_NAME.get(), HEADER_VALUES.get()) { + for guard_value in guard_values { + let stele = state.archive().stelae.get(guard_value); + if let Some(guarded_stele) = stele { + let shared_state = state::init_shared(guarded_stele)?; + let mut stelae_scope = web::scope(""); + stelae_scope = stelae_scope.guard(guard::Header(guard_name, guard_value)); + app = app.service( + stelae_scope + .app_data(web::Data::new(shared_state)) + .wrap(TracingLogger::::new()) + .configure(|cfg| { + register_root_routes(cfg, guarded_stele).unwrap_or_else(|_| { + tracing::error!( + "Failed to initialize routes for Stele: {}", + guarded_stele.get_qualified_name() + ); + process::exit(1); + }); + }), + ); + } + } + } + } else { + tracing::info!("Initializing app"); + let root = state.archive().get_root()?; + let shared_state = state::init_shared(root)?; + app = app.service( + web::scope("") + .app_data(web::Data::new(shared_state)) + .wrap(TracingLogger::::new()) + .configure(|cfg| { + register_routes(cfg, state).unwrap_or_else(|_| { + tracing::error!( + // TODO: error handling + "Failed to initialize routes for root Stele: {}", + root.get_qualified_name() + ); + process::exit(1); + }); + }), + ); + }; + Ok(app) +} + +/// Registers all dynamic routes for the given Archive +/// Each current document routes consists of two dynamic segments: `{prefix}/{tail}`. +/// prefix: the first part of the request uri, used to determine which dependent Stele to serve. +/// tail: the remaining glob pattern path of the request uri. +/// # Arguments +/// * `cfg` - The Actix `ServiceConfig` +/// * `state` - The application state +/// # Errors +/// Will error if unable to register routes (e.g. if git repository cannot be opened) +#[allow(clippy::iter_over_hash_type)] +fn register_routes(cfg: &mut web::ServiceConfig, state: &T) -> anyhow::Result<()> { + for stele in state.archive().stelae.values() { + if let Some(repositories) = stele.repositories.as_ref() { + if stele.is_root() { + continue; + } + register_dependent_routes(cfg, stele, repositories)?; + } + } + let root = state.archive().get_root()?; + register_root_routes(cfg, root)?; + Ok(()) +} + +/// Register routes for the root Stele +/// Root Stele is the Stele specified in config.toml +/// # Arguments +/// * `cfg` - The Actix `ServiceConfig` +/// * `stele` - The root Stele +/// # Errors +/// Will error if unable to register routes (e.g. if git repository cannot be opened) +fn register_root_routes(cfg: &mut web::ServiceConfig, stele: &Stele) -> anyhow::Result<()> { + let mut root_scope: Scope = web::scope(""); + if let Some(repositories) = stele.repositories.as_ref() { + let sorted_repositories = repositories.get_sorted_repositories(); + for repository in sorted_repositories { + let custom = &repository.custom; + let repo_state = state::init_repo(repository, stele)?; + for route in custom.routes.iter().flat_map(|routes| routes.iter()) { + let actix_route = format!("/{{tail:{}}}", &route); + root_scope = root_scope.service( + web::resource(actix_route.as_str()) + .route(web::get().to(serve)) + .app_data(web::Data::new(repo_state.clone())), + ); + } + if let Some(underscore_scope) = custom.scope.as_ref() { + let actix_underscore_scope = web::scope(underscore_scope.as_str()).service( + web::scope("").service( + web::resource("/{tail:.*}") + .route(web::get().to(serve)) + .app_data(web::Data::new(repo_state.clone())), + ), + ); + cfg.service(actix_underscore_scope); + } + } + cfg.service(root_scope); + } + Ok(()) +} + +/// Register routes for dependent Stele +/// Dependent Stele are all Steles' specified in the root Stele's `dependencies.json` config file. +/// # Arguments +/// * `cfg` - The Actix `ServiceConfig` +/// * `stele` - The root Stele +/// * `repositories` - Data repositories of the dependent Stele +/// # Errors +/// Will error if unable to register routes (e.g. if git repository cannot be opened) +fn register_dependent_routes( + cfg: &mut web::ServiceConfig, + stele: &Stele, + repositories: &Repositories, +) -> anyhow::Result<()> { + let sorted_repositories = repositories.get_sorted_repositories(); + for scope in repositories.scopes.iter().flat_map(|scopes| scopes.iter()) { + let scope_str = format!("/{{prefix:{}}}", &scope.as_str()); + let mut actix_scope = web::scope(scope_str.as_str()); + for repository in &sorted_repositories { + let custom = &repository.custom; + let repo_state = state::init_repo(repository, stele)?; + for route in custom.routes.iter().flat_map(|routes| routes.iter()) { + if route.starts_with('_') { + // Ignore routes in dependent Stele that start with underscore + // These routes are handled by the root Stele. + continue; + } + let actix_route = format!("/{{tail:{}}}", &route); + actix_scope = actix_scope.service( + web::resource(actix_route.as_str()) + .route(web::get().to(serve)) + .app_data(web::Data::new(repo_state.clone())), + ); + } + } + cfg.service(actix_scope); + } + Ok(()) +} diff --git a/src/server/api/serve/mod.rs b/src/server/api/serve/mod.rs new file mode 100644 index 0000000..58ab55a --- /dev/null +++ b/src/server/api/serve/mod.rs @@ -0,0 +1,67 @@ +//! API endpoint for serving current documents from Stele repositories. +#![allow(clippy::infinite_loop)] +use actix_web::{web, HttpRequest, HttpResponse, Responder}; +use lazy_static::lazy_static; +use regex::Regex; + +use crate::utils::{git::Repo, http::get_contenttype}; + +use super::state::{RepoData as RepoState, Shared as SharedState}; +/// Most-recent git commit +const HEAD_COMMIT: &str = "HEAD"; + +#[allow(clippy::expect_used)] +/// Remove leading and trailing `/`s from the `path` string. +fn clean_path(path: &str) -> String { + lazy_static! { + static ref RE: Regex = Regex::new("(?:^/*|/*$)").expect("Failed to compile regex!?!"); + } + RE.replace_all(path, "").to_string() +} + +/// Serve current document +#[allow(clippy::future_not_send)] +pub async fn serve( + req: HttpRequest, + shared: web::Data, + data: web::Data, +) -> impl Responder { + let prefix = req + .match_info() + .get("prefix") + .unwrap_or_default() + .to_owned(); + let tail = req.match_info().get("tail").unwrap_or_default().to_owned(); + let mut path = format!("{prefix}/{tail}"); + path = clean_path(&path); + let contenttype = get_contenttype(&path); + let blob = find_current_blob(&data.repo, &shared, &path); + match blob { + Ok(content) => HttpResponse::Ok().insert_header(contenttype).body(content), + Err(error) => { + tracing::debug!("{path}: {error}",); + HttpResponse::BadRequest().into() + } + } +} + +/// Find the latest blob for the given path from the given repo +/// Latest blob is found by looking at the HEAD commit +#[allow(clippy::panic_in_result_fn, clippy::unreachable)] +#[tracing::instrument(name = "Finding document", skip(repo, shared))] +fn find_current_blob(repo: &Repo, shared: &SharedState, path: &str) -> anyhow::Result> { + let blob = repo.get_bytes_at_path(HEAD_COMMIT, path); + match blob { + Ok(content) => Ok(content), + Err(error) => { + if let Some(fallback) = shared.fallback.as_ref() { + let fallback_blob = fallback.repo.get_bytes_at_path(HEAD_COMMIT, path); + return fallback_blob.map_or_else( + |err| anyhow::bail!("No fallback blob found - {}", err.to_string()), + Ok, + ); + } + anyhow::bail!("No fallback repo - {}", error.to_string()) + } + } +} diff --git a/src/server/api/state.rs b/src/server/api/state.rs new file mode 100644 index 0000000..9932e36 --- /dev/null +++ b/src/server/api/state.rs @@ -0,0 +1,138 @@ +//! Centralized state management for the Actix web server +use std::{fmt, path::PathBuf}; + +use crate::{ + db, + stelae::{archive::Archive, stele::Stele, types::repositories::Repository}, + utils::{archive::get_name_parts, git}, +}; +use git2::Repository as GitRepository; + +/// Global, read-only state +pub trait Global { + /// Fully initialized Stelae archive + fn archive(&self) -> &Archive; + /// Database connection + fn db(&self) -> &db::DatabaseConnection; +} + +/// Application state +#[derive(Debug, Clone)] +pub struct App { + /// Fully initialized Stelae archive + pub archive: Archive, + /// Database connection + pub db: db::DatabaseConnection, +} + +impl Global for App { + fn archive(&self) -> &Archive { + &self.archive + } + + fn db(&self) -> &db::DatabaseConnection { + &self.db + } +} + +/// Repository to serve +pub struct RepoData { + /// git2 wrapper repository pointing to the repo in the archive. + pub repo: git::Repo, + ///Latest or historical + pub serve: String, +} + +/// Shared, read-only app state +pub struct Shared { + /// Repository to fall back to if the current one is not found + pub fallback: Option, +} + +impl fmt::Debug for RepoData { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + write!( + formatter, + "Repo for {} in the archive at {}", + self.repo.name, + self.repo.path.display() + ) + } +} + +impl fmt::Debug for Shared { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let fb = &self.fallback; + match fb.as_ref() { + Some(fallback) => write!( + formatter, + "Repo for {} in the archive at {}", + fallback.repo.name, + fallback.repo.path.display() + ), + None => write!(formatter, "No fallback repo"), + } + } +} + +#[allow(clippy::missing_trait_methods)] +impl Clone for RepoData { + fn clone(&self) -> Self { + Self { + repo: self.repo.clone(), + serve: self.serve.clone(), + } + } +} + +#[allow(clippy::missing_trait_methods)] +impl Clone for Shared { + fn clone(&self) -> Self { + Self { + fallback: self.fallback.clone(), + } + } +} + +/// Initialize the data repository used in the Actix route +/// Each Actix route has its own data repository +/// +/// # Errors +/// Will error if unable to initialize the data repository +pub fn init_repo(repo: &Repository, stele: &Stele) -> anyhow::Result { + let custom = &repo.custom; + let (org, name) = get_name_parts(&repo.name)?; + let mut repo_path = stele.archive_path.to_string_lossy().into_owned(); + repo_path = format!("{repo_path}/{org}/{name}"); + Ok(RepoData { + repo: git::Repo { + archive_path: stele.archive_path.to_string_lossy().to_string(), + path: PathBuf::from(&repo_path), + org, + name, + repo: GitRepository::open(&repo_path)?, + }, + serve: custom.serve.clone(), + }) +} + +/// Initialize the shared application state +/// Currently shared application state consists of: +/// - fallback: used as a data repository to resolve data when no other url matches the request +/// # Returns +/// Returns a `SharedState` object +/// # Errors +/// Will error if unable to open the git repo for the fallback data repository +pub fn init_shared(stele: &Stele) -> anyhow::Result { + let fallback = stele + .get_fallback_repo() + .map(|repo| { + let (org, name) = get_name_parts(&repo.name)?; + Ok::(RepoData { + repo: git::Repo::new(&stele.archive_path, &org, &name)?, + serve: repo.custom.serve.clone(), + }) + }) + .transpose()?; + Ok(Shared { fallback }) +} diff --git a/src/server/api/versions/mod.rs b/src/server/api/versions/mod.rs index d5ada86..f43e025 100644 --- a/src/server/api/versions/mod.rs +++ b/src/server/api/versions/mod.rs @@ -12,12 +12,13 @@ use crate::{ }, DatabaseConnection, }, - server::app::{AppState, GlobalState}, stelae::archive::Archive, }; use self::response::messages; +use super::state::{App as AppState, Global as GlobalState}; + /// Name of the current publication. pub const CURRENT_PUBLICATION_NAME: &str = "Current"; /// Name of the current version. diff --git a/src/server/app.rs b/src/server/app.rs index 061968c..fc196c6 100644 --- a/src/server/app.rs +++ b/src/server/app.rs @@ -6,174 +6,18 @@ clippy::module_name_repetitions )] use crate::db; -use crate::server::routes::register_api; +use crate::server::api::state::App as AppState; use crate::stelae::archive::Archive; -use crate::stelae::types::repositories::{Repositories, Repository}; -use crate::utils::archive::get_name_parts; -use crate::utils::git::Repo; -use crate::utils::http::get_contenttype; -use crate::{server::tracing::StelaeRootSpanBuilder, stelae::stele::Stele}; use actix_web::dev::{ServiceRequest, ServiceResponse}; -use actix_web::{guard, web, App, Error, HttpRequest, HttpResponse, HttpServer, Responder, Scope}; -use git2::Repository as GitRepository; -use lazy_static::lazy_static; -use regex::Regex; -use std::{fmt, io, path::PathBuf, process}; -use tracing_actix_web::TracingLogger; +use actix_web::{App, Error, HttpServer}; + +use std::{io, path::PathBuf, process}; use actix_http::body::MessageBody; use actix_service::ServiceFactory; -use std::sync::OnceLock; - -/// Name of the header to guard current documents -static HEADER_NAME: OnceLock = OnceLock::new(); -/// Values of the header to guard current documents -static HEADER_VALUES: OnceLock> = OnceLock::new(); - -/// Most-recent git commit -const HEAD_COMMIT: &str = "HEAD"; - -#[allow(clippy::expect_used)] -/// Remove leading and trailing `/`s from the `path` string. -fn clean_path(path: &str) -> String { - lazy_static! { - static ref RE: Regex = Regex::new("(?:^/*|/*$)").expect("Failed to compile regex!?!"); - } - RE.replace_all(path, "").to_string() -} - -/// Global, read-only state -pub trait GlobalState { - /// Fully initialized Stelae archive - fn archive(&self) -> &Archive; - /// Database connection - fn db(&self) -> &db::DatabaseConnection; -} - -/// Application state -#[derive(Debug, Clone)] -pub struct AppState { - /// Fully initialized Stelae archive - pub archive: Archive, - /// Database connection - pub db: db::DatabaseConnection, -} - -impl GlobalState for AppState { - fn archive(&self) -> &Archive { - &self.archive - } - - fn db(&self) -> &db::DatabaseConnection { - &self.db - } -} - -/// Git repository to serve -struct RepoState { - /// git2 repository pointing to the repo in the archive. - repo: Repo, - ///Latest or historical - serve: String, -} - -/// Shared, read-only app state -pub struct SharedState { - /// Repository to fall back to if the current one is not found - fallback: Option, -} - -impl fmt::Debug for RepoState { - fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { - write!( - formatter, - "Repo for {} in the archive at {}", - self.repo.name, - self.repo.path.display() - ) - } -} - -impl fmt::Debug for SharedState { - fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { - let fb = &self.fallback; - match fb.as_ref() { - Some(fallback) => write!( - formatter, - "Repo for {} in the archive at {}", - fallback.repo.name, - fallback.repo.path.display() - ), - None => write!(formatter, "No fallback repo"), - } - } -} - -#[allow(clippy::missing_trait_methods)] -impl Clone for RepoState { - fn clone(&self) -> Self { - Self { - repo: self.repo.clone(), - serve: self.serve.clone(), - } - } -} - -#[allow(clippy::missing_trait_methods)] -impl Clone for SharedState { - fn clone(&self) -> Self { - Self { - fallback: self.fallback.clone(), - } - } -} - -/// Serve current document -#[allow(clippy::future_not_send)] -async fn serve( - req: HttpRequest, - shared: web::Data, - data: web::Data, -) -> impl Responder { - let prefix = req - .match_info() - .get("prefix") - .unwrap_or_default() - .to_owned(); - let tail = req.match_info().get("tail").unwrap_or_default().to_owned(); - let mut path = format!("{prefix}/{tail}"); - path = clean_path(&path); - let contenttype = get_contenttype(&path); - let blob = find_current_blob(&data.repo, &shared, &path); - match blob { - Ok(content) => HttpResponse::Ok().insert_header(contenttype).body(content), - Err(error) => { - tracing::debug!("{path}: {error}",); - HttpResponse::BadRequest().into() - } - } -} -/// Find the latest blob for the given path from the given repo -/// Latest blob is found by looking at the HEAD commit -#[allow(clippy::panic_in_result_fn, clippy::unreachable)] -#[tracing::instrument(name = "Finding document", skip(repo, shared))] -fn find_current_blob(repo: &Repo, shared: &SharedState, path: &str) -> anyhow::Result> { - let blob = repo.get_bytes_at_path(HEAD_COMMIT, path); - match blob { - Ok(content) => Ok(content), - Err(error) => { - if let Some(fallback) = shared.fallback.as_ref() { - let fallback_blob = fallback.repo.get_bytes_at_path(HEAD_COMMIT, path); - return fallback_blob.map_or_else( - |err| anyhow::bail!("No fallback blob found - {}", err.to_string()), - Ok, - ); - } - anyhow::bail!("No fallback repo - {}", error.to_string()) - } - } -} +use super::api::state::Global; +use crate::server::api::routes; /// Serve documents in a Stelae archive. #[actix_web::main] @@ -224,7 +68,7 @@ pub async fn serve_archive( /// * `state` - The application state /// # Errors /// Will error if unable to initialize the application -pub fn init_app( +pub fn init_app( state: &T, ) -> anyhow::Result< App< @@ -237,219 +81,6 @@ pub fn init_app( >, >, > { - let config = state.archive().get_config()?; - let stelae_guard = config - .headers - .and_then(|headers| headers.current_documents_guard); - - stelae_guard.map_or_else( - || { - tracing::info!("Initializing app"); - let root = state.archive().get_root()?; - let shared_state = init_shared_app_state(root)?; - Ok(App::new().service( - web::scope("") - .app_data(web::Data::new(shared_state)) - .wrap(TracingLogger::::new()) - .configure(|cfg| { - register_api(cfg, state); - register_routes(cfg, state).unwrap_or_else(|_| { - tracing::error!( - "Failed to initialize routes for root Stele: {}", - root.get_qualified_name() - ); - process::exit(1); - }); - }), - )) - }, - |guard| { - tracing::info!( - "Initializing guarded current documents with header: {}", - guard - ); - HEADER_NAME.get_or_init(|| guard); - HEADER_VALUES.get_or_init(|| { - state - .archive() - .stelae - .keys() - .map(ToString::to_string) - .collect() - }); - - let mut app = App::new(); - if let (Some(guard_name), Some(guard_values)) = (HEADER_NAME.get(), HEADER_VALUES.get()) - { - for guard_value in guard_values { - let stele = state.archive().stelae.get(guard_value); - if let Some(guarded_stele) = stele { - let shared_state = init_shared_app_state(guarded_stele)?; - let mut stelae_scope = web::scope(""); - stelae_scope = stelae_scope.guard(guard::Header(guard_name, guard_value)); - app = app.service( - stelae_scope - .app_data(web::Data::new(shared_state)) - .wrap(TracingLogger::::new()) - .configure(|cfg| { - register_api(cfg, state); - register_root_routes(cfg, guarded_stele).unwrap_or_else(|_| { - tracing::error!( - "Failed to initialize routes for Stele: {}", - guarded_stele.get_qualified_name() - ); - process::exit(1); - }); - }), - ); - } - } - } - Ok(app) - }, - ) -} - -/// Initialize the data repository used in the Actix route -/// Each Actix route has its own data repository -/// -/// # Errors -/// Will error if unable to initialize the data repository -fn init_repo_state(repo: &Repository, stele: &Stele) -> anyhow::Result { - let custom = &repo.custom; - let (org, name) = get_name_parts(&repo.name)?; - let mut repo_path = stele.archive_path.to_string_lossy().into_owned(); - repo_path = format!("{repo_path}/{org}/{name}"); - Ok(RepoState { - repo: Repo { - archive_path: stele.archive_path.to_string_lossy().to_string(), - path: PathBuf::from(&repo_path), - org, - name, - repo: GitRepository::open(&repo_path)?, - }, - serve: custom.serve.clone(), - }) -} - -/// Registers all routes for the given Archive -/// Each current document routes consists of two dynamic segments: `{prefix}/{tail}`. -/// prefix: the first part of the request uri, used to determine which dependent Stele to serve. -/// tail: the remaining glob pattern path of the request uri. -/// # Arguments -/// * `cfg` - The Actix `ServiceConfig` -/// * `state` - The application state -/// # Errors -/// Will error if unable to register routes (e.g. if git repository cannot be opened) -#[allow(clippy::iter_over_hash_type)] -fn register_routes(cfg: &mut web::ServiceConfig, state: &T) -> anyhow::Result<()> { - for stele in state.archive().stelae.values() { - if let Some(repositories) = stele.repositories.as_ref() { - if stele.is_root() { - continue; - } - register_dependent_routes(cfg, stele, repositories)?; - } - } - let root = state.archive().get_root()?; - register_root_routes(cfg, root)?; - Ok(()) -} - -/// Initialize the shared application state -/// Currently shared application state consists of: -/// - fallback: used as a data repository to resolve data when no other url matches the request -/// # Returns -/// Returns a `SharedState` object -/// # Errors -/// Will error if unable to open the git repo for the fallback data repository -pub fn init_shared_app_state(stele: &Stele) -> anyhow::Result { - let fallback = stele - .get_fallback_repo() - .map(|repo| { - let (org, name) = get_name_parts(&repo.name)?; - Ok::(RepoState { - repo: Repo::new(&stele.archive_path, &org, &name)?, - serve: repo.custom.serve.clone(), - }) - }) - .transpose()?; - Ok(SharedState { fallback }) -} - -/// Register routes for the root Stele -/// Root Stele is the Stele specified in config.toml -/// # Arguments -/// * `cfg` - The Actix `ServiceConfig` -/// * `stele` - The root Stele -/// # Errors -/// Will error if unable to register routes (e.g. if git repository cannot be opened) -fn register_root_routes(cfg: &mut web::ServiceConfig, stele: &Stele) -> anyhow::Result<()> { - let mut root_scope: Scope = web::scope(""); - if let Some(repositories) = stele.repositories.as_ref() { - let sorted_repositories = repositories.get_sorted_repositories(); - for repository in sorted_repositories { - let custom = &repository.custom; - let repo_state = init_repo_state(repository, stele)?; - for route in custom.routes.iter().flat_map(|routes| routes.iter()) { - let actix_route = format!("/{{tail:{}}}", &route); - root_scope = root_scope.service( - web::resource(actix_route.as_str()) - .route(web::get().to(serve)) - .app_data(web::Data::new(repo_state.clone())), - ); - } - if let Some(underscore_scope) = custom.scope.as_ref() { - let actix_underscore_scope = web::scope(underscore_scope.as_str()).service( - web::scope("").service( - web::resource("/{tail:.*}") - .route(web::get().to(serve)) - .app_data(web::Data::new(repo_state.clone())), - ), - ); - cfg.service(actix_underscore_scope); - } - } - cfg.service(root_scope); - } - Ok(()) -} - -/// Register routes for dependent Stele -/// Dependent Stele are all Steles' specified in the root Stele's `dependencies.json` config file. -/// # Arguments -/// * `cfg` - The Actix `ServiceConfig` -/// * `stele` - The root Stele -/// * `repositories` - Data repositories of the dependent Stele -/// # Errors -/// Will error if unable to register routes (e.g. if git repository cannot be opened) -fn register_dependent_routes( - cfg: &mut web::ServiceConfig, - stele: &Stele, - repositories: &Repositories, -) -> anyhow::Result<()> { - let sorted_repositories = repositories.get_sorted_repositories(); - for scope in repositories.scopes.iter().flat_map(|scopes| scopes.iter()) { - let scope_str = format!("/{{prefix:{}}}", &scope.as_str()); - let mut actix_scope = web::scope(scope_str.as_str()); - for repository in &sorted_repositories { - let custom = &repository.custom; - let repo_state = init_repo_state(repository, stele)?; - for route in custom.routes.iter().flat_map(|routes| routes.iter()) { - if route.starts_with('_') { - // Ignore routes in dependent Stele that start with underscore - // These routes are handled by the root Stele. - continue; - } - let actix_route = format!("/{{tail:{}}}", &route); - actix_scope = actix_scope.service( - web::resource(actix_route.as_str()) - .route(web::get().to(serve)) - .app_data(web::Data::new(repo_state.clone())), - ); - } - } - cfg.service(actix_scope); - } - Ok(()) + let app = routes::register_app(App::new(), state)?; + Ok(app) } diff --git a/src/server/mod.rs b/src/server/mod.rs index 0fccdab..c287916 100644 --- a/src/server/mod.rs +++ b/src/server/mod.rs @@ -6,5 +6,4 @@ pub mod api; pub mod app; pub mod errors; pub mod git; -pub mod routes; pub mod tracing; diff --git a/src/server/routes.rs b/src/server/routes.rs deleted file mode 100644 index 79f7dc5..0000000 --- a/src/server/routes.rs +++ /dev/null @@ -1,26 +0,0 @@ -//! A central place to register App routes. -use actix_web::web; - -use super::{api::versions::versions, app::GlobalState}; - -/// Central place to register _api/ routes. -pub fn register_api(cfg: &mut web::ServiceConfig, state: &T) { - cfg.service( - web::scope("/_api").service( - web::scope("/versions") - .service( - web::resource( - "/_publication/{publication}/_compare/{date}/{compare_date}/{path:.*}", - ) - .to(versions), - ) - .service(web::resource("/_publication/{publication}/_date/{date}").to(versions)) - .service(web::resource("/_publication/{publication}").to(versions)) - .service(web::resource("/_publication/{publication}/{path:.*}").to(versions)) - .service(web::resource("/_compare/{date}/{compare_date}/{path:.*}").to(versions)) - .service(web::resource("/_date/{date}/{path:.*}").to(versions)) - .service(web::resource("/{path:.*}").to(versions)), - ), - ) - .app_data(web::Data::new(state.clone())); -} diff --git a/tests/common/mod.rs b/tests/common/mod.rs index c86c538..e13afd4 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -9,7 +9,8 @@ use actix_web::{ use anyhow::Result; use std::path::{Path, PathBuf}; use std::sync::Once; -use stelae::{db, server::app::GlobalState}; +use stelae::db; +use stelae::server::api::state::Global; use tempfile::Builder; static INIT: Once = Once::new(); @@ -36,7 +37,7 @@ pub struct TestAppState { archive: Archive, } -impl GlobalState for TestAppState { +impl Global for TestAppState { fn archive(&self) -> &Archive { &self.archive } From 3194010430ea8a27bf3491ecb48a7f8f002a1eaf Mon Sep 17 00:00:00 2001 From: n-dusan Date: Mon, 20 May 2024 16:14:33 +0200 Subject: [PATCH 14/15] refact(api): split init code into two separate functions Add `initialize_guarded_dynamic_routes` and `initialize_dynamic_routes` functions which, depending on whether the guarded header exists, initialize the app with or without the guarded header. --- src/server/api/routes.rs | 203 ++++++++++++++++++++++----------------- 1 file changed, 116 insertions(+), 87 deletions(-) diff --git a/src/server/api/routes.rs b/src/server/api/routes.rs index 5b685f8..e740093 100644 --- a/src/server/api/routes.rs +++ b/src/server/api/routes.rs @@ -40,17 +40,7 @@ pub fn register_app< >( mut app: App, state: &T, -) -> anyhow::Result< - App< - impl ServiceFactory< - ServiceRequest, - Response = ServiceResponse, - Config = (), - InitError = (), - Error = Error, - >, - >, -> { +) -> anyhow::Result> { app = app .service( web::scope("/_api").service( @@ -73,8 +63,8 @@ pub fn register_app< ) .app_data(web::Data::new(state.clone())); - let initialized_app = register_dynamic_routes(app, state)?; - Ok(initialized_app) + app = register_dynamic_routes(app, state)?; + Ok(app) } /// Initialize all dynamic routes for the given Archive. @@ -84,92 +74,131 @@ pub fn register_app< /// /// # Errors /// Errors if unable to register dynamic routes (e.g. if git repository cannot be opened) -fn register_dynamic_routes( - mut app: App< - impl ServiceFactory< - ServiceRequest, - Response = ServiceResponse, - Config = (), - InitError = (), - Error = Error, - >, +fn register_dynamic_routes< + T: MessageBody, + U: ServiceFactory< + ServiceRequest, + Response = ServiceResponse, + Config = (), + InitError = (), + Error = Error, >, +>( + mut app: App, state: &impl Global, -) -> anyhow::Result< - App< - impl ServiceFactory< - ServiceRequest, - Response = ServiceResponse, - Config = (), - InitError = (), - Error = Error, - >, - >, -> { +) -> anyhow::Result> { let config = state.archive().get_config()?; let stelae_guard = config .headers .and_then(|headers| headers.current_documents_guard); if let Some(guard) = stelae_guard { - tracing::info!( - "Initializing guarded current documents with header: {}", - guard - ); - HEADER_NAME.get_or_init(|| guard); - HEADER_VALUES.get_or_init(|| { - state - .archive() - .stelae - .keys() - .map(ToString::to_string) - .collect() - }); + app = initialize_guarded_dynamic_routes(guard, app, state)?; + } else { + app = initialize_dynamic_routes(app, state)?; + }; + Ok(app) +} - if let (Some(guard_name), Some(guard_values)) = (HEADER_NAME.get(), HEADER_VALUES.get()) { - for guard_value in guard_values { - let stele = state.archive().stelae.get(guard_value); - if let Some(guarded_stele) = stele { - let shared_state = state::init_shared(guarded_stele)?; - let mut stelae_scope = web::scope(""); - stelae_scope = stelae_scope.guard(guard::Header(guard_name, guard_value)); - app = app.service( - stelae_scope - .app_data(web::Data::new(shared_state)) - .wrap(TracingLogger::::new()) - .configure(|cfg| { - register_root_routes(cfg, guarded_stele).unwrap_or_else(|_| { - tracing::error!( - "Failed to initialize routes for Stele: {}", - guarded_stele.get_qualified_name() - ); - process::exit(1); - }); - }), - ); - } +/// Initialize all guarded dynamic routes for the given Archive. +/// Routes are guarded by a header value specified in the config.toml file. +/// +/// # Errors +/// Errors if unable to register dynamic routes (e.g. if git repository cannot be opened) +fn initialize_guarded_dynamic_routes< + T: MessageBody, + U: ServiceFactory< + ServiceRequest, + Response = ServiceResponse, + Config = (), + InitError = (), + Error = Error, + >, +>( + guard: String, + mut app: App, + state: &impl Global, +) -> anyhow::Result> { + tracing::info!( + "Initializing guarded current documents with header: {}", + guard + ); + HEADER_NAME.get_or_init(|| guard); + HEADER_VALUES.get_or_init(|| { + state + .archive() + .stelae + .keys() + .map(ToString::to_string) + .collect() + }); + + if let (Some(guard_name), Some(guard_values)) = (HEADER_NAME.get(), HEADER_VALUES.get()) { + for guard_value in guard_values { + let stele = state.archive().stelae.get(guard_value); + if let Some(guarded_stele) = stele { + let shared_state = state::init_shared(guarded_stele)?; + let mut stelae_scope = web::scope(""); + stelae_scope = stelae_scope.guard(guard::Header(guard_name, guard_value)); + app = app.service( + stelae_scope + .app_data(web::Data::new(shared_state)) + .wrap(TracingLogger::::new()) + .configure(|cfg| { + register_root_routes(cfg, guarded_stele).unwrap_or_else(|_| { + tracing::error!( + "Failed to initialize routes for Stele: {}", + guarded_stele.get_qualified_name() + ); + process::exit(1); + }); + }), + ); } } } else { - tracing::info!("Initializing app"); - let root = state.archive().get_root()?; - let shared_state = state::init_shared(root)?; - app = app.service( - web::scope("") - .app_data(web::Data::new(shared_state)) - .wrap(TracingLogger::::new()) - .configure(|cfg| { - register_routes(cfg, state).unwrap_or_else(|_| { - tracing::error!( - // TODO: error handling - "Failed to initialize routes for root Stele: {}", - root.get_qualified_name() - ); - process::exit(1); - }); - }), - ); - }; + let err_msg = "Failed to initialize guarded routes. Header name or values not found."; + tracing::error!(err_msg); + anyhow::bail!(err_msg); + } + Ok(app) +} + +/// Initialize all dynamic routes for the given Archive. +/// +/// # Errors +/// Errors if unable to register dynamic routes (e.g. if git repository cannot be opened) +fn initialize_dynamic_routes< + T: MessageBody, + U: ServiceFactory< + ServiceRequest, + Response = ServiceResponse, + Config = (), + InitError = (), + Error = Error, + >, +>( + mut app: App, + state: &impl Global, +) -> anyhow::Result> { + tracing::info!("Initializing app"); + let root = state.archive().get_root()?; + let shared_state = state::init_shared(root)?; + app = app.service( + web::scope("") + .app_data(web::Data::new(shared_state)) + .wrap(TracingLogger::::new()) + .configure(|cfg| { + register_routes(cfg, state).unwrap_or_else(|_| { + tracing::error!( + // TODO: error handling + "Failed to initialize routes for root Stele: {}", + root.get_qualified_name() + ); + process::exit(1); + }); + }), + ); Ok(app) } From 8e1b6a3bc8657fd1c1b9166421e7f93028885371 Mon Sep 17 00:00:00 2001 From: n-dusan Date: Mon, 20 May 2024 16:15:50 +0200 Subject: [PATCH 15/15] refact(api): use `?` instead of `.ok` where possible For queries which we want to exit out of earlier, use `?`. However, I've left .ok for queries which should continue executing (so expecting a null value), instead of propagating an error to the caller where not expected. --- src/db/models/document_change.rs | 2 +- src/db/models/library_change.rs | 2 +- src/db/statements/queries.rs | 27 ++++++++---------------- src/server/api/versions/mod.rs | 35 ++++++++++++++------------------ 4 files changed, 26 insertions(+), 40 deletions(-) diff --git a/src/db/models/document_change.rs b/src/db/models/document_change.rs index ce44ac2..b91b889 100644 --- a/src/db/models/document_change.rs +++ b/src/db/models/document_change.rs @@ -7,7 +7,7 @@ use super::version::Version; #[async_trait] pub trait Manager { /// Find one document materialized path by url. - async fn find_doc_mpath_by_url(&self, url: &str) -> anyhow::Result>; + async fn find_doc_mpath_by_url(&self, url: &str) -> anyhow::Result; /// All dates on which given document changed. async fn find_all_document_versions_by_mpath_and_publication( &self, diff --git a/src/db/models/library_change.rs b/src/db/models/library_change.rs index 6b87d98..2e3ea27 100644 --- a/src/db/models/library_change.rs +++ b/src/db/models/library_change.rs @@ -7,7 +7,7 @@ use super::version::Version; #[async_trait] pub trait Manager { /// Find one library materialized path by url. - async fn find_lib_mpath_by_url(&self, url: &str) -> anyhow::Result>; + async fn find_lib_mpath_by_url(&self, url: &str) -> anyhow::Result; /// All dates on which given documents within a collection changed. async fn find_all_collection_versions_by_mpath_and_publication( &self, diff --git a/src/db/statements/queries.rs b/src/db/statements/queries.rs index 9b30f81..54cb529 100644 --- a/src/db/statements/queries.rs +++ b/src/db/statements/queries.rs @@ -17,10 +17,7 @@ use crate::db::DatabaseKind; /// /// # Errors /// Errors if can't establish a connection to the database. -pub async fn find_stele_by_name( - conn: &DatabaseConnection, - name: &str, -) -> anyhow::Result> { +pub async fn find_stele_by_name(conn: &DatabaseConnection, name: &str) -> anyhow::Result { let statement = " SELECT * FROM stele @@ -32,8 +29,7 @@ pub async fn find_stele_by_name( sqlx::query_as::<_, Stele>(statement) .bind(name) .fetch_one(&mut *connection) - .await - .ok() + .await? } }; Ok(row) @@ -272,7 +268,7 @@ pub async fn find_last_inserted_publication_version_by_publication_and_stele( LIMIT 1 "; let row = match conn.kind { - DatabaseKind::Sqlite => { + DatabaseKind::Sqlite | DatabaseKind::Postgres => { let mut connection = conn.pool.acquire().await?; sqlx::query_as::<_, PublicationVersion>(statement) .bind(publication) @@ -281,9 +277,6 @@ pub async fn find_last_inserted_publication_version_by_publication_and_stele( .await .ok() } - DatabaseKind::Postgres => { - unimplemented!(); - } }; Ok(row) } @@ -294,7 +287,7 @@ impl document_change::Manager for DatabaseConnection { /// /// # Errors /// Errors if can't establish a connection to the database. - async fn find_doc_mpath_by_url(&self, url: &str) -> anyhow::Result> { + async fn find_doc_mpath_by_url(&self, url: &str) -> anyhow::Result { let statement = " SELECT doc_mpath FROM document_change @@ -307,11 +300,10 @@ impl document_change::Manager for DatabaseConnection { sqlx::query_as::<_, (String,)>(statement) .bind(url) .fetch_one(&mut *connection) - .await - .ok() + .await? } }; - Ok(row.map(|(doc_mpath,)| doc_mpath)) + Ok(row.0) } /// All dates on which given document changed. @@ -418,7 +410,7 @@ impl library_change::Manager for DatabaseConnection { /// /// # Errors /// Errors if can't establish a connection to the database. - async fn find_lib_mpath_by_url(&self, url: &str) -> anyhow::Result> { + async fn find_lib_mpath_by_url(&self, url: &str) -> anyhow::Result { let statement = " SELECT library_mpath FROM library_change @@ -431,11 +423,10 @@ impl library_change::Manager for DatabaseConnection { sqlx::query_as::<_, (String,)>(statement) .bind(url) .fetch_one(&mut *connection) - .await - .ok() + .await? } }; - Ok(row.map(|(library_mpath,)| library_mpath)) + Ok(row.0) } /// All dates on which documents from this collection changed. /// diff --git a/src/server/api/versions/mod.rs b/src/server/api/versions/mod.rs index f43e025..ee69b00 100644 --- a/src/server/api/versions/mod.rs +++ b/src/server/api/versions/mod.rs @@ -150,10 +150,8 @@ async fn publication_versions( url: String, ) -> Vec { let mut versions = vec![]; - let doc_mpath = document_change::Manager::find_doc_mpath_by_url(db, &url) - .await - .unwrap_or_default(); - if let Some(mpath) = doc_mpath { + let doc_mpath = document_change::Manager::find_doc_mpath_by_url(db, &url).await; + if let Ok(mpath) = doc_mpath { let doc_versions = document_change::Manager::find_all_document_versions_by_mpath_and_publication( db, @@ -163,22 +161,19 @@ async fn publication_versions( .await .unwrap_or_default(); versions = doc_versions.into_iter().map(Into::into).collect(); - return versions; - } - - let lib_mpath = library_change::Manager::find_lib_mpath_by_url(db, &url) - .await - .unwrap_or_default(); - if let Some(mpath) = lib_mpath { - let coll_versions = - library_change::Manager::find_all_collection_versions_by_mpath_and_publication( - db, - &mpath, - &publication.name, - ) - .await - .unwrap_or_default(); - versions = coll_versions.into_iter().map(Into::into).collect(); + } else { + let lib_mpath = library_change::Manager::find_lib_mpath_by_url(db, &url).await; + if let Ok(mpath) = lib_mpath { + let coll_versions = + library_change::Manager::find_all_collection_versions_by_mpath_and_publication( + db, + &mpath, + &publication.name, + ) + .await + .unwrap_or_default(); + versions = coll_versions.into_iter().map(Into::into).collect(); + } } versions }