From d178443d261988cb2d2565843a41e9d0ace64c92 Mon Sep 17 00:00:00 2001 From: shouya <526598+shouya@users.noreply.github.com> Date: Wed, 18 Sep 2024 17:42:28 +0900 Subject: [PATCH] feat: htmx-based web ui (#147) This new web ui is written with htmx and maud for the goal of making the ui self-contained in the rust project. Which means it requires no additional build-step for front-end. htmx does most of the heavy lifting so the new ui is smaller and hopefully easier to maintain than the old. Currently, the new ui has mostly reached feature parity with the old inspector ui. New features can be expected in the future. The new ui is enabled by default. You can disable it the same way by customizing `RSS_FUNNEL_INSPECTOR_UI` environment variable. For compatibility reasons, the old ui is still available through the `/_inspector` route. This route will be removed in a future release. Any feedback on the new ui is welcome. --- Cargo.lock | 208 ++++++++++++++-- Cargo.toml | 5 +- scripts/watch-dev.sh | 9 + src/cli.rs | 4 +- src/client.rs | 35 ++- src/feed.rs | 9 + src/filter.rs | 78 ++++-- src/filter/js.rs | 10 +- src/filter/merge.rs | 12 +- src/filter_pipeline.rs | 15 +- src/server.rs | 9 +- src/server/endpoint.rs | 42 +++- src/server/feed_service.rs | 5 +- src/server/index.rs | 1 + src/server/inspector.rs | 11 +- src/server/web.rs | 111 +++++++++ src/server/web/endpoint.rs | 428 ++++++++++++++++++++++++++++++++ src/server/web/list.rs | 120 +++++++++ src/server/web/login.rs | 117 +++++++++ src/source.rs | 125 +++++++++- src/util.rs | 2 +- static/common.css | 149 +++++++++++ static/common.js | 0 static/endpoint.css | 198 +++++++++++++++ static/endpoint.js | 44 ++++ static/list.css | 0 static/login.css | 22 ++ static/login.js | 14 ++ static/sprite.svg | 85 +++++++ static/svg/book.svg | 1 + static/svg/caret-right.svg | 1 + static/svg/code.svg | 1 + static/svg/copy.svg | 1 + static/svg/external-link.svg | 1 + static/svg/file-description.svg | 1 + static/svg/json.svg | 1 + static/svg/loader.svg | 1 + 37 files changed, 1760 insertions(+), 116 deletions(-) create mode 100755 scripts/watch-dev.sh create mode 100644 src/server/index.rs create mode 100644 src/server/web.rs create mode 100644 src/server/web/endpoint.rs create mode 100644 src/server/web/list.rs create mode 100644 src/server/web/login.rs create mode 100644 static/common.css create mode 100644 static/common.js create mode 100644 static/endpoint.css create mode 100644 static/endpoint.js create mode 100644 static/list.css create mode 100644 static/login.css create mode 100644 static/login.js create mode 100644 static/sprite.svg create mode 100644 static/svg/book.svg create mode 100644 static/svg/caret-right.svg create mode 100644 static/svg/code.svg create mode 100644 static/svg/copy.svg create mode 100644 static/svg/external-link.svg create mode 100644 static/svg/file-description.svg create mode 100644 static/svg/json.svg create mode 100644 static/svg/loader.svg diff --git a/Cargo.lock b/Cargo.lock index e2c0d0a..2f2a7b9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -45,6 +45,19 @@ version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0942ffc6dcaadf03badf6e6a2d0228460359d5e34b57ccdc720b7382dfbd5ec5" +[[package]] +name = "ammonia" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ab99eae5ee58501ab236beb6f20f6ca39be615267b014899c89b2f0bc18a459" +dependencies = [ + "html5ever 0.27.0", + "maplit", + "once_cell", + "tendril", + "url", +] + [[package]] name = "android-tzdata" version = "0.1.1" @@ -285,9 +298,9 @@ checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" [[package]] name = "base64" -version = "0.22.0" +version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9475866fec1451be56a3c2400fd081ff546538961565ccb5b7142cbd22bc7a51" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" [[package]] name = "bindgen" @@ -440,7 +453,7 @@ dependencies = [ "anstream", "anstyle", "clap_lex", - "strsim 0.11.0", + "strsim 0.11.1", ] [[package]] @@ -579,8 +592,18 @@ version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b750cb3417fd1b327431a470f388520309479ab0bf5e323505daf0290cd3850" dependencies = [ - "darling_core", - "darling_macro", + "darling_core 0.14.4", + "darling_macro 0.14.4", +] + +[[package]] +name = "darling" +version = "0.20.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f63b86c8a8826a49b8c21f08a2d07338eec8d900540f8630dc76284be802989" +dependencies = [ + "darling_core 0.20.10", + "darling_macro 0.20.10", ] [[package]] @@ -597,17 +620,42 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "darling_core" +version = "0.20.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95133861a8032aaea082871032f5815eb9e98cef03fa916ab4500513994df9e5" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim 0.11.1", + "syn 2.0.52", +] + [[package]] name = "darling_macro" version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4aab4dbc9f7611d8b55048a3a16d2d010c2c8334e46304b40ac1cc14bf3b48e" dependencies = [ - "darling_core", + "darling_core 0.14.4", "quote", "syn 1.0.109", ] +[[package]] +name = "darling_macro" +version = "0.20.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806" +dependencies = [ + "darling_core 0.20.10", + "quote", + "syn 2.0.52", +] + [[package]] name = "deranged" version = "0.3.11" @@ -615,6 +663,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4" dependencies = [ "powerfmt", + "serde", ] [[package]] @@ -632,7 +681,7 @@ version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c11bdc11a0c47bc7d37d582b5285da6849c96681023680b906673c5707af7b0f" dependencies = [ - "darling", + "darling 0.14.4", "proc-macro2", "quote", "syn 1.0.109", @@ -697,14 +746,14 @@ dependencies = [ [[package]] name = "duration-str" -version = "0.7.1" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8bb6a301a95ba86fa0ebaf71d49ae4838c51f8b84cb88ed140dfb66452bb3c4" +checksum = "709d653e7c92498eb29fb86a2a6f0f3502b97530f33aedb32ef848d4d28b31a3" dependencies = [ - "nom", "rust_decimal", "serde", "thiserror", + "winnow 0.6.18", ] [[package]] @@ -1028,6 +1077,12 @@ version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + [[package]] name = "home" version = "0.5.9" @@ -1045,12 +1100,26 @@ checksum = "bea68cab48b8459f17cf1c944c67ddc572d272d9f2b274140f223ecb1da4a3b7" dependencies = [ "log", "mac", - "markup5ever", + "markup5ever 0.11.0", "proc-macro2", "quote", "syn 1.0.109", ] +[[package]] +name = "html5ever" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c13771afe0e6e846f1e67d038d4cb29998a6779f93c809212e4e9c32efd244d4" +dependencies = [ + "log", + "mac", + "markup5ever 0.12.1", + "proc-macro2", + "quote", + "syn 2.0.52", +] + [[package]] name = "htmlescape" version = "0.3.1" @@ -1219,6 +1288,7 @@ checksum = "7b0b929d511467233429c45a44ac1dcaa21ba0f5ba11e4879e6ed28ddb4f9df4" dependencies = [ "equivalent", "hashbrown 0.14.3", + "serde", ] [[package]] @@ -1381,6 +1451,12 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4" +[[package]] +name = "maplit" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d" + [[package]] name = "markup5ever" version = "0.11.0" @@ -1395,14 +1471,28 @@ dependencies = [ "tendril", ] +[[package]] +name = "markup5ever" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16ce3abbeba692c8b8441d036ef91aea6df8da2c6b6e21c7e14d3c18e526be45" +dependencies = [ + "log", + "phf 0.11.2", + "phf_codegen 0.11.2", + "string_cache", + "string_cache_codegen", + "tendril", +] + [[package]] name = "markup5ever_rcdom" version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9521dd6750f8e80ee6c53d65e2e4656d7de37064f3a7a5d2d11d05df93839c2" dependencies = [ - "html5ever", - "markup5ever", + "html5ever 0.26.0", + "markup5ever 0.11.0", "tendril", "xml5ever", ] @@ -1419,6 +1509,30 @@ version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" +[[package]] +name = "maud" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df518b75016b4289cdddffa1b01f2122f4a49802c93191f3133f6dc2472ebcaa" +dependencies = [ + "axum-core", + "http", + "itoa 1.0.10", + "maud_macros", +] + +[[package]] +name = "maud_macros" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa453238ec218da0af6b11fc5978d3b5c3a45ed97b722391a2a11f3306274e18" +dependencies = [ + "proc-macro-error", + "proc-macro2", + "quote", + "syn 2.0.52", +] + [[package]] name = "memchr" version = "2.7.1" @@ -1650,6 +1764,16 @@ dependencies = [ "phf_shared 0.10.0", ] +[[package]] +name = "phf_codegen" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8d39688d359e6b34654d328e262234662d16cc0f60ec8dcbe5e718709342a5a" +dependencies = [ + "phf_generator 0.11.2", + "phf_shared 0.11.2", +] + [[package]] name = "phf_generator" version = "0.8.0" @@ -1949,7 +2073,7 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e56596e20a6d3cf715182d9b6829220621e6e985cec04d00410cee29821b4220" dependencies = [ - "html5ever", + "html5ever 0.26.0", "lazy_static", "markup5ever_rcdom", "regex", @@ -2129,12 +2253,13 @@ dependencies = [ name = "rss-funnel" version = "0.1.4" dependencies = [ + "ammonia", "async-trait", "atom_syndication", "axum", "axum-extra", "axum-macros", - "base64 0.22.0", + "base64 0.22.1", "blake3", "chrono", "clap", @@ -2144,13 +2269,14 @@ dependencies = [ "encoding_rs", "futures", "glob-match", - "html5ever", + "html5ever 0.26.0", "htmlescape", "http", "itertools", "lazy_static", "lol_html", "lru", + "maud", "mime", "notify", "paste", @@ -2165,6 +2291,7 @@ dependencies = [ "scraper", "serde", "serde_json", + "serde_with", "serde_yaml", "thiserror", "tokio", @@ -2356,7 +2483,7 @@ dependencies = [ "cssparser 0.31.2", "ego-tree", "getopts", - "html5ever", + "html5ever 0.26.0", "once_cell", "selectors 0.25.0", "tendril", @@ -2471,6 +2598,36 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_with" +version = "3.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cecfa94848272156ea67b2b1a53f20fc7bc638c4a46d2f8abde08f05f4b857" +dependencies = [ + "base64 0.22.1", + "chrono", + "hex", + "indexmap 1.9.3", + "indexmap 2.2.5", + "serde", + "serde_derive", + "serde_json", + "serde_with_macros", + "time", +] + +[[package]] +name = "serde_with_macros" +version = "3.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8fee4991ef4f274617a51ad4af30519438dacb2f56ac773b08a1922ff743350" +dependencies = [ + "darling 0.20.10", + "proc-macro2", + "quote", + "syn 2.0.52", +] + [[package]] name = "serde_yaml" version = "0.9.32" @@ -2615,9 +2772,9 @@ checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" [[package]] name = "strsim" -version = "0.11.0" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ee073c9e4cd00e28217186dbe12796d692868f432bf2e97ee73bed0c56dfa01" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" [[package]] name = "subtle" @@ -2826,7 +2983,7 @@ checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421" dependencies = [ "indexmap 2.2.5", "toml_datetime", - "winnow", + "winnow 0.5.40", ] [[package]] @@ -3356,6 +3513,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "winnow" +version = "0.6.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68a9bda4691f099d435ad181000724da8e5899daa10713c2d432552b9ccd3a6f" +dependencies = [ + "memchr", +] + [[package]] name = "winreg" version = "0.50.0" @@ -3374,7 +3540,7 @@ checksum = "4034e1d05af98b51ad7214527730626f019682d797ba38b51689212118d8e650" dependencies = [ "log", "mac", - "markup5ever", + "markup5ever 0.11.0", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index cbd91a7..c264460 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -35,7 +35,7 @@ blake3 = "1.5.1" clap = { version = "4.5.1", features = ["derive", "env"] } serde = { version = "1.0.197", features = ["derive", "rc"] } serde_yaml = "0.9.32" -duration-str = { version = "0.7.1", default-features = false, features = ["serde"] } +duration-str = { version = "0.11.2", default-features = false, features = ["serde"] } # webserver axum-macros = "0.4.1" @@ -87,6 +87,9 @@ glob-match = "0.2.1" # Logging tracing = { version = "0.1.40"} tracing-subscriber = "0.3.18" +maud = { version = "0.26.0", features = ["axum"] } +ammonia = "4.0.0" +serde_with = "3.9.0" [patch.crates-io] ego-tree = { git = "https://github.com/shouya/ego-tree.git" } diff --git a/scripts/watch-dev.sh b/scripts/watch-dev.sh new file mode 100755 index 0000000..19ccf39 --- /dev/null +++ b/scripts/watch-dev.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +trap "trap - SIGTERM && kill -- -$$" SIGINT SIGTERM EXIT + +cargo watch -x build -s 'touch /tmp/.trigger' & +cargo watch -w /tmp/.trigger -d0 -s 'target/debug/rss-funnel -c ~/.config/rss-funnel-dev/funnel.yaml server' & + +wait + diff --git a/src/cli.rs b/src/cli.rs index db964fe..28e74ab 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -50,7 +50,7 @@ struct TestConfig { source: Option, /// Limit the first N filter steps to run #[clap(long, short)] - limit_filters: Option, + filter_skip: Option, /// Limit the number of items in the feed #[clap(long, short('n'))] limit_posts: Option, @@ -66,7 +66,7 @@ impl TestConfig { fn to_endpoint_param(&self) -> server::EndpointParam { server::EndpointParam::new( self.source.as_ref().cloned(), - self.limit_filters, + self.filter_skip, self.limit_posts, self.base.clone(), ) diff --git a/src/client.rs b/src/client.rs index c9e4ec0..e7b4d13 100644 --- a/src/client.rs +++ b/src/client.rs @@ -22,43 +22,50 @@ struct HttpFixture { content: String, } +#[serde_with::skip_serializing_none] #[derive( JsonSchema, Serialize, Deserialize, Debug, Clone, Default, PartialEq, Eq, Hash, )] pub struct ClientConfig { /// The "user-agent" header to send with requests - user_agent: Option, + #[serde(default)] + pub user_agent: Option, /// The "accept" header to send with requests - accept: Option, + #[serde(default)] + pub accept: Option, /// The "cookie" header to send with requests (Deprecated, specify "cookie" field instead) - set_cookie: Option, + #[serde(default)] + pub set_cookie: Option, /// The "cookie" header to send with requests - cookie: Option, + #[serde(default)] + pub cookie: Option, /// The "referer" header to send with requests - referer: Option, - /// The maximum number of cached responses - cache_size: Option, + #[serde(default)] + pub referer: Option, /// Ignore tls error #[serde(default)] - accept_invalid_certs: bool, + pub accept_invalid_certs: bool, + /// The maximum number of cached responses + #[serde(default)] + pub cache_size: Option, /// The maximum time a response is kept in the cache (Format: "4s", /// 10m", "1h", "1d") #[serde(default)] #[serde(deserialize_with = "duration_str::deserialize_option_duration")] #[schemars(with = "String")] - cache_ttl: Option, + pub cache_ttl: Option, /// Request timeout (Format: "4s", "10m", "1h", "1d") #[serde(deserialize_with = "duration_str::deserialize_option_duration")] #[schemars(with = "String")] - timeout: Option, + pub timeout: Option, /// Sometimes the feed doesn't report a correct content type, so we /// need to override it. #[serde(default)] - assume_content_type: Option, + pub assume_content_type: Option, /// The proxy to use for requests /// (Format: "http://user:pass@host:port", "socks5://user:pass@host:port") #[serde(default)] - proxy: Option, + pub proxy: Option, } impl ClientConfig { @@ -124,6 +131,10 @@ impl ClientConfig { ); Ok(client) } + + pub fn to_yaml(&self) -> Result { + Ok(serde_yaml::to_string(self)?) + } } pub struct Client { diff --git a/src/feed.rs b/src/feed.rs index 05b72c5..4f149f3 100644 --- a/src/feed.rs +++ b/src/feed.rs @@ -40,6 +40,15 @@ pub enum FeedFormat { Atom, } +impl FeedFormat { + pub fn as_str(&self) -> &'static str { + match self { + FeedFormat::Rss => "rss", + FeedFormat::Atom => "atom", + } + } +} + impl Feed { pub fn format(&self) -> FeedFormat { match self { diff --git a/src/filter.rs b/src/filter.rs index b506fe5..420a647 100644 --- a/src/filter.rs +++ b/src/filter.rs @@ -1,22 +1,23 @@ -mod convert; -mod full_text; -mod highlight; -mod html; -mod image_proxy; -mod js; -mod limit; -mod magnet; -mod merge; -mod note; -mod sanitize; -mod select; -mod simplify_html; - -use std::collections::HashMap; +pub(crate) mod convert; +pub(crate) mod full_text; +pub(crate) mod highlight; +pub(crate) mod html; +pub(crate) mod image_proxy; +pub(crate) mod js; +pub(crate) mod limit; +pub(crate) mod magnet; +pub(crate) mod merge; +pub(crate) mod note; +pub(crate) mod sanitize; +pub(crate) mod select; +pub(crate) mod simplify_html; + +use std::collections::{HashMap, HashSet}; use std::sync::Arc; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; +use serde_with::{formats::CommaSeparator, serde_as, StringWithSeparator}; use url::Url; use crate::{ @@ -24,6 +25,25 @@ use crate::{ util::{ConfigError, Error, Result}, }; +#[serde_as] +#[derive(Clone, Debug, Deserialize)] +#[serde(transparent)] +pub struct FilterSkip { + #[serde_as(as = "StringWithSeparator::")] + indices: HashSet, +} + +impl FilterSkip { + pub(crate) fn upto(n: usize) -> Self { + let indices = (0..n).collect::>(); + Self { indices } + } + + pub fn allows_filter(&self, index: usize) -> bool { + !self.indices.contains(&index) + } +} + #[derive(Clone)] pub struct FilterContext { /// The base URL of the application. Used to construct absolute URLs @@ -31,7 +51,7 @@ pub struct FilterContext { base: Option, /// The maximum number of filters to run on this pipeline - limit_filters: Option, + filter_skip: Option, /// The extra query parameters passed to the endpoint extra_queries: HashMap, @@ -42,15 +62,11 @@ impl FilterContext { pub fn new() -> Self { Self { base: None, - limit_filters: None, + filter_skip: None, extra_queries: HashMap::new(), } } - pub fn limit_filters(&self) -> Option { - self.limit_filters - } - pub fn base(&self) -> Option<&Url> { self.base.as_ref() } @@ -63,8 +79,8 @@ impl FilterContext { &self.extra_queries } - pub fn set_limit_filters(&mut self, limit: usize) { - self.limit_filters = Some(limit); + pub fn set_filter_skip(&mut self, filter_skip: FilterSkip) { + self.filter_skip = Some(filter_skip); } pub fn set_base(&mut self, base: Url) { @@ -74,7 +90,7 @@ impl FilterContext { pub fn subcontext(&self) -> Self { Self { base: self.base.clone(), - limit_filters: None, + filter_skip: None, extra_queries: self.extra_queries.clone(), } } @@ -82,10 +98,18 @@ impl FilterContext { pub fn from_param(param: &crate::server::EndpointParam) -> Self { Self { base: param.base().cloned(), - limit_filters: param.limit_filters(), + filter_skip: param.filter_skip().cloned(), extra_queries: param.extra_queries().clone(), } } + + pub fn allows_filter(&self, index: usize) -> bool { + if let Some(f) = &self.filter_skip { + f.allows_filter(index) + } else { + true + } + } } #[async_trait::async_trait] @@ -187,6 +211,10 @@ macro_rules! define_filters { } } + pub fn to_yaml(&self) -> Result { + Ok(serde_yaml::to_string(self)?) + } + pub fn name(&self) -> &'static str { match self { $(FilterConfig::$variant(_) => paste::paste! {stringify!([<$variant:snake>])},)* diff --git a/src/filter/js.rs b/src/filter/js.rs index fff5b8d..733504e 100644 --- a/src/filter/js.rs +++ b/src/filter/js.rs @@ -17,7 +17,7 @@ use super::{FeedFilter, FeedFilterConfig, FilterContext}; /// See JavaScript API. pub struct JsConfig { - code: String, + pub code: String, } #[derive( @@ -30,7 +30,7 @@ pub struct JsConfig { ///

/// Example: modify_post: post.title += " (modified)"; pub struct ModifyPostConfig { - code: String, + pub code: String, } #[derive( @@ -43,7 +43,7 @@ pub struct ModifyPostConfig { ///

/// Example: modify_feed: feed.title.value = "Modified Feed"; pub struct ModifyFeedConfig { - code: String, + pub code: String, } pub struct JsFilter { @@ -83,8 +83,8 @@ impl FeedFilterConfig for JsConfig { async fn build(self) -> Result { let runtime = Runtime::new().await?; - runtime.eval(&self.code).await?; - runtime.eval(MODIFY_POSTS_CODE).await?; + let () = runtime.eval(&self.code).await?; + let () = runtime.eval(MODIFY_POSTS_CODE).await?; Ok(Self::Filter { runtime }) } diff --git a/src/filter/merge.rs b/src/filter/merge.rs index 9779670..bc0eefa 100644 --- a/src/filter/merge.rs +++ b/src/filter/merge.rs @@ -7,7 +7,7 @@ use serde::{Deserialize, Serialize}; use crate::client::{Client, ClientConfig}; use crate::feed::Feed; use crate::filter_pipeline::{FilterPipeline, FilterPipelineConfig}; -use crate::source::{Source, SourceConfig}; +use crate::source::{SimpleSourceConfig, Source}; use crate::util::{ConfigError, Result, SingleOrVec}; use super::{FeedFilter, FeedFilterConfig, FilterContext}; @@ -28,7 +28,7 @@ pub enum MergeConfig { )] #[serde(transparent)] pub struct MergeSimpleConfig { - source: SingleOrVec, + pub source: SingleOrVec, } #[derive( @@ -36,16 +36,16 @@ pub struct MergeSimpleConfig { )] pub struct MergeFullConfig { /// Source configuration - source: SingleOrVec, + pub source: SingleOrVec, /// Number of concurrent requests to make for fetching multiple sources (default: 20) #[serde(default)] - parallelism: Option, + pub parallelism: Option, /// Client configuration #[serde(default)] - client: Option, + pub client: Option, /// Filters to apply to the merged feed #[serde(default)] - filters: Option, + pub filters: Option, } impl From for MergeFullConfig { diff --git a/src/filter_pipeline.rs b/src/filter_pipeline.rs index c301f60..4ff68be 100644 --- a/src/filter_pipeline.rs +++ b/src/filter_pipeline.rs @@ -14,7 +14,7 @@ use crate::{ )] #[serde(transparent)] pub struct FilterPipelineConfig { - filters: Vec, + pub filters: Vec, } impl From> for FilterPipelineConfig { @@ -88,16 +88,11 @@ impl Inner { mut context: FilterContext, mut feed: Feed, ) -> Result { - let limit_filters = context - .limit_filters() - .unwrap_or_else(|| self.num_filters()); - for filter in self.filters.iter().take(limit_filters) { - feed = filter.run(&mut context, feed).await?; + for (i, filter) in self.filters.iter().enumerate() { + if context.allows_filter(i) { + feed = filter.run(&mut context, feed).await?; + } } Ok(feed) } - - fn num_filters(&self) -> usize { - self.filters.len() - } } diff --git a/src/server.rs b/src/server.rs index b2e779b..4788992 100644 --- a/src/server.rs +++ b/src/server.rs @@ -5,6 +5,7 @@ pub mod image_proxy; #[cfg(feature = "inspector-ui")] mod inspector; mod watcher; +mod web; use std::{path::Path, sync::Arc}; @@ -120,7 +121,13 @@ impl ServerConfig { #[cfg(feature = "inspector-ui")] if self.inspector_ui { - app = app.nest("/", inspector::router()) + app = app + .nest("/", inspector::router()) + .nest("/_/", web::router()) + .route( + "/", + get(|| async { axum::response::Redirect::temporary("/_/") }), + ); } else { app = app.route("/", get(|| async { "rss-funnel is up and running!" })); } diff --git a/src/server/endpoint.rs b/src/server/endpoint.rs index 11df854..8913a66 100644 --- a/src/server/endpoint.rs +++ b/src/server/endpoint.rs @@ -20,7 +20,7 @@ use url::Url; use crate::client::{Client, ClientConfig}; use crate::feed::Feed; -use crate::filter::FilterContext; +use crate::filter::{FilterContext, FilterSkip}; use crate::filter_pipeline::{FilterPipeline, FilterPipelineConfig}; use crate::otf_filter::{OnTheFlyFilter, OnTheFlyFilterQuery}; use crate::source::{Source, SourceConfig}; @@ -65,6 +65,10 @@ impl EndpointConfig { pub async fn build(self) -> Result { EndpointService::from_config(self.config).await } + + pub(crate) fn source(&self) -> Option<&SourceConfig> { + self.config.source.as_ref() + } } #[derive( @@ -72,13 +76,13 @@ impl EndpointConfig { )] pub struct EndpointServiceConfig { #[serde(default)] - source: Option, + pub source: Option, #[serde(default)] - filters: FilterPipelineConfig, + pub filters: FilterPipelineConfig, #[serde(default)] - on_the_fly_filters: bool, + pub on_the_fly_filters: bool, #[serde(default)] - client: Option, + pub client: Option, } // Ideally I would implement this endpoint service to include a @@ -105,7 +109,7 @@ pub struct EndpointParam { source: Option, /// Only process the initial N filter steps #[serde(default)] - limit_filters: Option, + filter_skip: Option, /// Limit the number of items in the feed #[serde(default)] limit_posts: Option, @@ -121,16 +125,20 @@ pub struct EndpointParam { } impl EndpointParam { + pub fn source(&self) -> Option<&Url> { + self.source.as_ref() + } + pub const fn all_fields() -> &'static [&'static str] { - &["source", "limit_filters", "limit_posts"] + &["source", "filter_skip", "limit_posts"] } pub(crate) fn base(&self) -> Option<&Url> { self.base.as_ref() } - pub(crate) fn limit_filters(&self) -> Option { - self.limit_filters + pub(crate) fn filter_skip(&self) -> Option<&FilterSkip> { + self.filter_skip.as_ref() } pub(crate) fn extra_queries(&self) -> &HashMap { @@ -165,13 +173,13 @@ where impl EndpointParam { pub fn new( source: Option, - limit_filters: Option, + filter_skip: Option, limit_posts: Option, base: Option, ) -> Self { Self { source, - limit_filters, + filter_skip: filter_skip.map(FilterSkip::upto), limit_posts, base, query: None, @@ -227,6 +235,14 @@ impl EndpointService { self } + pub fn source(&self) -> &Option { + &self.source + } + + pub fn config(&self) -> &EndpointServiceConfig { + &self.config + } + async fn handle(self, mut req: Request) -> Result { // infallible let param: EndpointParam = req.extract_parts().await.unwrap(); @@ -269,8 +285,8 @@ impl EndpointService { .fetch_feed(&context, Some(&self.client)) .await .map_err(|e| Error::FetchSource(Box::new(e)))?; - if let Some(limit_filters) = param.limit_filters { - context.set_limit_filters(limit_filters); + if let Some(filter_skip) = param.filter_skip { + context.set_filter_skip(filter_skip); } if let Some(base) = param.base { context.set_base(base); diff --git a/src/server/feed_service.rs b/src/server/feed_service.rs index 94caf25..afe0dc1 100644 --- a/src/server/feed_service.rs +++ b/src/server/feed_service.rs @@ -87,7 +87,10 @@ impl FeedService { let mut buffer = [0u8; 32]; OsRng.fill_bytes(&mut buffer); - let session_id = format!("{:x?}", buffer); + let mut session_id = String::with_capacity(32 * 2); + for byte in buffer.iter() { + session_id.push_str(&format!("{:02x}", byte)); + } let mut inner = self.inner.write().await; inner.session_id = Some(session_id.clone()); diff --git a/src/server/index.rs b/src/server/index.rs new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/src/server/index.rs @@ -0,0 +1 @@ + diff --git a/src/server/inspector.rs b/src/server/inspector.rs index cbca649..fade56c 100644 --- a/src/server/inspector.rs +++ b/src/server/inspector.rs @@ -1,7 +1,7 @@ use std::collections::HashMap; use axum::extract::Query; -use axum::response::{IntoResponse, Redirect, Response}; +use axum::response::{IntoResponse, Response}; use axum::Json; use axum::{ routing::{get, post}, @@ -32,15 +32,6 @@ pub fn router() -> Router { .route("/_inspector/config", get(config_handler)) .route("/_inspector/filter_schema", get(filter_schema_handler)) .route("/_inspector/preview", get(preview_handler)) - .route("/", get(index_handler)) -} - -async fn index_handler(auth: Option) -> impl IntoResponse { - if auth.is_some() { - Redirect::temporary("/_inspector/index.html") - } else { - Redirect::temporary("/_inspector/login.html") - } } async fn inspector_page_handler(_auth: Auth) -> impl IntoResponse { diff --git a/src/server/web.rs b/src/server/web.rs new file mode 100644 index 0000000..62d2d97 --- /dev/null +++ b/src/server/web.rs @@ -0,0 +1,111 @@ +mod endpoint; +mod list; +mod login; + +use std::borrow::Cow; + +use axum::{ + extract::{rejection::QueryRejection, Path, Query}, + response::{IntoResponse, Redirect, Response}, + routing, Extension, Router, +}; +use http::StatusCode; +use login::Auth; +use maud::{html, Markup}; + +use super::{feed_service::FeedService, EndpointParam}; + +#[derive(rust_embed::RustEmbed)] +#[folder = "static/"] +#[include = "*.js"] +#[include = "*.css"] +struct Asset; + +impl Asset { + fn get_content(name: &str) -> Cow<'static, str> { + let file = ::get(name).unwrap(); + match file.data { + Cow::Borrowed(data) => String::from_utf8_lossy(data), + Cow::Owned(data) => String::from_utf8_lossy(&data).into_owned().into(), + } + } +} + +pub fn router() -> Router { + Router::new() + .route("/", routing::get(handle_home)) + .route( + "/login", + routing::get(login::handle_login_page).post(login::handle_login), + ) + .route("/logout", routing::get(login::handle_logout)) + // requires login + .route("/endpoints", routing::get(handle_endpoint_list)) + .route("/endpoint/:path", routing::get(handle_endpoint)) + .route("/sprite.svg", routing::get(handle_sprite)) +} + +async fn handle_sprite() -> impl IntoResponse { + let svg = include_str!("../../static/sprite.svg"); + (StatusCode::OK, [("Content-Type", "image/svg+xml")], svg) +} + +async fn handle_home(auth: Option) -> impl IntoResponse { + if auth.is_some() { + Redirect::temporary("/_/endpoints") + } else { + Redirect::temporary("/_/login") + } +} + +async fn handle_endpoint_list( + _: Auth, + Extension(service): Extension, +) -> Markup { + let root_config = service.root_config().await; + list::render_endpoint_list_page(&root_config) +} + +async fn handle_endpoint( + _: Auth, + Path(path): Path, + Extension(service): Extension, + param: Result, QueryRejection>, +) -> Result { + let endpoint = service.get_endpoint(&path).await.ok_or_else(|| { + (StatusCode::NOT_FOUND, format!("Endpoint {path} not found")) + .into_response() + })?; + + let param = param.map(|q| q.0).map_err(|e| e.body_text()); + Ok(endpoint::render_endpoint_page(endpoint, path, param).await) +} + +fn header_libs_fragment() -> Markup { + html! { + script + src="https://unpkg.com/htmx.org@2.0.1/dist/htmx.min.js" + referrerpolicy="no-referrer" {} + } +} + +fn favicon() -> Markup { + html! { + link + rel="icon" + type="image/svg+xml" + href="data:image/svg+xml,%3Csvg xmlns=\"http://www.w3.org/2000/svg\" viewBox=\"-45 -50 140 140\"%3E%3Ccircle cx=\"306.7\" cy=\"343.7\" r=\"11.4\" fill=\"%23ff7b00\" transform=\"translate(-282 -331)\"/%3E%3Cpath fill=\"none\" stroke=\"%23ff7b00\" stroke-width=\"15\" d=\"M-3 16a29 29 0 1 1 56 0\"/%3E%3Cpath fill=\"none\" stroke=\"%23ff7b00\" stroke-width=\"15\" d=\"M-23 18a49 49 0 1 1 96-1\"/%3E%3Cpath fill=\"%23ff7b00\" d=\"m-24 29 98-1-1 10-40 28 1 19H17l1-20-42-27z\"/%3E%3C/svg%3E%0A"; + } +} + +fn sprite(icon: &str) -> Markup { + html! { + svg class="icon" xmlns="http://www.w3.org/2000/svg" width="20" height="20" { + use xlink:href=(format!("/_/sprite.svg#{icon}")); + } + } +} + +fn common_styles() -> Cow<'static, str> { + Asset::get_content("common.css") +} diff --git a/src/server/web/endpoint.rs b/src/server/web/endpoint.rs new file mode 100644 index 0000000..30d4cce --- /dev/null +++ b/src/server/web/endpoint.rs @@ -0,0 +1,428 @@ +use std::{borrow::Cow, collections::HashMap}; + +use either::Either; +use maud::{html, Markup, PreEscaped, DOCTYPE}; +use url::Url; + +use crate::{ + feed::{Feed, Post, PostPreview}, + server::{endpoint::EndpointService, web::sprite, EndpointParam}, + source::{FromScratch, Source}, +}; + +pub async fn render_endpoint_page( + endpoint: EndpointService, + path: String, + param: Result, +) -> Markup { + // render source control + let source = source_control_fragment(&path, endpoint.source(), ¶m); + + // render config + let config = render_config_fragment(&path, param.as_ref().ok(), &endpoint); + let config_tags = render_config_header_tags(&endpoint); + + // render feed preview + let feed = match param { + Ok(param) => fetch_and_render_feed(endpoint, param).await, + Err(e) => html! { + div .flash.error { + header { b { "Invalid request params" } } + p { (e) } + } + }, + }; + + html! { + (DOCTYPE) + head { + title { "RSS Funnel" } + meta charset="utf-8"; + (super::favicon()); + (super::header_libs_fragment()); + script { (PreEscaped(inline_script())) } + style { (PreEscaped(super::common_styles())) } + style { (PreEscaped(inline_styles())) } + link rel="stylesheet" + referrerpolicy="no-referrer" + href="https://unpkg.com/prismjs@v1.x/themes/prism.css"; + script + src="https://unpkg.com/prismjs@v1.x/components/prism-core.min.js" + referrerpolicy="no-referrer" {} + script + src="https://unpkg.com/prismjs@v1.x/plugins/autoloader/prism-autoloader.min.js" + referrerpolicy="no-referrer" {} + } + body { + header .header-bar { + button .back-button { + a href="/_/" { "Back" } + } + h2 { (path) } + button .copy-button title="Copy Endpoint URL" onclick="copyToClipboard()" { + (sprite("copy")) + } + } + + section .source-and-config { + @if let Some(source) = source { + section .source-control { + (source); + div.loading { (sprite("loader")) } + } + } + + details { + summary { + "Config"; + (config_tags) + } + section .config-section { + (config) + } + } + } + + main .feed-section { + (feed) + } + } + } +} + +fn source_control_fragment( + path: &str, + source: &Option, + param: &Result, +) -> Option { + match source { + None => Some(html! { + input + .hx-included.grow + type="text" + name="source" + placeholder="Source URL" + value=[param.as_ref().ok().and_then(|p| p.source()).map(|url| url.as_str())] + hx-get=(format!("/_/endpoint/{path}")) + hx-trigger="keyup changed delay:500ms" + hx-push-url="true" + hx-indicator=".loading" + hx-include=".hx-included" + hx-target="main" + hx-select="main" + {} + }), + Some(Source::AbsoluteUrl(url)) => Some(html! { + div title="Source" .source { (url) } + }), + Some(Source::RelativeUrl(url)) => Some(html! { + div title="Source" .source { (url) } + }), + Some(Source::Templated(templated)) => Some(html! { + div .source-template-container { + @let queries = param.as_ref().ok().map(|p| p.extra_queries()); + (source_template_fragment(templated, path, queries)); + } + }), + Some(Source::FromScratch(scratch)) => Some(from_scratch_fragment(scratch)), + } +} + +fn from_scratch_fragment(scratch: &FromScratch) -> Markup { + html! { + table { + tbody { + tr { + th { "Format" } + td { (scratch.format.as_str()) } + } + tr { + th { "Title" } + td { (scratch.title) } + } + @if let Some(link) = &scratch.link { + tr { + th { "Link" } + td { (link) } + } + } + @if let Some(description) = &scratch.description { + tr { + th { "Description" } + td { (description) } + } + } + } + } + } +} + +fn source_template_fragment( + templated: &crate::source::Templated, + path: &str, + queries: Option<&HashMap>, +) -> Markup { + html! { + @for fragment in templated.fragments() { + @match fragment { + Either::Left(plain) => span style="white-space: nowrap" { (plain) }, + Either::Right((name, Some(placeholder))) => { + @let value=queries.and_then(|q| q.get(name)); + @let default_value=placeholder.default.as_ref(); + @let value=value.or(default_value); + @let validation=placeholder.validation.as_ref(); + input + .source-template-placeholder.hx-included + name=(name) + placeholder=(name) + pattern=[validation] + value=[value] + hx-get=(format!("/_/endpoint/{path}")) + hx-trigger="keyup changed delay:500ms" + hx-push-url="true" + hx-include=".hx-included" + hx-indicator=".loading" + hx-target="main" + hx-select="main" + id={"placeholder-" (name)} + {} + } + Either::Right((name, None)) => { + span style="color: red" title="Placeholder not defined" { "${" (name) "}" } + } + } + } + } +} + +fn render_config_header_tags(endpoint: &EndpointService) -> Markup { + let config = endpoint.config(); + + html! { + @if config.on_the_fly_filters { + div .tag-container { + span .tag.otf title="On-the-fly filters enabled" { "OTF" } + } + } + } +} + +fn render_config_fragment( + path: &str, + param: Option<&EndpointParam>, + endpoint: &EndpointService, +) -> Markup { + let config = endpoint.config(); + let filter_enabled = |i| { + if let Some(f) = param.and_then(|p| p.filter_skip()) { + f.allows_filter(i) as u8 + } else { + true as u8 + } + }; + + html! { + @if let Some(client) = &config.client { + section { + header { b { "Custom client configuration:" } } + @if let Ok(yaml) = client.to_yaml() { + div .client-config { + pre { code .language-yaml { (yaml) } } + } + } + } + } + + @let filters = &config.filters; + @if filters.filters.is_empty() { + "No filters" + } @else { + div { + header { b { "Filters:" } } + ul #filter-list .hx-included + hx-vals="js:{...gatherFilterSkip()}" + hx-get=(format!("/_/endpoint/{path}")) + hx-trigger="click from:.filter-name" + hx-push-url="true" + hx-include=".hx-included" + hx-indicator=".loading" + hx-target="main" + hx-select="main" + { + @for (i, filter) in filters.filters.iter().enumerate() { + li { + div .filter-item.flex.flex-center { + span .filter-name title="Toggle" + data-enabled=(filter_enabled(i)) + onclick="this.dataset.enabled=1-+this.dataset.enabled" + data-index=(i) { + (filter.name()) + } + + (filter_doc(filter.name())); + + @if let Ok(yaml) = filter.to_yaml() { + div .filter-link {} + div .filter-definition { + pre { code .language-yaml { (yaml) } } + } + } + } + } + } + } + } + } + } +} + +async fn fetch_and_render_feed( + endpoint: EndpointService, + params: EndpointParam, +) -> Markup { + html! { + @match endpoint.run(params).await { + Ok(feed) => (render_feed(feed)), + Err(e) => { + div .flash.error { + header { b { "Failed to fetch feed" } } + p { (e.to_string()) } + } + } + } + } +} + +fn render_post(preview: PostPreview, post: Post) -> Markup { + let link_url = Url::parse(&preview.link).ok(); + let json = + serde_json::to_string_pretty(&post).unwrap_or_else(|e| e.to_string()); + let id = format!("entry-{}", rand_id()); + + html! { + article data-display-mode="rendered" data-folded="true" .post-entry { + header { + span .icon-container.fold-icon onclick="toggleFold(this)" title="Expand" { + (sprite("caret-right")) + } + span .icon-container.raw-icon onclick="toggleRaw(this)" title="HTML body" { + (sprite("code")) + } + span .icon-container.json-icon onclick="toggleJson(this)" title="JSON structure" { + (sprite("json")) + } + + div .flex style="margin-left: .5rem" { + span .entry-title.grow { (preview.title) } + (external_link(&preview.link)) + } + } + + section { + @if let Some(body) = &preview.body { + @let content = santize_html(body, link_url); + div id=(id) .entry-content.rendered { + template { + style { (PreEscaped("*{max-width: 100%;}")) } + (PreEscaped(content)) + } + script { (PreEscaped(format!("fillEntryContent('{id}')"))) } + } + div .entry-content.raw { + pre { code .language-html { (body) } } + } + } @else { + div id=(id) .entry-content.rendered { + "No body" + } + div .entry-content.raw { + pre { code .language-html { } } + } + } + + div .entry-content.json { + pre { code .language-json { (json) } } + } + } + + footer { + @if let Some(date) = preview.date { + time .inline datetime=(date.to_rfc3339()) { (date.to_rfc2822()) } + } + @if let Some(author) = &preview.author { + span .author { + (PreEscaped("By ")); + address .inline rel="author" { (author) } + } + } + } + } + } +} + +fn render_feed(mut feed: Feed) -> Markup { + let preview = feed.preview(); + let posts = feed.take_posts(); + + html! { + h3 .flex { + (preview.title); + (external_link(&preview.link)) + } + @if let Some(description) = &preview.description { + p { (description) } + } + p { (format!("Entries ({}):", preview.posts.len())) } + + @for (preview, post) in preview.posts.into_iter().zip(posts) { + (render_post(preview, post)) + } + } +} + +fn inline_styles() -> Cow<'static, str> { + super::Asset::get_content("endpoint.css") +} + +fn inline_script() -> Cow<'static, str> { + super::Asset::get_content("endpoint.js") +} + +// requires the container to have a `display: flex` style +fn external_link(url: &str) -> Markup { + html! { + a style="margin-left: 0.25rem;display:inline-flex;align-self:center" + href=(url) + title="Open external link" + target="_blank" + rel="noopener noreferrer" { + (sprite("external-link")) + } + } +} + +fn filter_doc(filter_name: &str) -> Markup { + html! { + a style="margin-left: 0.25rem;display:inline-flex;align-self:center" + href=(format!("https://github.com/shouya/rss-funnel/wiki/Filter-config#{}", filter_name)) + target="_blank" + rel="noopener noreferrer" + title="Documentation" { + (sprite("book")) + } + } +} + +fn santize_html(html: &str, base: Option) -> String { + use ammonia::UrlRelative; + let mut builder = ammonia::Builder::new(); + if let Some(base) = base { + builder.url_relative(UrlRelative::RewriteWithBase(base)); + } + builder.clean(html).to_string() +} + +fn rand_id() -> String { + use rand::Rng as _; + rand::thread_rng().gen::().to_string() +} diff --git a/src/server/web/list.rs b/src/server/web/list.rs new file mode 100644 index 0000000..ea575fb --- /dev/null +++ b/src/server/web/list.rs @@ -0,0 +1,120 @@ +use std::borrow::Cow; + +use maud::{html, Markup, PreEscaped, DOCTYPE}; +use url::Url; + +use crate::{cli::RootConfig, server::EndpointConfig, source::SourceConfig}; + +pub fn render_endpoint_list_page(root_config: &RootConfig) -> Markup { + html! { + (DOCTYPE) + head { + title { "RSS Funnel" } + meta charset="utf-8"; + (super::favicon()); + (super::header_libs_fragment()); + style { (PreEscaped(super::common_styles())) } + style { (PreEscaped(inline_styles())) } + } + body { + header .header-bar { + h2 { "RSS Funnel" } + } + + main { + ul { + @for endpoint in &root_config.endpoints { + (endpoint_list_entry_fragment(endpoint)) + } + } + } + } + } +} + +fn endpoint_list_entry_fragment(endpoint: &EndpointConfig) -> Markup { + html! { + li ."my-.5" { + p { + a href={"/_/endpoint/" (endpoint.path.trim_start_matches('/'))} { + (endpoint.path) + } + + // badges + span .tag-container { + @if endpoint.config.on_the_fly_filters { + span .tag.otf title="On-the-fly filters" { "OTF" } + } + + @let source = endpoint.source(); + (short_source_repr(source)) + } + } + + @if let Some(note) = &endpoint.note { + p { (note) } + } + } + } +} + +fn url_host(url: impl TryInto) -> Option { + let Ok(url) = url.try_into() else { + return None; + }; + + url.host_str().map(|s| s.to_owned()) +} + +fn url_path(url: impl TryInto) -> Option { + let Ok(url) = url.try_into() else { + return None; + }; + + Some(url.path().to_owned()) +} + +fn short_source_repr(source: Option<&SourceConfig>) -> Markup { + match source { + None => html! { + span .tag.dynamic { "dynamic" } + }, + Some(SourceConfig::Simple(url)) if url.starts_with("/") => { + let path = url_path(url.as_str()); + let path = path.map(|p| format!("/_/{p}")); + html! { + @if let Some(path) = path { + span .tag.local { + a href=(path) { "local" } + } + } @else { + span .tag.local title=(url) { + "local" + } + } + } + } + Some(SourceConfig::Simple(url)) => { + let host = url_host(url.as_str()).unwrap_or_else(|| "...".into()); + html! { + span .tag.simple { + a href=(url) { (host) } + } + } + } + Some(SourceConfig::FromScratch(_)) => { + html! { + span .tag.scratch title="Made from scratch" { "scratch" } + } + } + Some(SourceConfig::Templated(_source)) => { + html! { + span .tag.templated title="Templated source" { "templated" } + } + } + } +} + +fn inline_styles() -> Cow<'static, str> { + super::Asset::get_content("list.css") +} diff --git a/src/server/web/login.rs b/src/server/web/login.rs new file mode 100644 index 0000000..f657f27 --- /dev/null +++ b/src/server/web/login.rs @@ -0,0 +1,117 @@ +use std::borrow::Cow; + +use axum::{ + extract::FromRequestParts, + response::{IntoResponse, Redirect, Response}, + Extension, Form, +}; +use axum_extra::extract::CookieJar; +use http::request::Parts; +use maud::{html, PreEscaped, DOCTYPE}; + +use crate::server::feed_service::FeedService; + +// Put this in request context +pub struct Auth; + +pub async fn handle_login_page() -> impl IntoResponse { + html! { + (DOCTYPE); + head { + title { "Login - RSS Funnel" } + meta charset="utf-8"; + (super::favicon()); + (super::header_libs_fragment()); + style { (PreEscaped(inline_styles())) } + script { (PreEscaped(inline_scripts())) } + } + + body onload="setErrorMessage()" { + p #message .hidden {} + + form method="post" { + input type="text" name="username" placeholder="Username"; + input type="password" name="password" placeholder="Password"; + button type="submit" { "Login" } + } + footer { + p { + "Powered by "; + a href="https://github.com/shouya/rss-funnel" { "RSS Funnel" } + } + } + } + } +} + +#[async_trait::async_trait] +impl FromRequestParts for Auth { + type Rejection = Response; + + async fn from_request_parts( + parts: &mut Parts, + state: &S, + ) -> Result { + let feed_service: Extension = + Extension::from_request_parts(parts, state) + .await + .map_err(|e| e.into_response())?; + + if !feed_service.requires_auth().await { + return Ok(Auth); + } + + let cookie_jar = CookieJar::from_request_parts(parts, state) + .await + .map_err(|e| e.into_response())?; + + let session_id = cookie_jar + .get("session_id") + .ok_or_else(redir_login)? + .value(); + + if !feed_service.validate_session_id(session_id).await { + return Err(redir_login()); + } + + Ok(Auth) + } +} + +fn redir_login() -> Response { + Redirect::to("/_/login?login_required=1").into_response() +} + +pub async fn handle_logout(cookie_jar: CookieJar) -> impl IntoResponse { + let cookie_jar = cookie_jar.remove("session_id"); + (cookie_jar, Redirect::to("/_/login?logged_out=1")).into_response() +} + +#[derive(serde::Deserialize)] +pub struct HandleLoginParams { + username: String, + password: String, +} + +pub async fn handle_login( + cookie_jar: CookieJar, + Extension(feed_service): Extension, + Form(params): Form, +) -> Response { + let cookie_jar = cookie_jar.remove("session_id"); + match feed_service.login(¶ms.username, ¶ms.password).await { + Some(session_id) => { + let cookie_jar = cookie_jar.add(("session_id", session_id)); + (cookie_jar, Redirect::to("/_/endpoints")).into_response() + } + _ => Redirect::to("/_/login?bad_auth=1").into_response(), + } +} + +fn inline_styles() -> Cow<'static, str> { + super::Asset::get_content("login.css") +} + +fn inline_scripts() -> Cow<'static, str> { + super::Asset::get_content("login.js") +} diff --git a/src/source.rs b/src/source.rs index 9206766..f0b3845 100644 --- a/src/source.rs +++ b/src/source.rs @@ -1,5 +1,6 @@ use std::collections::{BTreeMap, HashMap}; +use either::Either; use regex::Regex; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; @@ -13,6 +14,10 @@ use crate::{ util::{ConfigError, Error, Result}, }; +lazy_static::lazy_static! { + static ref VAR_RE: Regex = Regex::new(r"\$\{(?\w+)\}").unwrap(); +} + #[derive( JsonSchema, Serialize, Deserialize, Clone, Debug, PartialEq, Eq, Hash, )] @@ -35,12 +40,17 @@ pub enum SourceConfig { Templated(Templated), } +#[derive( + JsonSchema, Serialize, Deserialize, Clone, Debug, PartialEq, Eq, Hash, +)] +pub struct SimpleSourceConfig(pub String); + #[derive( JsonSchema, Serialize, Deserialize, Clone, Debug, PartialEq, Eq, Hash, )] pub struct Templated { /// The url of the source - template: String, + pub template: String, /// The placeholders. The key is the placeholder name and the value /// defines the value of the placeholder. // using BTreeMap instead of HashMap only because it implements Hash @@ -50,15 +60,15 @@ pub struct Templated { #[derive( JsonSchema, Serialize, Deserialize, Clone, Debug, PartialEq, Eq, Hash, )] -struct Placeholder { +pub struct Placeholder { /// The default value of the placeholder. If not set, the placeholder /// is required. - default: Option, + pub default: Option, /// The regular expression that the placeholder must match. If not /// set, the placeholder can be any value. The validation is checked /// against the url-decoded value. - validation: Option, + pub validation: Option, } impl Templated { @@ -95,6 +105,36 @@ impl Templated { .try_into() .map_err(|e: ConfigError| e.into()) } + + // https://foo.bar/${name}/baz -> https://foo.bar + #[expect(unused)] + pub fn base(&self) -> Option { + let mut url = Url::parse(&self.template).ok()?; + let host = url.host_str()?; + if VAR_RE.is_match(host) { + return None; + } + url.set_fragment(None); + url.set_query(None); + url.set_path(""); + Some(url.to_string()) + } + + // used for rendering control + pub fn fragments( + &self, + ) -> impl Iterator)>> + '_ { + split_with_delimiter(&self.template, &VAR_RE).map(|e| match e { + Either::Left(s) => Either::Left(s), + Either::Right(cap) => { + // SAFETY: name is guaranteed to be Some because the regex is + // static. + let name = &cap.name("name").unwrap(); + let placeholder = self.placeholders.get(name.as_str()); + Either::Right((&self.template[name.range()], placeholder)) + } + }) + } } #[derive( @@ -127,6 +167,14 @@ impl From for Source { } } +impl TryFrom for Source { + type Error = ConfigError; + + fn try_from(config: SimpleSourceConfig) -> Result { + SourceConfig::Simple(config.0).try_into() + } +} + impl TryFrom for Source { type Error = ConfigError; @@ -167,10 +215,7 @@ fn validate_placeholders(config: &Templated) -> Result<(), ConfigError> { // Validation: all placeholder patterns in template must be // defined in placeholders - lazy_static::lazy_static! { - static ref RE: Regex = Regex::new(r"$\{(?\w+)\}").unwrap(); - } - for cap in RE.captures_iter(&config.template) { + for cap in VAR_RE.captures_iter(&config.template) { let name = &cap["name"]; if !config.placeholders.contains_key(name) { return Err(ConfigError::BadSourceTemplate(format!( @@ -234,6 +279,31 @@ impl Source { } } +fn split_with_delimiter<'a>( + s: &'a str, + re: &Regex, +) -> impl Iterator>> { + let mut list = Vec::new(); + let mut last = 0; + + for cap in re.captures_iter(s) { + // SAFETY: get(0) is guaranteed to be Some + let full = cap.get(0).unwrap(); + let seg = &s[last..full.start()]; + if !seg.is_empty() { + list.push(Either::Left(seg)); + } + list.push(Either::Right(cap)); + last = full.end(); + } + + let tail = &s[last..]; + if !tail.is_empty() { + list.push(Either::Left(tail)); + } + list.into_iter() +} + #[cfg(test)] mod test { use super::*; @@ -271,4 +341,43 @@ description: "A test feed" assert_eq!(feed.title(), "Test Feed"); assert_eq!(feed.format(), FeedFormat::Atom); } + + #[test] + fn test_template_source_segmentation() { + const YAML_CONFIG: &str = r#" +template: "https://example.com/${name1}/${name2}/feed.xml" +placeholders: + name1: + default: "default1" + name2: + default: "default2" +"#; + + let config: Templated = serde_yaml::from_str(YAML_CONFIG).unwrap(); + let fragments: Vec<_> = config.fragments().collect(); + assert_eq!(fragments.len(), 5); + assert_eq!(fragments[0], Either::Left("https://example.com/")); + assert_eq!( + fragments[1], + Either::Right(( + "name1", + Some(&Placeholder { + default: Some("default1".into()), + validation: None, + }) + )) + ); + assert_eq!(fragments[2], Either::Left("/")); + assert_eq!( + fragments[3], + Either::Right(( + "name2", + Some(&Placeholder { + default: Some("default2".into()), + validation: None, + }) + )) + ); + assert_eq!(fragments[4], Either::Left("/feed.xml")); + } } diff --git a/src/util.rs b/src/util.rs index a5b7a40..a11a0b1 100644 --- a/src/util.rs +++ b/src/util.rs @@ -135,7 +135,7 @@ pub enum Error { #[error("Source template placeholder unspecified: {0}")] MissingSourceTemplatePlaceholder(String), - #[error("Based URL not inferred, please refer to https://github.com/shouya/rss-funnel/wiki/App-base")] + #[error("Can't infer app base, please refer to https://github.com/shouya/rss-funnel/wiki/App-base")] BaseUrlNotInferred, #[error("{0}")] diff --git a/static/common.css b/static/common.css new file mode 100644 index 0000000..b7f787a --- /dev/null +++ b/static/common.css @@ -0,0 +1,149 @@ +* { + box-sizing: border-box; +} + +:root { + --fg: #000; + --bg: #fff; + --fg-active: #007bff; + --fg-muted: #333; + --bg-muted: #f5f5f5; + --bd-radius: 0.2rem; + --bd-muted: #ddd; + + --shadow: #777; + + --bg-otf: #f8d7da; + --bg-templated: #d4edda; + --bg-scratch: #cce5ff; + --bg-local: #fff3cd; + --bg-remote: #d6d8d9; +} + +body { + & > header, + section, + main { + margin-left: auto; + margin-right: auto; + max-width: 800px; + } + + & > section { + margin-bottom: 1rem; + margin-top: 1rem; + } +} + +.header-bar { + padding: 0.5rem 0; + border-bottom: 1.5px dotted; + display: flex; + align-items: center; + + h2 { + flex: 1; + margin: 0; + } +} + +article { + margin: 1rem; + padding: 0.5rem; + display: flex; + flex-direction: column; + gap: 0.2rem; + background-color: var(--bg); + border-radius: var(--bd-radius); + + & > header { + border-bottom: 1px solid; + display: flex; + max-width: 100%; + } + & > footer { + border-top: 1px solid; + display: flex; + flex-direction: row; + justify-content: space-between; + } + & > section { + max-width: 100%; + } +} + +button { + a { + color: inherit; + text-decoration: none; + } +} + +/* Utility classes */ + +.flex { + display: flex; +} + +.flex-center { + align-items: center; +} + +.grow { + flex: 1; +} + +.inline { + display: inline; +} + +.flash { + border-radius: var(--bd-radius); + padding: 0.5rem; + border: 1px solid; + + &.error { + color: red; + border-color: red; + } +} + +.tag-container { + margin: 0 0.5rem; + display: inline-flex; + gap: 0.2rem; + + .tag { + color: var(--fg-muted); + background-color: var(--bg-muted); + padding: 0.2rem; + border-radius: var(--bd-radius); + font-size: 0.8rem; + cursor: default; + + &.otf { + background-color: var(--bg-otf); + } + &.templated { + background-color: var(--bg-templated); + } + &.scratch { + background-color: var(--bg-scratch); + } + &.local { + background-color: var(--bg-local); + } + &.remote { + background-color: var(--bg-remote); + } + + a { + color: var(--fg); + text-decoration: none; + + &:hover { + color: var(--fg-active); + } + } + } +} diff --git a/static/common.js b/static/common.js new file mode 100644 index 0000000..e69de29 diff --git a/static/endpoint.css b/static/endpoint.css new file mode 100644 index 0000000..48d35be --- /dev/null +++ b/static/endpoint.css @@ -0,0 +1,198 @@ +.icon { + transition: all 0.2s; +} + +.post-entry { + margin-left: 0 !important; + margin-right: 0 !important; + + .icon-container { + display: inline-flex; + align-self: center; + margin: 0 0.2rem; + } + + &[data-folded="false"] { + .fold-icon > .icon { + transform: rotate(90deg); + } + } + &[data-folded="true"] { + header { + border: 0 !important; + margin-bottom: 0 !important; + padding-bottom: 0 !important; + } + + section, + footer { + display: none; + } + } + + .entry-title { + flex-grow: 1; + } + + .entry-content { + display: none; + } + &[data-display-mode="rendered"] { + .entry-content.rendered { + display: block; + overflow-x: scroll; + background-color: var(--bg); + } + } + &[data-display-mode="raw"] { + .raw-icon > .icon { + color: var(--fg-active); + } + .entry-content.raw { + display: block; + background-color: unset; + } + } + &[data-display-mode="json"] { + .json-icon > .icon { + color: var(--fg-active); + } + .entry-content.json { + display: block; + background-color: unset; + } + } +} + +.source { + font-family: monospace; +} + +@keyframes rotation { + from { + transform: rotate(0deg); + } + to { + transform: rotate(360deg); + } +} + +.loading { + display: none; + position: absolute; + right: 1rem; + align-items: center; + height: 100%; + + &.htmx-request { + display: flex; + animation: rotation 2s infinite linear; + } +} + +.filter-item { + position: relative; + margin-top: 0.5rem; + margin-bottom: 0.5rem; + + > .filter-name { + font-family: monospace; + color: var(--fg-active); + background-color: var(--bg-muted); + padding: 0.2rem 0.5rem; + border-radius: var(--bd-radius); + cursor: pointer; + } + + > .filter-name[data-enabled="0"] { + color: var(--fg-muted); + background-color: var(--bg-muted); + } + + > .filter-definition, + .filter-link { + display: none; + } + + &:hover > .filter-link { + display: inline-block; + border-top: 1px solid var(--bd-muted); + margin-left: 0.2rem; + height: 0; + width: 15rem; + vertical-align: middle; + } + + &:hover > .filter-definition { + display: block; + position: absolute; + left: 15rem; + top: 0; + z-index: 1; + border: 1px solid var(--bd-muted); + border-radius: var(--bd-radius); + box-shadow: 2px 2px 3px var(--shadow); + } +} + +.source-control { + background-color: var(--bg-muted); + padding: 1rem; + border-radius: var(--bd-radius); + display: flex; + position: relative; + align-items: center; + width: 100%; +} +.source-template-container { + display: flex; + position: relative; + align-items: baseline; + flex-wrap: wrap; + + .source-template-placeholder { + width: auto; + display: inline-block; + } +} + +main.feed-section { + background-color: var(--bg-muted); + padding: 1.5rem; + border-radius: var(--bd-radius); +} + +.header-bar { + > .back-button { + margin-right: 2rem; + + a:hover { + color: var(--bg-accent); + } + } + + > .copy-button { + > svg { + vertical-align: middle; + } + } +} + +pre[class*="language"] { + margin: 0 !important; +} + +.config-section { + padding: 1.5rem; +} + +.source-and-config { + margin-top: 1rem; + display: flex; + flex-direction: column; + gap: 1rem; +} + +summary { + cursor: pointer; +} diff --git a/static/endpoint.js b/static/endpoint.js new file mode 100644 index 0000000..e27579f --- /dev/null +++ b/static/endpoint.js @@ -0,0 +1,44 @@ +function toggleFold(element) { + const article = element.closest("article"); + article.dataset.folded = article.dataset.folded === "false"; +} + +function toggleRaw(element) { + const article = element.closest("article"); + article.dataset.folded = "false"; + article.dataset.displayMode = + article.dataset.displayMode === "raw" ? "rendered" : "raw"; +} + +function toggleJson(element) { + const article = element.closest("article"); + article.dataset.folded = "false"; + article.dataset.displayMode = + article.dataset.displayMode === "json" ? "rendered" : "json"; +} + +function gatherFilterSkip() { + const skipped = [...document.querySelectorAll(".filter-item > .filter-name")] + .filter((x) => !+x.dataset.enabled) + .map((x) => x.dataset.index) + .join(","); + if (skipped === "") { + return {}; + } else { + return { filter_skip: skipped }; + } +} + +function fillEntryContent(id) { + const parent = document.getElementById(id); + const shadowRoot = parent.attachShadow({ mode: "open" }); + const content = parent.querySelector("template").innerHTML; + parent.innerHTML = ""; + shadowRoot.innerHTML = content; +} + +function copyToClipboard() { + const url = window.location.href.replace(/\/_\/endpoint\//, "/"); + navigator.clipboard.writeText(url); + alert("Copied: " + url); +} diff --git a/static/list.css b/static/list.css new file mode 100644 index 0000000..e69de29 diff --git a/static/login.css b/static/login.css new file mode 100644 index 0000000..1da6155 --- /dev/null +++ b/static/login.css @@ -0,0 +1,22 @@ +body { + display: flex; + flex-direction: column; + align-items: center; +} + +form { + margin-top: 15px; + margin-bottom: 20px; + display: flex; + flex-direction: column; + width: 200px; + gap: 0.5rem; +} + +.hidden { + display: none; +} + +p#message { + color: red; +} diff --git a/static/login.js b/static/login.js new file mode 100644 index 0000000..8aefc56 --- /dev/null +++ b/static/login.js @@ -0,0 +1,14 @@ +function setErrorMessage() { + const message = document.getElementById("message"); + const search = window.location.search; + if (search.includes("bad_auth=1")) { + message.classList.remove("hidden"); + message.textContent = "Invalid username or password"; + } else if (search.includes("logged_out=1")) { + message.classList.remove("hidden"); + message.textContent = "You have been logged out"; + } else if (search.includes("login_required=1")) { + message.classList.remove("hidden"); + message.textContent = "You must be logged in to access that page"; + } +} diff --git a/static/sprite.svg b/static/sprite.svg new file mode 100644 index 0000000..61e02dc --- /dev/null +++ b/static/sprite.svg @@ -0,0 +1,85 @@ + + diff --git a/static/svg/book.svg b/static/svg/book.svg new file mode 100644 index 0000000..47aabb3 --- /dev/null +++ b/static/svg/book.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/static/svg/caret-right.svg b/static/svg/caret-right.svg new file mode 100644 index 0000000..1c0d309 --- /dev/null +++ b/static/svg/caret-right.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/static/svg/code.svg b/static/svg/code.svg new file mode 100644 index 0000000..e9be046 --- /dev/null +++ b/static/svg/code.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/static/svg/copy.svg b/static/svg/copy.svg new file mode 100644 index 0000000..def3234 --- /dev/null +++ b/static/svg/copy.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/static/svg/external-link.svg b/static/svg/external-link.svg new file mode 100644 index 0000000..0e456ca --- /dev/null +++ b/static/svg/external-link.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/static/svg/file-description.svg b/static/svg/file-description.svg new file mode 100644 index 0000000..e6a93af --- /dev/null +++ b/static/svg/file-description.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/static/svg/json.svg b/static/svg/json.svg new file mode 100644 index 0000000..8be4fe9 --- /dev/null +++ b/static/svg/json.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/static/svg/loader.svg b/static/svg/loader.svg new file mode 100644 index 0000000..b2f3a9a --- /dev/null +++ b/static/svg/loader.svg @@ -0,0 +1 @@ + \ No newline at end of file