Limit render bundle cache size (Maximkaaa#28)

* Fix vector tiles for web target

* Improve raster tiles caching

* Refactor vector tile layer with data providers

galileo/Cargo.toml

@@ -15,7 +15,8 @@ exclude = [
 
 [features]
 default = ["wgpu", "tokio", "image", "serde"]
-web = ["wasm-bindgen-futures", "serde"]
+web = ["wasm-bindgen-futures", "serde", "byte-conversion", "js-sys"]
+byte-conversion = ["serde_bytes", "js-sys"]
 
 [dependencies]
 cfg-if = "1"
@@ -35,6 +36,9 @@ serde = { version = "1.0", optional = true, features = ["std", "derive"] }
 web-time = "0.2"
 thiserror = "1.0"
 nalgebra = "0.32"
+quick_cache = "0.4"
+serde_bytes = { version = "0.11", optional = true }
+js-sys = { version = "0.3", optional = true }
 
 [target.'cfg(not(target_arch = "wasm32"))'.dependencies]
 tokio = { version = "1.28.2", features = ["macros", "rt", "rt-multi-thread" ], optional = true }
@@ -48,10 +52,9 @@ console_log = "1.0"
 wgpu = { version = "0.18", features = ["webgl"] }
 wasm-bindgen-futures = { version = "0.4", optional = true }
 wasm-bindgen = "0.2"
+wasm-bindgen-derive = { version = "0.2" }
 bincode = "1.3"
 serde-wasm-bindgen = "0.6"
-serde_bytes = "0.11"
-js-sys = { version = "0.3" }
 maybe-sync = {  version = "0.1", features = [] }
 web-sys = { version = "0.3", features = [
     "Document",

galileo/examples/vector_tiles.rs

@@ -1,10 +1,10 @@
 use galileo::bounding_box::BoundingBox;
 use galileo::control::{EventPropagation, MouseButton, UserEvent};
-use galileo::galileo_map::{MapBuilder, VectorTileProver};
+use galileo::galileo_map::{MapBuilder, VectorTileProvider};
 use galileo::layer::vector_tile_layer::style::VectorTileStyle;
 use galileo::layer::vector_tile_layer::VectorTileLayer;
 use galileo::lod::Lod;
-use galileo::tile_scheme::{TileScheme, VerticalDirection};
+use galileo::tile_scheme::{TileIndex, TileScheme, VerticalDirection};
 use galileo_types::cartesian::impls::point::Point2d;
 use galileo_types::geo::crs::Crs;
 use std::sync::{Arc, RwLock};
@@ -16,17 +16,16 @@ fn get_layer_style() -> Option<VectorTileStyle> {
 }
 
 thread_local!(
-    pub static LAYER: Arc<RwLock<VectorTileLayer<VectorTileProver>>> =
-        Arc::new(RwLock::new(VectorTileLayer::<VectorTileProver>::from_url(
-            |index| {
+    pub static LAYER: Arc<RwLock<VectorTileLayer<VectorTileProvider>>> =
+        Arc::new(RwLock::new(MapBuilder::create_vector_tile_layer(
+            |&index: &TileIndex| {
                 format!(
                     "https://d1zqyi8v6vm8p9.cloudfront.net/planet/{}/{}/{}.mvt",
                     index.z, index.x, index.y
                 )
             },
-            VectorTileStyle::default(),
-            None,
             tile_scheme(),
+            VectorTileStyle::default(),
         )));
 );
 

galileo/src/error.rs

@@ -1,4 +1,6 @@
 use galileo_mvt::error::GalileoMvtError;
+use image::ImageError;
+use std::io::Error;
 use thiserror::Error;
 
 #[derive(Debug, Error)]
@@ -9,6 +11,12 @@ pub enum GalileoError {
     Decoding(#[from] GalileoMvtError),
     #[error("wasm error: {0:?}")]
     Wasm(Option<String>),
+    #[error("item not found")]
+    NotFound,
+    #[error("image decode error: {0:?}")]
+    ImageDecode(#[from] ImageError),
+    #[error("{0}")]
+    Generic(String),
 }
 
 #[cfg(not(target_arch = "wasm32"))]
@@ -18,6 +26,12 @@ impl From<reqwest::Error> for GalileoError {
     }
 }
 
+impl From<std::io::Error> for GalileoError {
+    fn from(_value: Error) -> Self {
+        Self::IO
+    }
+}
+
 #[cfg(target_arch = "wasm32")]
 impl From<wasm_bindgen::JsValue> for GalileoError {
     fn from(value: wasm_bindgen::JsValue) -> Self {

galileo/src/galileo_map.rs

@@ -1,15 +1,18 @@
 use crate::control::custom::{CustomEventHandler, EventHandler};
 use crate::control::event_processor::EventProcessor;
 use crate::control::map::MapController;
+use crate::layer::data_provider::file_cache::FileCacheController;
+use crate::layer::data_provider::url_data_provider::UrlDataProvider;
+use crate::layer::data_provider::url_image_provider::{UrlImageProvider, UrlSource};
 use crate::layer::raster_tile::RasterTileLayer;
-use crate::layer::tile_provider::TileSource;
 use crate::layer::vector_tile_layer::style::VectorTileStyle;
+use crate::layer::vector_tile_layer::tile_provider::vt_processor::VtProcessor;
 use crate::layer::vector_tile_layer::VectorTileLayer;
 use crate::layer::Layer;
 use crate::map::Map;
 use crate::render::wgpu::WgpuRenderer;
 use crate::render::Renderer;
-use crate::tile_scheme::TileScheme;
+use crate::tile_scheme::{TileIndex, TileScheme};
 use crate::view::MapView;
 use crate::winit::{WinitInputHandler, WinitMessenger};
 use galileo_types::cartesian::size::Size;
@@ -21,10 +24,13 @@ use winit::event_loop::{ControlFlow, EventLoop};
 use winit::window::{Window, WindowBuilder};
 
 #[cfg(not(target_arch = "wasm32"))]
-pub type VectorTileProver =
-    crate::layer::vector_tile_layer::tile_provider::rayon_provider::RayonProvider;
+pub type VectorTileProvider =
+    crate::layer::vector_tile_layer::tile_provider::rayon_provider::RayonProvider<
+        UrlDataProvider<TileIndex, VtProcessor, FileCacheController>,
+    >;
+
 #[cfg(target_arch = "wasm32")]
-pub type VectorTileProver =
+pub type VectorTileProvider =
     crate::layer::vector_tile_layer::tile_provider::web_worker_provider::WebWorkerVectorTileProvider;
 
 pub struct GalileoMap {
@@ -198,30 +204,55 @@ impl MapBuilder {
 
     pub fn with_raster_tiles(
         mut self,
-        tile_source: impl TileSource + 'static,
+        tile_source: impl UrlSource<TileIndex> + 'static,
         tile_scheme: TileScheme,
     ) -> Self {
-        self.layers.push(Box::new(RasterTileLayer::from_url(
-            tile_source,
+        #[cfg(not(target = "wasm32-unknown-unknown"))]
+        let cache_controller = Some(FileCacheController::new(".tile_cache"));
+        #[cfg(target = "wasm32-unknown-unknown")]
+        let cache_controller = None;
+
+        let tile_provider = UrlImageProvider::new(tile_source, cache_controller);
+        self.layers.push(Box::new(RasterTileLayer::new(
             tile_scheme,
+            tile_provider,
             None,
         )));
         self
     }
 
+    pub fn create_vector_tile_layer(
+        tile_source: impl UrlSource<TileIndex> + 'static,
+        tile_scheme: TileScheme,
+        style: VectorTileStyle,
+    ) -> VectorTileLayer<VectorTileProvider> {
+        #[cfg(not(target_arch = "wasm32"))]
+        let tile_provider = VectorTileProvider::new(
+            None,
+            tile_scheme.clone(),
+            UrlDataProvider::new(
+                tile_source,
+                VtProcessor {},
+                FileCacheController::new(".tile_cache"),
+            ),
+        );
+
+        #[cfg(target_arch = "wasm32")]
+        let tile_provider = VectorTileProvider::new(4, None, tile_source, tile_scheme.clone());
+        VectorTileLayer::from_url(tile_provider, style, tile_scheme)
+    }
+
     pub fn with_vector_tiles(
         mut self,
-        tile_source: impl TileSource + 'static,
+        tile_source: impl UrlSource<TileIndex> + 'static,
         tile_scheme: TileScheme,
         style: VectorTileStyle,
     ) -> Self {
-        self.layers
-            .push(Box::new(VectorTileLayer::<VectorTileProver>::from_url(
-                tile_source,
-                style,
-                None,
-                tile_scheme,
-            )));
+        self.layers.push(Box::new(Self::create_vector_tile_layer(
+            tile_source,
+            tile_scheme,
+            style,
+        )));
         self
     }
 

galileo/src/cache.rs → galileo/src/layer/data_provider/file_cache.rs

@@ -1,34 +1,51 @@
+use crate::error::GalileoError;
+use crate::layer::data_provider::PersistentCacheController;
 use bytes::Bytes;
-use log::info;
+use log::debug;
 use std::path::{Path, PathBuf};
 
+const CACHE_FOLDER: &str = ".tile_cache";
+
 #[derive(Debug, Clone)]
 pub struct FileCacheController {
     folder_path: PathBuf,
 }
 
 impl Default for FileCacheController {
     fn default() -> Self {
-        Self::new()
+        Self::new(CACHE_FOLDER)
     }
 }
 
-impl FileCacheController {
-    pub fn new() -> Self {
-        let folder_path = get_folder_path();
-        ensure_folder_exists(&folder_path).unwrap();
-        Self { folder_path }
-    }
-
-    pub fn get_from_cache(&self, url: &str) -> Option<Bytes> {
-        let file_path = self.get_file_path(url);
+impl PersistentCacheController<str, Bytes> for FileCacheController {
+    fn get(&self, key: &str) -> Option<Bytes> {
+        let file_path = self.get_file_path(key);
         if let Ok(bytes) = std::fs::read(file_path) {
             Some(bytes.into())
         } else {
             None
         }
     }
 
+    fn insert(&self, key: &str, data: &Bytes) -> Result<(), GalileoError> {
+        let file_path = self.get_file_path(key);
+        ensure_folder_exists(file_path.parent().unwrap()).unwrap();
+        std::fs::write(&file_path, data)?;
+
+        debug!("Entry {key} saved to cache file {file_path:?}");
+
+        Ok(())
+    }
+}
+
+impl FileCacheController {
+    pub fn new(path: impl AsRef<Path>) -> Self {
+        ensure_folder_exists(path.as_ref()).unwrap();
+        Self {
+            folder_path: path.as_ref().into(),
+        }
+    }
+
     fn get_file_path(&self, url: &str) -> PathBuf {
         let stripped = if let Some(v) = url.strip_prefix("http://") {
             v
@@ -40,23 +57,8 @@ impl FileCacheController {
 
         self.folder_path.join(Path::new(stripped))
     }
-
-    pub fn save_to_cache(&self, url: &str, bytes: &Bytes) {
-        let file_path = self.get_file_path(url);
-        ensure_folder_exists(file_path.parent().unwrap()).unwrap();
-        match std::fs::write(&file_path, bytes) {
-            Ok(_) => info!("Url {url} saved to cache file {file_path:?}"),
-            Err(e) => info!("Failed to save {url} to cache: {e:?}"),
-        };
-    }
 }
 
 fn ensure_folder_exists(folder_path: &Path) -> std::io::Result<()> {
     std::fs::create_dir_all(folder_path)
 }
-
-const CACHE_FOLDER: &str = ".tile_cache";
-
-fn get_folder_path() -> PathBuf {
-    Path::new(CACHE_FOLDER).into()
-}

galileo/src/layer/data_provider/mod.rs

@@ -0,0 +1,55 @@
+pub mod file_cache;
+pub mod url_data_provider;
+pub mod url_image_provider;
+
+use crate::error::GalileoError;
+use bytes::Bytes;
+use maybe_sync::{MaybeSend, MaybeSync};
+use std::future::Future;
+
+pub trait DataProvider<Key, Data, Context>: MaybeSend + MaybeSync
+where
+    Key: MaybeSend + MaybeSync + ?Sized,
+    Context: MaybeSend + MaybeSync,
+{
+    fn load_raw(&self, key: &Key) -> impl Future<Output = Result<Bytes, GalileoError>> + MaybeSend;
+    fn decode(&self, bytes: Bytes, context: Context) -> Result<Data, GalileoError>;
+    fn load(
+        &self,
+        key: &Key,
+        context: Context,
+    ) -> impl Future<Output = Result<Data, GalileoError>> + MaybeSend {
+        async {
+            let raw = self.load_raw(key).await?;
+            self.decode(raw, context)
+        }
+    }
+}
+
+pub trait DataProcessor {
+    type Input;
+    type Output;
+    type Context;
+
+    fn process(
+        &self,
+        input: Self::Input,
+        context: Self::Context,
+    ) -> Result<Self::Output, GalileoError>;
+}
+
+pub trait PersistentCacheController<Key: ?Sized, Data> {
+    fn get(&self, key: &Key) -> Option<Data>;
+    fn insert(&self, key: &Key, data: &Data) -> Result<(), GalileoError>;
+}
+
+pub struct EmptyCache {}
+impl<Key: ?Sized, Data> PersistentCacheController<Key, Data> for EmptyCache {
+    fn get(&self, _key: &Key) -> Option<Data> {
+        None
+    }
+
+    fn insert(&self, _key: &Key, _data: &Data) -> Result<(), GalileoError> {
+        Ok(())
+    }
+}