diff --git a/foyer-intrusive/src/core/adapter.rs b/foyer-intrusive/src/core/adapter.rs index eb09afbd..3f98d74e 100644 --- a/foyer-intrusive/src/core/adapter.rs +++ b/foyer-intrusive/src/core/adapter.rs @@ -117,6 +117,8 @@ pub unsafe trait PriorityAdapter: Adapter { /// # Examples /// /// ``` +/// #![feature(offset_of)] +/// /// use foyer_intrusive::{intrusive_adapter, key_adapter}; /// use foyer_intrusive::core::adapter::{Adapter, KeyAdapter, Link}; /// use foyer_intrusive::core::pointer::Pointer; @@ -213,6 +215,8 @@ macro_rules! intrusive_adapter { /// # Examples /// /// ``` +/// #![feature(offset_of)] +/// /// use foyer_intrusive::{intrusive_adapter, key_adapter}; /// use foyer_intrusive::core::adapter::{Adapter, KeyAdapter, Link}; /// use foyer_intrusive::core::pointer::Pointer; @@ -282,6 +286,8 @@ macro_rules! key_adapter { /// # Examples /// /// ``` +/// #![feature(offset_of)] +/// /// use foyer_intrusive::{intrusive_adapter, priority_adapter}; /// use foyer_intrusive::core::adapter::{Adapter, PriorityAdapter, Link}; /// use foyer_intrusive::core::pointer::Pointer; diff --git a/foyer-intrusive/src/eviction/lfu.rs b/foyer-intrusive/src/eviction/lfu.rs index cd1ae9c1..dc62da52 100644 --- a/foyer-intrusive/src/eviction/lfu.rs +++ b/foyer-intrusive/src/eviction/lfu.rs @@ -468,7 +468,7 @@ where link } (Some(link_main), Some(link_tiny)) => { - // Eviction from tiny or main depending on whether the tiny handle woould be + // Eviction from tiny or main depending on whether the tiny handle would be // admitted to main cachce. If it would be, evict from main cache, otherwise // from tiny cache. if self.lfu.admit_to_main(link_main.raw(), link_tiny.raw()) { diff --git a/foyer-intrusive/src/lib.rs b/foyer-intrusive/src/lib.rs index 7ea663d5..4241d821 100644 --- a/foyer-intrusive/src/lib.rs +++ b/foyer-intrusive/src/lib.rs @@ -27,6 +27,8 @@ pub use memoffset::offset_of; /// # Examples /// /// ``` +/// #![feature(offset_of)] +/// /// use foyer_intrusive::container_of; /// /// struct S { x: u32, y: u32 }; diff --git a/foyer-memory/Cargo.toml b/foyer-memory/Cargo.toml index de144cba..ffe808fd 100644 --- a/foyer-memory/Cargo.toml +++ b/foyer-memory/Cargo.toml @@ -25,6 +25,7 @@ itertools = "0.12" libc = "0.2" parking_lot = "0.12" tokio = { workspace = true } + [dev-dependencies] bytesize = "1" clap = { version = "4", features = ["derive"] } diff --git a/foyer-memory/benches/bench_hit_ratio.rs b/foyer-memory/benches/bench_hit_ratio.rs index a78daad4..3cb51fa7 100644 --- a/foyer-memory/benches/bench_hit_ratio.rs +++ b/foyer-memory/benches/bench_hit_ratio.rs @@ -16,7 +16,8 @@ use std::sync::Arc; use ahash::RandomState; use foyer_memory::{ - Cache, DefaultCacheEventListener, FifoCacheConfig, FifoConfig, LfuCacheConfig, LfuConfig, LruCacheConfig, LruConfig, + Cache, DefaultCacheEventListener, FifoCacheConfig, FifoConfig, LfuCacheConfig, LfuConfig, LruCacheConfig, + LruConfig, S3FifoCacheConfig, S3FifoConfig, }; use rand::{distributions::Distribution, thread_rng}; @@ -32,32 +33,32 @@ const OBJECT_POOL_CAPACITY: usize = 16; inspired by pingora/tinyufo/benches/bench_hit_ratio.rs cargo bench --bench bench_hit_ratio -zif_exp, cache_size fifo lru lfu moka -0.90, 0.005 16.26% 19.22% 32.38% 33.46% -0.90, 0.01 22.55% 26.21% 38.55% 37.93% -0.90, 0.05 41.10% 45.60% 55.45% 55.26% -0.90, 0.1 51.11% 55.72% 63.83% 64.21% -0.90, 0.25 66.81% 71.17% 76.21% 77.14% -1.00, 0.005 26.64% 31.08% 44.14% 45.61% -1.00, 0.01 34.37% 39.13% 50.60% 50.67% -1.00, 0.05 54.06% 58.79% 66.79% 66.99% -1.00, 0.1 63.12% 67.58% 73.92% 74.38% -1.00, 0.25 76.14% 79.92% 83.61% 84.33% -1.05, 0.005 32.63% 37.68% 50.24% 51.77% -1.05, 0.01 40.95% 46.09% 56.75% 57.15% -1.05, 0.05 60.47% 65.06% 72.07% 72.36% -1.05, 0.1 68.96% 73.15% 78.52% 78.96% -1.05, 0.25 80.42% 83.76% 86.79% 87.42% -1.10, 0.005 39.02% 44.52% 56.29% 57.90% -1.10, 0.01 47.66% 52.99% 62.64% 63.23% -1.10, 0.05 66.60% 70.95% 76.94% 77.25% -1.10, 0.1 74.26% 78.09% 82.56% 82.93% -1.10, 0.25 84.15% 87.06% 89.54% 90.05% -1.50, 0.005 81.19% 85.28% 88.91% 89.94% -1.50, 0.01 86.91% 89.87% 92.24% 92.78% -1.50, 0.05 94.75% 96.04% 96.95% 97.07% -1.50, 0.1 96.65% 97.51% 98.06% 98.15% -1.50, 0.25 98.35% 98.81% 99.04% 99.09% +zif_exp, cache_size fifo lru lfu s3fifo moka +0.90, 0.005 16.24% 19.22% 32.37% 32.39% 33.50% +0.90, 0.01 22.55% 26.20% 38.54% 39.20% 37.92% +0.90, 0.05 41.05% 45.56% 55.37% 56.63% 55.25% +0.90, 0.1 51.06% 55.68% 63.82% 65.06% 64.20% +0.90, 0.25 66.81% 71.17% 76.21% 77.26% 77.12% +1.00, 0.005 26.62% 31.10% 44.16% 44.15% 45.62% +1.00, 0.01 34.38% 39.17% 50.63% 51.29% 50.72% +1.00, 0.05 54.04% 58.76% 66.79% 67.85% 66.89% +1.00, 0.1 63.15% 67.60% 73.93% 74.92% 74.38% +1.00, 0.25 76.18% 79.95% 83.63% 84.39% 84.38% +1.05, 0.005 32.67% 37.71% 50.26% 50.21% 51.85% +1.05, 0.01 40.97% 46.10% 56.74% 57.40% 57.09% +1.05, 0.05 60.44% 65.03% 72.04% 73.02% 72.28% +1.05, 0.1 68.93% 73.12% 78.49% 79.37% 79.00% +1.05, 0.25 80.38% 83.73% 86.78% 87.42% 87.41% +1.10, 0.005 39.02% 44.50% 56.26% 56.20% 57.90% +1.10, 0.01 47.60% 52.93% 62.61% 63.24% 63.05% +1.10, 0.05 66.59% 70.95% 76.92% 77.76% 77.27% +1.10, 0.1 74.24% 78.07% 82.54% 83.28% 83.00% +1.10, 0.25 84.18% 87.10% 89.57% 90.06% 90.08% +1.50, 0.005 81.17% 85.27% 88.90% 89.10% 89.89% +1.50, 0.01 86.91% 89.87% 92.25% 92.56% 92.79% +1.50, 0.05 94.77% 96.04% 96.96% 97.10% 97.07% +1.50, 0.1 96.65% 97.50% 98.06% 98.14% 98.15% +1.50, 0.25 98.36% 98.81% 99.04% 99.06% 99.09% */ fn cache_hit(cache: Arc>, keys: Arc>) -> f64 { let mut hit = 0; @@ -131,6 +132,21 @@ fn new_lfu_cache(capacity: usize) -> Arc> { Arc::new(Cache::lfu(config)) } +fn new_s3fifo_cache(capacity: usize) -> Arc> { + let config = S3FifoCacheConfig { + capacity, + shards: SHARDS, + eviction_config: S3FifoConfig { + small_queue_capacity_ratio: 0.1, + }, + object_pool_capacity: OBJECT_POOL_CAPACITY, + hash_builder: RandomState::default(), + event_listener: DefaultCacheEventListener::default(), + }; + + Arc::new(Cache::s3fifo(config)) +} + fn bench_one(zif_exp: f64, cache_size_percent: f64) { print!("{zif_exp:.2}, {cache_size_percent:4}\t\t\t"); let mut rng = thread_rng(); @@ -141,6 +157,7 @@ fn bench_one(zif_exp: f64, cache_size_percent: f64) { let fifo_cache = new_fifo_cache(cache_size); let lru_cache = new_lru_cache(cache_size); let lfu_cache = new_lfu_cache(cache_size); + let s3fifo_cache = new_s3fifo_cache(cache_size); let moka_cache = moka::sync::Cache::new(cache_size as u64); let mut keys = Vec::with_capacity(ITERATIONS); @@ -170,6 +187,12 @@ fn bench_one(zif_exp: f64, cache_size_percent: f64) { move || cache_hit(cache, keys) }); + let s3fifo_cache_hit_handle = std::thread::spawn({ + let cache = s3fifo_cache.clone(); + let keys = keys.clone(); + move || cache_hit(cache, keys) + }); + let moka_cache_hit_handle = std::thread::spawn({ let cache = moka_cache.clone(); let keys = keys.clone(); @@ -179,16 +202,18 @@ fn bench_one(zif_exp: f64, cache_size_percent: f64) { let fifo_hit_ratio = fifo_cache_hit_handle.join().unwrap(); let lru_hit_ratio = lru_cache_hit_handle.join().unwrap(); let lfu_hit_ratio = lfu_cache_hit_handle.join().unwrap(); + let s3fifo_hit_ratio = s3fifo_cache_hit_handle.join().unwrap(); let moka_hit_ratio = moka_cache_hit_handle.join().unwrap(); print!("{:.2}%\t\t", fifo_hit_ratio * 100.0); print!("{:.2}%\t\t", lru_hit_ratio * 100.0); print!("{:.2}%\t\t", lfu_hit_ratio * 100.0); + print!("{:.2}%\t\t", s3fifo_hit_ratio * 100.0); println!("{:.2}%", moka_hit_ratio * 100.0); } fn bench_zipf_hit() { - println!("zif_exp, cache_size\t\tfifo\t\tlru\t\tlfu\t\tmoka"); + println!("zif_exp, cache_size\t\tfifo\t\tlru\t\tlfu\t\ts3fifo\t\tmoka"); for zif_exp in [0.9, 1.0, 1.05, 1.1, 1.5] { for cache_capacity in [0.005, 0.01, 0.05, 0.1, 0.25] { bench_one(zif_exp, cache_capacity); diff --git a/foyer-memory/src/cache.rs b/foyer-memory/src/cache.rs index 5ca929aa..d1a59d69 100644 --- a/foyer-memory/src/cache.rs +++ b/foyer-memory/src/cache.rs @@ -24,6 +24,7 @@ use crate::{ fifo::{Fifo, FifoHandle}, lfu::{Lfu, LfuHandle}, lru::{Lru, LruHandle}, + s3fifo::{S3Fifo, S3FifoHandle}, }, generic::{CacheConfig, GenericCache, GenericCacheEntry, GenericEntry}, indexer::HashTableIndexer, @@ -56,6 +57,15 @@ pub type LfuCacheEntry, S = RandomStat pub type LfuEntry, S = RandomState> = GenericEntry, Lfu, HashTableIndexer>, L, S, ER>; +pub type S3FifoCache, S = RandomState> = + GenericCache, S3Fifo, HashTableIndexer>, L, S>; +pub type S3FifoCacheConfig, S = RandomState> = + CacheConfig, L, S>; +pub type S3FifoCacheEntry, S = RandomState> = + GenericCacheEntry, S3Fifo, HashTableIndexer>, L, S>; +pub type S3FifoEntry, S = RandomState> = + GenericEntry, S3Fifo, HashTableIndexer>, L, S, ER>; + pub enum CacheEntry where K: Key, @@ -66,6 +76,7 @@ where Fifo(FifoCacheEntry), Lru(LruCacheEntry), Lfu(LfuCacheEntry), + S3Fifo(S3FifoCacheEntry), } impl Clone for CacheEntry @@ -80,6 +91,7 @@ where Self::Fifo(entry) => Self::Fifo(entry.clone()), Self::Lru(entry) => Self::Lru(entry.clone()), Self::Lfu(entry) => Self::Lfu(entry.clone()), + Self::S3Fifo(entry) => Self::S3Fifo(entry.clone()), } } } @@ -98,6 +110,7 @@ where CacheEntry::Fifo(entry) => entry.deref(), CacheEntry::Lru(entry) => entry.deref(), CacheEntry::Lfu(entry) => entry.deref(), + CacheEntry::S3Fifo(entry) => entry.deref(), } } } @@ -138,6 +151,18 @@ where } } +impl From> for CacheEntry +where + K: Key, + V: Value, + L: CacheEventListener, + S: BuildHasher + Send + Sync + 'static, +{ + fn from(entry: S3FifoCacheEntry) -> Self { + Self::S3Fifo(entry) + } +} + impl CacheEntry where K: Key, @@ -150,6 +175,7 @@ where CacheEntry::Fifo(entry) => entry.key(), CacheEntry::Lru(entry) => entry.key(), CacheEntry::Lfu(entry) => entry.key(), + CacheEntry::S3Fifo(entry) => entry.key(), } } @@ -158,6 +184,7 @@ where CacheEntry::Fifo(entry) => entry.value(), CacheEntry::Lru(entry) => entry.value(), CacheEntry::Lfu(entry) => entry.value(), + CacheEntry::S3Fifo(entry) => entry.value(), } } @@ -166,6 +193,7 @@ where CacheEntry::Fifo(entry) => entry.context().clone().into(), CacheEntry::Lru(entry) => entry.context().clone().into(), CacheEntry::Lfu(entry) => entry.context().clone().into(), + CacheEntry::S3Fifo(entry) => entry.context().clone().into(), } } @@ -174,6 +202,7 @@ where CacheEntry::Fifo(entry) => entry.charge(), CacheEntry::Lru(entry) => entry.charge(), CacheEntry::Lfu(entry) => entry.charge(), + CacheEntry::S3Fifo(entry) => entry.charge(), } } @@ -182,6 +211,7 @@ where CacheEntry::Fifo(entry) => entry.refs(), CacheEntry::Lru(entry) => entry.refs(), CacheEntry::Lfu(entry) => entry.refs(), + CacheEntry::S3Fifo(entry) => entry.refs(), } } } @@ -196,6 +226,7 @@ where Fifo(Arc>), Lru(Arc>), Lfu(Arc>), + S3Fifo(Arc>), } impl Clone for Cache @@ -210,6 +241,7 @@ where Self::Fifo(cache) => Self::Fifo(cache.clone()), Self::Lru(cache) => Self::Lru(cache.clone()), Self::Lfu(cache) => Self::Lfu(cache.clone()), + Self::S3Fifo(cache) => Self::S3Fifo(cache.clone()), } } } @@ -233,11 +265,16 @@ where Self::Lfu(Arc::new(GenericCache::new(config))) } + pub fn s3fifo(config: S3FifoCacheConfig) -> Self { + Self::S3Fifo(Arc::new(GenericCache::new(config))) + } + pub fn insert(&self, key: K, value: V, charge: usize) -> CacheEntry { match self { Cache::Fifo(cache) => cache.insert(key, value, charge).into(), Cache::Lru(cache) => cache.insert(key, value, charge).into(), Cache::Lfu(cache) => cache.insert(key, value, charge).into(), + Cache::S3Fifo(cache) => cache.insert(key, value, charge).into(), } } @@ -252,6 +289,7 @@ where Cache::Fifo(cache) => cache.insert_with_context(key, value, charge, context).into(), Cache::Lru(cache) => cache.insert_with_context(key, value, charge, context).into(), Cache::Lfu(cache) => cache.insert_with_context(key, value, charge, context).into(), + Cache::S3Fifo(cache) => cache.insert_with_context(key, value, charge, context).into(), } } @@ -260,6 +298,7 @@ where Cache::Fifo(cache) => cache.remove(key), Cache::Lru(cache) => cache.remove(key), Cache::Lfu(cache) => cache.remove(key), + Cache::S3Fifo(cache) => cache.remove(key), } } @@ -268,6 +307,7 @@ where Cache::Fifo(cache) => cache.get(key).map(CacheEntry::from), Cache::Lru(cache) => cache.get(key).map(CacheEntry::from), Cache::Lfu(cache) => cache.get(key).map(CacheEntry::from), + Cache::S3Fifo(cache) => cache.get(key).map(CacheEntry::from), } } @@ -276,6 +316,7 @@ where Cache::Fifo(cache) => cache.clear(), Cache::Lru(cache) => cache.clear(), Cache::Lfu(cache) => cache.clear(), + Cache::S3Fifo(cache) => cache.clear(), } } @@ -284,6 +325,7 @@ where Cache::Fifo(cache) => cache.capacity(), Cache::Lru(cache) => cache.capacity(), Cache::Lfu(cache) => cache.capacity(), + Cache::S3Fifo(cache) => cache.capacity(), } } @@ -292,6 +334,7 @@ where Cache::Fifo(cache) => cache.usage(), Cache::Lru(cache) => cache.usage(), Cache::Lfu(cache) => cache.usage(), + Cache::S3Fifo(cache) => cache.usage(), } } @@ -300,6 +343,7 @@ where Cache::Fifo(cache) => cache.metrics(), Cache::Lru(cache) => cache.metrics(), Cache::Lfu(cache) => cache.metrics(), + Cache::S3Fifo(cache) => cache.metrics(), } } } @@ -315,6 +359,7 @@ where Fifo(FifoEntry), Lru(LruEntry), Lfu(LfuEntry), + S3Fifo(S3FifoEntry), } impl From> for Entry @@ -356,6 +401,19 @@ where } } +impl From> for Entry +where + K: Key + Clone, + V: Value, + ER: std::error::Error, + L: CacheEventListener, + S: BuildHasher + Send + Sync + 'static, +{ + fn from(entry: S3FifoEntry) -> Self { + Self::S3Fifo(entry) + } +} + impl Future for Entry where K: Key + Clone, @@ -371,6 +429,7 @@ where Entry::Fifo(entry) => entry.poll_unpin(cx).map(|res| res.map(CacheEntry::from)), Entry::Lru(entry) => entry.poll_unpin(cx).map(|res| res.map(CacheEntry::from)), Entry::Lfu(entry) => entry.poll_unpin(cx).map(|res| res.map(CacheEntry::from)), + Entry::S3Fifo(entry) => entry.poll_unpin(cx).map(|res| res.map(CacheEntry::from)), } } } @@ -423,6 +482,7 @@ where Cache::Fifo(cache) => Entry::from(cache.entry(key, f)), Cache::Lru(cache) => Entry::from(cache.entry(key, f)), Cache::Lfu(cache) => Entry::from(cache.entry(key, f)), + Cache::S3Fifo(cache) => Entry::from(cache.entry(key, f)), } } } @@ -436,7 +496,7 @@ mod tests { use rand::{rngs::StdRng, seq::SliceRandom, Rng, SeedableRng}; use super::*; - use crate::{FifoConfig, LfuConfig, LruConfig}; + use crate::{eviction::s3fifo::S3FifoConfig, FifoConfig, LfuConfig, LruConfig}; const CAPACITY: usize = 100; const SHARDS: usize = 4; @@ -485,6 +545,19 @@ mod tests { }) } + fn s3fifo() -> Cache { + Cache::s3fifo(S3FifoCacheConfig { + capacity: CAPACITY, + shards: SHARDS, + eviction_config: S3FifoConfig { + small_queue_capacity_ratio: 0.1, + }, + object_pool_capacity: OBJECT_POOL_CAPACITY, + hash_builder: RandomState::default(), + event_listener: DefaultCacheEventListener::default(), + }) + } + fn init_cache(cache: &Cache, rng: &mut StdRng) { let mut v = RANGE.collect_vec(); v.shuffle(rng); @@ -559,4 +632,9 @@ mod tests { async fn test_lfu_cache() { case(lfu()).await } + + #[tokio::test] + async fn test_s3fifo_cache() { + case(s3fifo()).await + } } diff --git a/foyer-memory/src/eviction/mod.rs b/foyer-memory/src/eviction/mod.rs index d18bd18b..d1ba0c9d 100644 --- a/foyer-memory/src/eviction/mod.rs +++ b/foyer-memory/src/eviction/mod.rs @@ -104,6 +104,7 @@ pub trait Eviction: Send + Sync + 'static { pub mod fifo; pub mod lfu; pub mod lru; +pub mod s3fifo; #[cfg(test)] pub mod test_utils; diff --git a/foyer-memory/src/eviction/s3fifo.rs b/foyer-memory/src/eviction/s3fifo.rs new file mode 100644 index 00000000..c9f54e8a --- /dev/null +++ b/foyer-memory/src/eviction/s3fifo.rs @@ -0,0 +1,420 @@ +// Copyright 2024 Foyer Project Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::{fmt::Debug, ptr::NonNull}; + +use foyer_intrusive::{ + collections::dlist::{Dlist, DlistLink}, + intrusive_adapter, +}; + +use crate::{ + eviction::Eviction, + handle::{BaseHandle, Handle}, + CacheContext, Key, Value, +}; + +#[derive(Debug, Clone)] +pub struct S3FifoContext; + +impl From for S3FifoContext { + fn from(_: CacheContext) -> Self { + Self + } +} + +impl From for CacheContext { + fn from(_: S3FifoContext) -> Self { + CacheContext::Default + } +} + +enum Queue { + None, + Main, + Small, +} + +pub struct S3FifoHandle +where + K: Key, + V: Value, +{ + link: DlistLink, + base: BaseHandle, + freq: u8, + queue: Queue, +} + +impl Debug for S3FifoHandle +where + K: Key, + V: Value, +{ + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("S3FifoHandle").finish() + } +} + +intrusive_adapter! { S3FifoHandleDlistAdapter = NonNull>: S3FifoHandle { link: DlistLink } where K: Key, V: Value } + +impl S3FifoHandle +where + K: Key, + V: Value, +{ + #[inline(always)] + pub fn inc(&mut self) { + self.freq = std::cmp::min(self.freq + 1, 3); + } + + #[inline(always)] + pub fn dec(&mut self) { + self.freq = self.freq.saturating_sub(1); + } + + #[inline(always)] + pub fn reset(&mut self) { + self.freq = 0; + } +} + +impl Handle for S3FifoHandle +where + K: Key, + V: Value, +{ + type Key = K; + type Value = V; + type Context = S3FifoContext; + + fn new() -> Self { + Self { + link: DlistLink::default(), + freq: 0, + base: BaseHandle::new(), + queue: Queue::None, + } + } + + fn init(&mut self, hash: u64, key: Self::Key, value: Self::Value, charge: usize, context: Self::Context) { + self.base.init(hash, key, value, charge, context); + } + + fn base(&self) -> &BaseHandle { + &self.base + } + + fn base_mut(&mut self) -> &mut BaseHandle { + &mut self.base + } +} + +#[derive(Debug, Clone)] +pub struct S3FifoConfig { + pub small_queue_capacity_ratio: f64, +} + +pub struct S3Fifo +where + K: Key, + V: Value, +{ + small_queue: Dlist>, + main_queue: Dlist>, + + small_capacity: usize, + + small_charges: usize, + main_charges: usize, +} + +impl S3Fifo +where + K: Key, + V: Value, +{ + unsafe fn evict(&mut self) -> Option as Eviction>::Handle>> { + if self.small_charges > self.small_capacity + && let Some(ptr) = self.evict_small() + { + Some(ptr) + } else { + self.evict_main() + } + } + + unsafe fn evict_small(&mut self) -> Option as Eviction>::Handle>> { + while let Some(mut ptr) = self.small_queue.pop_front() { + let handle = ptr.as_mut(); + if handle.freq > 1 { + self.main_queue.push_back(ptr); + handle.queue = Queue::Main; + self.small_charges -= handle.base().charge(); + self.main_charges += handle.base().charge(); + } else { + handle.queue = Queue::None; + handle.reset(); + self.small_charges -= handle.base().charge(); + return Some(ptr); + } + } + None + } + + unsafe fn evict_main(&mut self) -> Option as Eviction>::Handle>> { + while let Some(mut ptr) = self.main_queue.pop_front() { + let handle = ptr.as_mut(); + if handle.freq > 0 { + self.main_queue.push_back(ptr); + handle.dec(); + } else { + handle.queue = Queue::None; + self.main_charges -= handle.base.charge(); + return Some(ptr); + } + } + None + } +} + +impl Eviction for S3Fifo +where + K: Key, + V: Value, +{ + type Handle = S3FifoHandle; + type Config = S3FifoConfig; + + unsafe fn new(capacity: usize, config: &Self::Config) -> Self + where + Self: Sized, + { + let small_capacity = (capacity as f64 * config.small_queue_capacity_ratio) as usize; + Self { + small_queue: Dlist::new(), + main_queue: Dlist::new(), + small_capacity, + small_charges: 0, + main_charges: 0, + } + } + + unsafe fn push(&mut self, mut ptr: NonNull) { + let handle = ptr.as_mut(); + + self.small_queue.push_back(ptr); + handle.queue = Queue::Small; + self.small_charges += handle.base().charge(); + + handle.base_mut().set_in_eviction(true); + } + + unsafe fn pop(&mut self) -> Option> { + if let Some(mut ptr) = self.evict() { + let handle = ptr.as_mut(); + // `handle.queue` has already been set with `evict()` + handle.base_mut().set_in_eviction(false); + Some(ptr) + } else { + debug_assert!(self.is_empty()); + None + } + } + + unsafe fn reinsert(&mut self, _: NonNull) {} + + unsafe fn access(&mut self, ptr: NonNull) { + let mut ptr = ptr; + ptr.as_mut().inc(); + } + + unsafe fn remove(&mut self, mut ptr: NonNull) { + let handle = ptr.as_mut(); + + match handle.queue { + Queue::None => unreachable!(), + Queue::Main => { + let p = self + .main_queue + .iter_mut_from_raw(ptr.as_mut().link.raw()) + .remove() + .unwrap_unchecked(); + debug_assert_eq!(p, ptr); + + handle.queue = Queue::None; + handle.base_mut().set_in_eviction(false); + + self.main_charges -= handle.base().charge(); + } + Queue::Small => { + let p = self + .small_queue + .iter_mut_from_raw(ptr.as_mut().link.raw()) + .remove() + .unwrap_unchecked(); + debug_assert_eq!(p, ptr); + + handle.queue = Queue::None; + handle.base_mut().set_in_eviction(false); + + self.small_charges -= handle.base().charge(); + } + } + } + + unsafe fn clear(&mut self) -> Vec> { + let mut res = Vec::with_capacity(self.len()); + while let Some(mut ptr) = self.small_queue.pop_front() { + let handle = ptr.as_mut(); + handle.base_mut().set_in_eviction(false); + handle.queue = Queue::None; + res.push(ptr); + } + while let Some(mut ptr) = self.main_queue.pop_front() { + let handle = ptr.as_mut(); + handle.base_mut().set_in_eviction(false); + handle.queue = Queue::None; + res.push(ptr); + } + res + } + + unsafe fn len(&self) -> usize { + self.small_queue.len() + self.main_queue.len() + } + + unsafe fn is_empty(&self) -> bool { + self.small_queue.is_empty() && self.main_queue.is_empty() + } +} + +unsafe impl Send for S3Fifo +where + K: Key, + V: Value, +{ +} +unsafe impl Sync for S3Fifo +where + K: Key, + V: Value, +{ +} + +#[cfg(test)] +mod tests { + use std::ops::Range; + + use itertools::Itertools; + + use super::*; + use crate::eviction::test_utils::TestEviction; + + impl TestEviction for S3Fifo + where + K: Key + Clone, + V: Value + Clone, + { + fn dump(&self) -> Vec<(::Key, ::Value)> { + self.small_queue + .iter() + .chain(self.main_queue.iter()) + .map(|handle| (handle.base().key().clone(), handle.base().value().clone())) + .collect_vec() + } + } + + type TestS3Fifo = S3Fifo; + type TestS3FifoHandle = S3FifoHandle; + + fn assert_test_s3fifo(s3fifo: &TestS3Fifo, small: Vec, main: Vec) { + let mut s = s3fifo + .dump() + .into_iter() + .map(|(k, v)| { + assert_eq!(k, v); + k + }) + .collect_vec(); + assert_eq!(s.len(), s3fifo.small_queue.len() + s3fifo.main_queue.len()); + let m = s.split_off(s3fifo.small_queue.len()); + assert_eq!((&s, &m), (&small, &main)); + assert_eq!(s3fifo.small_charges, s.len()); + } + + fn assert_count(ptrs: &[NonNull], range: Range, count: u8) { + unsafe { + ptrs[range].iter().for_each(|ptr| assert_eq!(ptr.as_ref().freq, count)); + } + } + + #[test] + fn test_lfu() { + unsafe { + let ptrs = (0..100) + .map(|i| { + let mut handle = Box::new(TestS3FifoHandle::new()); + handle.init(i, i, i, 1, S3FifoContext); + NonNull::new_unchecked(Box::into_raw(handle)) + }) + .collect_vec(); + + // window: 2, probation: 2, protected: 6 + let config = S3FifoConfig { + small_queue_capacity_ratio: 0.25, + }; + let mut s3fifo = TestS3Fifo::new(8, &config); + + assert_eq!(s3fifo.small_capacity, 2); + + s3fifo.push(ptrs[0]); + s3fifo.push(ptrs[1]); + assert_test_s3fifo(&s3fifo, vec![0, 1], vec![]); + + s3fifo.push(ptrs[2]); + s3fifo.push(ptrs[3]); + assert_test_s3fifo(&s3fifo, vec![0, 1, 2, 3], vec![]); + + assert_count(&ptrs, 0..4, 0); + + (0..4).for_each(|i| s3fifo.access(ptrs[i])); + s3fifo.access(ptrs[1]); + s3fifo.access(ptrs[2]); + assert_count(&ptrs, 0..1, 1); + assert_count(&ptrs, 1..3, 2); + assert_count(&ptrs, 3..4, 1); + + let p0 = s3fifo.pop().unwrap(); + let p3 = s3fifo.pop().unwrap(); + assert_eq!(p0, ptrs[0]); + assert_eq!(p3, ptrs[3]); + assert_test_s3fifo(&s3fifo, vec![], vec![1, 2]); + assert_count(&ptrs, 0..1, 0); + assert_count(&ptrs, 1..3, 2); + assert_count(&ptrs, 3..4, 0); + + let p1 = s3fifo.pop().unwrap(); + assert_eq!(p1, ptrs[1]); + assert_test_s3fifo(&s3fifo, vec![], vec![2]); + assert_count(&ptrs, 0..4, 0); + + assert_eq!(s3fifo.clear(), [2].into_iter().map(|i| ptrs[i]).collect_vec()); + + for ptr in ptrs { + let _ = Box::from_raw(ptr.as_ptr()); + } + } + } +} diff --git a/foyer-memory/src/prelude.rs b/foyer-memory/src/prelude.rs index a789b8d1..5ae7c4d0 100644 --- a/foyer-memory/src/prelude.rs +++ b/foyer-memory/src/prelude.rs @@ -13,9 +13,9 @@ // limitations under the License. pub use crate::{ - cache::{Cache, CacheEntry, Entry, EntryState, FifoCacheConfig, LfuCacheConfig, LruCacheConfig}, + cache::{Cache, CacheEntry, Entry, EntryState, FifoCacheConfig, LfuCacheConfig, LruCacheConfig, S3FifoCacheConfig}, context::CacheContext, - eviction::{fifo::FifoConfig, lfu::LfuConfig, lru::LruConfig}, + eviction::{fifo::FifoConfig, lfu::LfuConfig, lru::LruConfig, s3fifo::S3FifoConfig}, listener::{CacheEventListener, DefaultCacheEventListener}, metrics::Metrics, }; diff --git a/foyer-workspace-hack/Cargo.toml b/foyer-workspace-hack/Cargo.toml index 43e9182e..153f7282 100644 --- a/foyer-workspace-hack/Cargo.toml +++ b/foyer-workspace-hack/Cargo.toml @@ -28,6 +28,7 @@ futures-executor = { version = "0.3" } futures-sink = { version = "0.3" } futures-util = { version = "0.3", default-features = false, features = ["async-await-macro", "channel", "io", "sink"] } hashbrown = { version = "0.14", features = ["raw"] } +itertools = { version = "0.12" } libc = { version = "0.2", features = ["extra_traits"] } parking_lot = { version = "0.12", features = ["arc_lock", "deadlock_detection"] } parking_lot_core = { version = "0.9", default-features = false, features = ["deadlock_detection"] } @@ -39,6 +40,7 @@ tracing-core = { version = "0.1" } [build-dependencies] cc = { version = "1", default-features = false, features = ["parallel"] } either = { version = "1", default-features = false, features = ["use_std"] } +itertools = { version = "0.12" } proc-macro2 = { version = "1" } quote = { version = "1" } syn = { version = "2", features = ["extra-traits", "full", "visit-mut"] }