From 5c007321221fd10a5e09cbab924cdd5823acfcd1 Mon Sep 17 00:00:00 2001 From: Levin Herr Date: Fri, 13 Dec 2024 13:05:09 +0100 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8=20Add=20DataProvider=20Cache=20(#3041?= =?UTF-8?q?)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/DataProviders/CachedHafas.php | 114 ++++++++++++++++++++ app/DataProviders/DataProviderBuilder.php | 6 +- app/Helpers/CacheKey.php | 48 ++++++++- app/Helpers/HCK.php | 1 + app/Providers/PrometheusServiceProvider.php | 26 +++++ config/trwl.php | 5 +- 6 files changed, 192 insertions(+), 8 deletions(-) create mode 100644 app/DataProviders/CachedHafas.php diff --git a/app/DataProviders/CachedHafas.php b/app/DataProviders/CachedHafas.php new file mode 100644 index 000000000..77a65f02a --- /dev/null +++ b/app/DataProviders/CachedHafas.php @@ -0,0 +1,114 @@ +remember( + $key, + now()->addMinutes(15), + function() use ($tripID, $lineName) { + return parent::fetchHafasTrip($tripID, $lineName); + }, + HCK::TRIPS_SUCCESS + ); + } + + public function getStations(string $query, int $results = 10): Collection { + $key = CacheKey::getHafasStationsKey($query); + + return $this->remember( + $key, + now()->addMinutes(15), + function() use ($query, $results) { + return parent::getStations($query, $results); + }, + HCK::LOCATIONS_SUCCESS + ); + } + + public function getDepartures(Station $station, Carbon $when, int $duration = 15, TravelType $type = null, bool $localtime = false): Collection { + $filterWhen = clone $when; + $when = clone $when; + $when->subMinutes(2); + // set cache when minutes to 0, 15, 30 or 45 + $when->minute = floor($when->minute / 15) * 15; + $when->second = 0; + + // set duration longer than 15 minutes + $duration = $duration < 15 ? 30 : $duration; + + $key = CacheKey::getHafasDeparturesKey($station->id, $when, $localtime); + + $departures = $this->remember( + $key, + now()->addMinutes(15), + function() use ($station, $when, $duration, $type, $localtime) { + return parent::getDepartures($station, $when, $duration, $type, $localtime); + }, + HCK::DEPARTURES_SUCCESS + ); + + // filter entries by when and duration + return $departures->filter(function($departure) use ($filterWhen, $duration) { + $depWhen = Carbon::parse($departure->when); + return $depWhen->between($filterWhen, $filterWhen->copy()->addMinutes($duration)); + }); + } + + public function getStationByRilIdentifier(string $rilIdentifier): ?Station { + $key = CacheKey::getHafasByRilIdentifierKey($rilIdentifier); + + return $this->remember( + $key, + now()->addMinutes(15), + function() use ($rilIdentifier) { + return parent::getStationByRilIdentifier($rilIdentifier); + }, + HCK::STATIONS_SUCCESS + ); + } + + public function getStationsByFuzzyRilIdentifier(string $rilIdentifier): ?Collection { + $key = CacheKey::getHafasStationsFuzzyKey($rilIdentifier); + + return $this->remember( + $key, + now()->addMinutes(15), + function() use ($rilIdentifier) { + return parent::getStationsByFuzzyRilIdentifier($rilIdentifier); + }, + HCK::STATIONS_SUCCESS + ); + } + + private function remember(string $key, Carbon $expires, callable $callback, ?string $ident = null): mixed { + if (Cache::has($key)) { + CacheKey::increment(CacheKey::getHafasCacheHitKey($ident)); + return Cache::get($key); + } + + try { + $result = $callback(); + CacheKey::increment(CacheKey::getHafasCacheSetKey($ident)); + Cache::put($key, $result, $expires); + return $result; + } catch (Throwable $e) { + Cache::put($key, null, $expires); + throw $e; + } + } +} diff --git a/app/DataProviders/DataProviderBuilder.php b/app/DataProviders/DataProviderBuilder.php index ff11dbecf..0f14fea79 100644 --- a/app/DataProviders/DataProviderBuilder.php +++ b/app/DataProviders/DataProviderBuilder.php @@ -4,7 +4,11 @@ class DataProviderBuilder { - public function build(): DataProviderInterface { + public function build(?bool $cache = null): DataProviderInterface { + if ($cache === true || ($cache === null && config('trwl.cache.hafas'))) { + return new CachedHafas(); + } + return new Hafas(); } } diff --git a/app/Helpers/CacheKey.php b/app/Helpers/CacheKey.php index 4a08c23ff..8c73b842b 100644 --- a/app/Helpers/CacheKey.php +++ b/app/Helpers/CacheKey.php @@ -19,13 +19,51 @@ class CacheKey public const string LEADERBOARD_GLOBAL_DISTANCE = 'LeaderboardGlobalDistance'; // dynamic keys - private const string LEADERBOARD_FRIENDS = 'LeaderboardFriends'; - private const string LEADERBOARD_MONTH = 'LeaderboardMonth'; - private const string STATISTICS_GLOBAL = 'StatisticsGlobal'; + private const string LEADERBOARD_FRIENDS = 'LeaderboardFriends'; + private const string LEADERBOARD_MONTH = 'LeaderboardMonth'; + private const string STATISTICS_GLOBAL = 'StatisticsGlobal'; + private const string HAFAS_TRIP = '_HafasTrip_%s_%s'; + private const string HAFAS_STATIONS = '_HafasStations'; + private const string HAFAS_DEPARTURES = '_HafasDepartures_%d_%s_%s'; + private const string HAFAFS_STATION_RIL = '_HafasStationRil'; + private const string HAFAS_STATIONS_FUZZY = '_HafasStationsFuzzy'; + private const string HAFAS_CACHE_HIT = '_HafasCacheHit_%s'; + private const string HAFAS_CACHE_SET = '_HafasCacheSet_%s'; // formatting keys - private const string FOR = '%s-for-%s'; - private const string FROM_TO = '%s-from-%s-to-%s'; + private const string FOR = '%s-for-%s'; + private const string FROM_TO = '%s-from-%s-to-%s'; + + public static function getHafasCacheHitKey(string $key): string { + $key = str_replace('monitoring-counter-', '', $key); + return sprintf(self::HAFAS_CACHE_HIT, $key); + } + + public static function getHafasCacheSetKey(string $key): string { + $key = str_replace('monitoring-counter-', '', $key); + return sprintf(self::HAFAS_CACHE_SET, $key); + } + + public static function getHafasTripKey(string $tripId, string $lineName): string { + $tripId = sha1($tripId); + return sprintf(self::HAFAS_TRIP, $tripId, $lineName); + } + + public static function getHafasStationsKey(string $query): string { + return sprintf(self::FOR, self::HAFAS_STATIONS, $query); + } + + public static function getHafasDeparturesKey(string $stationId, Carbon $when, bool $localtime): string { + return sprintf(self::HAFAS_DEPARTURES, $stationId, $when->toTimeString(), $localtime ? 'local' : 'utc'); + } + + public static function getHafasByRilIdentifierKey(string $rilIdentifier): string { + return sprintf(self::FOR, self::HAFAFS_STATION_RIL, $rilIdentifier); + } + + public static function getHafasStationsFuzzyKey(string $rilIdentifier): string { + return sprintf(self::FOR, self::HAFAS_STATIONS_FUZZY, $rilIdentifier); + } public static function getFriendsLeaderboardKey(int $userId): string { return sprintf(self::FOR, self::LEADERBOARD_FRIENDS, $userId); diff --git a/app/Helpers/HCK.php b/app/Helpers/HCK.php index 80afc9e6f..951417fc6 100644 --- a/app/Helpers/HCK.php +++ b/app/Helpers/HCK.php @@ -33,6 +33,7 @@ public static function getFailures(): array { return [ self::DEPARTURES_FAILURE => 'Departures', self::TRIPS_FAILURE => 'Trips', + self::TRIPS_502 => 'Trips502', self::STOPS_FAILURE => 'Stops', self::STATIONS_FAILURE => 'Stations', self::LOCATIONS_FAILURE => 'Locations', diff --git a/app/Providers/PrometheusServiceProvider.php b/app/Providers/PrometheusServiceProvider.php index 0c78a9c19..be0ec1c67 100644 --- a/app/Providers/PrometheusServiceProvider.php +++ b/app/Providers/PrometheusServiceProvider.php @@ -115,6 +115,32 @@ public function register() { return $this->getHafasByType(HCK::getSuccesses()); }); + Prometheus::addGauge("hafas_cache_hits") + ->helpText("How many hafas requests have been served from cache?") + ->labels(["request_name"]) + ->value(function() { + $values = []; + foreach (HCK::getSuccesses() as $key => $name) { + $key = CacheKey::getHafasCacheHitKey($key); + $values[$name] = Cache::get($key, 0); + } + + return array_map(fn($value, $key) => [$value, [$key]], $values, array_keys($values)); + }); + + Prometheus::addGauge("hafas_cache_sets") + ->helpText("How many hafas requests have been stored in cache?") + ->labels(["request_name"]) + ->value(function() { + $values = []; + foreach (HCK::getSuccesses() as $key => $name) { + $key = CacheKey::getHafasCacheSetKey($key); + $values[$name] = Cache::get($key, 0); + } + + return array_map(fn($value, $key) => [$value, [$key]], $values, array_keys($values)); + }); + Prometheus::addGauge("completed_jobs_count") ->helpText("How many jobs are done? Old items from queue monitor table are deleted after 7 days.") ->labels(["job_name", "status", "queue"]) diff --git a/config/trwl.php b/config/trwl.php index 6d8de23ba..afb183b2a 100644 --- a/config/trwl.php +++ b/config/trwl.php @@ -12,7 +12,7 @@ 'mastodon_timeout_seconds' => env("MASTODON_TIMEOUT_SECONDS", 5), # Brouter - 'brouter' => env('BROUTER', true), + 'brouter' => env('BROUTER', true), 'brouter_url' => env('BROUTER_URL', 'https://brouter.de/'), 'brouter_timeout' => env('BROUTER_TIMEOUT', 10), @@ -55,7 +55,8 @@ ], 'cache' => [ 'global-statistics-retention-seconds' => env('GLOBAL_STATISTICS_CACHE_RETENTION_SECONDS', 60 * 60), - 'leaderboard-retention-seconds' => env('LEADERBOARD_CACHE_RETENTION_SECONDS', 5 * 60) + 'leaderboard-retention-seconds' => env('LEADERBOARD_CACHE_RETENTION_SECONDS', 5 * 60), + 'hafas' => env('HAFAS_CACHE', false), ], 'year_in_review' => [ 'alert' => env('YEAR_IN_REVIEW_ALERT', false),