feat: add more route caching and functionality offline, improved loca…

…ldb typings (#1505)
julianpoy · Feb 2, 2025 · 720ca3f · 720ca3f
2 parents 2a3ed65 + e48eadc
commit 720ca3f
Show file tree

Hide file tree

Showing 39 changed files with 941 additions and 242 deletions.
diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -71,6 +71,7 @@
     "openai": "^4.48.1",
     "p-debounce": "^2.1.0",
     "p-limit": "^3.1.0",
+    "p-throttle": "^7.0.0",
     "pdfmake": "^0.2.15",
     "pg": "8.11.4",
     "pg-hstore": "^2.3.4",

diff --git a/packages/frontend/src/app/app.component.ts b/packages/frontend/src/app/app.component.ts
@@ -184,9 +184,12 @@ export class AppComponent {
     (window as any).appLoaded = true;
 
     (window as any).swRegistration?.update();
-    setInterval(() => {
-      (window as any).swRegistration?.update();
-    }, SW_UPDATE_CHECK_INTERVAL_MINUTES);
+    setInterval(
+      () => {
+        (window as any).swRegistration?.update();
+      },
+      SW_UPDATE_CHECK_INTERVAL_MINUTES * 60 * 1000,
+    );
   }
 
   initEventListeners() {

diff --git a/packages/frontend/src/app/utils/SearchManager.ts b/packages/frontend/src/app/utils/SearchManager.ts
@@ -4,7 +4,12 @@ import MiniSearch, {
   type SearchResult,
 } from "minisearch";
 import { IDBPDatabase } from "idb";
-import { KVStoreKeys, ObjectStoreName } from "./localDb";
+import {
+  getKvStoreEntry,
+  KVStoreKeys,
+  ObjectStoreName,
+  RSLocalDB,
+} from "./localDb";
 import type { RecipeSummary } from "@recipesage/prisma";
 
 /**
@@ -39,7 +44,7 @@ export class SearchManager {
   private saveTimeout: NodeJS.Timeout | undefined;
   private maxSaveTimeout: NodeJS.Timeout | undefined;
 
-  constructor(private localDb: IDBPDatabase) {
+  constructor(private localDb: IDBPDatabase<RSLocalDB>) {
     this.miniSearch = new MiniSearch(this.miniSearchOptions);
   }
 
@@ -54,16 +59,13 @@ export class SearchManager {
   async populateFromLocalDb() {
     performance.mark("startIndexLoad");
 
-    const indexRecord = await this.localDb.get(
-      ObjectStoreName.KV,
-      KVStoreKeys.RecipeSearchIndex,
-    );
+    const indexRecord = await getKvStoreEntry(KVStoreKeys.RecipeSearchIndex);
 
     if (!indexRecord) return;
 
     try {
       this.miniSearch = MiniSearch.loadJSON(
-        indexRecord.value,
+        indexRecord,
         this.miniSearchOptions,
       );
 
@@ -168,19 +170,10 @@ export class SearchManager {
     clearTimeout(this.saveTimeout);
     clearTimeout(this.maxSaveTimeout);
 
-    if (
-      await this.localDb.get(ObjectStoreName.KV, KVStoreKeys.RecipeSearchIndex)
-    ) {
-      await this.localDb.put(ObjectStoreName.KV, {
-        key: KVStoreKeys.RecipeSearchIndex,
-        value: JSON.stringify(this.miniSearch),
-      });
-    } else {
-      await this.localDb.add(ObjectStoreName.KV, {
-        key: KVStoreKeys.RecipeSearchIndex,
-        value: JSON.stringify(this.miniSearch),
-      });
-    }
+    await this.localDb.put(ObjectStoreName.KV, {
+      key: KVStoreKeys.RecipeSearchIndex,
+      value: JSON.stringify(this.miniSearch),
+    });
   }
 
   async destroy(): Promise<void> {

diff --git a/packages/frontend/src/app/utils/SyncManager.ts b/packages/frontend/src/app/utils/SyncManager.ts
@@ -1,30 +1,31 @@
 import { IDBPDatabase } from "idb";
+import pThrottle from "p-throttle";
 import type { SearchManager } from "./SearchManager";
 import { trpcClient as trpc } from "./trpcClient";
-import { ObjectStoreName } from "./localDb";
+import { KVStoreKeys, ObjectStoreName, RSLocalDB } from "./localDb";
 import { appIdbStorageManager } from "./appIdbStorageManager";
 import { SW_BROADCAST_CHANNEL_NAME } from "./SW_BROADCAST_CHANNEL_NAME";
 
-const waitFor = (time: number): Promise<void> => {
-  return new Promise((resolve) => {
-    setTimeout(resolve, time);
-  });
-};
-
 const broadcastChannel = new BroadcastChannel(SW_BROADCAST_CHANNEL_NAME);
 
 const ENABLE_VERBOSE_SYNC_LOGGING = false;
 
 const SYNC_BATCH_SIZE = 200;
 
 /**
- * How long to wait between syncing each recipe
+ * We cannot exceed the rate limit of the API (is limited per-IP),
+ * so we throttle to 4 requests/sec to allow the browser some buffer as well in case
+ * the user is doing activities during a sync.
+ * Due to OPTIONS requests, the number of requests that actually go through will be doubled
  */
-const SYNC_BATCH_RATE_LIMIT_WAIT_MS = 250;
+const throttle = pThrottle({
+  limit: 3,
+  interval: 1000,
+});
 
 export class SyncManager {
   constructor(
-    private localDb: IDBPDatabase,
+    private localDb: IDBPDatabase<RSLocalDB>,
     private searchManager: SearchManager,
   ) {
     broadcastChannel.addEventListener("message", (event) => {
@@ -58,15 +59,12 @@ export class SyncManager {
 
     try {
       await this.syncRecipes();
-      await waitFor(SYNC_BATCH_RATE_LIMIT_WAIT_MS);
-
       await this.syncLabels();
-      await waitFor(SYNC_BATCH_RATE_LIMIT_WAIT_MS);
-
       await this.syncShoppingLists();
-      await waitFor(SYNC_BATCH_RATE_LIMIT_WAIT_MS);
-
       await this.syncMealPlans();
+      await this.syncMyUserProfile();
+      await this.syncMyFriends();
+      await this.syncMyStats();
 
       performance.mark("endSync");
       const measure = performance.measure("syncTime", "startSync", "endSync");
@@ -77,17 +75,20 @@ export class SyncManager {
   }
 
   async syncRecipe(recipeId: string): Promise<void> {
-    const recipe = await trpc.recipes.getRecipe.query({
-      id: recipeId,
-    });
+    const recipe = await throttle(() =>
+      trpc.recipes.getRecipe.query({
+        id: recipeId,
+      }),
+    )();
 
     await this.localDb.put(ObjectStoreName.Recipes, recipe);
     await this.searchManager.indexRecipe(recipe);
   }
 
   async syncRecipes(): Promise<void> {
-    const allVisibleRecipesManifest =
-      await trpc.recipes.getAllVisibleRecipesManifest.query();
+    const allVisibleRecipesManifest = await throttle(() =>
+      trpc.recipes.getAllVisibleRecipesManifest.query(),
+    )();
     const serverRecipeIds = allVisibleRecipesManifest.reduce(
       (acc, el) => acc.add(el.id),
       new Set<string>(),
@@ -136,29 +137,31 @@ export class SyncManager {
     while (remainingRecipeIdsToSync.length) {
       const ids = remainingRecipeIdsToSync.splice(0, SYNC_BATCH_SIZE);
 
-      const recipes = await trpc.recipes.getRecipesByIds.query({
-        ids,
-      });
+      const recipes = await throttle(() =>
+        trpc.recipes.getRecipesByIds.query({
+          ids,
+        }),
+      )();
 
       for (const recipe of recipes) {
         await this.localDb.put(ObjectStoreName.Recipes, recipe);
         await this.searchManager.indexRecipe(recipe);
       }
-
-      await waitFor(SYNC_BATCH_RATE_LIMIT_WAIT_MS);
     }
   }
 
   async syncLabels() {
-    const allLabels = await trpc.labels.getAllVisibleLabels.query();
+    const allLabels = await throttle(() =>
+      trpc.labels.getAllVisibleLabels.query(),
+    )();
     await this.localDb.clear(ObjectStoreName.Labels);
     for (const label of allLabels) {
       await this.localDb.put(ObjectStoreName.Labels, label);
     }
 
-    await waitFor(SYNC_BATCH_RATE_LIMIT_WAIT_MS);
-
-    const labelGroups = await trpc.labelGroups.getLabelGroups.query();
+    const labelGroups = await throttle(() =>
+      trpc.labelGroups.getLabelGroups.query(),
+    )();
     await this.localDb.clear(ObjectStoreName.LabelGroups);
     for (const labelGroup of labelGroups) {
       await this.localDb.put(ObjectStoreName.LabelGroups, labelGroup);
@@ -170,19 +173,56 @@ export class SyncManager {
   }
 
   async syncShoppingLists() {
-    const shoppingLists =
-      await trpc.shoppingLists.getShoppingListsWithItems.query();
+    const shoppingLists = await throttle(() =>
+      trpc.shoppingLists.getShoppingListsWithItems.query(),
+    )();
     await this.localDb.clear(ObjectStoreName.ShoppingLists);
     for (const shoppingList of shoppingLists) {
       await this.localDb.put(ObjectStoreName.ShoppingLists, shoppingList);
     }
   }
 
   async syncMealPlans() {
-    const mealPlans = await trpc.mealPlans.getMealPlansWithItems.query();
+    const mealPlans = await throttle(() =>
+      trpc.mealPlans.getMealPlansWithItems.query(),
+    )();
     await this.localDb.clear(ObjectStoreName.MealPlans);
     for (const mealPlan of mealPlans) {
       await this.localDb.put(ObjectStoreName.MealPlans, mealPlan);
     }
   }
+
+  async syncMyUserProfile() {
+    const myProfile = await throttle(() => trpc.users.getMe.query())();
+    await this.localDb.put(ObjectStoreName.KV, {
+      key: KVStoreKeys.MyUserProfile,
+      value: myProfile,
+    });
+  }
+
+  async syncMyFriends() {
+    const myFriends = await throttle(() => trpc.users.getMyFriends.query())();
+    await this.localDb.put(ObjectStoreName.KV, {
+      key: KVStoreKeys.MyFriends,
+      value: myFriends,
+    });
+
+    const userProfiles = [
+      myFriends.friends,
+      myFriends.incomingRequests,
+      myFriends.outgoingRequests,
+    ].flat();
+
+    for (const userProfile of userProfiles) {
+      await this.localDb.put(ObjectStoreName.UserProfiles, userProfile);
+    }
+  }
+
+  async syncMyStats() {
+    const myStats = await throttle(() => trpc.users.getMyStats.query())();
+    await this.localDb.put(ObjectStoreName.KV, {
+      key: KVStoreKeys.MyStats,
+      value: myStats,
+    });
+  }
 }
diff --git a/packages/frontend/src/app/utils/appIdbStorageManager.ts b/packages/frontend/src/app/utils/appIdbStorageManager.ts
@@ -1,12 +1,16 @@
-import { getLocalDb, KVStoreKeys, ObjectStoreName } from "./localDb";
+import {
+  getKvStoreEntry,
+  getLocalDb,
+  KVStoreKeys,
+  ObjectStoreName,
+} from "./localDb";
 import type { SessionDTO } from "@recipesage/prisma";
 
 export class AppIdbStorageManager {
   async getSession(): Promise<SessionDTO | null> {
-    const localDb = await getLocalDb();
-    const session = await localDb.get(ObjectStoreName.KV, KVStoreKeys.Session);
+    const session = await getKvStoreEntry(KVStoreKeys.Session);
 
-    return session?.value || null;
+    return session || null;
   }
 
   async setSession(session: SessionDTO): Promise<void> {
@@ -33,12 +37,8 @@ export class AppIdbStorageManager {
   }
 
   async getLastSessionUserId(): Promise<string | null> {
-    const localDb = await getLocalDb();
-    const record = await localDb.get(
-      ObjectStoreName.KV,
-      KVStoreKeys.LastSessionUserId,
-    );
-    return record?.value || null;
+    const record = await getKvStoreEntry(KVStoreKeys.LastSessionUserId);
+    return record || null;
   }
 
   async deleteAllData(): Promise<void> {
@@ -49,6 +49,8 @@ export class AppIdbStorageManager {
     await localDb.clear(ObjectStoreName.LabelGroups);
     await localDb.clear(ObjectStoreName.ShoppingLists);
     await localDb.clear(ObjectStoreName.MealPlans);
+    await localDb.clear(ObjectStoreName.AssistantMessages);
+    await localDb.clear(ObjectStoreName.Jobs);
   }
 }