From a9a5dea0c251ec18dd2405b87f4424352046f44e Mon Sep 17 00:00:00 2001
From: Ben Hollis <ben@benhollis.net>
Date: Wed, 13 Nov 2024 00:02:13 -0800
Subject: [PATCH] Add some delays to bulk actions to try and avoid throttling

---
 api/routes/import.ts        | 3 ++-
 api/stately/bulk-queries.ts | 3 ++-
 api/utils.ts                | 4 ++++
 3 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/api/routes/import.ts b/api/routes/import.ts
index b6b0c3b..f05857a 100644
--- a/api/routes/import.ts
+++ b/api/routes/import.ts
@@ -32,7 +32,7 @@ import { importSearches } from '../stately/searches-queries.js';
 import { convertToStatelyItem } from '../stately/settings-queries.js';
 import { batches } from '../stately/stately-utils.js';
 import { importTriumphs } from '../stately/triumphs-queries.js';
-import { badRequest, subtractObject } from '../utils.js';
+import { badRequest, delay, subtractObject } from '../utils.js';
 import { deleteAllData } from './delete-all-data.js';
 
 export const importHandler = asyncHandler(async (req, res) => {
@@ -265,6 +265,7 @@ export async function statelyImport(
   // OK now put them in as fast as we can
   for (const batch of batches(items)) {
     await client.putBatch(...batch);
+    await delay(100); // give it some time to flush
   }
 
   return numTriumphs;
diff --git a/api/stately/bulk-queries.ts b/api/stately/bulk-queries.ts
index cb81570..9cfbd32 100644
--- a/api/stately/bulk-queries.ts
+++ b/api/stately/bulk-queries.ts
@@ -4,7 +4,7 @@ import { ExportResponse } from '../shapes/export.js';
 import { DestinyVersion } from '../shapes/general.js';
 import { ProfileResponse } from '../shapes/profile.js';
 import { defaultSettings } from '../shapes/settings.js';
-import { subtractObject } from '../utils.js';
+import { delay, subtractObject } from '../utils.js';
 import { client } from './client.js';
 import { AnyItem } from './generated/index.js';
 import { convertItemAnnotation, keyFor as tagKeyFor } from './item-annotations-queries.js';
@@ -76,6 +76,7 @@ async function deleteAllDataForProfile(
   // Then delete them all. We're not in a transaction!
   for (const batch of batches(keys)) {
     await client.del(...batch);
+    await delay(100); // give it some time to flush
   }
   return response;
 }
diff --git a/api/utils.ts b/api/utils.ts
index 915ef04..a858e13 100644
--- a/api/utils.ts
+++ b/api/utils.ts
@@ -148,3 +148,7 @@ export function subtractObject<T extends object>(obj: Partial<T>, defaults: T):
   }
   return result;
 }
+
+export function delay(ms: number) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}