User import

api/db/item-annotations-queries.ts

@@ -46,7 +46,7 @@ export async function getAllItemAnnotationsForUser(
     ItemAnnotationRow & { platform_membership_id: string; destiny_version: DestinyVersion }
   >({
     name: 'get_all_item_annotations',
-    text: 'SELECT platform_membership_id, destiny_version, inventory_item_id, tag, notes, variant, crafted_date FROM item_annotations WHERE membership_id = $1',
+    text: 'SELECT platform_membership_id, destiny_version, inventory_item_id, tag, notes, variant, crafted_date FROM item_annotations WHERE inventory_item_id != 0 and membership_id = $1',
     values: [bungieMembershipId],
   });
   return results.rows.map((row) => ({

api/db/migration-state-queries.ts

@@ -27,6 +27,14 @@ interface MigrationStateRow {
   last_error: string | null;
 }
 
+export async function getUsersToMigrate(client: ClientBase): Promise<number[]> {
+  const results = await client.query<MigrationStateRow>({
+    name: 'get_users_to_migrate',
+    text: 'select membership_id from migration_state where state != 3 limit 1000',
+  });
+  return results.rows.map((row) => row.membership_id);
+}
+
 export async function getMigrationState(
   client: ClientBase,
   bungieMembershipId: number,
@@ -41,7 +49,7 @@ export async function getMigrationState(
   } else {
     return {
       bungieMembershipId,
-      state: MigrationState.Postgres,
+      state: MigrationState.Stately,
       lastStateChangeAt: 0,
       attemptCount: 0,
     };
@@ -206,7 +214,7 @@ export async function doMigration(
     });
     metrics.increment('migration.finish.count');
   } catch (e) {
-    console.error('Stately migration failed', e);
+    console.error(`Stately migration failed for ${bungieMembershipId}`, e);
     await transaction(async (client) => {
       await abortMigrationToStately(
         client,

api/routes/import.ts

@@ -1,25 +1,12 @@
 import { isEmpty } from 'es-toolkit/compat';
 import asyncHandler from 'express-async-handler';
-import { readTransaction, transaction } from '../db/index.js';
-import { updateItemAnnotation } from '../db/item-annotations-queries.js';
-import { updateItemHashTag } from '../db/item-hash-tags-queries.js';
-import { updateLoadout } from '../db/loadouts-queries.js';
-import {
-  doMigration,
-  getDesiredMigrationState,
-  getMigrationState,
-  MigrationState,
-} from '../db/migration-state-queries.js';
-import { importSearch } from '../db/searches-queries.js';
-import { replaceSettings } from '../db/settings-queries.js';
-import { trackTriumph } from '../db/triumphs-queries.js';
-import { ApiApp } from '../shapes/app.js';
+import { readTransaction } from '../db/index.js';
+import { doMigration, getMigrationState, MigrationState } from '../db/migration-state-queries.js';
 import { ExportResponse } from '../shapes/export.js';
 import { DestinyVersion } from '../shapes/general.js';
 import { ImportResponse } from '../shapes/import.js';
 import { ItemAnnotation, ItemHashTag } from '../shapes/item-annotations.js';
 import { Loadout } from '../shapes/loadouts.js';
-import { SearchType } from '../shapes/search.js';
 import { defaultSettings, Settings } from '../shapes/settings.js';
 import { UserInfo } from '../shapes/user.js';
 import { deleteAllDataForUser } from '../stately/bulk-queries.js';
@@ -33,11 +20,9 @@ import { convertToStatelyItem } from '../stately/settings-queries.js';
 import { batches } from '../stately/stately-utils.js';
 import { importTriumphs } from '../stately/triumphs-queries.js';
 import { badRequest, delay, subtractObject } from '../utils.js';
-import { deleteAllData } from './delete-all-data.js';
 
 export const importHandler = asyncHandler(async (req, res) => {
   const { bungieMembershipId, profileIds } = req.user as UserInfo;
-  const { id: appId } = req.dimApp as ApiApp;
 
   // Support only new API exports
   const importData = req.body as ExportResponse;
@@ -60,12 +45,6 @@ export const importHandler = asyncHandler(async (req, res) => {
     getMigrationState(client, bungieMembershipId),
   );
 
-  // this is a great time to do the migration
-  const desiredMigrationState = await getDesiredMigrationState(migrationState);
-  const shouldMigrateToStately =
-    desiredMigrationState === MigrationState.Stately &&
-    migrationState.state !== desiredMigrationState;
-
   let numTriumphs = 0;
   const importToStately = async () => {
     numTriumphs = await statelyImport(
@@ -82,22 +61,7 @@ export const importHandler = asyncHandler(async (req, res) => {
 
   switch (migrationState.state) {
     case MigrationState.Postgres:
-      if (shouldMigrateToStately) {
-        await doMigration(bungieMembershipId, importToStately, async (client) =>
-          deleteAllData(client, bungieMembershipId),
-        );
-      } else {
-        numTriumphs = await pgImport(
-          bungieMembershipId,
-          appId,
-          settings,
-          loadouts,
-          itemAnnotations,
-          triumphs,
-          searches,
-          itemHashTags,
-        );
-      }
+      await doMigration(bungieMembershipId, importToStately);
       break;
     case MigrationState.Stately:
       await importToStately();
@@ -138,90 +102,6 @@ export function extractImportData(importData: ExportResponse) {
   };
 }
 
-async function pgImport(
-  bungieMembershipId: number,
-  appId: string,
-  settings: Partial<Settings>,
-  loadouts: PlatformLoadout[],
-  itemAnnotations: PlatformItemAnnotation[],
-  triumphs: ExportResponse['triumphs'],
-  searches: ExportResponse['searches'],
-  itemHashTags: ItemHashTag[],
-): Promise<number> {
-  let numTriumphs = 0;
-  await transaction(async (client) => {
-    await deleteAllData(client, bungieMembershipId);
-
-    // TODO: pass a list of keys that are being set to default?
-    await replaceSettings(client, appId, bungieMembershipId, settings);
-
-    // TODO: query first so we can delete after?
-    for (const loadout of loadouts) {
-      // For now, ignore ancient loadouts
-      if (!loadout.platformMembershipId || !loadout.destinyVersion) {
-        continue;
-      }
-      await updateLoadout(
-        client,
-        appId,
-        bungieMembershipId,
-        loadout.platformMembershipId,
-        loadout.destinyVersion,
-        loadout,
-      );
-    }
-
-    // TODO: query first so we can delete after?
-    for (const annotation of itemAnnotations) {
-      await updateItemAnnotation(
-        client,
-        appId,
-        bungieMembershipId,
-        annotation.platformMembershipId,
-        annotation.destinyVersion,
-        annotation,
-      );
-    }
-
-    for (const tag of itemHashTags) {
-      await updateItemHashTag(client, appId, bungieMembershipId, tag);
-    }
-
-    if (Array.isArray(triumphs)) {
-      for (const triumphData of triumphs) {
-        if (Array.isArray(triumphData?.triumphs)) {
-          for (const triumph of triumphData.triumphs) {
-            trackTriumph(
-              client,
-              appId,
-              bungieMembershipId,
-              triumphData.platformMembershipId,
-              triumph,
-            );
-            numTriumphs++;
-          }
-        }
-      }
-    }
-
-    for (const search of searches) {
-      importSearch(
-        client,
-        appId,
-        bungieMembershipId,
-        search.destinyVersion,
-        search.search.query,
-        search.search.saved,
-        search.search.lastUsage,
-        search.search.usageCount,
-        search.search.type ?? SearchType.Item,
-      );
-    }
-  });
-
-  return numTriumphs;
-}
-
 export async function statelyImport(
   bungieMembershipId: number,
   platformMembershipIds: string[],
@@ -271,7 +151,6 @@ export async function statelyImport(
   // Put the settings in first since it's in a different group
   await client.put({
     item: settingsItem,
-    mustNotExist: true,
   });
   // OK now put them in as fast as we can
   for (const batch of batches(items)) {

api/routes/update.ts

@@ -154,7 +154,7 @@ export const updateHandler = asyncHandler(async (req, res) => {
       triumphs,
       searches,
       itemHashTags,
-      !migrationState.lastError,
+      migrationState.attemptCount > 0 || Boolean(migrationState.lastError),
     );
   };
 

api/stately/init/migrate-users.ts

@@ -0,0 +1,34 @@
+import { chunk } from 'es-toolkit';
+import { readTransaction } from '../../db/index.js';
+import { getUsersToMigrate } from '../../db/migration-state-queries.js';
+import { delay } from '../../utils.js';
+import { migrateUser } from '../migrator/user.js';
+
+while (true) {
+  try {
+    const bungieMembershipIds = await readTransaction(async (client) => getUsersToMigrate(client));
+    if (bungieMembershipIds.length === 0) {
+      console.log('No users to migrate');
+      break;
+    }
+    for (const idChunk of chunk(bungieMembershipIds, 10)) {
+      await Promise.all(
+        idChunk.map(async (bungieMembershipId) => {
+          try {
+            await migrateUser(bungieMembershipId);
+            console.log(`Migrated user ${bungieMembershipId}`);
+          } catch (e) {
+            if (e instanceof Error) {
+              console.error(`Error migrating user ${bungieMembershipId}: ${e}`);
+            }
+          }
+        }),
+      );
+    }
+  } catch (e) {
+    if (e instanceof Error) {
+      console.error(`Error getting users to migrate: ${e}`);
+    }
+    await delay(1000);
+  }
+}

api/stately/loadouts-queries.ts

@@ -264,18 +264,22 @@ export function convertLoadoutCommonFieldsToStately(
   MessageInitShape<typeof LoadoutSchema> | MessageInitShape<typeof LoadoutShareSchema>,
   'id' | '$typeName'
 > {
-  return {
+  const out = {
     destinyVersion,
     profileId: BigInt(platformMembershipId),
-    name: loadout.name,
+    name: loadout.name || 'Unnamed',
     classType: loadout.classType as number,
     equipped: (loadout.equipped || []).map(convertLoadoutItemToStately),
     unequipped: (loadout.unequipped || []).map(convertLoadoutItemToStately),
     notes: loadout.notes,
     parameters: convertLoadoutParametersToStately(loadout.parameters),
-    createdAt: BigInt(loadout.createdAt ?? 0n),
-    lastUpdatedAt: BigInt(loadout.lastUpdatedAt ?? 0n),
+    createdAt: BigInt(loadout.createdAt ? new Date(loadout.createdAt).getTime() : 0n),
+    lastUpdatedAt: BigInt(loadout.lastUpdatedAt ? new Date(loadout.lastUpdatedAt).getTime() : 0n),
   };
+  if (out.lastUpdatedAt < out.createdAt) {
+    out.lastUpdatedAt = out.createdAt;
+  }
+  return out;
 }
 
 function convertLoadoutItemToStately(item: LoadoutItem): StatelyLoadoutItem {
@@ -316,7 +320,7 @@ export function convertLoadoutParametersToStately(
       modsByBucket: modsByBucket
         ? Object.entries(modsByBucket).map(([bucketHash, modHashes]) => ({
             bucketHash: Number(bucketHash),
-            modHashes,
+            modHashes: modHashes.filter((h) => Number.isInteger(h)),
           }))
         : undefined,
     };
@@ -328,8 +332,8 @@ export function statConstraintsToStately(statConstraints: StatConstraint[] | und
   return statConstraints && statConstraints.length > 0
     ? statConstraints.map((c) => ({
         statHash: c.statHash,
-        minTier: Math.max(0, c.minTier ?? 0),
-        maxTier: Math.min(c.maxTier ?? 10, 10),
+        minTier: Math.max(0, Math.floor(c.minTier ?? 0)),
+        maxTier: Math.min(Math.ceil(c.maxTier ?? 10), 10),
       }))
     : [];
 }