Improve vault start pnl query. (#2664)

(cherry picked from commit 81ec99c)

Author: vincentwschau

indexer/packages/postgres/src/stores/vault-pnl-ticks-view.ts

@@ -62,3 +62,27 @@ export async function getVaultsPnl(
 
   return result.rows;
 }
+
+export async function getLatestVaultPnl(): Promise<PnlTicksFromDatabase[]> {
+  const result: {
+    rows: PnlTicksFromDatabase[],
+  } = await knexReadReplica.getConnection().raw(
+    `
+    SELECT DISTINCT ON ("subaccountId")
+      "id",
+      "subaccountId",
+      "equity",
+      "totalPnl",
+      "netTransfers",
+      "createdAt",
+      "blockHeight",
+      "blockTime"
+    FROM ${VAULT_HOURLY_PNL_VIEW}
+    ORDER BY "subaccountId", "blockTime" DESC;
+    `,
+  ) as unknown as {
+    rows: PnlTicksFromDatabase[],
+  };
+
+  return result.rows;
+}

indexer/services/comlink/__tests__/controllers/api/v4/vault-controller.test.ts

@@ -24,6 +24,7 @@ import { getFixedRepresentation, sendRequest } from '../../../helpers/helpers';
 import { DateTime, Settings } from 'luxon';
 import Big from 'big.js';
 import config from '../../../../src/config';
+import { clearVaultStartPnl, startVaultStartPnlCache } from '../../../../src/caches/vault-start-pnl';
 
 describe('vault-controller#V4', () => {
   const latestBlockHeight: string = '25';
@@ -131,6 +132,7 @@ describe('vault-controller#V4', () => {
       await dbHelpers.clearData();
       await VaultPnlTicksView.refreshDailyView();
       await VaultPnlTicksView.refreshHourlyView();
+      clearVaultStartPnl();
       config.VAULT_PNL_HISTORY_HOURS = vaultPnlHistoryHoursPrev;
       config.VAULT_LATEST_PNL_TICK_WINDOW_HOURS = vaultPnlLastPnlWindowPrev;
       config.VAULT_PNL_START_DATE = vaultPnlStartDatePrev;
@@ -653,6 +655,7 @@ describe('vault-controller#V4', () => {
     }
     await VaultPnlTicksView.refreshDailyView();
     await VaultPnlTicksView.refreshHourlyView();
+    await startVaultStartPnlCache();
 
     return createdTicks;
   }

indexer/services/comlink/src/caches/vault-start-pnl.ts

@@ -0,0 +1,33 @@
+import { NodeEnv } from '@dydxprotocol-indexer/base';
+import {
+  PnlTicksFromDatabase,
+  PnlTicksTable,
+} from '@dydxprotocol-indexer/postgres';
+import _ from 'lodash';
+
+import { getVaultMapping, getVaultPnlStartDate } from '../lib/helpers';
+import { VaultMapping } from '../types';
+
+let vaultStartPnl: PnlTicksFromDatabase[] = [];
+
+export async function startVaultStartPnlCache(): Promise<void> {
+  const vaultMapping: VaultMapping = await getVaultMapping();
+  vaultStartPnl = await PnlTicksTable.getLatestPnlTick(
+    _.keys(vaultMapping),
+    // Add a buffer of 10 minutes to get the first PnL tick for PnL data as PnL ticks aren't
+    // created exactly on the hour.
+    getVaultPnlStartDate().plus({ minutes: 10 }),
+  );
+}
+
+export function getVaultStartPnl(): PnlTicksFromDatabase[] {
+  return vaultStartPnl;
+}
+
+export function clearVaultStartPnl(): void {
+  if (process.env.NODE_ENV !== NodeEnv.TEST) {
+    throw Error('cannot clear vault start pnl cache outside of test environment');
+  }
+
+  vaultStartPnl = [];
+}

indexer/services/comlink/src/controllers/api/v4/vault-controller.ts

@@ -1,7 +1,6 @@
-import { logger, stats } from '@dydxprotocol-indexer/base';
+import { stats } from '@dydxprotocol-indexer/base';
 import {
   PnlTicksFromDatabase,
-  PnlTicksTable,
   perpetualMarketRefresher,
   PerpetualMarketFromDatabase,
   USDC_ASSET_ID,
@@ -22,7 +21,6 @@ import {
   BlockFromDatabase,
   FundingIndexUpdatesTable,
   PnlTickInterval,
-  VaultTable,
   VaultFromDatabase,
   MEGAVAULT_SUBACCOUNT_ID,
   TransferFromDatabase,
@@ -42,10 +40,13 @@ import {
 } from 'tsoa';
 
 import { getReqRateLimiter } from '../../../caches/rate-limiters';
+import { getVaultStartPnl } from '../../../caches/vault-start-pnl';
 import config from '../../../config';
 import {
   aggregateHourlyPnlTicks,
   getSubaccountResponse,
+  getVaultMapping,
+  getVaultPnlStartDate,
   handleControllerError,
 } from '../../../lib/helpers';
 import { rateLimiterMiddleware } from '../../../lib/rate-limit';
@@ -108,7 +109,7 @@ class VaultController extends Controller {
       getVaultPositions(vaultSubaccounts),
       BlockTable.getLatest(),
       getMainSubaccountEquity(),
-      getLatestPnlTick(vaultSubaccountIdsWithMainSubaccount, _.values(vaultSubaccounts)),
+      getLatestPnlTick(_.values(vaultSubaccounts)),
       getFirstMainVaultTransferDateTime(),
     ]);
     stats.timing(
@@ -162,7 +163,7 @@ class VaultController extends Controller {
       getVaultSubaccountPnlTicks(_.keys(vaultSubaccounts), getResolution(resolution)),
       getVaultPositions(vaultSubaccounts),
       BlockTable.getLatest(),
-      getLatestPnlTicks(_.keys(vaultSubaccounts)),
+      getLatestPnlTicks(),
     ]);
     const latestTicksBySubaccountId: Dictionary<PnlTicksFromDatabase> = _.keyBy(
       latestTicks,
@@ -348,27 +349,13 @@ async function getVaultSubaccountPnlTicks(
     windowSeconds = config.VAULT_PNL_HISTORY_HOURS * 60 * 60; // hours to seconds
   }
 
-  const [
-    pnlTicks,
-    adjustByPnlTicks,
-  ] : [
-    PnlTicksFromDatabase[],
-    PnlTicksFromDatabase[],
-  ] = await Promise.all([
-    VaultPnlTicksView.getVaultsPnl(
-      resolution,
-      windowSeconds,
-      getVaultPnlStartDate(),
-    ),
-    PnlTicksTable.getLatestPnlTick(
-      vaultSubaccountIds,
-      // Add a buffer of 10 minutes to get the first PnL tick for PnL data as PnL ticks aren't
-      // created exactly on the hour.
-      getVaultPnlStartDate().plus({ minutes: 10 }),
-    ),
-  ]);
+  const pnlTicks: PnlTicksFromDatabase[] = await VaultPnlTicksView.getVaultsPnl(
+    resolution,
+    windowSeconds,
+    getVaultPnlStartDate(),
+  );
 
-  return adjustVaultPnlTicks(pnlTicks, adjustByPnlTicks);
+  return adjustVaultPnlTicks(pnlTicks, getVaultStartPnl());
 }
 
 async function getVaultPositions(
@@ -559,60 +546,26 @@ function getPnlTicksWithCurrentTick(
   return pnlTicks.concat([currentTick]);
 }
 
-export async function getLatestPnlTicks(
-  vaultSubaccountIds: string[],
-): Promise<PnlTicksFromDatabase[]> {
-  const [
-    latestPnlTicks,
-    adjustByPnlTicks,
-  ] : [
-    PnlTicksFromDatabase[],
-    PnlTicksFromDatabase[],
-  ] = await Promise.all([
-    PnlTicksTable.getLatestPnlTick(
-      vaultSubaccountIds,
-      DateTime.now().toUTC(),
-    ),
-    PnlTicksTable.getLatestPnlTick(
-      vaultSubaccountIds,
-      // Add a buffer of 10 minutes to get the first PnL tick for PnL data as PnL ticks aren't
-      // created exactly on the hour.
-      getVaultPnlStartDate().plus({ minutes: 10 }),
-    ),
-  ]);
+export async function getLatestPnlTicks(): Promise<PnlTicksFromDatabase[]> {
+  const latestPnlTicks: PnlTicksFromDatabase[] = await VaultPnlTicksView.getLatestVaultPnl();
   const adjustedPnlTicks: PnlTicksFromDatabase[] = adjustVaultPnlTicks(
     latestPnlTicks,
-    adjustByPnlTicks,
+    getVaultStartPnl(),
   );
   return adjustedPnlTicks;
 }
 
 export async function getLatestPnlTick(
-  vaultSubaccountIds: string[],
   vaults: VaultFromDatabase[],
 ): Promise<PnlTicksFromDatabase | undefined> {
-  const [
-    pnlTicks,
-    adjustByPnlTicks,
-  ] : [
-    PnlTicksFromDatabase[],
-    PnlTicksFromDatabase[],
-  ] = await Promise.all([
-    VaultPnlTicksView.getVaultsPnl(
-      PnlTickInterval.hour,
-      config.VAULT_LATEST_PNL_TICK_WINDOW_HOURS * 60 * 60,
-      getVaultPnlStartDate(),
-    ),
-    PnlTicksTable.getLatestPnlTick(
-      vaultSubaccountIds,
-      // Add a buffer of 10 minutes to get the first PnL tick for PnL data as PnL ticks aren't
-      // created exactly on the hour.
-      getVaultPnlStartDate().plus({ minutes: 10 }),
-    ),
-  ]);
+  const pnlTicks: PnlTicksFromDatabase[] = await VaultPnlTicksView.getVaultsPnl(
+    PnlTickInterval.hour,
+    config.VAULT_LATEST_PNL_TICK_WINDOW_HOURS * 60 * 60,
+    getVaultPnlStartDate(),
+  );
   const adjustedPnlTicks: PnlTicksFromDatabase[] = adjustVaultPnlTicks(
     pnlTicks,
-    adjustByPnlTicks,
+    getVaultStartPnl(),
   );
   // Aggregate and get pnl tick closest to the hour
   const aggregatedTicks: PnlTicksFromDatabase[] = aggregateVaultPnlTicks(
@@ -802,41 +755,4 @@ function adjustVaultPnlTicks(
   });
 }
 
-async function getVaultMapping(): Promise<VaultMapping> {
-  const vaults: VaultFromDatabase[] = await VaultTable.findAll(
-    {},
-    [],
-    {},
-  );
-  const vaultMapping: VaultMapping = _.zipObject(
-    vaults.map((vault: VaultFromDatabase): string => {
-      return SubaccountTable.uuid(vault.address, 0);
-    }),
-    vaults,
-  );
-  const validVaultMapping: VaultMapping = {};
-  for (const subaccountId of _.keys(vaultMapping)) {
-    const perpetual: PerpetualMarketFromDatabase | undefined = perpetualMarketRefresher
-      .getPerpetualMarketFromClobPairId(
-        vaultMapping[subaccountId].clobPairId,
-      );
-    if (perpetual === undefined) {
-      logger.warning({
-        at: 'VaultController#getVaultPositions',
-        message: `Vault clob pair id ${vaultMapping[subaccountId]} does not correspond to a ` +
-          'perpetual market.',
-        subaccountId,
-      });
-      continue;
-    }
-    validVaultMapping[subaccountId] = vaultMapping[subaccountId];
-  }
-  return validVaultMapping;
-}
-
-function getVaultPnlStartDate(): DateTime {
-  const startDate: DateTime = DateTime.fromISO(config.VAULT_PNL_START_DATE).toUTC();
-  return startDate;
-}
-
 export default router;

indexer/services/comlink/src/index.ts

@@ -5,6 +5,7 @@ import {
 } from '@dydxprotocol-indexer/base';
 import { perpetualMarketRefresher, liquidityTierRefresher } from '@dydxprotocol-indexer/postgres';
 
+import { startVaultStartPnlCache } from './caches/vault-start-pnl';
 import config from './config';
 import IndexV4 from './controllers/api/index-v4';
 import { connect as connectToRedis } from './helpers/redis/redis-controller';
@@ -42,6 +43,8 @@ async function start() {
   ]);
   wrapBackgroundTask(perpetualMarketRefresher.start(), true, 'startUpdatePerpetualMarkets');
   wrapBackgroundTask(liquidityTierRefresher.start(), true, 'startUpdateLiquidityTiers');
+  // Initialize cache for vault start PnL
+  await startVaultStartPnlCache();
 
   await connectToRedis();
   logger.info({

indexer/services/comlink/src/lib/helpers.ts

@@ -24,6 +24,7 @@ import {
   AssetFromDatabase,
   AssetColumns,
   MarketColumns,
+  VaultFromDatabase, VaultTable, perpetualMarketRefresher,
 } from '@dydxprotocol-indexer/postgres';
 import Big from 'big.js';
 import express from 'express';
@@ -47,6 +48,7 @@ import {
   PerpetualPositionWithFunding,
   Risk,
   SubaccountResponseObject,
+  VaultMapping,
 } from '../types';
 import { ZERO, ZERO_USDC_POSITION } from './constants';
 import { InvalidParamError, NotFoundError } from './errors';
@@ -720,3 +722,42 @@ export function aggregateHourlyPnlTicks(
     };
   });
 }
+
+/* ------- VAULT HELPERS ------- */
+
+export async function getVaultMapping(): Promise<VaultMapping> {
+  const vaults: VaultFromDatabase[] = await VaultTable.findAll(
+    {},
+    [],
+    {},
+  );
+  const vaultMapping: VaultMapping = _.zipObject(
+    vaults.map((vault: VaultFromDatabase): string => {
+      return SubaccountTable.uuid(vault.address, 0);
+    }),
+    vaults,
+  );
+  const validVaultMapping: VaultMapping = {};
+  for (const subaccountId of _.keys(vaultMapping)) {
+    const perpetual: PerpetualMarketFromDatabase | undefined = perpetualMarketRefresher
+      .getPerpetualMarketFromClobPairId(
+        vaultMapping[subaccountId].clobPairId,
+      );
+    if (perpetual === undefined) {
+      logger.warning({
+        at: 'get-vault-mapping',
+        message: `Vault clob pair id ${vaultMapping[subaccountId]} does not correspond to a ` +
+          'perpetual market.',
+        subaccountId,
+      });
+      continue;
+    }
+    validVaultMapping[subaccountId] = vaultMapping[subaccountId];
+  }
+  return validVaultMapping;
+}
+
+export function getVaultPnlStartDate(): DateTime {
+  const startDate: DateTime = DateTime.fromISO(config.VAULT_PNL_START_DATE).toUTC();
+  return startDate;
+}

indexer/services/comlink/src/types.ts

@@ -24,6 +24,7 @@ import {
   TradeType,
   TradingRewardAggregationPeriod,
   TransferType,
+  VaultFromDatabase,
 } from '@dydxprotocol-indexer/postgres';
 import { RedisOrder } from '@dydxprotocol-indexer/v4-protos';
 import Big from 'big.js';
@@ -691,6 +692,10 @@ export interface MegavaultHistoricalPnlRequest {
 
 export interface VaultsHistoricalPnlRequest extends MegavaultHistoricalPnlRequest {}
 
+export interface VaultMapping {
+  [subaccountId: string]: VaultFromDatabase,
+}
+
 /* ------- Affiliates Types ------- */
 export interface AffiliateMetadataRequest{
   address: string,