Merge pull request #211 from midday-ai/feature/fetch-rates

Add rates sync

apps/dashboard/package.json

@@ -16,7 +16,7 @@
     "@ai-sdk/openai": "^0.0.40",
     "@date-fns/utc": "^1.2.0",
     "@hookform/resolvers": "^3.9.0",
-    "@midday-ai/engine": "^0.1.0-alpha.20",
+    "@midday-ai/engine": "^0.1.0-alpha.22",
     "@midday/events": "workspace:*",
     "@midday/inbox": "workspace:*",
     "@midday/jobs": "workspace:*",

apps/engine/src/routes/rates/schema.ts

@@ -7,6 +7,9 @@ export const RatesSchema = z
         source: z.string().openapi({
           example: "USD",
         }),
+        date: z.string().openapi({
+          example: "2024-02-29",
+        }),
         rates: z.record(z.string(), z.number()).openapi({
           example: {
             EUR: 0.925393,

apps/engine/src/utils/rates.ts

@@ -37,6 +37,7 @@ export async function getRates() {
     )
     .map((rate) => rate.value)
     .map((value) => {
+      const date = Object.values(value).at(0);
       const currency = Object.keys(value).at(1);
 
       if (!currency) {
@@ -50,6 +51,7 @@ export async function getRates() {
 
       return {
         source: currency.toUpperCase(),
+        date,
         rates: transformKeysToUppercase(currencyData as Record<string, number>),
       };
     })

packages/jobs/package.json

@@ -17,7 +17,7 @@
     "@langchain/openai": "^0.2.5",
     "@midday/documents": "workspace:*",
     "@midday/import": "workspace:*",
-    "@midday-ai/engine": "^0.1.0-alpha.20",
+    "@midday-ai/engine": "^0.1.0-alpha.22",
     "@trigger.dev/react": "2.3.19",
     "@trigger.dev/resend": "2.3.19",
     "@trigger.dev/sdk": "2.3.19",

packages/jobs/src/index.ts

@@ -3,3 +3,4 @@ export * from "./inbox";
 export * from "./client";
 export * from "./constants";
 export * from "./bank";
+export * from "./rates";

packages/jobs/src/rates/index.ts

@@ -0,0 +1 @@
+export * from "./update";

packages/jobs/src/rates/update.ts

@@ -0,0 +1,35 @@
+import { cronTrigger } from "@trigger.dev/sdk";
+import { client, supabase } from "../client";
+import { engine } from "../utils/engine";
+import { processBatch } from "../utils/process";
+
+client.defineJob({
+  id: "exchange-rates-update",
+  name: "Exchange Rates - Update",
+  version: "0.1.1",
+  trigger: cronTrigger({
+    cron: "0 12 * * *",
+  }),
+  integrations: {
+    supabase,
+  },
+  run: async (_, io) => {
+    const rates = await engine.rates.list();
+
+    const data = rates.data.flatMap((rate) => {
+      return Object.entries(rate.rates).map(([target, value]) => ({
+        base: rate.source,
+        target: target,
+        rate: value,
+        updated_at: rate.date,
+      }));
+    });
+
+    await processBatch(data, 500, async (batch) => {
+      await io.supabase.client.from("exchange_rates").upsert(batch, {
+        onConflict: "base, target",
+        ignoreDuplicates: false,
+      });
+    });
+  },
+});

packages/jobs/src/transactions/initial-sync.ts

@@ -10,7 +10,7 @@ import { processBatch } from "../utils/process";
 import { getClassification, transformTransaction } from "../utils/transform";
 import { scheduler } from "./scheduler";
 
-const BATCH_LIMIT = 300;
+const BATCH_LIMIT = 500;
 
 client.defineJob({
   id: Jobs.TRANSACTIONS_INITIAL_SYNC,

packages/jobs/src/transactions/manual-sync.ts

@@ -9,7 +9,7 @@ import { parseAPIError } from "../utils/error";
 import { processBatch } from "../utils/process";
 import { getClassification, transformTransaction } from "../utils/transform";
 
-const BATCH_LIMIT = 300;
+const BATCH_LIMIT = 500;
 
 client.defineJob({
   id: Jobs.TRANSACTIONS_MANUAL_SYNC,

turbo.json

@@ -22,6 +22,8 @@
         "API_ROUTE_SECRET",
         "TELLER_CERTIFICATE",
         "TELLER_CERTIFICATE_PRIVATE_KEY",
+        "MIDDAY_ENGINE_ENVIRONMENT",
+        "MIDDAY_ENGINE_API_KEY",
         "PLAID_CLIENT_ID",
         "PLAID_SECRET",
         "GITHUB_TOKEN",