From 48f980e3f04b2e1a25a47fa4b8589e80e76f8b7a Mon Sep 17 00:00:00 2001 From: Aashish Dhakal <85501584+dhakalaashish@users.noreply.github.com> Date: Tue, 11 Jul 2023 22:21:11 +0545 Subject: [PATCH 1/3] Create sync-ro-users cron --- .../src/pages/api/cron/sync-ro-users.ts | 306 ++++++++++++++++++ apps/server/src/server/api/routers/cron.ts | 246 -------------- 2 files changed, 306 insertions(+), 246 deletions(-) create mode 100644 apps/server/src/pages/api/cron/sync-ro-users.ts delete mode 100644 apps/server/src/server/api/routers/cron.ts diff --git a/apps/server/src/pages/api/cron/sync-ro-users.ts b/apps/server/src/pages/api/cron/sync-ro-users.ts new file mode 100644 index 000000000..cdda6e792 --- /dev/null +++ b/apps/server/src/pages/api/cron/sync-ro-users.ts @@ -0,0 +1,306 @@ +// to execute this handler, access the endpoint: http://localhost:3000/api/cron/sync-ro-users +// Sync RO Users CRON job +// This cron job runs every day and syncs sites, projects, and profile data for RO users. + +import { type NextApiRequest, type NextApiResponse } from "next"; +import { prisma } from '../../../server/db' +import { env } from "../../../env.mjs"; +import { logger } from "../../../../src/server/logger"; +import { fetchAllProjectsWithSites } from "../../../../src/utils/fetch"; +import moment from 'moment'; +import { Prisma, Project } from "@prisma/client"; + +export default async function syncROUsers(req: NextApiRequest, res: NextApiResponse) { + // Verify the 'cron_key' in the request headers before proceeding + if (env.CRON_KEY) { + const cronKey = req.query['cron_key']; + if (!cronKey || cronKey !== env.CRON_KEY) { + res.status(403).json({ message: "Unauthorized: Invalid Cron Key" }); + return; + } + } + + // Fetch projects from PP API + const allProjectsPPWebApp = await fetchAllProjectsWithSites(); + + // Fetch RO Users from the database and their respective remoteIds + const ROUsers = await prisma.user.findMany({ + where: { + isPlanetRO: true, + }, + select: { + remoteId: true, + id: true, + }, + }); + + const userRemoteIdList = ROUsers.map(user => user.remoteId); + + // Create a map to associate remoteId with userId + const mapRemoteIdWithUserId = new Map(); + ROUsers.forEach(user => { + mapRemoteIdWithUserId.set(user.remoteId, user.id); + }); + + // Filter projects from PP API to include only those related to RO users + const projectsPP = allProjectsPPWebApp.filter(projectPP => + userRemoteIdList.includes(projectPP.tpo.id) + ); + + // Fetch corresponding projects for the RO users from the database + const userIdList = ROUsers.map(user => user.id); + const projectsFA = await prisma.project.findMany({ + where: { + userId: { + in: userIdList + } + } + }); + + // Identify projects in the database that are not present in PP API + const projectsIdsFA = projectsFA.map((project) => project.id); + const deleteFAProjectIds = projectsIdsFA.filter( + (projectId) => !projectsPP.some((project) => project.id === projectId) + ); + + // Soft delete sites associated with these projects and delete these projects + if (deleteFAProjectIds.length) { + await prisma.$transaction(async (prisma) => { + await prisma.site.updateMany({ + where: { + projectId: { + in: deleteFAProjectIds, + }, + }, + data: { + deletedAt: new Date(), + projectId: null, + }, + }); + logger(`Soft deleted sites and deleted projects with ids: ${deleteFAProjectIds.join(", ")}`, 'info',); + await prisma.project.deleteMany({ + where: { + id: { + in: deleteFAProjectIds, + }, + }, + }); + }); + } + + // Identify new projects from PP API that are not present in the database + const newProjectsPP = projectsPP.filter( + (projectPP) => !projectsFA.some((projectFA) => projectFA.id === projectPP.id) + ); + + // Add those projects to the database, and all the sites inside of it. + if (newProjectsPP.length > 0) { + // Prepare the projects and sites data for bulk creation + const newProjectData: Project[] = []; + const newSiteData: Prisma.SiteCreateManyInput[] = []; + + for (const projectPP of newProjectsPP) { + const { id: projectId, name: projectName, slug: projectSlug, lastUpdated: projectLastUpdated, sites: sitesFromPP } = projectPP; + const tpoId = projectPP.tpo.id; + const userId = mapRemoteIdWithUserId.get(tpoId); + + // Add the new project to the array for bulk creation + newProjectData.push({ + id: projectId, + name: projectName, + slug: projectSlug, + lastUpdated: new Date(), + userId: userId, + }); + + // Iterate through the sites of the new project + for (const siteFromPP of sitesFromPP) { + const { geometry, properties } = siteFromPP; + const { id: siteIdFromPP, lastUpdated: siteLastUpdatedFromPP } = properties; + + if (geometry && geometry.type) { + // Add the new site to the array for bulk creation + newSiteData.push({ + id: siteIdFromPP, + type: geometry.type, + geometry: geometry, + radius: 0, // Use the actual radius value if available + projectId: projectId, + lastUpdated: new Date(), + userId: userId, + }); + } else { + // Handle the case where geometry or type is null + console.log(`Skipping site with id ${siteIdFromPP} due to null geometry or type.`); + } + } + } + + // Add the new projects and sites to the database in a transaction + await prisma.$transaction(async (prisma) => { + const createdProjects = await prisma.project.createMany({ + data: newProjectData, + }); + const createdSites = await prisma.site.createMany({ + data: newSiteData, + }); + }) + } + + + // Fetch all sites from the database for the projects in projectsPP + const sitesFA = await prisma.site.findMany({ + where: { + projectId: { + in: projectsPP.map((project) => project.id), + }, + }, + }); + + const newProjectsFA = await prisma.project.findMany({ + where: { + userId: { + in: userIdList + } + } + }); + + // Create a list of site IDs from PP + const ppSiteIdList: string[] = []; + for (const projectPP of projectsPP) { + for (const siteFromPP of projectPP.sites) { + ppSiteIdList.push(siteFromPP.properties.id); + } + } + + // Perform bulk creations, bulk updates, and bulk deletions for sites + await prisma.$transaction(async (prisma) => { + const createPromises = []; + const updatePromises = []; + const deletePromises = []; + + // Identify sites in the database that are not present in PP API and soft delete them + for (const siteFA of sitesFA) { + if (!ppSiteIdList.includes(siteFA.id)) { + deletePromises.push( + prisma.site.update({ + where: { + id: siteFA.id, + }, + data: { + projectId: null, + deletedAt: new Date(), + }, + }) + ); + } + } + logger(`Deleting ${deletePromises.length} sites not present in the PP API`, 'info',); + + // For each project in PP API, identify sites in the database that need to be updated or created + for (const projectPP of projectsPP) { + const { + sites: sitesFromPPProject, + id: projectId, + lastUpdated: projectLastUpdated, + name: projectNameFormPP, + slug: projectSlugFormPP, + } = projectPP; + const projectLastUpdatedFormPP = moment(projectLastUpdated, 'YYYY-MM-DD HH:mm:ss').toDate(); // Convert to Date object + const projectFromDatabase = newProjectsFA.find((project) => project.id === projectId); + + // If the project has been updated in the PP API, update the project and its sites in the database + if (projectFromDatabase!.lastUpdated?.getTime() !== projectLastUpdatedFormPP.getTime()) { + updatePromises.push( + prisma.project.update({ + where: { + id: projectId, + }, + data: { + lastUpdated: projectLastUpdatedFormPP, + name: projectNameFormPP, + slug: projectSlugFormPP, + }, + }) + ); + + const tpoId = projectPP.tpo.id; + const userId = mapRemoteIdWithUserId.get(tpoId); + + const siteIdsFromPP = sitesFromPPProject.map((site) => site.properties.id); + + for (const siteFromPP of sitesFromPPProject) { + const { geometry, properties } = siteFromPP; + const { id: siteIdFromPP, lastUpdated: siteLastUpdated } = properties; + const siteLastUpdatedFromPP = moment(siteLastUpdated.date, siteLastUpdated.timezone).utc().toDate(); + + if (geometry && geometry.type) { + const siteFromDatabase = sitesFA.find((site) => site.id === siteIdFromPP); + + const radius = 0; + + // If the site does not exist in the database, create a new site + if (!siteFromDatabase) { + createPromises.push( + prisma.site.create({ + data: { + id: siteIdFromPP, + type: geometry.type, + geometry: geometry, + radius: radius, + userId: userId, + projectId: projectId, + lastUpdated: new Date(), + }, + }) + ); + // If the site exists in the database but has been updated in the PP API, update the site in the database + } else if (siteFromDatabase.lastUpdated?.getTime() !== siteLastUpdatedFromPP.getTime()) { + updatePromises.push( + prisma.site.update({ + where: { + id: siteIdFromPP, + }, + data: { + type: geometry.type, + geometry: geometry, + radius: radius, + lastUpdated: siteLastUpdatedFromPP, + }, + }) + ); + } + } else { + // Handle the case where geometry or type is null + logger(`Skipping site with id ${siteIdFromPP} due to null geometry or type.`, 'info',); + } + } + } + } + + // Execute all promises + const createResults = await Promise.all(createPromises); + const updateResults = await Promise.all(updatePromises); + const deleteResults = await Promise.all(deletePromises); + + const createCount = createResults.length; // Number of created items + const updateCount = updateResults.length; // Number of updated items + const deleteCount = deleteResults.length; // Number of deleted items + + logger(`Created ${createCount} items.`, 'info',); + logger(`Updated ${updateCount} items.`, 'info',); + logger(`Deleted ${deleteCount} items.`, 'info',); + + res.status(200).json({ + message: "Success! Data has been synced for RO Users!", + status: 200, + results: { created: createCount, updated: updateCount, deleted: deleteCount }, + }); + }).catch(error => { + logger(`Error in transaction: ${error}`, "error"); + res.status(500).json({ + message: "An error occurred while syncing data for RO Users.", + status: 500 + }); + }); +} diff --git a/apps/server/src/server/api/routers/cron.ts b/apps/server/src/server/api/routers/cron.ts deleted file mode 100644 index 793d28ee6..000000000 --- a/apps/server/src/server/api/routers/cron.ts +++ /dev/null @@ -1,246 +0,0 @@ -import { TRPCError } from "@trpc/server"; -import { createTRPCRouter, publicProcedure } from "../trpc"; -import { fetchAllProjectsWithSites } from "../../../utils/fetch" -import { subtractDays } from "../../../utils/date"; - -// TODO: test all three procedures -export const cronRouter = createTRPCRouter({ - - // TODO: debug the variables when fetched from pp - syncProjectsAndSitesForAllROUsers: publicProcedure - .mutation(async ({ ctx }) => { - // Get all the projects from PP - const projectsFromPP = await fetchAllProjectsWithSites(); - // Get all projects from DB, and only ROs have projects, normal user cannot make projects - const projectsFromDB = await ctx.prisma.project.findMany(); - // Filter PP list to only contain projects that are in DB - const ppListFiltered = projectsFromPP.filter((projectFromPP) => - projectsFromDB.some((projectFromDB) => projectFromDB.id === projectFromPP.id) - ); - - // Check for projects in DB that are not in PP and delete them - const dbProjectsIds = projectsFromDB.map((project) => project.id); - const projectsIdsToDelete = dbProjectsIds.filter( - (projectId) => !ppListFiltered.some((project) => project.id === projectId) - ); - - if (projectsIdsToDelete.length) { - await ctx.prisma.$transaction(async (prisma) => { - await prisma.site.updateMany({ - where: { - projectId: { - in: projectsIdsToDelete, - }, - }, - data: { - deletedAt: new Date(), - projectId: null, - }, - }); - - await prisma.project.deleteMany({ - where: { - id: { - in: projectsIdsToDelete, - }, - }, - }); - }); - } - - // Create a mapping of project IDs to project lastUpdated values from PP - const ppProjectLastUpdatedMap = new Map(); - for (const projectFromPP of ppListFiltered) { - ppProjectLastUpdatedMap.set(projectFromPP.id, projectFromPP.lastUpdated); - } - - // Fetch all sites from the DB for the projects in ppListFiltered - const dbSites = await ctx.prisma.site.findMany({ - where: { - projectId: { - in: ppListFiltered.map((project) => project.id), - }, - }, - }); - - // Create a mapping of site IDs to site lastUpdated values from PP - const ppSiteLastUpdatedMap = new Map(); - for (const projectFromPP of ppListFiltered) { - for (const siteFromPP of projectFromPP.sites) { - ppSiteLastUpdatedMap.set(siteFromPP.properties.id, siteFromPP.properties.lastUpdated.date); - } - } - - // Perform bulk creations, bulk updates, and bulk deletions for sites - await ctx.prisma.$transaction(async (prisma) => { - const createPromises = []; - const updatePromises = []; - const deletePromises = []; - - for (const dbSite of dbSites) { - if (!ppSiteLastUpdatedMap.has(dbSite.id)) { - // Site not found in PP, delete it - deletePromises.push( - prisma.site.update({ - where: { - id: dbSite.id, - }, - data: { - projectId: null, - deletedAt: new Date(), - }, - }) - ); - } - } - - for (const projectFromPP of ppListFiltered) { - const { - sites: sitesFromPPProject, - id: projectId, - lastUpdated: projectLastUpdatedFormPP, - name: projectNameFormPP, - slug: projectSlugFormPP, - } = projectFromPP; - - const projectFromDatabase = projectsFromDB.find((project) => project.id === projectId); - - if (projectFromDatabase.lastUpdated !== projectLastUpdatedFormPP) { - // Project exists and last updated has changed, update the entire project and sites - updatePromises.push( - prisma.project.update({ - where: { - id: projectId, - }, - data: { - lastUpdated: projectLastUpdatedFormPP, - name: projectNameFormPP, - slug: projectSlugFormPP, - }, - }) - ); - - const tpoId = projectFromPP.tpo.id; - const siteIdsFromPP = sitesFromPPProject.map((site) => site.properties.id); - - for (const siteFromPP of sitesFromPPProject) { - const { geometry, properties } = siteFromPP; - const { id: siteIdFromPP, lastUpdated: siteLastUpdatedFromPP } = properties; - - if (geometry && geometry.type) { - const siteFromDatabase = dbSites.find((site) => site.id === siteIdFromPP); - - const radius = 0; - - if (!siteFromDatabase) { - // Site does not exist in the database, create a new site - createPromises.push( - prisma.site.create({ - data: { - id: siteIdFromPP, - type: geometry.type, - geometry: geometry, - radius: radius, - userId: tpoId, - projectId: projectId, - lastUpdated: siteLastUpdatedFromPP.date, - }, - }) - ); - } else if (siteFromDatabase.lastUpdated !== siteLastUpdatedFromPP.date) { - // Site exists in the database but last updated has changed, update the site - updatePromises.push( - prisma.site.update({ - where: { - id: siteIdFromPP, - }, - data: { - type: geometry.type, - geometry: geometry, - radius: radius, - lastUpdated: siteLastUpdatedFromPP.date, - }, - }) - ); - } - } else { - // Handle the case where geometry or type is null - console.log(`Skipping site with id ${siteIdFromPP} due to null geometry or type.`); - } - } - } - } - const createResults = await Promise.all(createPromises); - const updateResults = await Promise.all(updatePromises); - const deleteResults = await Promise.all(deletePromises); - - const createCount = createResults.length; // Number of created items - const updateCount = updateResults.length; // Number of updated items - const deleteCount = deleteResults.length; // Number of deleted items - - console.log('Create Count:', createCount); - console.log('Update Count:', updateCount); - console.log('Delete Count:', deleteCount); - - return { created: createCount, updated: updateCount, deleted: deleteCount }; - }); - }), - - - - permanentlyDeleteUsers: publicProcedure - .mutation(async ({ ctx }) => { - await ctx.prisma.$transaction(async (prisma) => { - const usersToDelete = await prisma.user.findMany({ - where: { - deletedAt: { - not: null, - lt: subtractDays(new Date(), 7), - }, - }, - select: { - id: true, - }, - }); - - const userIdsToDelete = usersToDelete.map((user) => user.id); - - if (userIdsToDelete.length > 0) { - await prisma.site.deleteMany({ - where: { - userId: { - in: userIdsToDelete, - }, - }, - }); - - await prisma.alertMethod.deleteMany({ - where: { - userId: { - in: userIdsToDelete, - }, - }, - }); - - await prisma.project.deleteMany({ - where: { - userId: { - in: userIdsToDelete, - }, - }, - }); - - await prisma.user.deleteMany({ - where: { - id: { - in: userIdsToDelete, - }, - }, - }); - } - }); - - return { success: true }; - }), - -}) \ No newline at end of file From 6802d4bb52d39929020abbebcee251a37839f4fb Mon Sep 17 00:00:00 2001 From: Aashish Dhakal <85501584+dhakalaashish@users.noreply.github.com> Date: Wed, 12 Jul 2023 21:02:25 +0545 Subject: [PATCH 2/3] Clean Code, Upgrade prisma version Add type and null check to cron Clean code --- apps/server/package.json | 6 +- .../src/pages/api/cron/sync-ro-users.ts | 147 +++++++++--------- apps/server/src/server/api/root.ts | 2 - yarn.lock | 36 ++--- 4 files changed, 96 insertions(+), 95 deletions(-) diff --git a/apps/server/package.json b/apps/server/package.json index 1ca7ab8c6..7dfd1eb84 100644 --- a/apps/server/package.json +++ b/apps/server/package.json @@ -22,7 +22,7 @@ "dependencies": { "@logtail/node": "^0.4.0", "@planet-sdk/common": "^0.1.11", - "@prisma/client": "^4.15.0", + "@prisma/client": "^5.0.0", "@sentry/nextjs": "^7.51.2", "@sentry/profiling-node": "^0.3.0", "@tanstack/react-query": "^4.20.2", @@ -42,7 +42,7 @@ "nodemailer": "^6.9.1", "pg-promise": "^11.4.3", "phone": "^3.1.37", - "prisma": "^4.15.0", + "prisma": "^5.0.0", "react-cookie": "^4.1.1", "react-lottie": "^1.2.3", "react-map-gl": "7.1.0-beta.3", @@ -60,7 +60,7 @@ "@typescript-eslint/parser": "^5.53.0", "eslint": "^8.34.0", "eslint-config-next": "^13.2.1", - "prisma": "^4.15.0", + "prisma": "^5.0.0", "tsconfig": "*", "typescript": "^5.0.3" }, diff --git a/apps/server/src/pages/api/cron/sync-ro-users.ts b/apps/server/src/pages/api/cron/sync-ro-users.ts index cdda6e792..31df807d9 100644 --- a/apps/server/src/pages/api/cron/sync-ro-users.ts +++ b/apps/server/src/pages/api/cron/sync-ro-users.ts @@ -9,6 +9,7 @@ import { logger } from "../../../../src/server/logger"; import { fetchAllProjectsWithSites } from "../../../../src/utils/fetch"; import moment from 'moment'; import { Prisma, Project } from "@prisma/client"; +import { type TreeProjectExtended } from '@planet-sdk/common' export default async function syncROUsers(req: NextApiRequest, res: NextApiResponse) { // Verify the 'cron_key' in the request headers before proceeding @@ -21,7 +22,7 @@ export default async function syncROUsers(req: NextApiRequest, res: NextApiRespo } // Fetch projects from PP API - const allProjectsPPWebApp = await fetchAllProjectsWithSites(); + const allProjectsPPWebApp: TreeProjectExtended[] = await fetchAllProjectsWithSites(); // Fetch RO Users from the database and their respective remoteIds const ROUsers = await prisma.user.findMany({ @@ -63,7 +64,7 @@ export default async function syncROUsers(req: NextApiRequest, res: NextApiRespo (projectId) => !projectsPP.some((project) => project.id === projectId) ); - // Soft delete sites associated with these projects and delete these projects + // Dissociate sites associated with these projects and delete the projects if (deleteFAProjectIds.length) { await prisma.$transaction(async (prisma) => { await prisma.site.updateMany({ @@ -73,11 +74,10 @@ export default async function syncROUsers(req: NextApiRequest, res: NextApiRespo }, }, data: { - deletedAt: new Date(), projectId: null, }, }); - logger(`Soft deleted sites and deleted projects with ids: ${deleteFAProjectIds.join(", ")}`, 'info',); + logger(`Deleted projects with ids: ${deleteFAProjectIds.join(", ")}`, 'info',); await prisma.project.deleteMany({ where: { id: { @@ -113,25 +113,28 @@ export default async function syncROUsers(req: NextApiRequest, res: NextApiRespo userId: userId, }); + // If sitesFromPP is not undefined, and its length is greater than 0, // Iterate through the sites of the new project - for (const siteFromPP of sitesFromPP) { - const { geometry, properties } = siteFromPP; - const { id: siteIdFromPP, lastUpdated: siteLastUpdatedFromPP } = properties; - - if (geometry && geometry.type) { - // Add the new site to the array for bulk creation - newSiteData.push({ - id: siteIdFromPP, - type: geometry.type, - geometry: geometry, - radius: 0, // Use the actual radius value if available - projectId: projectId, - lastUpdated: new Date(), - userId: userId, - }); - } else { - // Handle the case where geometry or type is null - console.log(`Skipping site with id ${siteIdFromPP} due to null geometry or type.`); + if (sitesFromPP && sitesFromPP.length > 0) { + for (const siteFromPP of sitesFromPP) { + const { geometry, properties } = siteFromPP; + const { id: siteIdFromPP, lastUpdated: siteLastUpdatedFromPP } = properties; + + if (geometry && geometry.type) { + // Add the new site to the array for bulk creation + newSiteData.push({ + id: siteIdFromPP, + type: geometry.type, + geometry: geometry, + radius: 0, + projectId: projectId, + lastUpdated: new Date(), + userId: userId, + }); + } else { + // Handle the case where geometry or type is null + logger(`Skipping site with id ${siteIdFromPP} due to null geometry or type.`, 'info',); + } } } } @@ -168,8 +171,10 @@ export default async function syncROUsers(req: NextApiRequest, res: NextApiRespo // Create a list of site IDs from PP const ppSiteIdList: string[] = []; for (const projectPP of projectsPP) { - for (const siteFromPP of projectPP.sites) { - ppSiteIdList.push(siteFromPP.properties.id); + if (projectPP.sites && projectPP.sites.length > 0) { // Checking if sites is not null before accessing it + for (const siteFromPP of projectPP.sites) { + ppSiteIdList.push(siteFromPP.properties.id); + } } } @@ -227,52 +232,52 @@ export default async function syncROUsers(req: NextApiRequest, res: NextApiRespo const tpoId = projectPP.tpo.id; const userId = mapRemoteIdWithUserId.get(tpoId); - const siteIdsFromPP = sitesFromPPProject.map((site) => site.properties.id); - - for (const siteFromPP of sitesFromPPProject) { - const { geometry, properties } = siteFromPP; - const { id: siteIdFromPP, lastUpdated: siteLastUpdated } = properties; - const siteLastUpdatedFromPP = moment(siteLastUpdated.date, siteLastUpdated.timezone).utc().toDate(); - - if (geometry && geometry.type) { - const siteFromDatabase = sitesFA.find((site) => site.id === siteIdFromPP); - - const radius = 0; - - // If the site does not exist in the database, create a new site - if (!siteFromDatabase) { - createPromises.push( - prisma.site.create({ - data: { - id: siteIdFromPP, - type: geometry.type, - geometry: geometry, - radius: radius, - userId: userId, - projectId: projectId, - lastUpdated: new Date(), - }, - }) - ); - // If the site exists in the database but has been updated in the PP API, update the site in the database - } else if (siteFromDatabase.lastUpdated?.getTime() !== siteLastUpdatedFromPP.getTime()) { - updatePromises.push( - prisma.site.update({ - where: { - id: siteIdFromPP, - }, - data: { - type: geometry.type, - geometry: geometry, - radius: radius, - lastUpdated: siteLastUpdatedFromPP, - }, - }) - ); + if (sitesFromPPProject && sitesFromPPProject.length > 0) { + for (const siteFromPP of sitesFromPPProject) { + const { geometry, properties } = siteFromPP; + const { id: siteIdFromPP, lastUpdated: siteLastUpdated } = properties; + const siteLastUpdatedFromPP = moment(siteLastUpdated.date, siteLastUpdated.timezone).utc().toDate(); + + if (geometry && geometry.type) { + const siteFromDatabase = sitesFA.find((site) => site.id === siteIdFromPP); + + const radius = 0; + + // If the site does not exist in the database, create a new site + if (!siteFromDatabase) { + createPromises.push( + prisma.site.create({ + data: { + id: siteIdFromPP, + type: geometry.type, + geometry: geometry, + radius: radius, + userId: userId, + projectId: projectId, + lastUpdated: new Date(), + }, + }) + ); + // If the site exists in the database but has been updated in the PP API, update the site in the database + } else if (siteFromDatabase.lastUpdated?.getTime() !== siteLastUpdatedFromPP.getTime()) { + updatePromises.push( + prisma.site.update({ + where: { + id: siteIdFromPP, + }, + data: { + type: geometry.type, + geometry: geometry, + radius: radius, + lastUpdated: siteLastUpdatedFromPP, + }, + }) + ); + } + } else { + // Handle the case where geometry or type is null + logger(`Skipping site with id ${siteIdFromPP} due to null geometry or type.`, 'info',); } - } else { - // Handle the case where geometry or type is null - logger(`Skipping site with id ${siteIdFromPP} due to null geometry or type.`, 'info',); } } } @@ -287,9 +292,7 @@ export default async function syncROUsers(req: NextApiRequest, res: NextApiRespo const updateCount = updateResults.length; // Number of updated items const deleteCount = deleteResults.length; // Number of deleted items - logger(`Created ${createCount} items.`, 'info',); - logger(`Updated ${updateCount} items.`, 'info',); - logger(`Deleted ${deleteCount} items.`, 'info',); + logger(`Created ${createCount} items. Updated ${updateCount} items. Deleted ${deleteCount} items.`, 'info',); res.status(200).json({ message: "Success! Data has been synced for RO Users!", diff --git a/apps/server/src/server/api/root.ts b/apps/server/src/server/api/root.ts index 1c53d389a..dfad74a71 100644 --- a/apps/server/src/server/api/root.ts +++ b/apps/server/src/server/api/root.ts @@ -3,7 +3,6 @@ import { siteRouter } from "../../server/api/routers/site"; import { alertMethodRouter } from "../../server/api/routers/alertMethod"; import { alertRouter } from "../../server/api/routers/alert"; import { userRouter } from "./routers/user"; -import { cronRouter } from "./routers/cron"; import { projectRouter } from "./routers/project"; import { geoEventProviderRouter } from "./routers/geoEventProvider"; @@ -17,7 +16,6 @@ export const appRouter = createTRPCRouter({ alertMethod: alertMethodRouter, alert: alertRouter, user: userRouter, - cron: cronRouter, project: projectRouter, geoEventProvider: geoEventProviderRouter, }); diff --git a/yarn.lock b/yarn.lock index 9c7f030d9..38d5816e4 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1813,22 +1813,22 @@ dependencies: "@types/geojson" "^7946.0.10" -"@prisma/client@^4.15.0": - version "4.16.1" - resolved "https://registry.yarnpkg.com/@prisma/client/-/client-4.16.1.tgz#030bf59ee51f223bae2a8e7c49827528756cf03a" - integrity sha512-CoDHu7Bt+NuDo40ijoeHP79EHtECsPBTy3yte5Yo3op8TqXt/kV0OT5OrsWewKvQGKFMHhYQ+ePed3zzjYdGAw== +"@prisma/client@^5.0.0": + version "5.0.0" + resolved "https://registry.yarnpkg.com/@prisma/client/-/client-5.0.0.tgz#9f0cd4164f4ffddb28bb1811c27eb7fa1e01a087" + integrity sha512-XlO5ELNAQ7rV4cXIDJUNBEgdLwX3pjtt9Q/RHqDpGf43szpNJx2hJnggfFs7TKNx0cOFsl6KJCSfqr5duEU/bQ== dependencies: - "@prisma/engines-version" "4.16.0-66.b20ead4d3ab9e78ac112966e242ded703f4a052c" + "@prisma/engines-version" "4.17.0-26.6b0aef69b7cdfc787f822ecd7cdc76d5f1991584" -"@prisma/engines-version@4.16.0-66.b20ead4d3ab9e78ac112966e242ded703f4a052c": - version "4.16.0-66.b20ead4d3ab9e78ac112966e242ded703f4a052c" - resolved "https://registry.yarnpkg.com/@prisma/engines-version/-/engines-version-4.16.0-66.b20ead4d3ab9e78ac112966e242ded703f4a052c.tgz#54fd17f9a9080e13e2f75613fd35afb7875e3715" - integrity sha512-tMWAF/qF00fbUH1HB4Yjmz6bjh7fzkb7Y3NRoUfMlHu6V+O45MGvqwYxqwBjn1BIUXkl3r04W351D4qdJjrgvA== +"@prisma/engines-version@4.17.0-26.6b0aef69b7cdfc787f822ecd7cdc76d5f1991584": + version "4.17.0-26.6b0aef69b7cdfc787f822ecd7cdc76d5f1991584" + resolved "https://registry.yarnpkg.com/@prisma/engines-version/-/engines-version-4.17.0-26.6b0aef69b7cdfc787f822ecd7cdc76d5f1991584.tgz#b36eda5620872d3fac810c302a7e46cf41daa033" + integrity sha512-HHiUF6NixsldsP3JROq07TYBLEjXFKr6PdH8H4gK/XAoTmIplOJBCgrIUMrsRAnEuGyRoRLXKXWUb943+PFoKQ== -"@prisma/engines@4.16.1": - version "4.16.1" - resolved "https://registry.yarnpkg.com/@prisma/engines/-/engines-4.16.1.tgz#ee487620dc5135fd175ac7494b1c60c9f12c1e4b" - integrity sha512-gpZG0kGGxfemgvK/LghHdBIz+crHkZjzszja94xp4oytpsXrgt/Ice82MvPsWMleVIniKuARrowtsIsim0PFJQ== +"@prisma/engines@5.0.0": + version "5.0.0" + resolved "https://registry.yarnpkg.com/@prisma/engines/-/engines-5.0.0.tgz#5249650eabe77c458c90f2be97d8210353c2e22e" + integrity sha512-kyT/8fd0OpWmhAU5YnY7eP31brW1q1YrTGoblWrhQJDiN/1K+Z8S1kylcmtjqx5wsUGcP1HBWutayA/jtyt+sg== "@react-native-async-storage/async-storage@^1.18.0": version "1.18.2" @@ -8726,12 +8726,12 @@ pretty-format@^29.0.0, pretty-format@^29.5.0: ansi-styles "^5.0.0" react-is "^18.0.0" -prisma@^4.15.0: - version "4.16.1" - resolved "https://registry.yarnpkg.com/prisma/-/prisma-4.16.1.tgz#c6d723a4326138a72489098a6c39a698a670fbbf" - integrity sha512-C2Xm7yxHxjFjjscBEW4tmoraPHH/Vyu/A0XABdbaFtoiOZARsxvOM7rwc2iZ0qVxbh0bGBGBWZUSXO/52/nHBQ== +prisma@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/prisma/-/prisma-5.0.0.tgz#f6571c46dc2478172cb7bc1bb62d74026a2c2630" + integrity sha512-KYWk83Fhi1FH59jSpavAYTt2eoMVW9YKgu8ci0kuUnt6Dup5Qy47pcB4/TLmiPAbhGrxxSz7gsSnJcCmkyPANA== dependencies: - "@prisma/engines" "4.16.1" + "@prisma/engines" "5.0.0" process-nextick-args@~2.0.0: version "2.0.1" From fa139602224b34c7b43e8c07878a54bfc430f09e Mon Sep 17 00:00:00 2001 From: Aashish Dhakal <85501584+dhakalaashish@users.noreply.github.com> Date: Mon, 17 Jul 2023 16:59:30 +0545 Subject: [PATCH 3/3] Finalize cron sync-ro-users --- .../src/pages/api/cron/sync-ro-users.ts | 310 ++++++++++-------- 1 file changed, 176 insertions(+), 134 deletions(-) diff --git a/apps/server/src/pages/api/cron/sync-ro-users.ts b/apps/server/src/pages/api/cron/sync-ro-users.ts index 31df807d9..63658f090 100644 --- a/apps/server/src/pages/api/cron/sync-ro-users.ts +++ b/apps/server/src/pages/api/cron/sync-ro-users.ts @@ -2,6 +2,7 @@ // Sync RO Users CRON job // This cron job runs every day and syncs sites, projects, and profile data for RO users. + import { type NextApiRequest, type NextApiResponse } from "next"; import { prisma } from '../../../server/db' import { env } from "../../../env.mjs"; @@ -21,10 +22,14 @@ export default async function syncROUsers(req: NextApiRequest, res: NextApiRespo } } + let createCount = 0; + let updateCount = 0; + let deleteCount = 0; + // Fetch projects from PP API const allProjectsPPWebApp: TreeProjectExtended[] = await fetchAllProjectsWithSites(); - // Fetch RO Users from the database and their respective remoteIds + // Fetch RO Users from the Firealert database and select their ids and remoteIds const ROUsers = await prisma.user.findMany({ where: { isPlanetRO: true, @@ -35,20 +40,19 @@ export default async function syncROUsers(req: NextApiRequest, res: NextApiRespo }, }); - const userRemoteIdList = ROUsers.map(user => user.remoteId); - // Create a map to associate remoteId with userId - const mapRemoteIdWithUserId = new Map(); + const map_userRemoteId_to_userId = new Map(); ROUsers.forEach(user => { - mapRemoteIdWithUserId.set(user.remoteId, user.id); + map_userRemoteId_to_userId.set(user.remoteId, user.id); }); // Filter projects from PP API to include only those related to RO users - const projectsPP = allProjectsPPWebApp.filter(projectPP => + const userRemoteIdList = ROUsers.map(user => user.remoteId); + const projectsPP1 = allProjectsPPWebApp.filter(projectPP => userRemoteIdList.includes(projectPP.tpo.id) ); - // Fetch corresponding projects for the RO users from the database + // Fetch all projects for the RO users from the Firealert database const userIdList = ROUsers.map(user => user.id); const projectsFA = await prisma.project.findMany({ where: { @@ -58,51 +62,47 @@ export default async function syncROUsers(req: NextApiRequest, res: NextApiRespo } }); - // Identify projects in the database that are not present in PP API + // Filter projects from the pp web api, depending on the project.id const projectsIdsFA = projectsFA.map((project) => project.id); - const deleteFAProjectIds = projectsIdsFA.filter( - (projectId) => !projectsPP.some((project) => project.id === projectId) + const projectsPP2 = allProjectsPPWebApp.filter(projectPP => + projectsIdsFA.includes(projectPP.id) ); - // Dissociate sites associated with these projects and delete the projects - if (deleteFAProjectIds.length) { - await prisma.$transaction(async (prisma) => { - await prisma.site.updateMany({ - where: { - projectId: { - in: deleteFAProjectIds, - }, - }, - data: { - projectId: null, - }, - }); - logger(`Deleted projects with ids: ${deleteFAProjectIds.join(", ")}`, 'info',); - await prisma.project.deleteMany({ - where: { - id: { - in: deleteFAProjectIds, - }, - }, - }); - }); - } - - // Identify new projects from PP API that are not present in the database - const newProjectsPP = projectsPP.filter( + // Combine projectsPP1 and projectsPP2 without duplicates + const projectsPP = projectsPP1.concat(projectsPP2.filter((projectPP2) => + !projectsPP1.some((projectPP1) => projectPP1.id === projectPP2.id) + )); + // Identify projects that are in PP Webapp that are not present in the FA database + const projectsInPP_not_in_FA = projectsPP.filter( (projectPP) => !projectsFA.some((projectFA) => projectFA.id === projectPP.id) ); + const sitesThatAreOrWereOnceRemote = await prisma.site.findMany({ + where: { + userId: { + in: userIdList, + }, + }, + }); + + const ids_sitesThatAreOrWereOnceRemote = sitesThatAreOrWereOnceRemote.map(site => site.id) + const map_ids_sitesThatAreOrWereOnceRemote_to_remoteId = new Map(); + sitesThatAreOrWereOnceRemote.forEach(site => { + map_ids_sitesThatAreOrWereOnceRemote_to_remoteId.set(site.remoteId, site.id); + }); + // Add those projects to the database, and all the sites inside of it. - if (newProjectsPP.length > 0) { + if (projectsInPP_not_in_FA.length > 0) { // Prepare the projects and sites data for bulk creation const newProjectData: Project[] = []; const newSiteData: Prisma.SiteCreateManyInput[] = []; + // Prepare an array of promises for site updates + const updateSitePromises = []; - for (const projectPP of newProjectsPP) { - const { id: projectId, name: projectName, slug: projectSlug, lastUpdated: projectLastUpdated, sites: sitesFromPP } = projectPP; + for (const projectPP of projectsInPP_not_in_FA) { + const { id: projectId, name: projectName, slug: projectSlug, sites: sitesFromPP } = projectPP; const tpoId = projectPP.tpo.id; - const userId = mapRemoteIdWithUserId.get(tpoId); + const userId = map_userRemoteId_to_userId.get(tpoId); // Add the new project to the array for bulk creation newProjectData.push({ @@ -118,40 +118,62 @@ export default async function syncROUsers(req: NextApiRequest, res: NextApiRespo if (sitesFromPP && sitesFromPP.length > 0) { for (const siteFromPP of sitesFromPP) { const { geometry, properties } = siteFromPP; - const { id: siteIdFromPP, lastUpdated: siteLastUpdatedFromPP } = properties; + const { id: remoteId_PP } = properties; + const siteId_mapped_from_remoteId = map_ids_sitesThatAreOrWereOnceRemote_to_remoteId.get(remoteId_PP) + // Check if geometry and geometry.type exists if (geometry && geometry.type) { - // Add the new site to the array for bulk creation - newSiteData.push({ - id: siteIdFromPP, - type: geometry.type, - geometry: geometry, - radius: 0, - projectId: projectId, - lastUpdated: new Date(), - userId: userId, - }); + // If site already existed before and was soft deleted from webapp, link that site with its corresponding project + if (ids_sitesThatAreOrWereOnceRemote.includes(siteId_mapped_from_remoteId)) { + // Prepare the site for bulk update, during update make the projectId null, and deletedAt as null. + updateSitePromises.push( + prisma.site.update({ + where: { + id: siteId_mapped_from_remoteId + }, + data: { + projectId: projectId, + deletedAt: null + } + }) + ); + } else { + // Add the new site to the array for bulk creation + newSiteData.push({ + remoteId: remoteId_PP, + type: geometry.type, + geometry: geometry, + radius: 0, + projectId: projectId, + lastUpdated: new Date(), + userId: userId, + }); + } } else { // Handle the case where geometry or type is null - logger(`Skipping site with id ${siteIdFromPP} due to null geometry or type.`, 'info',); + logger(`Skipping site with id ${remoteId_PP} due to null geometry or type.`, 'info',); } } } } // Add the new projects and sites to the database in a transaction - await prisma.$transaction(async (prisma) => { - const createdProjects = await prisma.project.createMany({ - data: newProjectData, - }); - const createdSites = await prisma.site.createMany({ - data: newSiteData, - }); - }) + const createdProjects = await prisma.project.createMany({ + data: newProjectData, + }); + const createdSites = await prisma.site.createMany({ + data: newSiteData, + }); + // Await all update promises + const updateResults = await Promise.all(updateSitePromises); + + createCount = createCount + createdProjects.count + createdSites.count + updateCount = updateCount + updateResults.length } // Fetch all sites from the database for the projects in projectsPP + // Only sites of public projects should be fetched. const sitesFA = await prisma.site.findMany({ where: { projectId: { @@ -160,47 +182,60 @@ export default async function syncROUsers(req: NextApiRequest, res: NextApiRespo }, }); + const mapSiteRemoteId_to_SiteId = new Map(); + sitesFA.forEach(site => { + mapSiteRemoteId_to_SiteId.set(site.remoteId, site.id); + }); + + + // Refetch projects from database to also include projects that were just created + // Only public projects should be fetched. const newProjectsFA = await prisma.project.findMany({ where: { - userId: { - in: userIdList + id: { + in: projectsPP.map((project) => project.id), } } }); + const remoteIdsList_SiteFA = sitesFA.map(siteFA => siteFA.remoteId) as string[] + // Create a list of site IDs from PP - const ppSiteIdList: string[] = []; + const remoteIdsList_PP: string[] = []; for (const projectPP of projectsPP) { if (projectPP.sites && projectPP.sites.length > 0) { // Checking if sites is not null before accessing it for (const siteFromPP of projectPP.sites) { - ppSiteIdList.push(siteFromPP.properties.id); + remoteIdsList_PP.push(siteFromPP.properties.id); } } } + // Find the remoteIds, which is present in remoteIdsList_PP but not in remoteIdsList_SiteFA + const remoteIdsInFA_NotInPP = remoteIdsList_SiteFA.filter(remoteId => !remoteIdsList_PP.includes(remoteId)); + + const siteIdsInFA_NotInPP = sitesFA.filter(site => remoteIdsInFA_NotInPP.includes(site.remoteId as string)).map(site => site.id); + // Perform bulk creations, bulk updates, and bulk deletions for sites - await prisma.$transaction(async (prisma) => { + try { const createPromises = []; const updatePromises = []; const deletePromises = []; - // Identify sites in the database that are not present in PP API and soft delete them - for (const siteFA of sitesFA) { - if (!ppSiteIdList.includes(siteFA.id)) { - deletePromises.push( - prisma.site.update({ - where: { - id: siteFA.id, - }, - data: { - projectId: null, - deletedAt: new Date(), + // Identify sites in the database that are not present in PP API and dissociate those sites from the projects + if (siteIdsInFA_NotInPP.length > 0) { + deletePromises.push( + prisma.site.updateMany({ + where: { + id: { + in: siteIdsInFA_NotInPP, }, - }) - ); - } + }, + data: { + projectId: null, + }, + })) + logger(`Soft Deleting ${siteIdsInFA_NotInPP.length} sites not present in the Webapp`, 'info',); } - logger(`Deleting ${deletePromises.length} sites not present in the PP API`, 'info',); // For each project in PP API, identify sites in the database that need to be updated or created for (const projectPP of projectsPP) { @@ -228,56 +263,63 @@ export default async function syncROUsers(req: NextApiRequest, res: NextApiRespo }, }) ); + } - const tpoId = projectPP.tpo.id; - const userId = mapRemoteIdWithUserId.get(tpoId); - - if (sitesFromPPProject && sitesFromPPProject.length > 0) { - for (const siteFromPP of sitesFromPPProject) { - const { geometry, properties } = siteFromPP; - const { id: siteIdFromPP, lastUpdated: siteLastUpdated } = properties; - const siteLastUpdatedFromPP = moment(siteLastUpdated.date, siteLastUpdated.timezone).utc().toDate(); - - if (geometry && geometry.type) { - const siteFromDatabase = sitesFA.find((site) => site.id === siteIdFromPP); - - const radius = 0; - - // If the site does not exist in the database, create a new site - if (!siteFromDatabase) { - createPromises.push( - prisma.site.create({ - data: { - id: siteIdFromPP, - type: geometry.type, - geometry: geometry, - radius: radius, - userId: userId, - projectId: projectId, - lastUpdated: new Date(), - }, - }) - ); - // If the site exists in the database but has been updated in the PP API, update the site in the database - } else if (siteFromDatabase.lastUpdated?.getTime() !== siteLastUpdatedFromPP.getTime()) { - updatePromises.push( - prisma.site.update({ - where: { - id: siteIdFromPP, - }, - data: { - type: geometry.type, - geometry: geometry, - radius: radius, - lastUpdated: siteLastUpdatedFromPP, - }, - }) - ); - } - } else { - // Handle the case where geometry or type is null - logger(`Skipping site with id ${siteIdFromPP} due to null geometry or type.`, 'info',); + const tpoId = projectPP.tpo.id; + const userId = map_userRemoteId_to_userId.get(tpoId); + + // If the project has sites + if (sitesFromPPProject && sitesFromPPProject.length > 0) { + for (const siteFromPP of sitesFromPPProject) { + const { geometry, properties } = siteFromPP; + const { id: remoteId_fromPP, lastUpdated: siteLastUpdated, name: siteName } = properties; + const siteLastUpdatedFromPP = moment(siteLastUpdated.date.split('.')[0], 'YYYY-MM-DD HH:mm:ss').utc().toDate(); + + // Check if the site is valid + if (geometry && geometry.type) { + const siteId_in_FADatabase = map_ids_sitesThatAreOrWereOnceRemote_to_remoteId.get(remoteId_fromPP) + const radius = 0; + let siteFromDatabase; + // Check if the site is already in database + if (siteId_in_FADatabase) { + siteFromDatabase = sitesFA.find((site) => site.id === siteId_in_FADatabase); } + // If the site does not exist in the database, create a new site + if (!siteFromDatabase) { + createPromises.push( + prisma.site.create({ + data: { + remoteId: remoteId_fromPP, + name: siteName, + type: geometry.type, + geometry: geometry, + radius: radius, + userId: userId, + projectId: projectId, + lastUpdated: new Date(), + }, + }) + ); + // else if the site exists in the FA database, and has been updated in the webapp, update the site in the database + } else if (siteFromDatabase.lastUpdated?.getTime() !== siteLastUpdatedFromPP.getTime()) { + updatePromises.push( + prisma.site.update({ + where: { + id: siteFromDatabase.id, + }, + data: { + type: geometry.type, + geometry: geometry, + radius: radius, + name: siteName, + lastUpdated: siteLastUpdatedFromPP, + }, + }) + ); + } + } else { + // Handle the case where geometry or type is null + logger(`Skipping site with id ${remoteId_fromPP} due to null geometry or type.`, 'info',); } } } @@ -288,9 +330,9 @@ export default async function syncROUsers(req: NextApiRequest, res: NextApiRespo const updateResults = await Promise.all(updatePromises); const deleteResults = await Promise.all(deletePromises); - const createCount = createResults.length; // Number of created items - const updateCount = updateResults.length; // Number of updated items - const deleteCount = deleteResults.length; // Number of deleted items + createCount = createCount + createResults.length; // Number of created items + updateCount = updateCount + updateResults.length; // Number of updated items + deleteCount = deleteCount + deleteResults.length; // Number of deleted items logger(`Created ${createCount} items. Updated ${updateCount} items. Deleted ${deleteCount} items.`, 'info',); @@ -299,11 +341,11 @@ export default async function syncROUsers(req: NextApiRequest, res: NextApiRespo status: 200, results: { created: createCount, updated: updateCount, deleted: deleteCount }, }); - }).catch(error => { + } catch (error) { logger(`Error in transaction: ${error}`, "error"); res.status(500).json({ message: "An error occurred while syncing data for RO Users.", status: 500 }); - }); -} + } +} \ No newline at end of file